def start_processing(self): data_path = '/Developments/NCLUni/pupil_crowd4Jul16/recordings/2016_07_29/003Original/CrowdOldMethod'#self.rec_path # Set user_dir to data_path so all related plugins save to the same folder as the recordings self.g_pool.user_dir = data_path # Manage plugins plugin_by_index = [Recorder]+calibration_plugins+gaze_mapping_plugins name_by_index = [p.__name__ for p in plugin_by_index] plugin_by_name = dict(zip(name_by_index,plugin_by_index)) self.g_pool.user_settings_path = os.path.join(data_path[:data_path.index('recordings')], 'capture_settings') ''' Step 1: when possible detect all pupil positions ''' # pupil_list = self.get_pupil_list(crowd_all=True) pupil_list = self.get_pupil_list_from_csv(data_path) # pupil_list = [] if pupil_list: # create events variable that should sent to plugins events = {'pupil_positions':pupil_list,'gaze_positions':[]} # get world settings user_settings_world_path = os.path.join(self.g_pool.user_settings_path,'user_settings_world') user_settings_world = Persistent_Dict(user_settings_world_path) default_plugins = [('Recorder',{})] simple_gaze_mapper = [('Simple_Gaze_Mapper',{})] manual_calibration_plugin = [('Manual_Marker_Calibration',{})] self.g_pool.plugins = Plugin_List(self.g_pool,plugin_by_name,user_settings_world.get('loaded_plugins',default_plugins)+manual_calibration_plugin) # self.g_pool.plugins.add(simple_gaze_mapper) self.g_pool.pupil_confidence_threshold = user_settings_world.get('pupil_confidence_threshold',.6) self.g_pool.detection_mapping_mode = user_settings_world.get('detection_mapping_mode','2d') ''' Step 2: before calculating gaze positions we shall process calibration data For calibration we need pupil_list (in events variable) and ref_list - ref_list contains all frames of detected marker Using manual_marker_calibration plugin use plugin.update to pass pupil_list and world frames for marker detection However, pupil_list is by this point fully detected. Thus, we shall do the following: First iteration: send events with all pupil_list with first world frame to manual_marker_calibration plugin.update Following iterations: send empty [] pupil_list with next world frame to manual_marker_calibration plugin.update ''' # self.calibrate(events, data_path, user_settings_world, crowd_all=True) self.calibrate_from_csv(pupil_list, data_path) # self.calibrate_from_user_calibration_data_file() ''' Step 3: calculate gaze positions passe events to gaze mapper plugin without the world frame ''' for p in self.g_pool.plugins: if 'Simple_Gaze_Mapper' in p.class_name: p.update(None,events) break save_object(events,os.path.join(data_path, "pupil_data")) # timestamps_path = os.path.join(data_path, "world_timestamps.npy") # timestamps = np.load(timestamps_path) # pupil_positions_by_frame = None # pupil_positions_by_frame = correlate_data(pupil_list,timestamps) else: logger.warning("No eye data found") # Again, remove all plugins except recorder for p in self.g_pool.plugins: if p.class_name != 'Recorder': p.alive = False
def __init__(self,g_pool,gui_settings={'pos':(220,200),'size':(300,300),'iconified':False}): super(Offline_Marker_Detector, self).__init__() self.g_pool = g_pool self.gui_settings = gui_settings self.order = .2 # all markers that are detected in the most recent frame self.markers = [] # all registered surfaces if g_pool.app == 'capture': raise Exception('For Player only.') #in player we load from the rec_dir: but we have a couple options: self.surface_definitions = Persistent_Dict(os.path.join(g_pool.rec_dir,'surface_definitions')) if self.load('offline_square_marker_surfaces',[]) != []: logger.debug("Found ref surfaces defined or copied in previous session.") self.surfaces = [Offline_Reference_Surface(self.g_pool,saved_definition=d,gaze_positions_by_frame=self.g_pool.positions_by_frame) for d in self.load('offline_square_marker_surfaces',[]) if isinstance(d,dict)] elif self.load('realtime_square_marker_surfaces',[]) != []: logger.debug("Did not find ref surfaces def created or used by the user in player from earlier session. Loading surfaces defined during capture.") self.surfaces = [Offline_Reference_Surface(self.g_pool,saved_definition=d,gaze_positions_by_frame=self.g_pool.positions_by_frame) for d in self.load('realtime_square_marker_surfaces',[]) if isinstance(d,dict)] else: logger.debug("No surface defs found. Please define using GUI.") self.surfaces = [] # ui mode settings self.mode = c_int(0) # edit surfaces self.edit_surfaces = [] #detector vars self.robust_detection = c_bool(1) self.aperture = c_int(11) self.min_marker_perimeter = 80 #check if marker cache is available from last session self.persistent_cache = Persistent_Dict(os.path.join(g_pool.rec_dir,'square_marker_cache')) self.cache = Cache_List(self.persistent_cache.get('marker_cache',[False for _ in g_pool.timestamps])) logger.debug("Loaded marker cache %s / %s frames had been searched before"%(len(self.cache)-self.cache.count(False),len(self.cache)) ) self.init_marker_cacher() #debug vars self.show_surface_idx = c_int(0) self.recent_pupil_positions = [] self.img_shape = None self.img = None
def __init__(self,g_pool,mode="Show Markers and Surfaces",min_marker_perimeter = 100,invert_image=False,robust_detection=True): super(Offline_Surface_Tracker, self).__init__(g_pool,mode,min_marker_perimeter,robust_detection) self.order = .2 if g_pool.app == 'capture': raise Exception('For Player only.') self.marker_cache_version = 2 self.min_marker_perimeter_cacher = 20 #find even super small markers. The surface locater will filter using min_marker_perimeter #check if marker cache is available from last session self.persistent_cache = Persistent_Dict(os.path.join(g_pool.rec_dir,'square_marker_cache')) version = self.persistent_cache.get('version',0) cache = self.persistent_cache.get('marker_cache',None) if cache is None: self.cache = Cache_List([False for _ in g_pool.timestamps]) self.persistent_cache['version'] = self.marker_cache_version elif version != self.marker_cache_version: self.persistent_cache['version'] = self.marker_cache_version self.cache = Cache_List([False for _ in g_pool.timestamps]) logger.debug("Marker cache version missmatch. Rebuilding marker cache.") else: self.cache = Cache_List(cache) logger.debug("Loaded marker cache {} / {} frames had been searched before".format(len(self.cache)-self.cache.count(False),len(self.cache)) ) self.init_marker_cacher() for s in self.surfaces: s.init_cache(self.cache,self.camera_calibration,self.min_marker_perimeter,self.min_id_confidence) self.recalculate()
def __init__(self,g_pool,mode="Show Markers and Frames"): super(Offline_Marker_Detector, self).__init__(g_pool) self.order = .2 # all markers that are detected in the most recent frame self.markers = [] # all registered surfaces if g_pool.app == 'capture': raise Exception('For Player only.') #in player we load from the rec_dir: but we have a couple options: self.surface_definitions = None self.surfaces = None self.load_surface_definitions_from_file() # ui mode settings self.mode = mode self.min_marker_perimeter = 20 #if we make this a slider we need to invalidate the cache on change. # edit surfaces self.edit_surfaces = [] #check if marker cache is available from last session self.persistent_cache = Persistent_Dict(os.path.join(g_pool.rec_dir,'square_marker_cache')) self.cache = Cache_List(self.persistent_cache.get('marker_cache',[False for _ in g_pool.timestamps])) logger.debug("Loaded marker cache %s / %s frames had been searched before"%(len(self.cache)-self.cache.count(False),len(self.cache)) ) self.init_marker_cacher() #debug vars self.show_surface_idx = c_int(0) self.img_shape = None self.img = None
def __init__(self,g_pool,atb_pos=(320,220)): super(Marker_Detector, self).__init__() self.g_pool = g_pool self.order = .2 # all markers that are detected in the most recent frame self.markers = [] # all registered surfaces self.surface_definitions = Persistent_Dict(os.path.join(g_pool.user_dir,'surface_definitions') ) self.surfaces = [Reference_Surface(saved_definition=d) for d in self.load('realtime_square_marker_surfaces',[]) if isinstance(d,dict)] # edit surfaces self.surface_edit_mode = c_bool(0) self.edit_surfaces = [] #detector vars self.robust_detection = c_bool(1) self.aperture = c_int(11) self.min_marker_perimeter = 80 #debug vars self.draw_markers = c_bool(0) self.show_surface_idx = c_int(0) self.recent_pupil_positions = [] self.img_shape = None atb_label = "marker detection" self._bar = atb.Bar(name =self.__class__.__name__, label=atb_label, help="marker detection parameters", color=(50, 150, 50), alpha=100, text='light', position=atb_pos,refresh=.3, size=(300, 300)) self.update_bar_markers()
def load_surface_definitions_from_file(self): # all registered surfaces self.surface_definitions = Persistent_Dict(os.path.join(self.g_pool.user_dir, "surface_definitions")) self.surfaces = [ Reference_Surface(saved_definition=d) for d in self.surface_definitions.get("realtime_square_marker_surfaces", []) if isinstance(d, dict) ]
def __init__(self, g_pool, mode="Show markers and frames", min_marker_perimeter=40): super(Marker_Detector, self).__init__(g_pool) self.order = 0.2 # all markers that are detected in the most recent frame self.markers = [] # load camera intrinsics try: camera_calibration = load_object(os.path.join(self.g_pool.user_dir, "camera_calibration")) except: self.camera_intrinsics = None else: same_name = camera_calibration["camera_name"] == self.g_pool.capture.name same_resolution = camera_calibration["resolution"] == self.g_pool.capture.frame_size if same_name and same_resolution: logger.info("Loaded camera calibration. 3D marker tracking enabled.") K = camera_calibration["camera_matrix"] dist_coefs = camera_calibration["dist_coefs"] resolution = camera_calibration["resolution"] self.camera_intrinsics = K, dist_coefs, resolution else: logger.info( "Loaded camera calibration but camera name and/or resolution has changed. Please re-calibrate." ) self.camera_intrinsics = None # all registered surfaces self.surface_definitions = Persistent_Dict(os.path.join(g_pool.user_dir, "surface_definitions")) self.surfaces = [ Reference_Surface(saved_definition=d) for d in self.surface_definitions.get("realtime_square_marker_surfaces", []) if isinstance(d, dict) ] # edit surfaces self.edit_surfaces = [] # plugin state self.mode = mode self.running = True self.robust_detection = 1 self.aperture = 11 self.min_marker_perimeter = min_marker_perimeter self.locate_3d = False # debug vars self.draw_markers = 0 self.show_surface_idx = 0 self.img_shape = None self.menu = None self.button = None self.add_button = None
def __init__(self,g_pool,mode="Show Markers and Frames"): super(Offline_Marker_Detector, self).__init__(g_pool) self.order = .2 # all markers that are detected in the most recent frame self.markers = [] # all registered surfaces if g_pool.app == 'capture': raise Exception('For Player only.') #in player we load from the rec_dir: but we have a couple options: self.surface_definitions = Persistent_Dict(os.path.join(g_pool.rec_dir,'surface_definitions')) if self.surface_definitions.get('offline_square_marker_surfaces',[]) != []: logger.debug("Found ref surfaces defined or copied in previous session.") self.surfaces = [Offline_Reference_Surface(self.g_pool,saved_definition=d) for d in self.surface_definitions.get('offline_square_marker_surfaces',[]) if isinstance(d,dict)] elif self.surface_definitions.get('realtime_square_marker_surfaces',[]) != []: logger.debug("Did not find ref surfaces def created or used by the user in player from earlier session. Loading surfaces defined during capture.") self.surfaces = [Offline_Reference_Surface(self.g_pool,saved_definition=d) for d in self.surface_definitions.get('realtime_square_marker_surfaces',[]) if isinstance(d,dict)] else: logger.debug("No surface defs found. Please define using GUI.") self.surfaces = [] # ui mode settings self.mode = mode self.min_marker_perimeter = 20 #if we make this a slider we need to invalidate the cache on change. # edit surfaces self.edit_surfaces = [] #check if marker cache is available from last session self.persistent_cache = Persistent_Dict(os.path.join(g_pool.rec_dir,'square_marker_cache')) self.cache = Cache_List(self.persistent_cache.get('marker_cache',[False for _ in g_pool.timestamps])) logger.debug("Loaded marker cache %s / %s frames had been searched before"%(len(self.cache)-self.cache.count(False),len(self.cache)) ) self.init_marker_cacher() #debug vars self.show_surface_idx = c_int(0) self.img_shape = None self.img = None
def load_surface_definitions_from_file(self): self.surface_definitions = Persistent_Dict(os.path.join(self.g_pool.rec_dir,'surface_definitions')) if self.surface_definitions.get('offline_square_marker_surfaces',[]) != []: logger.debug("Found ref surfaces defined or copied in previous session.") self.surfaces = [Offline_Reference_Surface(self.g_pool,saved_definition=d) for d in self.surface_definitions.get('offline_square_marker_surfaces',[]) if isinstance(d,dict)] elif self.surface_definitions.get('realtime_square_marker_surfaces',[]) != []: logger.debug("Did not find ref surfaces def created or used by the user in player from earlier session. Loading surfaces defined during capture.") self.surfaces = [Offline_Reference_Surface(self.g_pool,saved_definition=d) for d in self.surface_definitions.get('realtime_square_marker_surfaces',[]) if isinstance(d,dict)] else: logger.debug("No surface defs found. Please define using GUI.") self.surfaces = []
def __init__(self, g_pool): super(Canny_Detector, self).__init__() self.g_pool = g_pool # load session persistent settings self.session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_detector') ) # coarse pupil filter params self.coarse_detection = self.session_settings.get('coarse_detection',True) self.coarse_filter_min = 100 self.coarse_filter_max = 400 # canny edge detection params self.blur = 1 self.canny_thresh = 159 self.canny_ratio= 2 self.canny_aperture = 5 # edge intensity filter params self.intensity_range = self.session_settings.get('intensity_range',17) self.bin_thresh = 0 # contour prefilter params self.min_contour_size = self.session_settings.get('min_contour_size',60) #ellipse filter params self.inital_ellipse_fit_threshhold = 1.8 self.min_ratio = .3 self.pupil_min = self.session_settings.get('pupil_min',40.) self.pupil_max = self.session_settings.get('pupil_max',150.) self.target_size= 100.0 self.strong_perimeter_ratio_range = .8, 1.1 self.strong_area_ratio_range = .6,1.1 self.final_perimeter_ratio_range = self.session_settings.get("final_perimeter_ratio_range",[.6, 1.2]) self.strong_prior = None #detector dignostics #confidance in the mesurement 0(bad) to 1 (perfect) # in this case we take the support ratio capped at 1. (uncapped if the pupil comes from prior) self.confidence = 0.0 self.confidence_hist = [] # GUI settings self.advanced_controls_menu = None #debug window self.suggested_size = 640,480 self._window = None self.window_should_open = False self.window_should_close = False #debug settings self.should_sleep = False
def __init__(self, g_pool, mode="Show Markers and Surfaces", min_marker_perimeter=100, invert_image=False, robust_detection=True): super(Offline_Surface_Tracker, self).__init__(g_pool, mode, min_marker_perimeter, robust_detection) self.order = .2 if g_pool.app == 'capture': raise Exception('For Player only.') self.marker_cache_version = 2 self.min_marker_perimeter_cacher = 20 #find even super small markers. The surface locater will filter using min_marker_perimeter #check if marker cache is available from last session self.persistent_cache = Persistent_Dict( os.path.join(g_pool.rec_dir, 'square_marker_cache')) version = self.persistent_cache.get('version', 0) cache = self.persistent_cache.get('marker_cache', None) if cache is None: self.cache = Cache_List([False for _ in g_pool.timestamps]) self.persistent_cache['version'] = self.marker_cache_version elif version != self.marker_cache_version: self.persistent_cache['version'] = self.marker_cache_version self.cache = Cache_List([False for _ in g_pool.timestamps]) logger.debug( "Marker cache version missmatch. Rebuilding marker cache.") else: self.cache = Cache_List(cache) logger.debug( "Loaded marker cache %s / %s frames had been searched before" % (len(self.cache) - self.cache.count(False), len(self.cache))) self.init_marker_cacher() for s in self.surfaces: s.init_cache(self.cache, self.camera_calibration, self.min_marker_perimeter, self.min_id_confidence) self.recalculate()
def __init__(self, g_pool, mode="Show Markers and Frames"): super(Offline_Marker_Detector, self).__init__(g_pool) self.order = .2 # all markers that are detected in the most recent frame self.markers = [] # all registered surfaces if g_pool.app == 'capture': raise Exception('For Player only.') #in player we load from the rec_dir: but we have a couple options: self.surface_definitions = None self.surfaces = None self.load_surface_definitions_from_file() # ui mode settings self.mode = mode self.min_marker_perimeter = 20 #if we make this a slider we need to invalidate the cache on change. # edit surfaces self.edit_surfaces = [] #check if marker cache is available from last session self.persistent_cache = Persistent_Dict( os.path.join(g_pool.rec_dir, 'square_marker_cache')) self.cache = Cache_List( self.persistent_cache.get('marker_cache', [False for _ in g_pool.timestamps])) logger.debug( "Loaded marker cache %s / %s frames had been searched before" % (len(self.cache) - self.cache.count(False), len(self.cache))) self.init_marker_cacher() #debug vars self.show_surface_idx = c_int(0) self.img_shape = None self.img = None
def __init__(self, g_pool,fullscreen=True,marker_scale=1.0,sample_duration=45): super().__init__(g_pool) self.detected = False self.screen_marker_state = 0. self.sample_duration = sample_duration # number of frames to sample per site self.fixation_boost = sample_duration/2. self.lead_in = 25 #frames of marker shown before starting to sample self.lead_out = 5 #frames of markers shown after sampling is donw self.black_duration = 15 self.session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_screen_calibration') ) self.active_site = None self.sites = [] self.display_pos = None self.on_position = False self.screen_markers = [[],[],[],[],[],[], []] self.encode_markers() self.markers = [] self.pos = None self.marker_scale = marker_scale self._window = None self.menu = None self.button = None self.fullscreen = fullscreen self.clicks_to_close = 5 self.glfont = fontstash.Context() self.glfont.add_font('opensans',get_opensans_font_path()) self.glfont.set_size(32) self.glfont.set_color_float((0.2,0.5,0.9,1.0)) self.glfont.set_align_string(v_align='center') # UI Platform tweaks if system() == 'Linux': self.window_position_default = (0, 0) elif system() == 'Windows': self.window_position_default = (8, 31) else: self.window_position_default = (0, 0)
def __init__(self,g_pool,mode="Show markers and frames"): super(Marker_Detector, self).__init__(g_pool) self.order = .2 # all markers that are detected in the most recent frame self.markers = [] #load camera intrinsics try: K = np.load(os.path.join(self.g_pool.user_dir,'camera_matrix.npy')) dist_coef = np.load(os.path.join(self.g_pool.user_dir,"dist_coefs.npy")) img_size = np.load(os.path.join(self.g_pool.user_dir,"camera_resolution.npy")) self.camera_intrinsics = K, dist_coefs, img_size except: self.camera_intrinsics = None # all registered surfaces self.surface_definitions = Persistent_Dict(os.path.join(g_pool.user_dir,'surface_definitions') ) self.surfaces = [Reference_Surface(saved_definition=d) for d in self.surface_definitions.get('realtime_square_marker_surfaces',[]) if isinstance(d,dict)] # edit surfaces self.edit_surfaces = [] #plugin state self.mode = mode self.running = True self.robust_detection = 1 self.aperture = 11 self.min_marker_perimeter = 80 self.locate_3d = False #debug vars self.draw_markers = 0 self.show_surface_idx = 0 self.img_shape = None self.menu= None self.button= None self.add_button = None self.screen = np.array([[0,0],[1280,0],[1280,720],[0,720]], dtype=np.float32)
def __init__(self, g_pool,fullscreen=True,marker_scale=1.0,sample_duration=40): super().__init__(g_pool,fullscreen,marker_scale) #result calculation variables: self.fov = 90. #taken from c930e specsheet, confirmed though mesurement within ~10deg. self.res = np.sqrt(self.g_pool.capture.frame_size[0]**2+self.g_pool.capture.frame_size[1]**2) self.outlier_thresh = 5. self.accuracy = 0 self.precision = 0 self.session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_screen_calibration') ) try: self.pt_cloud = np.load(os.path.join(self.g_pool.user_dir,'accuracy_test_pt_cloud.npy')) gaze,ref = self.pt_cloud[:,0:2],self.pt_cloud[:,2:4] error_lines = np.array([[g,r] for g,r in zip(gaze,ref)]) self.error_lines = error_lines.reshape(-1,2) except Exception: self.error_lines = None self.pt_cloud = None
def load_marker_cache(self): #check if marker cache is available from last session self.persistent_cache = Persistent_Dict(os.path.join(self.g_pool.rec_dir,'square_marker_cache')) version = self.persistent_cache.get('version',0) cache = self.persistent_cache.get('marker_cache',None) if cache is None: self.cache = Cache_List([False for _ in self.g_pool.timestamps]) self.persistent_cache['version'] = self.marker_cache_version self.persistent_cache['inverted_markers'] = self.invert_image elif version != self.marker_cache_version: self.persistent_cache['version'] = self.marker_cache_version self.invert_image = self.persistent_cache.get('inverted_markers',False) self.cache = Cache_List([False for _ in self.g_pool.timestamps]) logger.debug("Marker cache version missmatch. Rebuilding marker cache.") else: self.cache = Cache_List(cache) #we overwrite the inverted_image setting from init with the one save in the marker cache. self.invert_image = self.persistent_cache.get('inverted_markers',False) logger.debug("Loaded marker cache {} / {} frames had been searched before".format(len(self.cache)-self.cache.count(False),len(self.cache)) )
def __init__(self, g_pool, customSettings = None): super(Glint_Detector, self).__init__() self.g_pool = g_pool if not customSettings: self.session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_glint_detector') ) else: self.session_settings = customSettings self.glint_dist = self.session_settings.get('glint_dist', 3.0) self.glint_thres = self.session_settings.get('glint_thres', 5) self.glint_min = self.session_settings.get('glint_min',50) self.glint_max = self.session_settings.get('glint_max',750) self.dilate = self.session_settings.get('dilate',0) self.prev_glint = (0,0,0) self.prev_timestamp = 0 self.vel_sum = 0. self.vel_n = 0.001 #debug window self.suggested_size = 640,480 self._window = None self.window_should_open = False self.window_should_close = False
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_version, debug): # general imports from time import sleep import logging from glob import glob from time import time, strftime, localtime # networking import zmq import zmq_tools import numpy as np # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify", )) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.NOTSET) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) try: from background_helper import IPC_Logging_Task_Proxy IPC_Logging_Task_Proxy.push_url = ipc_push_url from tasklib.background.patches import IPCLoggingPatch IPCLoggingPatch.ipc_push_url = ipc_push_url # imports from file_methods import Persistent_Dict, next_export_sub_dir # display import glfw # check versions for our own depedencies as they are fast-changing from pyglui import __version__ as pyglui_version from pyglui import ui, cygl from pyglui.cygl.utils import Named_Texture, RGBA import gl_utils # capture from video_capture import File_Source # helpers/utils from version_utils import VersionFormat from methods import normalize, denormalize, delta_t, get_system_info import player_methods as pm from pupil_recording import PupilRecording from csv_utils import write_key_value_file # Plug-ins from plugin import Plugin, Plugin_List, import_runtime_plugins from plugin_manager import Plugin_Manager from vis_circle import Vis_Circle from vis_cross import Vis_Cross from vis_polyline import Vis_Polyline from vis_light_points import Vis_Light_Points from vis_watermark import Vis_Watermark from vis_fixation import Vis_Fixation from seek_control import Seek_Control from surface_tracker import Surface_Tracker_Offline # from marker_auto_trim_marks import Marker_Auto_Trim_Marks from fixation_detector import Offline_Fixation_Detector from log_display import Log_Display from annotations import Annotation_Player from raw_data_exporter import Raw_Data_Exporter from log_history import Log_History from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection from gaze_producer.gaze_from_recording import GazeFromRecording from gaze_producer.gaze_from_offline_calibration import ( GazeFromOfflineCalibration, ) from system_graphs import System_Graphs from system_timelines import System_Timelines from blink_detection import Offline_Blink_Detection from audio_playback import Audio_Playback from video_export.plugins.imotions_exporter import iMotions_Exporter from video_export.plugins.eye_video_exporter import Eye_Video_Exporter from video_export.plugins.world_video_exporter import World_Video_Exporter from head_pose_tracker.offline_head_pose_tracker import ( Offline_Head_Pose_Tracker, ) from video_capture import File_Source from video_overlay.plugins import Video_Overlay, Eye_Overlay from pupil_recording import ( assert_valid_recording_type, InvalidRecordingException, ) assert VersionFormat(pyglui_version) >= VersionFormat( "1.27"), "pyglui out of date, please upgrade to newest version" process_was_interrupted = False def interrupt_handler(sig, frame): import traceback trace = traceback.format_stack(f=frame) logger.debug(f"Caught signal {sig} in:\n" + "".join(trace)) nonlocal process_was_interrupted process_was_interrupted = True signal.signal(signal.SIGINT, interrupt_handler) runtime_plugins = import_runtime_plugins( os.path.join(user_dir, "plugins")) system_plugins = [ Log_Display, Seek_Control, Plugin_Manager, System_Graphs, System_Timelines, Audio_Playback, ] user_plugins = [ Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross, Vis_Watermark, Eye_Overlay, Video_Overlay, Offline_Fixation_Detector, Offline_Blink_Detection, Surface_Tracker_Offline, Raw_Data_Exporter, Annotation_Player, Log_History, Pupil_From_Recording, Offline_Pupil_Detection, GazeFromRecording, GazeFromOfflineCalibration, World_Video_Exporter, iMotions_Exporter, Eye_Video_Exporter, Offline_Head_Pose_Tracker, ] + runtime_plugins plugins = system_plugins + user_plugins # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal hdpi_factor if w == 0 or h == 0: return hdpi_factor = glfw.getHDPIFactor(window) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor window_size = w, h g_pool.camera_render_size = w - int( icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(*window_size) g_pool.gui.collect_menus() for p in g_pool.plugins: p.on_window_resize(window, *g_pool.camera_render_size) def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) pos = x, y pos = normalize(pos, g_pool.camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_pos(pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, count, paths): paths = [paths[x].decode("utf-8") for x in range(count)] for path in paths: try: assert_valid_recording_type(path) _restart_with_recording(path) return except InvalidRecordingException as err: logger.debug(str(err)) for plugin in g_pool.plugins: if plugin.on_drop(paths): break def _restart_with_recording(rec_dir): logger.debug("Starting new session with '{}'".format(rec_dir)) ipc_pub.notify({ "subject": "player_drop_process.should_start", "rec_dir": rec_dir }) glfw.glfwSetWindowShouldClose(g_pool.main_window, True) tick = delta_t() def get_dt(): return next(tick) recording = PupilRecording(rec_dir) meta_info = recording.meta_info # log info about Pupil Platform and Platform in player.log logger.info("Application Version: {}".format(app_version)) logger.info("System Info: {}".format(get_system_info())) logger.debug(f"Debug flag: {debug}") icon_bar_width = 50 window_size = None hdpi_factor = 1.0 # create container for globally scoped vars g_pool = SimpleNamespace() g_pool.app = "player" g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.plugin_by_name = {p.__name__: p for p in plugins} g_pool.camera_render_size = None video_path = recording.files().core().world().videos()[0].resolve() File_Source( g_pool, timing="external", source_path=video_path, buffered_decoding=True, fill_gaps=True, ) # load session persistent settings session_settings = Persistent_Dict( os.path.join(user_dir, "user_settings_player")) if VersionFormat(session_settings.get("version", "0.0")) != app_version: logger.info( "Session setting are a different version of this app. I will not use those." ) session_settings.clear() width, height = g_pool.capture.frame_size width += icon_bar_width width, height = session_settings.get("window_size", (width, height)) window_pos = session_settings.get("window_position", window_position_default) window_name = f"Pupil Player: {meta_info.recording_name} - {rec_dir}" glfw.glfwInit() main_window = glfw.glfwCreateWindow(width, height, window_name, None, None) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window def set_scale(new_scale): g_pool.gui_user_scale = new_scale window_size = ( g_pool.camera_render_size[0] + int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor), glfw.glfwGetFramebufferSize(main_window)[1], ) logger.warning(icon_bar_width * g_pool.gui_user_scale * hdpi_factor) glfw.glfwSetWindowSize(main_window, *window_size) g_pool.version = app_version g_pool.timestamps = g_pool.capture.timestamps g_pool.get_timestamp = lambda: 0.0 g_pool.user_dir = user_dir g_pool.rec_dir = rec_dir g_pool.meta_info = meta_info g_pool.min_data_confidence = session_settings.get( "min_data_confidence", MIN_DATA_CONFIDENCE_DEFAULT) g_pool.min_calibration_confidence = session_settings.get( "min_calibration_confidence", MIN_CALIBRATION_CONFIDENCE_DEFAULT) # populated by producers g_pool.pupil_positions = pm.PupilDataBisector() g_pool.gaze_positions = pm.Bisector() g_pool.fixations = pm.Affiliator() g_pool.eye_movements = pm.Affiliator() def set_data_confidence(new_confidence): g_pool.min_data_confidence = new_confidence notification = {"subject": "min_data_confidence_changed"} notification["_notify_time_"] = time() + 0.8 g_pool.ipc_pub.notify(notification) def do_export(_): left_idx = g_pool.seek_control.trim_left right_idx = g_pool.seek_control.trim_right export_range = left_idx, right_idx + 1 # exclusive range.stop export_ts_window = pm.exact_window(g_pool.timestamps, (left_idx, right_idx)) export_dir = os.path.join(g_pool.rec_dir, "exports") export_dir = next_export_sub_dir(export_dir) os.makedirs(export_dir) logger.info('Created export dir at "{}"'.format(export_dir)) export_info = { "Player Software Version": str(g_pool.version), "Data Format Version": meta_info.min_player_version, "Export Date": strftime("%d.%m.%Y", localtime()), "Export Time": strftime("%H:%M:%S", localtime()), "Frame Index Range:": g_pool.seek_control.get_frame_index_trim_range_string(), "Relative Time Range": g_pool.seek_control.get_rel_time_trim_range_string(), "Absolute Time Range": g_pool.seek_control.get_abs_time_trim_range_string(), } with open(os.path.join(export_dir, "export_info.csv"), "w") as csv: write_key_value_file(csv, export_info) notification = { "subject": "should_export", "range": export_range, "ts_window": export_ts_window, "export_dir": export_dir, } g_pool.ipc_pub.notify(notification) def reset_restart(): logger.warning("Resetting all settings and restarting Player.") glfw.glfwSetWindowShouldClose(main_window, True) ipc_pub.notify({"subject": "clear_settings_process.should_start"}) ipc_pub.notify({ "subject": "player_process.should_start", "rec_dir": rec_dir, "delay": 2.0, }) def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is open the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0) g_pool.menubar = ui.Scrolling_Menu("Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos="left") g_pool.iconbar = ui.Scrolling_Menu("Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden") g_pool.timelines = ui.Container((0, 0), (0, 0), (0, 0)) g_pool.timelines.horizontal_constraint = g_pool.menubar g_pool.user_timelines = ui.Timeline_Menu("User Timelines", pos=(0.0, -150.0), size=(0.0, 0.0), header_pos="headline") g_pool.user_timelines.color = RGBA(a=0.0) g_pool.user_timelines.collapsed = True # add container that constaints itself to the seekbar height vert_constr = ui.Container((0, 0), (0, -50.0), (0, 0)) vert_constr.append(g_pool.user_timelines) g_pool.timelines.append(vert_constr) def set_window_size(): f_width, f_height = g_pool.capture.frame_size f_width += int(icon_bar_width * g_pool.gui.scale) glfw.glfwSetWindowSize(main_window, f_width, f_height) general_settings = ui.Growing_Menu("General", header_pos="headline") general_settings.append(ui.Button("Reset window size", set_window_size)) general_settings.append( ui.Selector( "gui_user_scale", g_pool, setter=set_scale, selection=[0.8, 0.9, 1.0, 1.1, 1.2] + list(np.arange(1.5, 5.1, 0.5)), label="Interface Size", )) general_settings.append( ui.Info_Text( f"Minimum Player Version: {meta_info.min_player_version}")) general_settings.append( ui.Info_Text(f"Player Version: {g_pool.version}")) general_settings.append( ui.Info_Text( f"Recording Software: {meta_info.recording_software_name}")) general_settings.append( ui.Info_Text( f"Recording Software Version: {meta_info.recording_software_version}" )) general_settings.append( ui.Info_Text( "High level data, e.g. fixations, or visualizations only consider gaze data that has an equal or higher confidence than the minimum data confidence." )) general_settings.append( ui.Slider( "min_data_confidence", g_pool, setter=set_data_confidence, step=0.05, min=0.0, max=1.0, label="Minimum data confidence", )) general_settings.append( ui.Button("Restart with default settings", reset_restart)) g_pool.menubar.append(general_settings) icon = ui.Icon( "collapsed", general_settings, label=chr(0xE8B8), on_val=False, off_val=True, setter=toggle_general_settings, label_font="pupil_icons", ) icon.tooltip = "General Settings" g_pool.iconbar.append(icon) user_plugin_separator = ui.Separator() user_plugin_separator.order = 0.35 g_pool.iconbar.append(user_plugin_separator) g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100), (100, -100)) g_pool.export_button = ui.Thumb( "export", label=chr(0xE2C5), getter=lambda: False, setter=do_export, hotkey="e", label_font="pupil_icons", ) g_pool.quickbar.extend([g_pool.export_button]) g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.timelines) g_pool.gui.append(g_pool.iconbar) g_pool.gui.append(g_pool.quickbar) # we always load these plugins default_plugins = [ ("Plugin_Manager", {}), ("Seek_Control", {}), ("Log_Display", {}), ("Raw_Data_Exporter", {}), ("Vis_Polyline", {}), ("Vis_Circle", {}), ("System_Graphs", {}), ("System_Timelines", {}), ("World_Video_Exporter", {}), ("Pupil_From_Recording", {}), ("GazeFromRecording", {}), ("Audio_Playback", {}), ] g_pool.plugins = Plugin_List( g_pool, session_settings.get("loaded_plugins", default_plugins)) # Manually add g_pool.capture to the plugin list g_pool.plugins._plugins.append(g_pool.capture) g_pool.plugins._plugins.sort(key=lambda p: p.order) g_pool.capture.init_ui() general_settings.insert( -1, ui.Text_Input( "rel_time_trim_section", getter=g_pool.seek_control.get_rel_time_trim_range_string, setter=g_pool.seek_control.set_rel_time_trim_range_string, label="Relative time range to export", ), ) general_settings.insert( -1, ui.Text_Input( "frame_idx_trim_section", getter=g_pool.seek_control.get_frame_index_trim_range_string, setter=g_pool.seek_control.set_frame_index_trim_range_string, label="Frame index range to export", ), ) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) toggle_general_settings(True) g_pool.gui.configuration = session_settings.get("ui_config", {}) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() # trigger on_resize on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) def handle_notifications(n): subject = n["subject"] if subject == "start_plugin": g_pool.plugins.add(g_pool.plugin_by_name[n["name"]], args=n.get("args", {})) elif subject.startswith("meta.should_doc"): ipc_pub.notify({ "subject": "meta.doc", "actor": g_pool.app, "doc": player.__doc__ }) for p in g_pool.plugins: if (p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify): ipc_pub.notify({ "subject": "meta.doc", "actor": p.class_name, "doc": p.on_notify.__doc__, }) while (not glfw.glfwWindowShouldClose(main_window) and not process_was_interrupted): # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) events = {} # report time between now and the last loop interation events["dt"] = get_dt() # pupil and gaze positions are added by their respective producer plugins events["pupil"] = [] events["gaze"] = [] # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() glfw.glfwMakeContextCurrent(main_window) glfw.glfwPollEvents() # render visual feedback from loaded plugins if gl_utils.is_window_visible(main_window): gl_utils.glViewport(0, 0, *g_pool.camera_render_size) g_pool.capture.gl_display() for p in g_pool.plugins: p.gl_display() gl_utils.glViewport(0, 0, *window_size) try: clipboard = glfw.glfwGetClipboardString( main_window).decode() except AttributeError: # clipbaord is None, might happen on startup clipboard = "" g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard and user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.glfwSetClipboardString(main_window, user_input.clipboard.encode()) for b in user_input.buttons: button, action, mods = b x, y = glfw.glfwGetCursorPos(main_window) pos = x * hdpi_factor, y * hdpi_factor pos = normalize(pos, g_pool.camera_render_size) pos = denormalize(pos, g_pool.capture.frame_size) for plugin in g_pool.plugins: if plugin.on_click(pos, button, action): break for key, scancode, action, mods in user_input.keys: for plugin in g_pool.plugins: if plugin.on_key(key, scancode, action, mods): break for char_ in user_input.chars: for plugin in g_pool.plugins: if plugin.on_char(char_): break # present frames at appropriate speed g_pool.seek_control.wait(events["frame"].timestamp) glfw.glfwSwapBuffers(main_window) session_settings["loaded_plugins"] = g_pool.plugins.get_initializers() session_settings["min_data_confidence"] = g_pool.min_data_confidence session_settings[ "min_calibration_confidence"] = g_pool.min_calibration_confidence session_settings["gui_scale"] = g_pool.gui_user_scale session_settings["ui_config"] = g_pool.gui.configuration session_settings["window_position"] = glfw.glfwGetWindowPos( main_window) session_settings["version"] = str(g_pool.version) session_window_size = glfw.glfwGetWindowSize(main_window) if 0 not in session_window_size: session_settings["window_size"] = session_window_size session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) except Exception: import traceback trace = traceback.format_exc() logger.error("Process Player crashed with trace:\n{}".format(trace)) finally: logger.info("Process shutting down.") ipc_pub.notify({"subject": "player_process.stopped"}) sleep(1.0)
class Canny_Detector(Pupil_Detector): """a Pupil detector based on Canny_Edges""" def __init__(self,g_pool): super(Canny_Detector, self).__init__() # load session persistent settings self.session_settings =Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_detector') ) # coase pupil filter params self.coarse_detection = c_bool(self.load('coarse_detection',True)) self.coarse_filter_min = 100 self.coarse_filter_max = 400 # canny edge detection params self.blur = 1 self.canny_thresh = 159 self.canny_ratio= 2 self.canny_aperture = 5 # edge intensity filter params self.intensity_range = c_int(self.load('intensity_range',11)) self.bin_thresh = c_int(0) # contour prefilter params self.min_contour_size = c_int(self.load('min_contour_size',80)) #ellipse filter params self.inital_ellipse_fit_threshhold = 1.8 self.min_ratio = .3 self.pupil_min = c_float(self.load('pupil_min',40.)) self.pupil_max = c_float(self.load('pupil_max',150.)) self.target_size= c_float(100.) self.strong_perimeter_ratio_range = .8, 1.1 self.strong_area_ratio_range = .6,1.1 self.final_perimeter_ratio_range = self.load("final_perimeter_ratio_range",[.6, 1.2]) self.strong_prior = None #detector dignostics #confidance in the mesurement 0(bad) to 1 (perfect) # in this case we take the support ratio capped at 1. (uncapped if the pupil comes from prior) self.confidence = c_float(0) self.confidence_hist = [] #debug window self.suggested_size = 640,480 self._window = None self.window_should_open = False self.window_should_close = False #debug settings self.should_sleep = False def load(self, var_name, default): return self.session_settings.get(var_name,default) def save(self, var_name, var): self.session_settings[var_name] = var def detect(self,frame,user_roi,visualize=False): u_r = user_roi if self.window_should_open: self.open_window((frame.img.shape[1],frame.img.shape[0])) if self.window_should_close: self.close_window() if self._window: debug_img = np.zeros(frame.img.shape,frame.img.dtype) #get the user_roi img = frame.img r_img = img[u_r.view] # bias_field = preproc.EstimateBias(r_img) # r_img = preproc.Unbias(r_img, bias_field) r_img = preproc.GaussBlur(r_img) r_img = preproc.RobustRescale(r_img) frame.img[u_r.view] = r_img gray_img = cv2.cvtColor(r_img,cv2.COLOR_BGR2GRAY) # coarse pupil detection if self.coarse_detection.value: integral = cv2.integral(gray_img) integral = np.array(integral,dtype=c_float) x,y,w,response = eye_filter(integral,self.coarse_filter_min,self.coarse_filter_max) p_r = Roi(gray_img.shape) if w>0: p_r.set((y,x,y+w,x+w)) else: p_r.set((0,0,-1,-1)) else: p_r = Roi(gray_img.shape) p_r.set((0,0,None,None)) w = img.shape[0]/2 coarse_pupil_width = w/2. padding = coarse_pupil_width/4. pupil_img = gray_img[p_r.view] # binary thresholding of pupil dark areas hist = cv2.calcHist([pupil_img],[0],None,[256],[0,256]) #(images, channels, mask, histSize, ranges[, hist[, accumulate]]) bins = np.arange(hist.shape[0]) spikes = bins[hist[:,0]>40] # every intensity seen in more than 40 pixels if spikes.shape[0] >0: lowest_spike = spikes.min() highest_spike = spikes.max() else: lowest_spike = 200 highest_spike = 255 offset = self.intensity_range.value spectral_offset = 5 if visualize: # display the histogram sx,sy = 100,1 colors = ((0,0,255),(255,0,0),(255,255,0),(255,255,255)) h,w,chan = img.shape hist *= 1./hist.max() # normalize for display for i,h in zip(bins,hist[:,0]): c = colors[1] cv2.line(img,(w,int(i*sy)),(w-int(h*sx),int(i*sy)),c) cv2.line(img,(w,int(lowest_spike*sy)),(int(w-.5*sx),int(lowest_spike*sy)),colors[0]) cv2.line(img,(w,int((lowest_spike+offset)*sy)),(int(w-.5*sx),int((lowest_spike+offset)*sy)),colors[2]) cv2.line(img,(w,int((highest_spike)*sy)),(int(w-.5*sx),int((highest_spike)*sy)),colors[0]) cv2.line(img,(w,int((highest_spike- spectral_offset )*sy)),(int(w-.5*sx),int((highest_spike - spectral_offset)*sy)),colors[3]) # create dark and spectral glint masks self.bin_thresh.value = lowest_spike binary_img = bin_thresholding(pupil_img,image_upper=lowest_spike + offset) kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (7,7)) cv2.dilate(binary_img, kernel,binary_img, iterations=2) spec_mask = bin_thresholding(pupil_img, image_upper=highest_spike - spectral_offset) cv2.erode(spec_mask, kernel,spec_mask, iterations=1) kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (9,9)) #open operation to remove eye lashes pupil_img = cv2.morphologyEx(pupil_img, cv2.MORPH_OPEN, kernel) if self.blur > 1: pupil_img = cv2.medianBlur(pupil_img,self.blur.value) edges = cv2.Canny(pupil_img, self.canny_thresh, self.canny_thresh*self.canny_ratio, apertureSize= self.canny_aperture) # remove edges in areas not dark enough and where the glint is (spectral refelction from IR leds) edges = cv2.min(edges, spec_mask) edges = cv2.min(edges,binary_img) overlay = img[u_r.view][p_r.view] if visualize: b,g,r = overlay[:,:,0],overlay[:,:,1],overlay[:,:,2] g[:] = cv2.max(g,edges) b[:] = cv2.max(b,binary_img) b[:] = cv2.min(b,spec_mask) # draw a frame around the automatic pupil ROI in overlay. overlay[::2,0] = 255 #yeay numpy broadcasting overlay[::2,-1]= 255 overlay[0,::2] = 255 overlay[-1,::2]= 255 # draw a frame around the area we require the pupil center to be. overlay[padding:-padding:4,padding] = 255 overlay[padding:-padding:4,-padding]= 255 overlay[padding,padding:-padding:4] = 255 overlay[-padding,padding:-padding:4]= 255 if visualize: c = (100.,frame.img.shape[0]-100.) e_max = ((c),(self.pupil_max.value,self.pupil_max.value),0) e_recent = ((c),(self.target_size.value,self.target_size.value),0) e_min = ((c),(self.pupil_min.value,self.pupil_min.value),0) cv2.ellipse(frame.img,e_min,(0,0,255),1) cv2.ellipse(frame.img,e_recent,(0,255,0),1) cv2.ellipse(frame.img,e_max,(0,0,255),1) #get raw edge pix for later raw_edges = cv2.findNonZero(edges) def ellipse_true_support(e,raw_edges): a,b = e[1][0]/2.,e[1][1]/2. # major minor radii of candidate ellipse ellipse_circumference = np.pi*abs(3*(a+b)-np.sqrt(10*a*b+3*(a**2+b**2))) distances = dist_pts_ellipse(e,raw_edges) support_pixels = raw_edges[distances<=1.3] # support_ratio = support_pixel.shape[0]/ellipse_circumference return support_pixels,ellipse_circumference # if we had a good ellipse before ,let see if it is still a good first guess: if self.strong_prior: e = p_r.sub_vector(u_r.sub_vector(self.strong_prior[0])),self.strong_prior[1],self.strong_prior[2] self.strong_prior = None if raw_edges is not None: support_pixels,ellipse_circumference = ellipse_true_support(e,raw_edges) support_ratio = support_pixels.shape[0]/ellipse_circumference if support_ratio >= self.strong_perimeter_ratio_range[0]: refit_e = cv2.fitEllipse(support_pixels) if self._window: cv2.ellipse(debug_img,e,(255,100,100),thickness=4) cv2.ellipse(debug_img,refit_e,(0,0,255),thickness=1) e = refit_e self.strong_prior = u_r.add_vector(p_r.add_vector(e[0])),e[1],e[2] goodness = min(1.,support_ratio) pupil_ellipse = {} pupil_ellipse['confidence'] = goodness pupil_ellipse['ellipse'] = e pupil_ellipse['roi_center'] = e[0] pupil_ellipse['major'] = max(e[1]) pupil_ellipse['minor'] = min(e[1]) pupil_ellipse['apparent_pupil_size'] = max(e[1]) pupil_ellipse['axes'] = e[1] pupil_ellipse['angle'] = e[2] e_img_center =u_r.add_vector(p_r.add_vector(e[0])) norm_center = normalize(e_img_center,(frame.img.shape[1], frame.img.shape[0]),flip_y=True) pupil_ellipse['norm_pupil'] = norm_center pupil_ellipse['center'] = e_img_center pupil_ellipse['timestamp'] = frame.timestamp self.target_size.value = max(e[1]) self.confidence.value = goodness self.confidence_hist.append(goodness) self.confidence_hist[:-200]=[] if self._window: #draw a little animation of confidence cv2.putText(debug_img, 'good',(410,debug_img.shape[0]-100), cv2.FONT_HERSHEY_SIMPLEX,0.3,(255,100,100)) cv2.putText(debug_img, 'threshold',(410,debug_img.shape[0]-int(self.final_perimeter_ratio_range[0]*100)), cv2.FONT_HERSHEY_SIMPLEX,0.3,(255,100,100)) cv2.putText(debug_img, 'no detection',(410,debug_img.shape[0]-10), cv2.FONT_HERSHEY_SIMPLEX,0.3,(255,100,100)) lines = np.array([[[2*x,debug_img.shape[0]-int(100*y)],[2*x,debug_img.shape[0]]] for x,y in enumerate(self.confidence_hist)]) cv2.polylines(debug_img,lines,isClosed=False,color=(255,100,100)) self.gl_display_in_window(debug_img) return pupil_ellipse # from edges to contours contours, hierarchy = cv2.findContours(edges, mode=cv2.RETR_LIST, method=cv2.CHAIN_APPROX_NONE,offset=(0,0)) #TC89_KCOS # contours is a list containing array([[[108, 290]],[[111, 290]]], dtype=int32) shape=(number of points,1,dimension(2) ) ### first we want to filter out the bad stuff # to short good_contours = [c for c in contours if c.shape[0]>self.min_contour_size.value] # now we learn things about each contour through looking at the curvature. # For this we need to simplyfy the contour so that pt to pt angles become more meaningfull aprox_contours = [cv2.approxPolyDP(c,epsilon=1.5,closed=False) for c in good_contours] if self._window: x_shift = coarse_pupil_width*2 color = zip(range(0,250,15),range(0,255,15)[::-1],range(230,250)) split_contours = [] for c in aprox_contours: curvature = GetAnglesPolyline(c) # we split whenever there is a real kink (abs(curvature)<right angle) or a change in the genreal direction kink_idx = find_kink_and_dir_change(curvature,80) segs = split_at_corner_index(c,kink_idx) #TODO: split at shart inward turns for s in segs: if s.shape[0]>2: split_contours.append(s) if self._window: c = color.pop(0) color.append(c) s = s.copy() s[:,:,0] += debug_img.shape[1]-coarse_pupil_width*2 # s[:,:,0] += x_shift # x_shift += 5 cv2.polylines(debug_img,[s],isClosed=False,color=map(lambda x: x,c),thickness = 1,lineType=4)#cv2.CV_AA split_contours.sort(key=lambda x:-x.shape[0]) # print [x.shape[0]for x in split_contours] if len(split_contours) == 0: # not a single usefull segment found -> no pupil found self.confidence.value = 0 self.confidence_hist.append(0) if self._window: self.gl_display_in_window(debug_img) return {'timestamp':frame.timestamp,'norm_pupil':None} # removing stubs makes combinatorial search feasable split_contours = [c for c in split_contours if c.shape[0]>3] def ellipse_filter(e): in_center = padding < e[0][1] < pupil_img.shape[0]-padding and padding < e[0][0] < pupil_img.shape[1]-padding if in_center: is_round = min(e[1])/max(e[1]) >= self.min_ratio if is_round: right_size = self.pupil_min.value <= max(e[1]) <= self.pupil_max.value if right_size: return True return False def ellipse_on_blue(e): center_on_dark = binary_img[e[0][1],e[0][0]] return bool(center_on_dark) def ellipse_support_ratio(e,contours): a,b = e[1][0]/2.,e[1][1]/2. # major minor radii of candidate ellipse ellipse_area = np.pi*a*b ellipse_circumference = np.pi*abs(3*(a+b)-np.sqrt(10*a*b+3*(a**2+b**2))) actual_area = cv2.contourArea(cv2.convexHull(np.concatenate(contours))) actual_contour_length = sum([cv2.arcLength(c,closed=False) for c in contours]) area_ratio = actual_area / ellipse_area perimeter_ratio = actual_contour_length / ellipse_circumference #we assume here that the contour lies close to the ellipse boundary return perimeter_ratio,area_ratio def final_fitting(c,edges): #use the real edge pixels to fit, not the aproximated contours support_mask = np.zeros(edges.shape,edges.dtype) cv2.polylines(support_mask,c,isClosed=False,color=(255,255,255),thickness=2) # #draw into the suport mast with thickness 2 new_edges = cv2.min(edges, support_mask) new_contours = cv2.findNonZero(new_edges) if self._window: new_edges[new_edges!=0] = 255 overlay[:,:,1] = cv2.max(overlay[:,:,1], new_edges) overlay[:,:,2] = cv2.max(overlay[:,:,2], new_edges) overlay[:,:,2] = cv2.max(overlay[:,:,2], new_edges) new_e = cv2.fitEllipse(new_contours) return new_e,new_contours # finding poential candidates for ellipse seeds that describe the pupil. strong_seed_contours = [] weak_seed_contours = [] for idx, c in enumerate(split_contours): if c.shape[0] >=5: e = cv2.fitEllipse(c) # is this ellipse a plausible canditate for a pupil? if ellipse_filter(e): distances = dist_pts_ellipse(e,c) fit_variance = np.sum(distances**2)/float(distances.shape[0]) if fit_variance <= self.inital_ellipse_fit_threshhold: # how much ellipse is supported by this contour? perimeter_ratio,area_ratio = ellipse_support_ratio(e,[c]) # logger.debug('Ellipse no %s with perimeter_ratio: %s , area_ratio: %s'%(idx,perimeter_ratio,area_ratio)) if self.strong_perimeter_ratio_range[0]<= perimeter_ratio <= self.strong_perimeter_ratio_range[1] and self.strong_area_ratio_range[0]<= area_ratio <= self.strong_area_ratio_range[1]: strong_seed_contours.append(idx) if self._window: cv2.polylines(debug_img,[c],isClosed=False,color=(255,100,100),thickness=4) e = (e[0][0]+debug_img.shape[1]-coarse_pupil_width*4,e[0][1]),e[1],e[2] cv2.ellipse(debug_img,e,color=(255,100,100),thickness=3) else: weak_seed_contours.append(idx) if self._window: cv2.polylines(debug_img,[c],isClosed=False,color=(255,0,0),thickness=2) e = (e[0][0]+debug_img.shape[1]-coarse_pupil_width*4,e[0][1]),e[1],e[2] cv2.ellipse(debug_img,e,color=(255,0,0)) sc = np.array(split_contours) if strong_seed_contours: seed_idx = strong_seed_contours elif weak_seed_contours: seed_idx = weak_seed_contours if not (strong_seed_contours or weak_seed_contours): if self._window: self.gl_display_in_window(debug_img) self.confidence.value = 0 self.confidence_hist.append(0) return {'timestamp':frame.timestamp,'norm_pupil':None} # if self._window: # cv2.polylines(debug_img,[split_contours[i] for i in seed_idx],isClosed=False,color=(255,255,100),thickness=3) def ellipse_eval(contours): c = np.concatenate(contours) e = cv2.fitEllipse(c) d = dist_pts_ellipse(e,c) fit_variance = np.sum(d**2)/float(d.shape[0]) return fit_variance <= self.inital_ellipse_fit_threshhold solutions = pruning_quick_combine(split_contours,ellipse_eval,seed_idx,max_evals=1000,max_depth=5) solutions = filter_subsets(solutions) ratings = [] for s in solutions: e = cv2.fitEllipse(np.concatenate(sc[s])) if self._window: cv2.ellipse(debug_img,e,(0,150,100)) support_pixels,ellipse_circumference = ellipse_true_support(e,raw_edges) support_ratio = support_pixels.shape[0]/ellipse_circumference # TODO: refine the selection of final canditate if support_ratio >=self.final_perimeter_ratio_range[0] and ellipse_filter(e): ratings.append(support_pixels.shape[0]) if support_ratio >=self.strong_perimeter_ratio_range[0]: self.strong_prior = u_r.add_vector(p_r.add_vector(e[0])),e[1],e[2] if self._window: cv2.ellipse(debug_img,e,(0,255,255),thickness = 2) else: #not a valid solution, bad rating ratings.append(-1) # selected ellipse if max(ratings) == -1: #no good final ellipse found if self._window: self.gl_display_in_window(debug_img) self.confidence.value = 0 self.confidence_hist.append(0) return {'timestamp':frame.timestamp,'norm_pupil':None} best = solutions[ratings.index(max(ratings))] e = cv2.fitEllipse(np.concatenate(sc[best])) #final calculation of goodness of fit support_pixels,ellipse_circumference = ellipse_true_support(e,raw_edges) support_ratio = support_pixels.shape[0]/ellipse_circumference goodness = min(1.,support_ratio) #final fitting and return of result new_e,final_edges = final_fitting(sc[best],edges) size_dif = abs(1 - max(e[1])/max(new_e[1])) if ellipse_filter(new_e) and size_dif < .3: if self._window: cv2.ellipse(debug_img,new_e,(0,255,0)) e = new_e pupil_ellipse = {} pupil_ellipse['confidence'] = goodness pupil_ellipse['ellipse'] = e pupil_ellipse['pos_in_roi'] = e[0] pupil_ellipse['major'] = max(e[1]) pupil_ellipse['apparent_pupil_size'] = max(e[1]) pupil_ellipse['minor'] = min(e[1]) pupil_ellipse['axes'] = e[1] pupil_ellipse['angle'] = e[2] e_img_center =u_r.add_vector(p_r.add_vector(e[0])) norm_center = normalize(e_img_center,(frame.img.shape[1], frame.img.shape[0]),flip_y=True) pupil_ellipse['norm_pupil'] = norm_center pupil_ellipse['center'] = e_img_center pupil_ellipse['timestamp'] = frame.timestamp self.target_size.value = max(e[1]) self.confidence.value = goodness self.confidence_hist.append(goodness) self.confidence_hist[:-200]=[] if self._window: #draw a little animation of confidence cv2.putText(debug_img, 'good',(410,debug_img.shape[0]-100), cv2.FONT_HERSHEY_SIMPLEX,0.3,(255,100,100)) cv2.putText(debug_img, 'threshold',(410,debug_img.shape[0]-int(self.final_perimeter_ratio_range[0]*100)), cv2.FONT_HERSHEY_SIMPLEX,0.3,(255,100,100)) cv2.putText(debug_img, 'no detection',(410,debug_img.shape[0]-10), cv2.FONT_HERSHEY_SIMPLEX,0.3,(255,100,100)) lines = np.array([[[2*x,debug_img.shape[0]-int(100*y)],[2*x,debug_img.shape[0]]] for x,y in enumerate(self.confidence_hist)]) cv2.polylines(debug_img,lines,isClosed=False,color=(255,100,100)) self.gl_display_in_window(debug_img) return pupil_ellipse # Display and interface methods def set_final_perimeter_ratio_range(self,val): self.final_perimeter_ratio_range[0] = val def create_atb_bar(self,pos): self.bar = atb.Bar(name = "Canny_Pupil_Detector", label="Pupil_Detector", help="pupil detection parameters", color=(50, 50, 50), alpha=100, text='light', position=pos,refresh=.3, size=(200, 100)) self.bar.fps = c_float(10) self.bar.add_var("fps", self.bar.fps, min=1) self.bar.add_var("use coarse detection",self.coarse_detection,help="Disbale when you have trouble with detection when using Mascara.") self.bar.add_button("open debug window", self.toggle_window,help="Open a debug window that shows geeky visual feedback from the algorithm.") self.bar.add_var("pupil_intensity_range",self.intensity_range,help="Using alorithm view set this as low as possible but so that the pupil is always fully overlayed in blue.") self.bar.add_var("pupil_min",self.pupil_min,min=1,help="Setting good bounds will increase detection robustness. Use alorithm view to see.") self.bar.add_var("pupil_max",self.pupil_max,min=1,help="Setting good bounds will increase detection robustness. Use alorithm view to see.") self.bar.add_var("Pupil_Aparent_Size",self.target_size,readonly=True) self.bar.add_var("Contour min length",self.min_contour_size,help="Setting this low will make the alorithm try to connect even smaller arcs to find the pupil but cost you cpu time!") self.bar.add_var("confidece threshold",c_float(0),getter= lambda: self.final_perimeter_ratio_range[0], setter=self.set_final_perimeter_ratio_range,step=.05,min=0.,max=1. , help="Fraction of pupil boundry that has to be visible and detected for the resukt to be declared valid.") # self.bar.add_var("Pupil_Shade",self.bin_thresh, readonly=True) self.bar.add_var("confidence",self.confidence, readonly=True,help="The measure of confidence is a number between 0 and 1 of how sure the algorithm is about the detected pupil. We currenlty use the fraction of pupil boundry edge that is used as support for the ellipse result.") # self.bar.add_var("Image_Blur",self.blur, step=2,min=1,max=9) # self.bar.add_var("Canny_aparture",self.canny_aperture, step=2,min=3,max=7) # self.bar.add_var("canny_threshold",self.canny_thresh, step=1,min=0) # self.bar.add_var("Canny_ratio",self.canny_ratio, step=1,min=1) def toggle_window(self): if self._window: self.window_should_close = True else: self.window_should_open = True def open_window(self,size): if not self._window: if 0: #we are not fullscreening monitor = glfwGetMonitors()[self.monitor_idx.value] mode = glfwGetVideoMode(monitor) height,width= mode[0],mode[1] else: monitor = None height,width= size active_window = glfwGetCurrentContext() self._window = glfwCreateWindow(height, width, "Plugin Window", monitor=monitor, share=active_window) if not 0: glfwSetWindowPos(self._window,200,0) self.on_resize(self._window,height,width) #Register callbacks glfwSetWindowSizeCallback(self._window,self.on_resize) # glfwSetKeyCallback(self._window,self.on_key) glfwSetWindowCloseCallback(self._window,self.on_close) # gl_state settings glfwMakeContextCurrent(self._window) basic_gl_setup() # refresh speed settings glfwSwapInterval(0) glfwMakeContextCurrent(active_window) self.window_should_open = False # window calbacks def on_resize(self,window,w, h): active_window = glfwGetCurrentContext() glfwMakeContextCurrent(window) adjust_gl_view(w,h,window) glfwMakeContextCurrent(active_window) def on_close(self,window): self.window_should_close = True def close_window(self): if self._window: glfwDestroyWindow(self._window) self._window = None self.window_should_close = False def gl_display_in_window(self,img): active_window = glfwGetCurrentContext() glfwMakeContextCurrent(self._window) clear_gl_screen() # gl stuff that will show on your plugin window goes here make_coord_system_norm_based() draw_gl_texture(img,interpolation=False) glfwSwapBuffers(self._window) glfwMakeContextCurrent(active_window) def cleanup(self): self.save('intensity_range',self.intensity_range.value) self.save('pupil_min',self.pupil_min.value) self.save('pupil_max',self.pupil_max.value) self.save('min_contour_size',self.min_contour_size.value) self.save('final_perimeter_ratio_range',self.final_perimeter_ratio_range) self.session_settings.close()
def session(rec_dir): # Callback functions def on_resize(window,w, h): g_pool.gui.update_window(w,h) g_pool.gui.collect_menus() graph.adjust_size(w,h) adjust_gl_view(w,h) for p in g_pool.plugins: p.on_window_resize(window,w,h) def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key,scancode,action,mods) def on_char(window,char): g_pool.gui.update_char(char) def on_button(window,button, action, mods): g_pool.gui.update_button(button,action,mods) pos = glfwGetCursorPos(window) pos = normalize(pos,glfwGetWindowSize(window)) pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels for p in g_pool.plugins: p.on_click(pos,button,action) def on_pos(window,x, y): hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0]) g_pool.gui.update_mouse(x*hdpi_factor,y*hdpi_factor) def on_scroll(window,x,y): g_pool.gui.update_scroll(x,y*y_scroll_factor) def on_drop(window,count,paths): for x in range(count): new_rec_dir = paths[x] if is_pupil_rec_dir(new_rec_dir): logger.debug("Starting new session with '%s'"%new_rec_dir) global rec_dir rec_dir = new_rec_dir glfwSetWindowShouldClose(window,True) else: logger.error("'%s' is not a valid pupil recording"%new_rec_dir) tick = delta_t() def get_dt(): return next(tick) video_path = [f for f in glob(os.path.join(rec_dir,"world.*")) if f[-3:] in ('mp4','mkv','avi')][0] timestamps_path = os.path.join(rec_dir, "world_timestamps.npy") pupil_data_path = os.path.join(rec_dir, "pupil_data") #parse info.csv file meta_info_path = os.path.join(rec_dir,"info.csv") with open(meta_info_path) as info: meta_info = dict( ((line.strip().split('\t')) for line in info.readlines() ) ) rec_version = read_rec_version(meta_info) if rec_version >= VersionFormat('0.7.4'): pass if rec_version >= VersionFormat('0.7.3'): update_recording_0v73_to_current(rec_dir) elif rec_version >= VersionFormat('0.5'): update_recording_0v5_to_current(rec_dir) elif rec_version >= VersionFormat('0.4'): update_recording_0v4_to_current(rec_dir) elif rec_version >= VersionFormat('0.3'): update_recording_0v3_to_current(rec_dir) timestamps_path = os.path.join(rec_dir, "timestamps.npy") else: logger.Error("This recording is to old. Sorry.") return timestamps = np.load(timestamps_path) # Initialize capture cap = File_Capture(video_path,timestamps=list(timestamps)) # load session persistent settings session_settings = Persistent_Dict(os.path.join(user_dir,"user_settings")) if session_settings.get("version",VersionFormat('0.0')) < get_version(version_file): logger.info("Session setting are from older version of this app. I will not use those.") session_settings.clear() width,height = session_settings.get('window_size',cap.frame_size) window_pos = session_settings.get('window_position',(0,0)) main_window = glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - "+ rec_dir.split(os.path.sep)[-1], None, None) glfwSetWindowPos(main_window,window_pos[0],window_pos[1]) glfwMakeContextCurrent(main_window) cygl.utils.init() # load pupil_positions, gaze_positions pupil_data = load_object(pupil_data_path) pupil_list = pupil_data['pupil_positions'] gaze_list = pupil_data['gaze_positions'] # create container for globally scoped vars g_pool = Global_Container() g_pool.app = 'player' g_pool.binocular = meta_info.get('Eye Mode','monocular') == 'binocular' g_pool.version = get_version(version_file) g_pool.capture = cap g_pool.timestamps = timestamps g_pool.play = False g_pool.new_seek = True g_pool.user_dir = user_dir g_pool.rec_dir = rec_dir g_pool.rec_version = rec_version g_pool.meta_info = meta_info g_pool.pupil_positions_by_frame = correlate_data(pupil_list,g_pool.timestamps) g_pool.gaze_positions_by_frame = correlate_data(gaze_list,g_pool.timestamps) g_pool.fixations_by_frame = [[] for x in g_pool.timestamps] #populated by the fixation detector plugin def next_frame(_): try: cap.seek_to_frame(cap.get_frame_index()) except FileSeekError: logger.warning("Could not seek to next frame.") else: g_pool.new_seek = True def prev_frame(_): try: cap.seek_to_frame(cap.get_frame_index()-2) except FileSeekError: logger.warning("Could not seek to previous frame.") else: g_pool.new_seek = True def set_scale(new_scale): g_pool.gui.scale = new_scale g_pool.gui.collect_menus() def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def purge_plugins(): for p in g_pool.plugins: if p.__class__ in user_launchable_plugins: p.alive = False g_pool.plugins.clean() def do_export(_): export_range = slice(g_pool.trim_marks.in_mark,g_pool.trim_marks.out_mark) export_dir = os.path.join(g_pool.rec_dir,'exports','%s-%s'%(export_range.start,export_range.stop)) try: os.makedirs(export_dir) except OSError as e: if e.errno != errno.EEXIST: logger.error("Could not create export dir") raise e else: logger.warning("Previous export for range [%s-%s] already exsits - overwriting."%(export_range.start,export_range.stop)) else: logger.info('Created export dir at "%s"'%export_dir) notification = {'subject':'should_export','range':export_range,'export_dir':export_dir} g_pool.notifications.append(notification) g_pool.gui = ui.UI() g_pool.gui.scale = session_settings.get('gui_scale',1) g_pool.main_menu = ui.Growing_Menu("Settings",pos=(-350,20),size=(300,400)) g_pool.main_menu.append(ui.Button("Close Pupil Player",lambda:glfwSetWindowShouldClose(main_window,True))) g_pool.main_menu.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=0.75,max=2.5,label='Interface Size')) g_pool.main_menu.append(ui.Info_Text('Player Version: %s'%g_pool.version)) g_pool.main_menu.append(ui.Info_Text('Recording Version: %s'%rec_version)) g_pool.main_menu.append(ui.Selector('Open plugin', selection = user_launchable_plugins, labels = [p.__name__.replace('_',' ') for p in user_launchable_plugins], setter= open_plugin, getter = lambda: "Select to load")) g_pool.main_menu.append(ui.Button('Close all plugins',purge_plugins)) g_pool.main_menu.append(ui.Button('Reset window size',lambda: glfwSetWindowSize(main_window,cap.frame_size[0],cap.frame_size[1])) ) g_pool.quickbar = ui.Stretching_Menu('Quick Bar',(0,100),(120,-100)) g_pool.play_button = ui.Thumb('play',g_pool,label='Play',hotkey=GLFW_KEY_SPACE) g_pool.play_button.on_color[:] = (0,1.,.0,.8) g_pool.forward_button = ui.Thumb('forward',getter = lambda: False,setter= next_frame, hotkey=GLFW_KEY_RIGHT) g_pool.backward_button = ui.Thumb('backward',getter = lambda: False, setter = prev_frame, hotkey=GLFW_KEY_LEFT) g_pool.export_button = ui.Thumb('export',getter = lambda: False, setter = do_export, hotkey='e') g_pool.quickbar.extend([g_pool.play_button,g_pool.forward_button,g_pool.backward_button,g_pool.export_button]) g_pool.gui.append(g_pool.quickbar) g_pool.gui.append(g_pool.main_menu) #we always load these plugins system_plugins = [('Trim_Marks',{}),('Seek_Bar',{})] default_plugins = [('Log_Display',{}),('Scan_Path',{}),('Vis_Polyline',{}),('Vis_Circle',{}),('Video_Export_Launcher',{})] previous_plugins = session_settings.get('loaded_plugins',default_plugins) g_pool.notifications = [] g_pool.delayed_notifications = {} g_pool.plugins = Plugin_List(g_pool,plugin_by_name,system_plugins+previous_plugins) # Register callbacks main_window glfwSetFramebufferSizeCallback(main_window,on_resize) glfwSetKeyCallback(main_window,on_key) glfwSetCharCallback(main_window,on_char) glfwSetMouseButtonCallback(main_window,on_button) glfwSetCursorPosCallback(main_window,on_pos) glfwSetScrollCallback(main_window,on_scroll) glfwSetDropCallback(main_window,on_drop) #trigger on_resize on_resize(main_window, *glfwGetFramebufferSize(main_window)) g_pool.gui.configuration = session_settings.get('ui_config',{}) # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() #set up performace graphs: pid = os.getpid() ps = psutil.Process(pid) ts = cap.get_timestamp()-.03 cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20,110) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140,110) fps_graph.update_rate = 5 fps_graph.label = "%0.0f REC FPS" pupil_graph = graph.Bar_Graph(max_val=1.0) pupil_graph.pos = (260,110) pupil_graph.update_rate = 5 pupil_graph.label = "Confidence: %0.2f" while not glfwWindowShouldClose(main_window): #grab new frame if g_pool.play or g_pool.new_seek: g_pool.new_seek = False try: new_frame = cap.get_frame_nowait() except EndofVideoFileError: #end of video logic: pause at last frame. g_pool.play=False update_graph = True else: update_graph = False frame = new_frame.copy() events = {} #report time between now and the last loop interation events['dt'] = get_dt() #new positons we make a deepcopy just like the image is a copy. events['gaze_positions'] = deepcopy(g_pool.gaze_positions_by_frame[frame.index]) events['pupil_positions'] = deepcopy(g_pool.pupil_positions_by_frame[frame.index]) if update_graph: #update performace graphs for p in events['pupil_positions']: pupil_graph.add(p['confidence']) t = new_frame.timestamp if ts != t: dt,ts = t-ts,t fps_graph.add(1./dt) g_pool.play_button.status_text = str(frame.index) #always update the CPU graph cpu_graph.update() # publish delayed notifiactions when their time has come. for n in g_pool.delayed_notifications.values(): if n['_notify_time_'] < time(): del n['_notify_time_'] del g_pool.delayed_notifications[n['subject']] g_pool.notifications.append(n) # notify each plugin if there are new notifactions: while g_pool.notifications: n = g_pool.notifications.pop(0) for p in g_pool.plugins: p.on_notify(n) # allow each Plugin to do its work. for p in g_pool.plugins: p.update(frame,events) #check if a plugin need to be destroyed g_pool.plugins.clean() # render camera image glfwMakeContextCurrent(main_window) make_coord_system_norm_based() g_pool.image_tex.update_from_frame(frame) g_pool.image_tex.draw() make_coord_system_pixel_based(frame.img.shape) # render visual feedback from loaded plugins for p in g_pool.plugins: p.gl_display() graph.push_view() fps_graph.draw() cpu_graph.draw() pupil_graph.draw() graph.pop_view() g_pool.gui.update() #present frames at appropriate speed cap.wait(frame) glfwSwapBuffers(main_window) glfwPollEvents() session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['gui_scale'] = g_pool.gui.scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['window_size'] = glfwGetWindowSize(main_window) session_settings['window_position'] = glfwGetWindowPos(main_window) session_settings['version'] = g_pool.version session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() cap.close() g_pool.gui.terminate() glfwDestroyWindow(main_window)
def __init__(self, g_pool, gui_settings={ 'pos': (220, 200), 'size': (300, 300), 'iconified': False }): super(Offline_Marker_Detector, self).__init__() self.g_pool = g_pool self.gui_settings = gui_settings self.order = .2 # all markers that are detected in the most recent frame self.markers = [] # all registered surfaces if g_pool.app == 'capture': raise Exception('For Player only.') #in player we load from the rec_dir: but we have a couple options: self.surface_definitions = Persistent_Dict( os.path.join(g_pool.rec_dir, 'surface_definitions')) if self.load('offline_square_marker_surfaces', []) != []: logger.debug( "Found ref surfaces defined or copied in previous session.") self.surfaces = [ Offline_Reference_Surface( self.g_pool, saved_definition=d, gaze_positions_by_frame=self.g_pool.positions_by_frame) for d in self.load('offline_square_marker_surfaces', []) if isinstance(d, dict) ] elif self.load('realtime_square_marker_surfaces', []) != []: logger.debug( "Did not find ref surfaces def created or used by the user in player from earlier session. Loading surfaces defined during capture." ) self.surfaces = [ Offline_Reference_Surface( self.g_pool, saved_definition=d, gaze_positions_by_frame=self.g_pool.positions_by_frame) for d in self.load('realtime_square_marker_surfaces', []) if isinstance(d, dict) ] else: logger.debug("No surface defs found. Please define using GUI.") self.surfaces = [] # edit surfaces self.surface_edit_mode = c_bool(0) self.edit_surfaces = [] #detector vars self.robust_detection = c_bool(1) self.aperture = c_int(11) self.min_marker_perimeter = 80 #check if marker cache is available from last session self.persistent_cache = Persistent_Dict( os.path.join(g_pool.rec_dir, 'square_marker_cache')) self.cache = Cache_List( self.persistent_cache.get('marker_cache', [False for _ in g_pool.timestamps])) logger.debug( "Loaded marker cache %s / %s frames had been searched before" % (len(self.cache) - self.cache.count(False), len(self.cache))) self.init_marker_cacher() #debug vars self.draw_markers = c_bool(0) self.show_surface_idx = c_int(0) self.recent_pupil_positions = [] self.img_shape = None self.img = None
def world(g_pool,cap_src,cap_size): """world Creates a window, gl context. Grabs images from a capture. Receives Pupil coordinates from eye process[es] Can run various plug-ins. """ #manage plugins runtime_plugins = import_runtime_plugins(os.path.join(g_pool.user_dir,'plugins')) user_launchable_plugins = [Show_Calibration,Pupil_Server,Pupil_Sync,Marker_Detector]+runtime_plugins system_plugins = [Log_Display,Display_Recent_Gaze,Recorder] plugin_by_index = system_plugins+user_launchable_plugins+calibration_plugins+gaze_mapping_plugins name_by_index = [p.__name__ for p in plugin_by_index] plugin_by_name = dict(zip(name_by_index,plugin_by_index)) default_plugins = [('Log_Display',{}),('Dummy_Gaze_Mapper',{}),('Display_Recent_Gaze',{}), ('Screen_Marker_Calibration',{}),('Recorder',{})] # Callback functions def on_resize(window,w, h): if not g_pool.iconified: g_pool.gui.update_window(w,h) g_pool.gui.collect_menus() graph.adjust_size(w,h) adjust_gl_view(w,h) for p in g_pool.plugins: p.on_window_resize(window,w,h) def on_iconify(window,iconified): g_pool.iconified = iconified def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key,scancode,action,mods) def on_char(window,char): g_pool.gui.update_char(char) def on_button(window,button, action, mods): g_pool.gui.update_button(button,action,mods) pos = glfwGetCursorPos(window) pos = normalize(pos,glfwGetWindowSize(main_window)) pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels for p in g_pool.plugins: p.on_click(pos,button,action) def on_pos(window,x, y): hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0]) x,y = x*hdpi_factor,y*hdpi_factor g_pool.gui.update_mouse(x,y) def on_scroll(window,x,y): g_pool.gui.update_scroll(x,y*scroll_factor) def on_close(window): g_pool.quit.value = True logger.info('Process closing from window') tick = delta_t() def get_dt(): return next(tick) # load session persistent settings session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_world')) if session_settings.get("version",VersionFormat('0.0')) < g_pool.version: logger.info("Session setting are from older version of this app. I will not use those.") session_settings.clear() # Initialize capture cap = autoCreateCapture(cap_src, timebase=g_pool.timebase) default_settings = {'frame_size':cap_size,'frame_rate':24} previous_settings = session_settings.get('capture_settings',None) if previous_settings and previous_settings['name'] == cap.name: cap.settings = previous_settings else: cap.settings = default_settings # Test capture try: frame = cap.get_frame() except CameraCaptureError: logger.error("Could not retrieve image from capture") cap.close() return # any object we attach to the g_pool object *from now on* will only be visible to this process! # vars should be declared here to make them visible to the code reader. g_pool.update_textures = session_settings.get("update_textures",2) g_pool.iconified = False g_pool.capture = cap g_pool.pupil_confidence_threshold = session_settings.get('pupil_confidence_threshold',.6) g_pool.active_calibration_plugin = None def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def set_scale(new_scale): g_pool.gui.scale = new_scale g_pool.gui.collect_menus() #window and gl setup glfwInit() width,height = session_settings.get('window_size',(frame.width, frame.height)) main_window = glfwCreateWindow(width,height, "World") window_pos = session_settings.get('window_position',window_position_default) glfwSetWindowPos(main_window,window_pos[0],window_pos[1]) glfwMakeContextCurrent(main_window) cygl.utils.init() #setup GUI g_pool.gui = ui.UI() g_pool.gui.scale = session_settings.get('gui_scale',1) g_pool.sidebar = ui.Scrolling_Menu("Settings",pos=(-350,0),size=(0,0),header_pos='left') general_settings = ui.Growing_Menu('General') general_settings.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=1.,max=2.5,label='Interface size')) general_settings.append(ui.Button('Reset window size',lambda: glfwSetWindowSize(main_window,frame.width,frame.height)) ) general_settings.append(ui.Selector('Open plugin', selection = user_launchable_plugins, labels = [p.__name__.replace('_',' ') for p in user_launchable_plugins], setter= open_plugin, getter=lambda: "Select to load")) g_pool.sidebar.append(general_settings) advanced_settings = ui.Growing_Menu('Advanced') advanced_settings.append(ui.Selector('update_textures',g_pool,label="Update display",selection=range(3),labels=('No update','Gray','Color'))) advanced_settings.append(ui.Slider('pupil_confidence_threshold', g_pool,step = .01,min=0.,max=1.,label='Minimum pupil confidence')) advanced_settings.append(ui.Info_Text('Capture Version: %s'%g_pool.version)) general_settings.append(advanced_settings) g_pool.calibration_menu = ui.Growing_Menu('Calibration') g_pool.calibration_menu.append(ui.Selector('active_calibration_plugin',getter=lambda: g_pool.active_calibration_plugin.__class__, selection = calibration_plugins, labels = [p.__name__.replace('_',' ') for p in calibration_plugins], setter= open_plugin,label='Method')) g_pool.sidebar.append(g_pool.calibration_menu) g_pool.gui.append(g_pool.sidebar) g_pool.quickbar = ui.Stretching_Menu('Quick Bar',(0,100),(120,-100)) g_pool.gui.append(g_pool.quickbar) g_pool.gui.append(ui.Hot_Key("quit",setter=on_close,getter=lambda:True,label="X",hotkey=GLFW_KEY_ESCAPE)) g_pool.capture.init_gui(g_pool.sidebar) #plugins that are loaded based on user settings from previous session g_pool.notifications = [] g_pool.plugins = Plugin_List(g_pool,plugin_by_name,session_settings.get('loaded_plugins',default_plugins)) # Register callbacks main_window glfwSetFramebufferSizeCallback(main_window,on_resize) glfwSetWindowCloseCallback(main_window,on_close) glfwSetWindowIconifyCallback(main_window,on_iconify) glfwSetKeyCallback(main_window,on_key) glfwSetCharCallback(main_window,on_char) glfwSetMouseButtonCallback(main_window,on_button) glfwSetCursorPosCallback(main_window,on_pos) glfwSetScrollCallback(main_window,on_scroll) # gl_state settings basic_gl_setup() g_pool.image_tex = create_named_texture(frame.img.shape) update_named_texture(g_pool.image_tex,frame.img) # refresh speed settings glfwSwapInterval(0) #trigger setup of window and gl sizes on_resize(main_window, *glfwGetFramebufferSize(main_window)) #now the we have aproper window we can load the last gui configuration g_pool.gui.configuration = session_settings.get('ui_config',{}) #set up performace graphs: pid = os.getpid() ps = psutil.Process(pid) ts = frame.timestamp cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20,130) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140,130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" pupil_graph = graph.Bar_Graph(max_val=1.0) pupil_graph.pos = (260,130) pupil_graph.update_rate = 5 pupil_graph.label = "Confidence: %0.2f" # Event loop while not g_pool.quit.value: # Get an image from the grabber try: frame = cap.get_frame() except CameraCaptureError: logger.error("Capture from camera failed. Stopping.") break except EndofVideoFileError: logger.warning("Video file is done. Stopping") break #update performace graphs t = frame.timestamp dt,ts = t-ts,t try: fps_graph.add(1./dt) except ZeroDivisionError: pass cpu_graph.update() #a dictionary that allows plugins to post and read events events = {} #report time between now and the last loop interation events['dt'] = get_dt() #receive and map pupil positions recent_pupil_positions = [] while not g_pool.pupil_queue.empty(): p = g_pool.pupil_queue.get() recent_pupil_positions.append(p) pupil_graph.add(p['confidence']) events['pupil_positions'] = recent_pupil_positions # notify each plugin if there are new notifactions: while g_pool.notifications: n = g_pool.notifications.pop(0) for p in g_pool.plugins: p.on_notify(n) # allow each Plugin to do its work. for p in g_pool.plugins: p.update(frame,events) #check if a plugin need to be destroyed g_pool.plugins.clean() # render camera image glfwMakeContextCurrent(main_window) if g_pool.iconified: pass elif g_pool.update_textures == 2: update_named_texture(g_pool.image_tex,frame.img) elif g_pool.update_textures == 1: update_named_texture(g_pool.image_tex,frame.gray) make_coord_system_norm_based() draw_named_texture(g_pool.image_tex) make_coord_system_pixel_based((frame.height,frame.width,3)) # render visual feedback from loaded plugins for p in g_pool.plugins: p.gl_display() if not g_pool.iconified: graph.push_view() fps_graph.draw() cpu_graph.draw() pupil_graph.draw() graph.pop_view() g_pool.gui.update() glfwSwapBuffers(main_window) glfwPollEvents() glfwRestoreWindow(main_window) #need to do this for windows os session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['pupil_confidence_threshold'] = g_pool.pupil_confidence_threshold session_settings['gui_scale'] = g_pool.gui.scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['capture_settings'] = g_pool.capture.settings session_settings['window_size'] = glfwGetWindowSize(main_window) session_settings['window_position'] = glfwGetWindowPos(main_window) session_settings['update_textures'] = g_pool.update_textures session_settings['version'] = g_pool.version session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfwDestroyWindow(main_window) glfwTerminate() cap.close() logger.debug("Process done")
def eye(g_pool, cap_src, cap_size, pipe_to_world, eye_id=0): """ Creates a window, gl context. Grabs images from a capture. Streams Pupil coordinates into g_pool.pupil_queue """ # modify the root logger for this process logger = logging.getLogger() # remove inherited handlers logger.handlers = [] # create file handler which logs even debug messages fh = logging.FileHandler(os.path.join(g_pool.user_dir, 'eye%s.log' % eye_id), mode='w') # fh.setLevel(logging.DEBUG) # create console handler with a higher log level ch = logging.StreamHandler() ch.setLevel(logger.level + 10) # create formatter and add it to the handlers formatter = logging.Formatter( 'Eye' + str(eye_id) + ' Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s') fh.setFormatter(formatter) formatter = logging.Formatter( 'EYE' + str(eye_id) + ' Process [%(levelname)s] %(name)s : %(message)s') ch.setFormatter(formatter) # add the handlers to the logger logger.addHandler(fh) logger.addHandler(ch) # create logger for the context of this function logger = logging.getLogger(__name__) #UI Platform tweaks if platform.system() == 'Linux': scroll_factor = 10.0 window_position_default = (600, 300 * eye_id) elif platform.system() == 'Windows': scroll_factor = 1.0 window_position_default = (600, 31 + 300 * eye_id) else: scroll_factor = 1.0 window_position_default = (600, 300 * eye_id) # Callback functions def on_resize(window, w, h): if not g_pool.iconified: active_window = glfwGetCurrentContext() glfwMakeContextCurrent(window) g_pool.gui.update_window(w, h) graph.adjust_size(w, h) adjust_gl_view(w, h) glfwMakeContextCurrent(active_window) def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_char(window, char): g_pool.gui.update_char(char) def on_iconify(window, iconified): g_pool.iconified = iconified def on_button(window, button, action, mods): if g_pool.display_mode == 'roi': if action == GLFW_RELEASE and u_r.active_edit_pt: u_r.active_edit_pt = False return # if the roi interacts we dont what the gui to interact as well elif action == GLFW_PRESS: pos = glfwGetCursorPos(window) pos = normalize(pos, glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] pos = denormalize( pos, (frame.width, frame.height)) # Position in img pixels if u_r.mouse_over_edit_pt(pos, u_r.handle_size + 40, u_r.handle_size + 40): return # if the roi interacts we dont what the gui to interact as well g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): hdpi_factor = float( glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0]) g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor) if u_r.active_edit_pt: pos = normalize((x, y), glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] pos = denormalize(pos, (frame.width, frame.height)) u_r.move_vertex(u_r.active_pt_idx, pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_close(window): g_pool.quit.value = True logger.info('Process closing from window') # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, 'user_settings_eye%s' % eye_id)) if session_settings.get("version", VersionFormat('0.0')) < g_pool.version: logger.info( "Session setting are from older version of this app. I will not use those." ) session_settings.clear() # Initialize capture cap = autoCreateCapture(cap_src, timebase=g_pool.timebase) default_settings = {'frame_size': cap_size, 'frame_rate': 30} previous_settings = session_settings.get('capture_settings', None) if previous_settings and previous_settings['name'] == cap.name: cap.settings = previous_settings else: cap.settings = default_settings # Test capture try: frame = cap.get_frame() except CameraCaptureError: logger.error("Could not retrieve image from capture") cap.close() return #signal world that we are ready to go pipe_to_world.send('eye%s process ready' % eye_id) # any object we attach to the g_pool object *from now on* will only be visible to this process! # vars should be declared here to make them visible to the code reader. g_pool.iconified = False g_pool.capture = cap g_pool.flip = session_settings.get('flip', False) g_pool.display_mode = session_settings.get('display_mode', 'camera_image') g_pool.display_mode_info_text = { 'camera_image': "Raw eye camera image. This uses the least amount of CPU power", 'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be a small as possible but big enough to capture to pupil in its movements", 'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters with in the Pupil Detection menu below." } # g_pool.draw_pupil = session_settings.get('draw_pupil',True) u_r = UIRoi(frame.img.shape) u_r.set(session_settings.get('roi', u_r.get())) writer = None pupil_detector = Canny_Detector(g_pool) # UI callback functions def set_scale(new_scale): g_pool.gui.scale = new_scale g_pool.gui.collect_menus() def set_display_mode_info(val): g_pool.display_mode = val g_pool.display_mode_info.text = g_pool.display_mode_info_text[val] # Initialize glfw glfwInit() if g_pool.binocular: title = "Binocular eye %s" % eye_id else: title = 'Eye' width, height = session_settings.get('window_size', (frame.width, frame.height)) main_window = glfwCreateWindow(width, height, title, None, None) window_pos = session_settings.get('window_position', window_position_default) glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfwMakeContextCurrent(main_window) cygl_init() # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() g_pool.image_tex.update_from_frame(frame) glfwSwapInterval(0) #setup GUI g_pool.gui = ui.UI() g_pool.gui.scale = session_settings.get('gui_scale', 1) g_pool.sidebar = ui.Scrolling_Menu("Settings", pos=(-300, 0), size=(0, 0), header_pos='left') general_settings = ui.Growing_Menu('General') general_settings.append( ui.Slider('scale', g_pool.gui, setter=set_scale, step=.05, min=1., max=2.5, label='Interface Size')) general_settings.append( ui.Button( 'Reset window size', lambda: glfwSetWindowSize(main_window, frame.width, frame.height))) general_settings.append( ui.Selector('display_mode', g_pool, setter=set_display_mode_info, selection=['camera_image', 'roi', 'algorithm'], labels=['Camera Image', 'ROI', 'Algorithm'], label="Mode")) general_settings.append( ui.Switch('flip', g_pool, label='Flip image display')) g_pool.display_mode_info = ui.Info_Text( g_pool.display_mode_info_text[g_pool.display_mode]) general_settings.append(g_pool.display_mode_info) g_pool.sidebar.append(general_settings) g_pool.gui.append(g_pool.sidebar) # let the camera add its GUI g_pool.capture.init_gui(g_pool.sidebar) # let detector add its GUI pupil_detector.init_gui(g_pool.sidebar) # Register callbacks main_window glfwSetFramebufferSizeCallback(main_window, on_resize) glfwSetWindowCloseCallback(main_window, on_close) glfwSetWindowIconifyCallback(main_window, on_iconify) glfwSetKeyCallback(main_window, on_key) glfwSetCharCallback(main_window, on_char) glfwSetMouseButtonCallback(main_window, on_button) glfwSetCursorPosCallback(main_window, on_pos) glfwSetScrollCallback(main_window, on_scroll) #set the last saved window size on_resize(main_window, *glfwGetWindowSize(main_window)) # load last gui configuration g_pool.gui.configuration = session_settings.get('ui_config', {}) #set up performance graphs pid = os.getpid() ps = psutil.Process(pid) ts = frame.timestamp cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 130) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" #create a timer to control window update frequency window_update_timer = timer(1 / 60.) def window_should_update(): return next(window_update_timer) # Event loop while not g_pool.quit.value: # Get an image from the grabber try: frame = cap.get_frame() except CameraCaptureError: logger.error("Capture from Camera Failed. Stopping.") break except EndofVideoFileError: logger.warning("Video File is done. Stopping") break #update performace graphs t = frame.timestamp dt, ts = t - ts, t try: fps_graph.add(1. / dt) except ZeroDivisionError: pass cpu_graph.update() ### RECORDING of Eye Video (on demand) ### # Setup variables and lists for recording if pipe_to_world.poll(): command, raw_mode = pipe_to_world.recv() if command is not None: record_path = command logger.info("Will save eye video to: %s" % record_path) timestamps_path = os.path.join(record_path, "eye%s_timestamps.npy" % eye_id) if raw_mode and frame.jpeg_buffer: video_path = os.path.join(record_path, "eye%s.mp4" % eye_id) writer = JPEG_Writer(video_path, cap.frame_rate) else: video_path = os.path.join(record_path, "eye%s.mp4" % eye_id) writer = AV_Writer(video_path, cap.frame_rate) timestamps = [] else: logger.info("Done recording.") writer.release() writer = None np.save(timestamps_path, np.asarray(timestamps)) del timestamps if writer: writer.write_video_frame(frame) timestamps.append(frame.timestamp) # pupil ellipse detection result = pupil_detector.detect( frame, user_roi=u_r, visualize=g_pool.display_mode == 'algorithm') result['id'] = eye_id # stream the result g_pool.pupil_queue.put(result) # GL drawing if window_should_update(): if not g_pool.iconified: glfwMakeContextCurrent(main_window) clear_gl_screen() # switch to work in normalized coordinate space if g_pool.display_mode == 'algorithm': g_pool.image_tex.update_from_ndarray(frame.img) elif g_pool.display_mode in ('camera_image', 'roi'): g_pool.image_tex.update_from_ndarray(frame.gray) else: pass make_coord_system_norm_based(g_pool.flip) g_pool.image_tex.draw() # switch to work in pixel space make_coord_system_pixel_based((frame.height, frame.width, 3), g_pool.flip) if result['confidence'] > 0: if result.has_key('axes'): pts = cv2.ellipse2Poly((int( result['center'][0]), int(result['center'][1])), (int(result['axes'][0] / 2), int(result['axes'][1] / 2)), int(result['angle']), 0, 360, 15) cygl_draw_polyline(pts, 1, cygl_rgba(1., 0, 0, .5)) cygl_draw_points([result['center']], size=20, color=cygl_rgba(1., 0., 0., .5), sharpness=1.) # render graphs graph.push_view() fps_graph.draw() cpu_graph.draw() graph.pop_view() # render GUI g_pool.gui.update() #render the ROI if g_pool.display_mode == 'roi': u_r.draw(g_pool.gui.scale) #update screen glfwSwapBuffers(main_window) glfwPollEvents() # END while running # in case eye recording was still runnnig: Save&close if writer: logger.info("Done recording eye.") writer = None np.save(timestamps_path, np.asarray(timestamps)) glfwRestoreWindow(main_window) #need to do this for windows os # save session persistent settings session_settings['gui_scale'] = g_pool.gui.scale session_settings['roi'] = u_r.get() session_settings['flip'] = g_pool.flip session_settings['display_mode'] = g_pool.display_mode session_settings['ui_config'] = g_pool.gui.configuration session_settings['capture_settings'] = g_pool.capture.settings session_settings['window_size'] = glfwGetWindowSize(main_window) session_settings['window_position'] = glfwGetWindowPos(main_window) session_settings['version'] = g_pool.version session_settings.close() pupil_detector.cleanup() g_pool.gui.terminate() glfwDestroyWindow(main_window) glfwTerminate() cap.close() #flushing queue in case world process did not exit gracefully while not g_pool.pupil_queue.empty(): g_pool.pupil_queue.get() g_pool.pupil_queue.close() logger.debug("Process done")
def player_drop(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_version): # general imports import logging # networking import zmq import zmq_tools from time import sleep # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.INFO) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) try: import glfw import gl_utils from OpenGL.GL import glClearColor from version_utils import VersionFormat from file_methods import Persistent_Dict from pyglui.pyfontstash import fontstash from pyglui.ui import get_roboto_font_path import player_methods as pm import update_methods as um def on_drop(window, count, paths): nonlocal rec_dir rec_dir = paths[0].decode("utf-8") if rec_dir: if not pm.is_pupil_rec_dir(rec_dir): rec_dir = None # load session persistent settings session_settings = Persistent_Dict( os.path.join(user_dir, "user_settings_player") ) if VersionFormat(session_settings.get("version", "0.0")) != app_version: logger.info( "Session setting are from a different version of this app. I will not use those." ) session_settings.clear() w, h = session_settings.get("window_size", (1280, 720)) window_pos = session_settings.get("window_position", window_position_default) glfw.glfwInit() glfw.glfwWindowHint(glfw.GLFW_RESIZABLE, 0) window = glfw.glfwCreateWindow(w, h, "Pupil Player") glfw.glfwWindowHint(glfw.GLFW_RESIZABLE, 1) glfw.glfwMakeContextCurrent(window) glfw.glfwSetWindowPos(window, window_pos[0], window_pos[1]) glfw.glfwSetDropCallback(window, on_drop) glfont = fontstash.Context() glfont.add_font("roboto", get_roboto_font_path()) glfont.set_align_string(v_align="center", h_align="middle") glfont.set_color_float((0.2, 0.2, 0.2, 0.9)) gl_utils.basic_gl_setup() glClearColor(0.5, 0.5, 0.5, 0.0) text = "Drop a recording directory onto this window." tip = "(Tip: You can drop a recording directory onto the app icon.)" # text = "Please supply a Pupil recording directory as first arg when calling Pupil Player." while not glfw.glfwWindowShouldClose(window): fb_size = glfw.glfwGetFramebufferSize(window) hdpi_factor = glfw.getHDPIFactor(window) gl_utils.adjust_gl_view(*fb_size) if rec_dir: if pm.is_pupil_rec_dir(rec_dir): logger.info("Starting new session with '{}'".format(rec_dir)) text = "Updating recording format." tip = "This may take a while!" else: logger.error("'{}' is not a valid pupil recording".format(rec_dir)) tip = "Oops! That was not a valid recording." rec_dir = None gl_utils.clear_gl_screen() glfont.set_blur(10.5) glfont.set_color_float((0.0, 0.0, 0.0, 1.0)) glfont.set_size(w / 25.0 * hdpi_factor) glfont.draw_text(w / 2 * hdpi_factor, 0.3 * h * hdpi_factor, text) glfont.set_size(w / 30.0 * hdpi_factor) glfont.draw_text(w / 2 * hdpi_factor, 0.4 * h * hdpi_factor, tip) glfont.set_blur(0.96) glfont.set_color_float((1.0, 1.0, 1.0, 1.0)) glfont.set_size(w / 25.0 * hdpi_factor) glfont.draw_text(w / 2 * hdpi_factor, 0.3 * h * hdpi_factor, text) glfont.set_size(w / 30.0 * hdpi_factor) glfont.draw_text(w / 2 * hdpi_factor, 0.4 * h * hdpi_factor, tip) glfw.glfwSwapBuffers(window) if rec_dir: try: um.update_recording_to_recent(rec_dir) except AssertionError as err: logger.error(str(err)) rec_dir = None else: glfw.glfwSetWindowShouldClose(window, True) glfw.glfwPollEvents() session_settings["window_position"] = glfw.glfwGetWindowPos(window) session_settings.close() glfw.glfwDestroyWindow(window) if rec_dir: ipc_pub.notify( {"subject": "player_process.should_start", "rec_dir": rec_dir} ) except: import traceback trace = traceback.format_exc() logger.error("Process player_drop crashed with trace:\n{}".format(trace)) finally: sleep(1.0)
def main(): import argparse from textwrap import dedent from file_methods import Persistent_Dict def show_progess(jobs): no_jobs = len(jobs) width = 80 full = width / no_jobs string = "" for j in jobs: try: p = int(width * j.current_frame.value / float(j.frames_to_export.value * no_jobs)) except Exception: p = 0 string += "[" + p * "|" + (full - p) * "-" + "]" sys.stdout.write("\r" + string) sys.stdout.flush() """Batch process recordings to produce visualizations Using simple_circle as the default visualizations Steps: - User Supplies: Directory that contains many recording(s) dirs or just one recordings dir - We walk the user supplied directory to get all data folders - Data is the list we feed to our multiprocessed - Error check -- do we have required files in each dir?: world.avi, gaze_positions.npy, timestamps.npy - Result: world_viz.avi within each original data folder """ parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description=dedent("""\ *************************************************** Batch process recordings to produce visualizations The default visualization will use simple_circle Usage Example: python batch_exporter.py -d /path/to/folder-with-many-recordings -s ~/Pupil_Player/settings/user_settings -e ~/my_export_dir Arguments: -d : Specify a recording directory. This could have one or many recordings contained within it. We will recurse into the dir. -s : Specify path to Pupil Player user_settings file to use last used vizualization settings. -e : Specify export directory if you dont want the export saved within each recording dir. -p : Export a 120 frame preview only. ***************************************************\ """), ) parser.add_argument("-d", "--rec-dir", required=True) parser.add_argument("-s", "--settings-file", required=True) parser.add_argument("-e", "--export-to-dir", default=False) parser.add_argument("-c", "--basic-color", default="red") parser.add_argument("-p", "--preview", action="store_true") if len(sys.argv) == 1: print(parser.description) return args = parser.parse_args() # get the top level data folder from terminal argument data_dir = args.rec_dir if args.settings_file and os.path.isfile(args.settings_file): session_settings = Persistent_Dict( os.path.splitext(args.settings_file)[0]) # these are loaded based on user settings plugin_initializers = session_settings.get("loaded_plugins", []) session_settings.close() else: logger.error("Setting file not found or valid") return if args.export_to_dir: export_dir = args.export_to_dir if os.path.isdir(export_dir): logger.info("Exporting all vids to {}".format(export_dir)) else: logger.error("Exporting dir is not valid {}".format(export_dir)) return else: export_dir = None logger.info("Exporting into the recording dirs.") if args.preview: preview = True logger.info("Exporting first 120frames only") else: preview = False class Temp(object): pass recording_dirs = get_recording_dirs(data_dir) # start multiprocessing engine n_cpu = mp.cpu_count() logger.info( "Using a maximum of {} CPUs to process visualizations in parallel...". format(n_cpu)) jobs = [] outfiles = set() for d in recording_dirs: j = Temp() logger.info("Adding new export: {}".format(d)) j.should_terminate = mp.Value(c_bool, 0) j.frames_to_export = mp.Value(c_int, 0) j.current_frame = mp.Value(c_int, 0) j.data_dir = d j.user_dir = None j.start_frame = None if preview: j.end_frame = 30 else: j.end_frame = None j.plugin_initializers = plugin_initializers[:] if export_dir: # make a unique name created from rec_session and dir name rec_session, rec_dir = d.rsplit(os.path.sep, 2)[1:] out_name = rec_session + "_" + rec_dir + ".mp4" j.out_file_path = os.path.join(os.path.expanduser(export_dir), out_name) if j.out_file_path in outfiles: logger.error( "This export setting would try to save {} at least twice pleace rename dirs to prevent this." .format(j.out_file_path)) return outfiles.add(j.out_file_path) logger.info("Exporting to: {}".format(j.out_file_path)) else: j.out_file_path = None j.args = ( j.should_terminate, j.frames_to_export, j.current_frame, j.data_dir, j.user_dir, j.start_frame, j.end_frame, j.plugin_initializers, j.out_file_path, None, ) jobs.append(j) todo = jobs[:] workers = [ Export_Process(target=export, args=todo.pop(0).args) for i in range(min(len(todo), n_cpu)) ] for w in workers: w.start() working = True while working: # cannot use pool as it does not allow shared memory working = False for i in range(len(workers)): if workers[i].is_alive(): working = True else: if todo: workers[i] = Export_Process(target=export, args=todo.pop(0).args) workers[i].start() working = True show_progess(jobs) time.sleep(0.25) print("\n")
def world(pupil_queue, timebase, lauchner_pipe, eye_pipes, eyes_are_alive, user_dir, version, cap_src): """world Creates a window, gl context. Grabs images from a capture. Receives Pupil coordinates from eye process[es] Can run various plug-ins. """ import logging # Set up root logger for this process before doing imports of logged modules. logger = logging.getLogger() logger.setLevel(logging.INFO) # create file handler which logs even debug messages fh = logging.FileHandler(os.path.join(user_dir, 'world.log'), mode='w') fh.setLevel(logger.level) # create console handler with a higher log level ch = logging.StreamHandler() ch.setLevel(logger.level + 10) # create formatter and add it to the handlers formatter = logging.Formatter( 'World Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s') fh.setFormatter(formatter) formatter = logging.Formatter( 'WORLD Process [%(levelname)s] %(name)s : %(message)s') ch.setFormatter(formatter) # add the handlers to the logger logger.addHandler(fh) logger.addHandler(ch) #silence noisy modules logging.getLogger("OpenGL").setLevel(logging.ERROR) logging.getLogger("libav").setLevel(logging.ERROR) # create logger for the context of this function logger = logging.getLogger(__name__) # We deferr the imports becasue of multiprocessing. # Otherwise the world process each process also loads the other imports. # This is not harmfull but unnessasary. #general imports from time import time import numpy as np #display import glfw from pyglui import ui, graph, cygl from pyglui.cygl.utils import Named_Texture from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen, make_coord_system_pixel_based, make_coord_system_norm_based #check versions for our own depedencies as they are fast-changing from pyglui import __version__ as pyglui_version assert pyglui_version >= '0.8' #monitoring import psutil # helpers/utils from file_methods import Persistent_Dict from methods import normalize, denormalize, delta_t from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError from version_utils import VersionFormat import audio # Plug-ins from plugin import Plugin_List, import_runtime_plugins from calibration_routines import calibration_plugins, gaze_mapping_plugins from recorder import Recorder from show_calibration import Show_Calibration from display_recent_gaze import Display_Recent_Gaze from pupil_server import Pupil_Server from pupil_sync import Pupil_Sync from marker_detector import Marker_Detector from log_display import Log_Display from annotations import Annotation_Capture # create logger for the context of this function #UI Platform tweaks if platform.system() == 'Linux': scroll_factor = 10.0 window_position_default = (0, 0) elif platform.system() == 'Windows': scroll_factor = 1.0 window_position_default = (8, 31) else: scroll_factor = 1.0 window_position_default = (0, 0) #g_pool holds variables for this process g_pool = Global_Container() # make some constants avaiable g_pool.user_dir = user_dir g_pool.version = version g_pool.app = 'capture' g_pool.pupil_queue = pupil_queue g_pool.timebase = timebase # g_pool.lauchner_pipe = lauchner_pipe g_pool.eye_pipes = eye_pipes g_pool.eyes_are_alive = eyes_are_alive #manage plugins runtime_plugins = import_runtime_plugins( os.path.join(g_pool.user_dir, 'plugins')) user_launchable_plugins = [ Show_Calibration, Pupil_Server, Pupil_Sync, Marker_Detector, Annotation_Capture ] + runtime_plugins system_plugins = [Log_Display, Display_Recent_Gaze, Recorder] plugin_by_index = system_plugins + user_launchable_plugins + calibration_plugins + gaze_mapping_plugins name_by_index = [p.__name__ for p in plugin_by_index] plugin_by_name = dict(zip(name_by_index, plugin_by_index)) default_plugins = [('Log_Display', {}), ('Dummy_Gaze_Mapper', {}), ('Display_Recent_Gaze', {}), ('Screen_Marker_Calibration', {}), ('Recorder', {})] # Callback functions def on_resize(window, w, h): if not g_pool.iconified: g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() graph.adjust_size(w, h) adjust_gl_view(w, h) for p in g_pool.plugins: p.on_window_resize(window, w, h) def on_iconify(window, iconified): g_pool.iconified = iconified def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_char(window, char): g_pool.gui.update_char(char) def on_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) pos = glfw.glfwGetCursorPos(window) pos = normalize(pos, glfw.glfwGetWindowSize(main_window)) pos = denormalize( pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels for p in g_pool.plugins: p.on_click(pos, button, action) def on_pos(window, x, y): hdpi_factor = float( glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) tick = delta_t() def get_dt(): return next(tick) # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, 'user_settings_world')) if session_settings.get("version", VersionFormat('0.0')) < g_pool.version: logger.info( "Session setting are from older version of this app. I will not use those." ) session_settings.clear() # Initialize capture cap = autoCreateCapture(cap_src, timebase=g_pool.timebase) default_settings = {'frame_size': (1280, 720), 'frame_rate': 30} previous_settings = session_settings.get('capture_settings', None) if previous_settings and previous_settings['name'] == cap.name: cap.settings = previous_settings else: cap.settings = default_settings # Test capture try: frame = cap.get_frame() except CameraCaptureError: logger.error("Could not retrieve image from capture") cap.close() lauchner_pipe.send("Exit") return g_pool.iconified = False g_pool.capture = cap g_pool.pupil_confidence_threshold = session_settings.get( 'pupil_confidence_threshold', .6) g_pool.detection_mapping_mode = session_settings.get( 'detection_mapping_mode', '2d') g_pool.active_calibration_plugin = None audio.audio_mode = session_settings.get('audio_mode', audio.default_audio_mode) def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def set_scale(new_scale): g_pool.gui.scale = new_scale g_pool.gui.collect_menus() def launch_eye_process(eye_id, blocking=False): if eyes_are_alive[eye_id].value: logger.error("Eye%s process already running." % eye_id) return lauchner_pipe.send(eye_id) eye_pipes[eye_id].send( ('Set_Detection_Mapping_Mode', g_pool.detection_mapping_mode)) if blocking: #wait for ready message from eye to sequentialize startup eye_pipes[eye_id].send('Ping') eye_pipes[eye_id].recv() logger.warning('Eye %s process started.' % eye_id) def stop_eye_process(eye_id, blocking=False): if eyes_are_alive[eye_id].value: eye_pipes[eye_id].send('Exit') if blocking: raise NotImplementedError() def start_stop_eye(eye_id, make_alive): if make_alive: launch_eye_process(eye_id) else: stop_eye_process(eye_id) def set_detection_mapping_mode(new_mode): if new_mode == '2d': for p in g_pool.plugins: if "Vector_Gaze_Mapper" in p.class_name: logger.warning( "The gaze mapper is not supported in 2d mode. Please recalibrate." ) p.alive = False g_pool.plugins.clean() for alive, pipe in zip(g_pool.eyes_are_alive, g_pool.eye_pipes): if alive.value: pipe.send(('Set_Detection_Mapping_Mode', new_mode)) g_pool.detection_mapping_mode = new_mode #window and gl setup glfw.glfwInit() width, height = session_settings.get('window_size', (frame.width, frame.height)) main_window = glfw.glfwCreateWindow(width, height, "World") window_pos = session_settings.get('window_position', window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window #setup GUI g_pool.gui = ui.UI() g_pool.gui.scale = session_settings.get('gui_scale', 1) g_pool.sidebar = ui.Scrolling_Menu("Settings", pos=(-350, 0), size=(0, 0), header_pos='left') general_settings = ui.Growing_Menu('General') general_settings.append( ui.Slider('scale', g_pool.gui, setter=set_scale, step=.05, min=1., max=2.5, label='Interface size')) general_settings.append( ui.Button( 'Reset window size', lambda: glfw.glfwSetWindowSize( main_window, frame.width, frame.height))) general_settings.append( ui.Selector('audio_mode', audio, selection=audio.audio_modes)) general_settings.append( ui.Selector('detection_mapping_mode', g_pool, label='detection & mapping mode', setter=set_detection_mapping_mode, selection=['2d', '3d'])) general_settings.append( ui.Switch('eye0_process', label='Detect eye 0', setter=lambda alive: start_stop_eye(0, alive), getter=lambda: eyes_are_alive[0].value)) general_settings.append( ui.Switch('eye1_process', label='Detect eye 1', setter=lambda alive: start_stop_eye(1, alive), getter=lambda: eyes_are_alive[1].value)) general_settings.append( ui.Selector('Open plugin', selection=user_launchable_plugins, labels=[ p.__name__.replace('_', ' ') for p in user_launchable_plugins ], setter=open_plugin, getter=lambda: "Select to load")) general_settings.append( ui.Slider('pupil_confidence_threshold', g_pool, step=.01, min=0., max=1., label='Minimum pupil confidence')) general_settings.append( ui.Info_Text('Capture Version: %s' % g_pool.version)) g_pool.sidebar.append(general_settings) g_pool.calibration_menu = ui.Growing_Menu('Calibration') g_pool.sidebar.append(g_pool.calibration_menu) g_pool.gui.append(g_pool.sidebar) g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100)) g_pool.gui.append(g_pool.quickbar) g_pool.capture.init_gui(g_pool.sidebar) #plugins that are loaded based on user settings from previous session g_pool.notifications = [] g_pool.delayed_notifications = {} g_pool.plugins = Plugin_List( g_pool, plugin_by_name, session_settings.get('loaded_plugins', default_plugins)) #We add the calibration menu selector, after a calibration has been added: g_pool.calibration_menu.insert( 0, ui.Selector( 'active_calibration_plugin', getter=lambda: g_pool.active_calibration_plugin.__class__, selection=calibration_plugins, labels=[p.__name__.replace('_', ' ') for p in calibration_plugins], setter=open_plugin, label='Method')) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetWindowIconifyCallback(main_window, on_iconify) glfw.glfwSetKeyCallback(main_window, on_key) glfw.glfwSetCharCallback(main_window, on_char) glfw.glfwSetMouseButtonCallback(main_window, on_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() g_pool.image_tex.update_from_frame(frame) # refresh speed settings glfw.glfwSwapInterval(0) #trigger setup of window and gl sizes on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) #now the we have aproper window we can load the last gui configuration g_pool.gui.configuration = session_settings.get('ui_config', {}) #set up performace graphs: pid = os.getpid() ps = psutil.Process(pid) ts = frame.timestamp cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 130) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" pupil_graph = graph.Bar_Graph(max_val=1.0) pupil_graph.pos = (260, 130) pupil_graph.update_rate = 5 pupil_graph.label = "Confidence: %0.2f" if session_settings.get('eye1_process_alive', False): launch_eye_process(1, blocking=True) if session_settings.get('eye0_process_alive', True): launch_eye_process(0, blocking=False) # Event loop while not glfw.glfwWindowShouldClose(main_window): # Get an image from the grabber try: frame = cap.get_frame() except CameraCaptureError: logger.error("Capture from camera failed. Stopping.") break except EndofVideoFileError: logger.warning("Video file is done. Stopping") break #update performace graphs t = frame.timestamp dt, ts = t - ts, t try: fps_graph.add(1. / dt) except ZeroDivisionError: pass cpu_graph.update() #a dictionary that allows plugins to post and read events events = {} #report time between now and the last loop interation events['dt'] = get_dt() #receive and map pupil positions recent_pupil_positions = [] while not g_pool.pupil_queue.empty(): p = g_pool.pupil_queue.get() recent_pupil_positions.append(p) pupil_graph.add(p['confidence']) events['pupil_positions'] = recent_pupil_positions # publish delayed notifiactions when their time has come. for n in g_pool.delayed_notifications.values(): if n['_notify_time_'] < time(): del n['_notify_time_'] del g_pool.delayed_notifications[n['subject']] g_pool.notifications.append(n) # notify each plugin if there are new notifications: while g_pool.notifications: n = g_pool.notifications.pop(0) for p in g_pool.plugins: p.on_notify(n) # allow each Plugin to do its work. for p in g_pool.plugins: p.update(frame, events) #check if a plugin need to be destroyed g_pool.plugins.clean() # render camera image glfw.glfwMakeContextCurrent(main_window) if g_pool.iconified: pass else: g_pool.image_tex.update_from_frame(frame) make_coord_system_norm_based() g_pool.image_tex.draw() make_coord_system_pixel_based((frame.height, frame.width, 3)) # render visual feedback from loaded plugins for p in g_pool.plugins: p.gl_display() if not g_pool.iconified: graph.push_view() fps_graph.draw() cpu_graph.draw() pupil_graph.draw() graph.pop_view() g_pool.gui.update() glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() glfw.glfwRestoreWindow(main_window) #need to do this for windows os session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings[ 'pupil_confidence_threshold'] = g_pool.pupil_confidence_threshold session_settings['gui_scale'] = g_pool.gui.scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['capture_settings'] = g_pool.capture.settings session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos(main_window) session_settings['version'] = g_pool.version session_settings['eye0_process_alive'] = eyes_are_alive[0].value session_settings['eye1_process_alive'] = eyes_are_alive[1].value session_settings['detection_mapping_mode'] = g_pool.detection_mapping_mode session_settings['audio_mode'] = audio.audio_mode session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) glfw.glfwTerminate() cap.close() #shut down eye processes: stop_eye_process(0) stop_eye_process(1) #shut down laucher lauchner_pipe.send("Exit") logger.debug("world process done")
def main(): def show_progess(jobs): no_jobs = len(jobs) width = 80 full = width/no_jobs string = "" for j in jobs: try: p = int(width*j.current_frame.value/float(j.frames_to_export.value*no_jobs) ) except: p = 0 string += '['+ p*"|"+(full-p)*"-" + "]" sys.stdout.write("\r"+string) sys.stdout.flush() """Batch process recordings to produce visualizations Using simple_circle as the default visualizations Steps: - User Supplies: Directory that contains many recording(s) dirs or just one recordings dir - We walk the user supplied directory to get all data folders - Data is the list we feed to our multiprocessed - Error check -- do we have required files in each dir?: world.avi, gaze_positions.npy, timestamps.npy - Result: world_viz.avi within each original data folder """ parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, description=dedent('''\ *************************************************** Batch process recordings to produce visualizations The default visualization will use simple_circle Usage Example: python batch_exporter.py -d /path/to/folder-with-many-recordings -s ~/Pupil_Player/settings/user_settings -e ~/my_export_dir Arguments: -d : Specify a recording directory. This could have one or many recordings contained within it. We will recurse into the dir. -s : Specify path to Pupil Player user_settings file to use last used vizualization settings. -e : Specify export directory if you dont want the export saved within each recording dir. -p : Export a 120 frame preview only. ***************************************************\ ''')) parser.add_argument('-d', '--rec-dir',required=True) parser.add_argument('-s', '--settings-file',required=True) parser.add_argument('-e', '--export-to-dir',default=False) parser.add_argument('-c', '--basic-color',default='red') parser.add_argument('-p', '--preview', action='store_true') if len(sys.argv)==1: print parser.description return args = parser.parse_args() # get the top level data folder from terminal argument data_dir = args.rec_dir if args.settings_file and os.path.isfile(args.settings_file): session_settings = Persistent_Dict(os.path.splitext(args.settings_file)[0]) #these are loaded based on user settings plugin_initializers = session_settings.get('loaded_plugins',[]) session_settings.close() else: logger.error("Setting file not found or valid") return if args.export_to_dir: export_dir = args.export_to_dir if os.path.isdir(export_dir): logger.info("Exporting all vids to %s"%export_dir) else: logger.error("Exporting dir is not valid %s"%export_dir) return else: export_dir = None logger.info("Exporting into the recording dirs.") if args.preview: preview = True logger.info("Exporting first 120frames only") else: preview = False class Temp(object): pass recording_dirs = get_recording_dirs(data_dir) # start multiprocessing engine n_cpu = cpu_count() logger.info("Using a maximum of %s CPUs to process visualizations in parallel..." %n_cpu) jobs = [] outfiles = set() for d in recording_dirs: j = Temp() logger.info("Adding new export: %s"%d) j.should_terminate = Value(c_bool,0) j.frames_to_export = Value(c_int,0) j.current_frame = Value(c_int,0) j.data_dir = d j.user_dir = None j.start_frame= None if preview: j.end_frame = 30 else: j.end_frame = None j.plugin_initializers = plugin_initializers[:] if export_dir: #make a unique name created from rec_session and dir name rec_session, rec_dir = d.rsplit(os.path.sep,2)[1:] out_name = rec_session+"_"+rec_dir+".mp4" j.out_file_path = os.path.join(os.path.expanduser(export_dir),out_name) if j.out_file_path in outfiles: logger.error("This export setting would try to save %s at least twice pleace rename dirs to prevent this."%j.out_file_path) return outfiles.add(j.out_file_path) logger.info("Exporting to: %s"%j.out_file_path) else: j.out_file_path = None j.args = (j.should_terminate,j.frames_to_export,j.current_frame, j.data_dir, j.user_dir,j.start_frame,j.end_frame,j.plugin_initializers,j.out_file_path) jobs.append(j) todo = jobs[:] workers = [Export_Process(target=export,args=todo.pop(0).args) for i in range(min(len(todo),n_cpu))] for w in workers: w.start() working = True t = time.time() while working: #cannot use pool as it does not allow shared memory working = False for i in range(len(workers)): if workers[i].is_alive(): working = True else: if todo: workers[i] = Process(target=export,args=todo.pop(0).args) workers[i].start() working = True show_progess(jobs) time.sleep(.25) print '\n'
def eye( timebase, is_alive_flag, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version, eye_id, overwrite_cap_settings=None, ): """reads eye video and detects the pupil. Creates a window, gl context. Grabs images from a capture. Streams Pupil coordinates. Reacts to notifications: ``set_detection_mapping_mode``: Sets detection method ``eye_process.should_stop``: Stops the eye process ``recording.started``: Starts recording eye video ``recording.stopped``: Stops recording eye video ``frame_publishing.started``: Starts frame publishing ``frame_publishing.stopped``: Stops frame publishing Emits notifications: ``eye_process.started``: Eye process started ``eye_process.stopped``: Eye process stopped Emits data: ``pupil.<eye id>``: Pupil data for eye with id ``<eye id>`` ``frame.eye.<eye id>``: Eye frames with id ``<eye id>`` """ # We deferr the imports becasue of multiprocessing. # Otherwise the world process each process also loads the other imports. import zmq import zmq_tools zmq_ctx = zmq.Context() ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",)) # logging setup import logging logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.NOTSET) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) if is_alive_flag.value: # indicates eye process that this is a duplicated startup logger.warning("Aborting redundant eye process startup") return with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id, logger): # general imports import traceback import numpy as np import cv2 # display import glfw from pyglui import ui, graph, cygl from pyglui.cygl.utils import draw_points, RGBA, draw_polyline from pyglui.cygl.utils import Named_Texture from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen from gl_utils import make_coord_system_pixel_based from gl_utils import make_coord_system_norm_based from gl_utils import is_window_visible, glViewport from ui_roi import UIRoi # monitoring import psutil # helpers/utils from uvc import get_time_monotonic from file_methods import Persistent_Dict from version_utils import VersionFormat from methods import normalize, denormalize, timer from av_writer import JPEG_Writer, AV_Writer from ndsi import H264Writer from video_capture import source_classes from video_capture import manager_classes from background_helper import IPC_Logging_Task_Proxy IPC_Logging_Task_Proxy.push_url = ipc_push_url # Pupil detectors from pupil_detectors import Detector_2D, Detector_3D, Detector_Dummy pupil_detectors = { Detector_2D.__name__: Detector_2D, Detector_3D.__name__: Detector_3D, Detector_Dummy.__name__: Detector_Dummy, } # UI Platform tweaks if platform.system() == "Linux": scroll_factor = 10.0 window_position_default = (600, 300 * eye_id + 30) elif platform.system() == "Windows": scroll_factor = 10.0 window_position_default = (600, 90 + 300 * eye_id) else: scroll_factor = 1.0 window_position_default = (600, 300 * eye_id) icon_bar_width = 50 window_size = None camera_render_size = None hdpi_factor = 1.0 # g_pool holds variables for this process g_pool = SimpleNamespace() # make some constants avaiable g_pool.user_dir = user_dir g_pool.version = version g_pool.app = "capture" g_pool.process = "eye{}".format(eye_id) g_pool.timebase = timebase g_pool.ipc_pub = ipc_socket def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal camera_render_size nonlocal hdpi_factor active_window = glfw.glfwGetCurrentContext() glfw.glfwMakeContextCurrent(window) hdpi_factor = glfw.getHDPIFactor(window) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor window_size = w, h camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() for g in g_pool.graphs: g.scale = hdpi_factor g.adjust_window_size(w, h) adjust_gl_view(w, h) glfw.glfwMakeContextCurrent(active_window) def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_iconify(window, iconified): g_pool.iconified = iconified def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x *= hdpi_factor y *= hdpi_factor g_pool.gui.update_mouse(x, y) if g_pool.u_r.active_edit_pt: pos = normalize((x, y), camera_render_size) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] pos = denormalize(pos, g_pool.capture.frame_size) g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx, pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, count, paths): paths = [paths[x].decode("utf-8") for x in range(count)] plugins = (g_pool.capture_manager, g_pool.capture) # call `on_drop` callbacks until a plugin indicates # that it has consumed the event (by returning True) any(p.on_drop(paths) for p in plugins) # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, "user_settings_eye{}".format(eye_id)) ) if VersionFormat(session_settings.get("version", "0.0")) != g_pool.version: logger.info( "Session setting are from a different version of this app. I will not use those." ) session_settings.clear() g_pool.iconified = False g_pool.capture = None g_pool.capture_manager = None g_pool.flip = session_settings.get("flip", False) g_pool.display_mode = session_settings.get("display_mode", "camera_image") g_pool.display_mode_info_text = { "camera_image": "Raw eye camera image. This uses the least amount of CPU power", "roi": "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.", "algorithm": "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below.", } capture_manager_settings = session_settings.get( "capture_manager_settings", ("UVC_Manager", {}) ) manager_class_name, manager_settings = capture_manager_settings manager_class_by_name = {c.__name__: c for c in manager_classes} g_pool.capture_manager = manager_class_by_name[manager_class_name]( g_pool, **manager_settings ) if eye_id == 0: cap_src = ["Pupil Cam3 ID0", "Pupil Cam2 ID0", "Pupil Cam1 ID0", "HD-6000"] else: cap_src = ["Pupil Cam3 ID1", "Pupil Cam2 ID1", "Pupil Cam1 ID1"] # Initialize capture default_settings = ( "UVC_Source", {"preferred_names": cap_src, "frame_size": (320, 240), "frame_rate": 120}, ) capture_source_settings = overwrite_cap_settings or session_settings.get( "capture_settings", default_settings ) source_class_name, source_settings = capture_source_settings source_class_by_name = {c.__name__: c for c in source_classes} g_pool.capture = source_class_by_name[source_class_name]( g_pool, **source_settings ) assert g_pool.capture g_pool.u_r = UIRoi((g_pool.capture.frame_size[1], g_pool.capture.frame_size[0])) roi_user_settings = session_settings.get("roi") if roi_user_settings and tuple(roi_user_settings[-1]) == g_pool.u_r.get()[-1]: g_pool.u_r.set(roi_user_settings) pupil_detector_settings = session_settings.get("pupil_detector_settings", None) last_pupil_detector = pupil_detectors[ session_settings.get("last_pupil_detector", Detector_2D.__name__) ] g_pool.pupil_detector = last_pupil_detector(g_pool, pupil_detector_settings) def set_display_mode_info(val): g_pool.display_mode = val g_pool.display_mode_info.text = g_pool.display_mode_info_text[val] def set_detector(new_detector): g_pool.pupil_detector.deinit_ui() g_pool.pupil_detector.cleanup() g_pool.pupil_detector = new_detector(g_pool) g_pool.pupil_detector.init_ui() def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is open the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed # Initialize glfw glfw.glfwInit() title = "Pupil Capture - eye {}".format(eye_id) width, height = g_pool.capture.frame_size width *= 2 height *= 2 width += icon_bar_width width, height = session_settings.get("window_size", (width, height)) main_window = glfw.glfwCreateWindow(width, height, title, None, None) window_pos = session_settings.get("window_position", window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() # UI callback functions def set_scale(new_scale): g_pool.gui_user_scale = new_scale on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() g_pool.image_tex.update_from_ndarray(np.ones((1, 1), dtype=np.uint8) + 125) # setup GUI g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0) g_pool.menubar = ui.Scrolling_Menu( "Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos="left" ) g_pool.iconbar = ui.Scrolling_Menu( "Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden" ) g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.iconbar) general_settings = ui.Growing_Menu("General", header_pos="headline") general_settings.append( ui.Selector( "gui_user_scale", g_pool, setter=set_scale, selection=[0.8, 0.9, 1.0, 1.1, 1.2], label="Interface Size", ) ) def set_window_size(): f_width, f_height = g_pool.capture.frame_size f_width *= 2 f_height *= 2 f_width += int(icon_bar_width * g_pool.gui.scale) glfw.glfwSetWindowSize(main_window, f_width, f_height) def uroi_on_mouse_button(button, action, mods): if g_pool.display_mode == "roi": if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt: g_pool.u_r.active_edit_pt = False # if the roi interacts we dont want # the gui to interact as well return elif action == glfw.GLFW_PRESS: x, y = glfw.glfwGetCursorPos(main_window) # pos = normalize(pos, glfw.glfwGetWindowSize(main_window)) x *= hdpi_factor y *= hdpi_factor pos = normalize((x, y), camera_render_size) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] # Position in img pixels pos = denormalize( pos, g_pool.capture.frame_size ) # Position in img pixels if g_pool.u_r.mouse_over_edit_pt( pos, g_pool.u_r.handle_size, g_pool.u_r.handle_size ): # if the roi interacts we dont want # the gui to interact as well return general_settings.append(ui.Button("Reset window size", set_window_size)) general_settings.append(ui.Switch("flip", g_pool, label="Flip image display")) general_settings.append( ui.Selector( "display_mode", g_pool, setter=set_display_mode_info, selection=["camera_image", "roi", "algorithm"], labels=["Camera Image", "ROI", "Algorithm"], label="Mode", ) ) g_pool.display_mode_info = ui.Info_Text( g_pool.display_mode_info_text[g_pool.display_mode] ) general_settings.append(g_pool.display_mode_info) detector_selector = ui.Selector( "pupil_detector", getter=lambda: g_pool.pupil_detector.__class__, setter=set_detector, selection=[Detector_Dummy, Detector_2D, Detector_3D], labels=["disabled", "C++ 2d detector", "C++ 3d detector"], label="Detection method", ) general_settings.append(detector_selector) g_pool.menubar.append(general_settings) icon = ui.Icon( "collapsed", general_settings, label=chr(0xE8B8), on_val=False, off_val=True, setter=toggle_general_settings, label_font="pupil_icons", ) icon.tooltip = "General Settings" g_pool.iconbar.append(icon) toggle_general_settings(False) g_pool.pupil_detector.init_ui() g_pool.capture.init_ui() g_pool.capture_manager.init_ui() g_pool.writer = None def replace_source(source_class_name, source_settings): g_pool.capture.deinit_ui() g_pool.capture.cleanup() g_pool.capture = source_class_by_name[source_class_name]( g_pool, **source_settings ) g_pool.capture.init_ui() if g_pool.writer: logger.info("Done recording.") try: g_pool.writer.release() except RuntimeError: logger.error("No eye video recorded") g_pool.writer = None g_pool.replace_source = replace_source # for ndsi capture # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetWindowIconifyCallback(main_window, on_iconify) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) # load last gui configuration g_pool.gui.configuration = session_settings.get("ui_config", {}) # set up performance graphs pid = os.getpid() ps = psutil.Process(pid) ts = g_pool.get_timestamp() cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 50) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = "CPU %0.1f" fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 50) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" g_pool.graphs = [cpu_graph, fps_graph] # set the last saved window size on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) should_publish_frames = False frame_publish_format = "jpeg" frame_publish_format_recent_warning = False # create a timer to control window update frequency window_update_timer = timer(1 / 60) def window_should_update(): return next(window_update_timer) logger.warning("Process started.") frame = None # Event loop while not glfw.glfwWindowShouldClose(main_window): if notify_sub.new_data: t, notification = notify_sub.recv() subject = notification["subject"] if subject.startswith("eye_process.should_stop"): if notification["eye_id"] == eye_id: break elif subject == "set_detection_mapping_mode": if notification["mode"] == "3d": if not isinstance(g_pool.pupil_detector, Detector_3D): set_detector(Detector_3D) detector_selector.read_only = True elif notification["mode"] == "2d": if not isinstance(g_pool.pupil_detector, Detector_2D): set_detector(Detector_2D) detector_selector.read_only = False else: if not isinstance(g_pool.pupil_detector, Detector_Dummy): set_detector(Detector_Dummy) detector_selector.read_only = True elif subject == "recording.started": if notification["record_eye"] and g_pool.capture.online: record_path = notification["rec_path"] raw_mode = notification["compression"] logger.info("Will save eye video to: {}".format(record_path)) video_path = os.path.join( record_path, "eye{}.mp4".format(eye_id) ) if raw_mode and frame and g_pool.capture.jpeg_support: g_pool.writer = JPEG_Writer( video_path, g_pool.capture.frame_rate ) elif hasattr(g_pool.capture._recent_frame, "h264_buffer"): g_pool.writer = H264Writer( video_path, g_pool.capture.frame_size[0], g_pool.capture.frame_size[1], g_pool.capture.frame_rate, ) else: g_pool.writer = AV_Writer( video_path, g_pool.capture.frame_rate ) elif subject == "recording.stopped": if g_pool.writer: logger.info("Done recording.") try: g_pool.writer.release() except RuntimeError: logger.error("No eye video recorded") g_pool.writer = None elif subject.startswith("meta.should_doc"): ipc_socket.notify( { "subject": "meta.doc", "actor": "eye{}".format(eye_id), "doc": eye.__doc__, } ) elif subject.startswith("frame_publishing.started"): should_publish_frames = True frame_publish_format = notification.get("format", "jpeg") elif subject.startswith("frame_publishing.stopped"): should_publish_frames = False frame_publish_format = "jpeg" elif ( subject.startswith("start_eye_capture") and notification["target"] == g_pool.process ): replace_source(notification["name"], notification["args"]) elif notification["subject"].startswith("pupil_detector.set_property"): target_process = notification.get("target", g_pool.process) should_apply = target_process == g_pool.process if should_apply: try: property_name = notification["name"] property_value = notification["value"] if "2d" in notification["subject"]: g_pool.pupil_detector.set_2d_detector_property( property_name, property_value ) elif "3d" in notification["subject"]: if not isinstance(g_pool.pupil_detector, Detector_3D): raise ValueError( "3d properties are only available" " if 3d detector is active" ) g_pool.pupil_detector.set_3d_detector_property( property_name, property_value ) else: raise KeyError( "Notification subject does not " "specifiy detector type." ) logger.debug( "`{}` property set to {}".format( property_name, property_value ) ) except KeyError: logger.error("Malformed notification received") logger.debug(traceback.format_exc()) except (ValueError, TypeError): logger.error("Invalid property or value") logger.debug(traceback.format_exc()) elif notification["subject"].startswith( "pupil_detector.broadcast_properties" ): target_process = notification.get("target", g_pool.process) should_respond = target_process == g_pool.process if should_respond: props = g_pool.pupil_detector.get_detector_properties() properties_broadcast = { "subject": "pupil_detector.properties.{}".format(eye_id), **props, # add properties to broadcast } ipc_socket.notify(properties_broadcast) g_pool.capture.on_notify(notification) g_pool.capture_manager.on_notify(notification) # Get an image from the grabber event = {} g_pool.capture.recent_events(event) frame = event.get("frame") g_pool.capture_manager.recent_events(event) if frame: f_width, f_height = g_pool.capture.frame_size if (g_pool.u_r.array_shape[0], g_pool.u_r.array_shape[1]) != ( f_height, f_width, ): g_pool.pupil_detector.on_resolution_change( (g_pool.u_r.array_shape[1], g_pool.u_r.array_shape[0]), g_pool.capture.frame_size, ) g_pool.u_r = UIRoi((f_height, f_width)) if should_publish_frames: try: if frame_publish_format == "jpeg": data = frame.jpeg_buffer elif frame_publish_format == "yuv": data = frame.yuv_buffer elif frame_publish_format == "bgr": data = frame.bgr elif frame_publish_format == "gray": data = frame.gray assert data is not None except (AttributeError, AssertionError, NameError): if not frame_publish_format_recent_warning: frame_publish_format_recent_warning = True logger.warning( '{}s are not compatible with format "{}"'.format( type(frame), frame_publish_format ) ) else: frame_publish_format_recent_warning = False pupil_socket.send( { "topic": "frame.eye.{}".format(eye_id), "width": frame.width, "height": frame.height, "index": frame.index, "timestamp": frame.timestamp, "format": frame_publish_format, "__raw_data__": [data], } ) t = frame.timestamp dt, ts = t - ts, t try: fps_graph.add(1.0 / dt) except ZeroDivisionError: pass if g_pool.writer: g_pool.writer.write_video_frame(frame) # pupil ellipse detection result = g_pool.pupil_detector.detect( frame, g_pool.u_r, g_pool.display_mode == "algorithm" ) if result is not None: result["id"] = eye_id result["topic"] = "pupil.{}".format(eye_id) pupil_socket.send(result) cpu_graph.update() # GL drawing if window_should_update(): if is_window_visible(main_window): glfw.glfwMakeContextCurrent(main_window) clear_gl_screen() if frame: # switch to work in normalized coordinate space if g_pool.display_mode == "algorithm": g_pool.image_tex.update_from_ndarray(frame.img) elif g_pool.display_mode in ("camera_image", "roi"): g_pool.image_tex.update_from_ndarray(frame.gray) else: pass glViewport(0, 0, *camera_render_size) make_coord_system_norm_based(g_pool.flip) g_pool.image_tex.draw() f_width, f_height = g_pool.capture.frame_size make_coord_system_pixel_based((f_height, f_width, 3), g_pool.flip) if frame and result: if result["method"] == "3d c++": eye_ball = result["projected_sphere"] try: pts = cv2.ellipse2Poly( ( int(eye_ball["center"][0]), int(eye_ball["center"][1]), ), ( int(eye_ball["axes"][0] / 2), int(eye_ball["axes"][1] / 2), ), int(eye_ball["angle"]), 0, 360, 8, ) except ValueError as e: pass else: draw_polyline( pts, 2, RGBA(0.0, 0.9, 0.1, result["model_confidence"]), ) if result["confidence"] > 0: if "ellipse" in result: pts = cv2.ellipse2Poly( ( int(result["ellipse"]["center"][0]), int(result["ellipse"]["center"][1]), ), ( int(result["ellipse"]["axes"][0] / 2), int(result["ellipse"]["axes"][1] / 2), ), int(result["ellipse"]["angle"]), 0, 360, 15, ) confidence = result["confidence"] * 0.7 draw_polyline(pts, 1, RGBA(1.0, 0, 0, confidence)) draw_points( [result["ellipse"]["center"]], size=20, color=RGBA(1.0, 0.0, 0.0, confidence), sharpness=1.0, ) glViewport(0, 0, *camera_render_size) make_coord_system_pixel_based((f_height, f_width, 3), g_pool.flip) # render the ROI g_pool.u_r.draw(g_pool.gui.scale) if g_pool.display_mode == "roi": g_pool.u_r.draw_points(g_pool.gui.scale) glViewport(0, 0, *window_size) make_coord_system_pixel_based((*window_size[::-1], 3), g_pool.flip) # render graphs fps_graph.draw() cpu_graph.draw() # render GUI unused_elements = g_pool.gui.update() for butt in unused_elements.buttons: uroi_on_mouse_button(*butt) make_coord_system_pixel_based((*window_size[::-1], 3), g_pool.flip) g_pool.pupil_detector.visualize() # detector decides if we visualize or not # update screen glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() # END while running # in case eye recording was still runnnig: Save&close if g_pool.writer: logger.info("Done recording eye.") g_pool.writer = None glfw.glfwRestoreWindow(main_window) # need to do this for windows os # save session persistent settings session_settings["gui_scale"] = g_pool.gui_user_scale session_settings["roi"] = g_pool.u_r.get() session_settings["flip"] = g_pool.flip session_settings["display_mode"] = g_pool.display_mode session_settings["ui_config"] = g_pool.gui.configuration session_settings["capture_settings"] = ( g_pool.capture.class_name, g_pool.capture.get_init_dict(), ) session_settings["capture_manager_settings"] = ( g_pool.capture_manager.class_name, g_pool.capture_manager.get_init_dict(), ) session_settings["window_position"] = glfw.glfwGetWindowPos(main_window) session_settings["version"] = str(g_pool.version) session_settings[ "last_pupil_detector" ] = g_pool.pupil_detector.__class__.__name__ session_settings[ "pupil_detector_settings" ] = g_pool.pupil_detector.get_settings() session_window_size = glfw.glfwGetWindowSize(main_window) if 0 not in session_window_size: session_settings["window_size"] = session_window_size session_settings.close() g_pool.capture.deinit_ui() g_pool.capture_manager.deinit_ui() g_pool.pupil_detector.deinit_ui() g_pool.pupil_detector.cleanup() g_pool.capture_manager.cleanup() g_pool.capture.cleanup() glfw.glfwDestroyWindow(main_window) g_pool.gui.terminate() glfw.glfwTerminate() logger.info("Process shutting down.")
def eye(g_pool,cap_src,cap_size,rx_from_world,eye_id=0): """ Creates a window, gl context. Grabs images from a capture. Streams Pupil coordinates into g_pool.pupil_queue """ # modify the root logger for this process logger = logging.getLogger() # remove inherited handlers logger.handlers = [] # create file handler which logs even debug messages fh = logging.FileHandler(os.path.join(g_pool.user_dir,'eye%s.log'%eye_id),mode='w') # fh.setLevel(logging.DEBUG) # create console handler with a higher log level ch = logging.StreamHandler() ch.setLevel(logger.level+10) # create formatter and add it to the handlers formatter = logging.Formatter('Eye'+str(eye_id)+' Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s') fh.setFormatter(formatter) formatter = logging.Formatter('EYE'+str(eye_id)+' Process [%(levelname)s] %(name)s : %(message)s') ch.setFormatter(formatter) # add the handlers to the logger logger.addHandler(fh) logger.addHandler(ch) # create logger for the context of this function logger = logging.getLogger(__name__) #UI Platform tweaks if platform.system() == 'Linux': scroll_factor = 10.0 window_position_default = (600,300*eye_id) elif platform.system() == 'Windows': scroll_factor = 1.0 window_position_default = (600,31+300*eye_id) else: scroll_factor = 1.0 window_position_default = (600,300*eye_id) # Callback functions def on_resize(window,w, h): active_window = glfwGetCurrentContext() glfwMakeContextCurrent(window) hdpi_factor = glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0] w,h = w*hdpi_factor, h*hdpi_factor g_pool.gui.update_window(w,h) graph.adjust_size(w,h) adjust_gl_view(w,h) # for p in g_pool.plugins: # p.on_window_resize(window,w,h) glfwMakeContextCurrent(active_window) def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key,scancode,action,mods) def on_char(window,char): g_pool.gui.update_char(char) def on_button(window,button, action, mods): if g_pool.display_mode == 'roi': if action == GLFW_RELEASE and u_r.active_edit_pt: u_r.active_edit_pt = False return # if the roi interacts we dont what the gui to interact as well elif action == GLFW_PRESS: pos = glfwGetCursorPos(window) pos = normalize(pos,glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1-pos[0],1-pos[1] pos = denormalize(pos,(frame.width,frame.height)) # Position in img pixels if u_r.mouse_over_edit_pt(pos,u_r.handle_size+40,u_r.handle_size+40): return # if the roi interacts we dont what the gui to interact as well g_pool.gui.update_button(button,action,mods) def on_pos(window,x, y): hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0]) g_pool.gui.update_mouse(x*hdpi_factor,y*hdpi_factor) if u_r.active_edit_pt: pos = normalize((x,y),glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1-pos[0],1-pos[1] pos = denormalize(pos,(frame.width,frame.height) ) u_r.move_vertex(u_r.active_pt_idx,pos) def on_scroll(window,x,y): g_pool.gui.update_scroll(x,y*scroll_factor) def on_close(window): g_pool.quit.value = True logger.info('Process closing from window') # load session persistent settings session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_eye%s'%eye_id)) if session_settings.get("version",VersionFormat('0.0')) < g_pool.version: logger.info("Session setting are from older version of this app. I will not use those.") session_settings.clear() # Initialize capture cap = autoCreateCapture(cap_src, timebase=g_pool.timebase) cap.frame_size = cap_size cap.frame_rate = 90 #default cap.settings = session_settings.get('capture_settings',{}) # Test capture try: frame = cap.get_frame() except CameraCaptureError: logger.error("Could not retrieve image from capture") cap.close() return g_pool.capture = cap g_pool.flip = session_settings.get('flip',False) # any object we attach to the g_pool object *from now on* will only be visible to this process! # vars should be declared here to make them visible to the code reader. g_pool.window_size = session_settings.get('window_size',1.) g_pool.display_mode = session_settings.get('display_mode','camera_image') g_pool.display_mode_info_text = {'camera_image': "Raw eye camera image. This uses the least amount of CPU power", 'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be a small as possible but big enough to capture to pupil in its movements", 'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters with in the Pupil Detection menu below."} # g_pool.draw_pupil = session_settings.get('draw_pupil',True) u_r = UIRoi(frame.img.shape) u_r.set(session_settings.get('roi',u_r.get())) writer = None pupil_detector = Canny_Detector(g_pool) # UI callback functions def set_scale(new_scale): g_pool.gui.scale = new_scale g_pool.gui.collect_menus() def set_display_mode_info(val): g_pool.display_mode = val g_pool.display_mode_info.text = g_pool.display_mode_info_text[val] window_pos = session_settings.get('window_position',window_position_default) width,height = session_settings.get('window_size',(frame.width, frame.height)) # Initialize glfw glfwInit() if g_pool.binocular: title = "Binocular eye %s"%eye_id else: title = 'Eye' main_window = glfwCreateWindow(width,height, title, None, None) glfwMakeContextCurrent(main_window) cygl_init() # Register callbacks main_window glfwSetWindowSizeCallback(main_window,on_resize) glfwSetWindowCloseCallback(main_window,on_close) glfwSetKeyCallback(main_window,on_key) glfwSetCharCallback(main_window,on_char) glfwSetMouseButtonCallback(main_window,on_button) glfwSetCursorPosCallback(main_window,on_pos) glfwSetScrollCallback(main_window,on_scroll) # gl_state settings basic_gl_setup() g_pool.image_tex = create_named_texture(frame.img.shape) update_named_texture(g_pool.image_tex,frame.img) # refresh speed settings glfwSwapInterval(0) glfwSetWindowPos(main_window,window_pos[0],window_pos[1]) #setup GUI g_pool.gui = ui.UI() g_pool.gui.scale = session_settings.get('gui_scale',1) g_pool.sidebar = ui.Scrolling_Menu("Settings",pos=(-300,0),size=(0,0),header_pos='left') general_settings = ui.Growing_Menu('General') general_settings.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=1.,max=2.5,label='Interface Size')) general_settings.append(ui.Button('Reset window size',lambda: glfwSetWindowSize(main_window,frame.width,frame.height)) ) general_settings.append(ui.Selector('display_mode',g_pool,setter=set_display_mode_info,selection=['camera_image','roi','algorithm'], labels=['Camera Image', 'ROI', 'Algorithm'], label="Mode") ) general_settings.append(ui.Switch('flip',g_pool,label='Flip image display')) g_pool.display_mode_info = ui.Info_Text(g_pool.display_mode_info_text[g_pool.display_mode]) general_settings.append(g_pool.display_mode_info) g_pool.sidebar.append(general_settings) g_pool.gui.append(g_pool.sidebar) g_pool.gui.append(ui.Hot_Key("quit",setter=on_close,getter=lambda:True,label="X",hotkey=GLFW_KEY_ESCAPE)) # let the camera add its GUI g_pool.capture.init_gui(g_pool.sidebar) # let detector add its GUI pupil_detector.init_gui(g_pool.sidebar) # load last gui configuration g_pool.gui.configuration = session_settings.get('ui_config',{}) #set the last saved window size on_resize(main_window, *glfwGetWindowSize(main_window)) #set up performance graphs pid = os.getpid() ps = psutil.Process(pid) ts = frame.timestamp cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20,130) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140,130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" # Event loop while not g_pool.quit.value: # Get an image from the grabber try: frame = cap.get_frame() except CameraCaptureError: logger.error("Capture from Camera Failed. Stopping.") break except EndofVideoFileError: logger.warning("Video File is done. Stopping") break #update performace graphs t = frame.timestamp dt,ts = t-ts,t try: fps_graph.add(1./dt) except ZeroDivisionError: pass cpu_graph.update() ### RECORDING of Eye Video (on demand) ### # Setup variables and lists for recording if rx_from_world.poll(): command,raw_mode = rx_from_world.recv() if command is not None: record_path = command logger.info("Will save eye video to: %s"%record_path) video_path = os.path.join(record_path, "eye%s.mkv"%eye_id) timestamps_path = os.path.join(record_path, "eye%s_timestamps.npy"%eye_id) if raw_mode: writer = JPEG_Dumper(video_path) else: writer = CV_Writer(video_path,float(cap.frame_rate), cap.frame_size) timestamps = [] else: logger.info("Done recording.") writer.release() writer = None np.save(timestamps_path,np.asarray(timestamps)) del timestamps if writer: writer.write_video_frame(frame) timestamps.append(frame.timestamp) # pupil ellipse detection result = pupil_detector.detect(frame,user_roi=u_r,visualize=g_pool.display_mode == 'algorithm') result['id'] = eye_id # stream the result g_pool.pupil_queue.put(result) # GL drawing glfwMakeContextCurrent(main_window) clear_gl_screen() # switch to work in normalized coordinate space if g_pool.display_mode == 'algorithm': update_named_texture(g_pool.image_tex,frame.img) elif g_pool.display_mode in ('camera_image','roi'): update_named_texture(g_pool.image_tex,frame.gray) else: pass make_coord_system_norm_based(g_pool.flip) draw_named_texture(g_pool.image_tex) # switch to work in pixel space make_coord_system_pixel_based((frame.height,frame.width,3),g_pool.flip) if result['confidence'] >0: if result.has_key('axes'): pts = cv2.ellipse2Poly( (int(result['center'][0]),int(result['center'][1])), (int(result['axes'][0]/2),int(result['axes'][1]/2)), int(result['angle']),0,360,15) cygl_draw_polyline(pts,1,cygl_rgba(1.,0,0,.5)) cygl_draw_points([result['center']],size=20,color=cygl_rgba(1.,0.,0.,.5),sharpness=1.) # render graphs graph.push_view() fps_graph.draw() cpu_graph.draw() graph.pop_view() # render GUI g_pool.gui.update() #render the ROI if g_pool.display_mode == 'roi': u_r.draw(g_pool.gui.scale) #update screen glfwSwapBuffers(main_window) glfwPollEvents() # END while running # in case eye recording was still runnnig: Save&close if writer: logger.info("Done recording eye.") writer = None np.save(timestamps_path,np.asarray(timestamps)) # save session persistent settings session_settings['gui_scale'] = g_pool.gui.scale session_settings['roi'] = u_r.get() session_settings['flip'] = g_pool.flip session_settings['display_mode'] = g_pool.display_mode session_settings['ui_config'] = g_pool.gui.configuration session_settings['capture_settings'] = g_pool.capture.settings session_settings['window_size'] = glfwGetWindowSize(main_window) session_settings['window_position'] = glfwGetWindowPos(main_window) session_settings['version'] = g_pool.version session_settings.close() pupil_detector.cleanup() glfwDestroyWindow(main_window) glfwTerminate() cap.close() #flushing queue in case world process did not exit gracefully while not g_pool.pupil_queue.empty(): g_pool.pupil_queue.get() g_pool.pupil_queue.close() logger.debug("Process done")
def world( timebase, eye_procs_alive, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version, preferred_remote_port, hide_ui, ): """Reads world video and runs plugins. Creates a window, gl context. Grabs images from a capture. Maps pupil to gaze data Can run various plug-ins. Reacts to notifications: ``set_detection_mapping_mode`` ``eye_process.started`` ``start_plugin`` Emits notifications: ``eye_process.should_start`` ``eye_process.should_stop`` ``set_detection_mapping_mode`` ``world_process.started`` ``world_process.stopped`` ``recording.should_stop``: Emits on camera failure ``launcher_process.should_stop`` Emits data: ``gaze``: Gaze data from current gaze mapping plugin.`` ``*``: any other plugin generated data in the events that it not [dt,pupil,gaze]. """ # We defer the imports because of multiprocessing. # Otherwise the world process each process also loads the other imports. # This is not harmful but unnecessary. # general imports from time import sleep import logging # networking import zmq import zmq_tools # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify", )) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.NOTSET) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) def launch_eye_process(eye_id, delay=0): n = { "subject": "eye_process.should_start.{}".format(eye_id), "eye_id": eye_id, "delay": delay, } ipc_pub.notify(n) def stop_eye_process(eye_id): n = { "subject": "eye_process.should_stop.{}".format(eye_id), "eye_id": eye_id, "delay": 0.2, } ipc_pub.notify(n) def start_stop_eye(eye_id, make_alive): if make_alive: launch_eye_process(eye_id) else: stop_eye_process(eye_id) def set_detection_mapping_mode(new_mode): n = {"subject": "set_detection_mapping_mode", "mode": new_mode} ipc_pub.notify(n) try: from background_helper import IPC_Logging_Task_Proxy IPC_Logging_Task_Proxy.push_url = ipc_push_url from tasklib.background.patches import IPCLoggingPatch IPCLoggingPatch.ipc_push_url = ipc_push_url # display import glfw from version_utils import VersionFormat from pyglui import ui, cygl, __version__ as pyglui_version assert VersionFormat(pyglui_version) >= VersionFormat( "1.24"), "pyglui out of date, please upgrade to newest version" from pyglui.cygl.utils import Named_Texture import gl_utils # helpers/utils from file_methods import Persistent_Dict from methods import normalize, denormalize, delta_t, get_system_info, timer from uvc import get_time_monotonic logger.info("Application Version: {}".format(version)) logger.info("System Info: {}".format(get_system_info())) import audio # trigger pupil detector cpp build: import pupil_detectors del pupil_detectors # Plug-ins from plugin import ( Plugin, System_Plugin_Base, Plugin_List, import_runtime_plugins, ) from plugin_manager import Plugin_Manager from calibration_routines import ( calibration_plugins, gaze_mapping_plugins, Calibration_Plugin, Gaze_Mapping_Plugin, ) from fixation_detector import Fixation_Detector from recorder import Recorder from display_recent_gaze import Display_Recent_Gaze from time_sync import Time_Sync from pupil_remote import Pupil_Remote from pupil_groups import Pupil_Groups from surface_tracker import Surface_Tracker_Online from log_display import Log_Display from annotations import Annotation_Capture from log_history import Log_History from frame_publisher import Frame_Publisher from blink_detection import Blink_Detection from video_capture import ( source_classes, manager_classes, Base_Manager, Base_Source, ) from pupil_data_relay import Pupil_Data_Relay from remote_recorder import Remote_Recorder from audio_capture import Audio_Capture from accuracy_visualizer import Accuracy_Visualizer # from saccade_detector import Saccade_Detector from system_graphs import System_Graphs from camera_intrinsics_estimation import Camera_Intrinsics_Estimation from hololens_relay import Hololens_Relay from head_pose_tracker.online_head_pose_tracker import Online_Head_Pose_Tracker # UI Platform tweaks if platform.system() == "Linux": scroll_factor = 10.0 window_position_default = (30, 30) elif platform.system() == "Windows": scroll_factor = 10.0 window_position_default = (8, 90) else: scroll_factor = 1.0 window_position_default = (0, 0) process_was_interrupted = False def interrupt_handler(sig, frame): import traceback trace = traceback.format_stack(f=frame) logger.debug(f"Caught signal {sig} in:\n" + "".join(trace)) nonlocal process_was_interrupted process_was_interrupted = True signal.signal(signal.SIGINT, interrupt_handler) icon_bar_width = 50 window_size = None camera_render_size = None hdpi_factor = 1.0 # g_pool holds variables for this process they are accessible to all plugins g_pool = SimpleNamespace() g_pool.app = "capture" g_pool.process = "world" g_pool.user_dir = user_dir g_pool.version = version g_pool.timebase = timebase g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.eye_procs_alive = eye_procs_alive g_pool.preferred_remote_port = preferred_remote_port def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic # manage plugins runtime_plugins = import_runtime_plugins( os.path.join(g_pool.user_dir, "plugins")) user_plugins = [ Audio_Capture, Pupil_Groups, Frame_Publisher, Pupil_Remote, Time_Sync, Surface_Tracker_Online, Annotation_Capture, Log_History, Fixation_Detector, Blink_Detection, Remote_Recorder, Accuracy_Visualizer, Camera_Intrinsics_Estimation, Hololens_Relay, Online_Head_Pose_Tracker, ] system_plugins = ([ Log_Display, Display_Recent_Gaze, Recorder, Pupil_Data_Relay, Plugin_Manager, System_Graphs, ] + manager_classes + source_classes) plugins = (system_plugins + user_plugins + runtime_plugins + calibration_plugins + gaze_mapping_plugins) user_plugins += [ p for p in runtime_plugins if not isinstance( p, ( Base_Manager, Base_Source, System_Plugin_Base, Calibration_Plugin, Gaze_Mapping_Plugin, ), ) ] g_pool.plugin_by_name = {p.__name__: p for p in plugins} default_capture_settings = { "preferred_names": [ "Pupil Cam1 ID2", "Logitech Camera", "(046d:081d)", "C510", "B525", "C525", "C615", "C920", "C930e", ], "frame_size": (1280, 720), "frame_rate": 30, } default_plugins = [ ("UVC_Source", default_capture_settings), ("Pupil_Data_Relay", {}), ("UVC_Manager", {}), ("Log_Display", {}), ("Dummy_Gaze_Mapper", {}), ("Display_Recent_Gaze", {}), ("Screen_Marker_Calibration", {}), ("Recorder", {}), ("Pupil_Remote", {}), ("Accuracy_Visualizer", {}), ("Plugin_Manager", {}), ("System_Graphs", {}), ] # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal camera_render_size nonlocal hdpi_factor if w == 0 or h == 0: return hdpi_factor = glfw.getHDPIFactor(window) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor window_size = w, h camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(*window_size) g_pool.gui.collect_menus() for p in g_pool.plugins: p.on_window_resize(window, *camera_render_size) def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) pos = x, y pos = normalize(pos, camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_pos(pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, count, paths): paths = [paths[x].decode("utf-8") for x in range(count)] for plugin in g_pool.plugins: if plugin.on_drop(paths): break tick = delta_t() def get_dt(): return next(tick) # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, "user_settings_world")) if VersionFormat(session_settings.get("version", "0.0")) != g_pool.version: logger.info( "Session setting are from a different version of this app. I will not use those." ) session_settings.clear() g_pool.min_calibration_confidence = session_settings.get( "min_calibration_confidence", 0.8) g_pool.detection_mapping_mode = session_settings.get( "detection_mapping_mode", "3d") g_pool.active_calibration_plugin = None g_pool.active_gaze_mapping_plugin = None g_pool.capture = None audio.audio_mode = session_settings.get("audio_mode", audio.default_audio_mode) def handle_notifications(noti): subject = noti["subject"] if subject == "set_detection_mapping_mode": if noti["mode"] == "2d": if ("Vector_Gaze_Mapper" in g_pool.active_gaze_mapping_plugin.class_name): logger.warning( "The gaze mapper is not supported in 2d mode. Please recalibrate." ) g_pool.plugins.add( g_pool.plugin_by_name["Dummy_Gaze_Mapper"]) g_pool.detection_mapping_mode = noti["mode"] elif subject == "start_plugin": try: g_pool.plugins.add(g_pool.plugin_by_name[noti["name"]], args=noti.get("args", {})) except KeyError as err: logger.error(f"Attempt to load unknown plugin: {err}") elif subject == "stop_plugin": for p in g_pool.plugins: if p.class_name == noti["name"]: p.alive = False g_pool.plugins.clean() elif subject == "eye_process.started": noti = { "subject": "set_detection_mapping_mode", "mode": g_pool.detection_mapping_mode, } ipc_pub.notify(noti) elif subject == "set_min_calibration_confidence": g_pool.min_calibration_confidence = noti["value"] elif subject.startswith("meta.should_doc"): ipc_pub.notify({ "subject": "meta.doc", "actor": g_pool.app, "doc": world.__doc__ }) for p in g_pool.plugins: if (p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify): ipc_pub.notify({ "subject": "meta.doc", "actor": p.class_name, "doc": p.on_notify.__doc__, }) elif subject == "world_process.adapt_window_size": set_window_size() width, height = session_settings.get("window_size", (1280 + icon_bar_width, 720)) # window and gl setup glfw.glfwInit() if hide_ui: glfw.glfwWindowHint(glfw.GLFW_VISIBLE, 0) # hide window main_window = glfw.glfwCreateWindow(width, height, "Pupil Capture - World") window_pos = session_settings.get("window_position", window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window def set_scale(new_scale): g_pool.gui_user_scale = new_scale window_size = ( camera_render_size[0] + int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor), glfw.glfwGetFramebufferSize(main_window)[1], ) logger.warning(icon_bar_width * g_pool.gui_user_scale * hdpi_factor) glfw.glfwSetWindowSize(main_window, *window_size) def reset_restart(): logger.warning("Resetting all settings and restarting Capture.") glfw.glfwSetWindowShouldClose(main_window, True) ipc_pub.notify({"subject": "clear_settings_process.should_start"}) ipc_pub.notify({ "subject": "world_process.should_start", "delay": 2.0 }) def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is opened, the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed # setup GUI g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0) g_pool.menubar = ui.Scrolling_Menu("Settings", pos=(-400, 0), size=(-icon_bar_width, 0), header_pos="left") g_pool.iconbar = ui.Scrolling_Menu("Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden") g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100), (120, -100)) g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.iconbar) g_pool.gui.append(g_pool.quickbar) general_settings = ui.Growing_Menu("General", header_pos="headline") general_settings.append( ui.Selector( "gui_user_scale", g_pool, setter=set_scale, selection=[0.6, 0.8, 1.0, 1.2, 1.4], label="Interface size", )) def set_window_size(): f_width, f_height = g_pool.capture.frame_size f_width += int(icon_bar_width * g_pool.gui.scale) glfw.glfwSetWindowSize(main_window, f_width, f_height) on_resize(main_window, f_width, f_height) general_settings.append(ui.Button("Reset window size", set_window_size)) general_settings.append( ui.Selector("audio_mode", audio, selection=audio.audio_modes)) general_settings.append( ui.Selector( "detection_mapping_mode", g_pool, label="detection & mapping mode", setter=set_detection_mapping_mode, selection=["disabled", "2d", "3d"], )) general_settings.append( ui.Switch( "eye0_process", label="Detect eye 0", setter=lambda alive: start_stop_eye(0, alive), getter=lambda: eye_procs_alive[0].value, )) general_settings.append( ui.Switch( "eye1_process", label="Detect eye 1", setter=lambda alive: start_stop_eye(1, alive), getter=lambda: eye_procs_alive[1].value, )) general_settings.append( ui.Info_Text("Capture Version: {}".format(g_pool.version))) general_settings.append( ui.Button("Restart with default settings", reset_restart)) g_pool.menubar.append(general_settings) icon = ui.Icon( "collapsed", general_settings, label=chr(0xE8B8), on_val=False, off_val=True, setter=toggle_general_settings, label_font="pupil_icons", ) icon.tooltip = "General Settings" g_pool.iconbar.append(icon) user_plugin_separator = ui.Separator() user_plugin_separator.order = 0.35 g_pool.iconbar.append(user_plugin_separator) # plugins that are loaded based on user settings from previous session g_pool.plugins = Plugin_List( g_pool, session_settings.get("loaded_plugins", default_plugins)) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() toggle_general_settings(True) # now that we have a proper window we can load the last gui configuration g_pool.gui.configuration = session_settings.get("ui_config", {}) # create a timer to control window update frequency window_update_timer = timer(1 / 60) def window_should_update(): return next(window_update_timer) # trigger setup of window and gl sizes on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) if session_settings.get("eye1_process_alive", True): launch_eye_process(1, delay=0.6) if session_settings.get("eye0_process_alive", True): launch_eye_process(0, delay=0.3) ipc_pub.notify({"subject": "world_process.started"}) logger.warning("Process started.") # Event loop while (not glfw.glfwWindowShouldClose(main_window) and not process_was_interrupted): # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) # a dictionary that allows plugins to post and read events events = {} # report time between now and the last loop interation events["dt"] = get_dt() # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() # "blacklisted" events that were already sent del events["pupil"] del events["gaze"] # delete if exists. More expensive than del, so only use it when key might not exist events.pop("annotation", None) # send new events to ipc: if "frame" in events: del events["frame"] # send explicitly with frame publisher if "depth_frame" in events: del events["depth_frame"] if "audio_packets" in events: del events["audio_packets"] del events["dt"] # no need to send this for data in events.values(): assert isinstance(data, (list, tuple)) for d in data: ipc_pub.send(d) glfw.glfwMakeContextCurrent(main_window) # render visual feedback from loaded plugins glfw.glfwPollEvents() if window_should_update() and gl_utils.is_window_visible( main_window): gl_utils.glViewport(0, 0, *camera_render_size) for p in g_pool.plugins: p.gl_display() gl_utils.glViewport(0, 0, *window_size) try: clipboard = glfw.glfwGetClipboardString( main_window).decode() except AttributeError: # clipboard is None, might happen on startup clipboard = "" g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.glfwSetClipboardString(main_window, user_input.clipboard.encode()) for button, action, mods in user_input.buttons: x, y = glfw.glfwGetCursorPos(main_window) pos = x * hdpi_factor, y * hdpi_factor pos = normalize(pos, camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for plugin in g_pool.plugins: if plugin.on_click(pos, button, action): break for key, scancode, action, mods in user_input.keys: for plugin in g_pool.plugins: if plugin.on_key(key, scancode, action, mods): break for char_ in user_input.chars: for plugin in g_pool.plugins: if plugin.on_char(char_): break glfw.glfwSwapBuffers(main_window) session_settings["loaded_plugins"] = g_pool.plugins.get_initializers() session_settings["gui_scale"] = g_pool.gui_user_scale session_settings["ui_config"] = g_pool.gui.configuration session_settings["version"] = str(g_pool.version) session_settings["eye0_process_alive"] = eye_procs_alive[0].value session_settings["eye1_process_alive"] = eye_procs_alive[1].value session_settings[ "min_calibration_confidence"] = g_pool.min_calibration_confidence session_settings[ "detection_mapping_mode"] = g_pool.detection_mapping_mode session_settings["audio_mode"] = audio.audio_mode if not hide_ui: glfw.glfwRestoreWindow( main_window) # need to do this for windows os session_settings["window_position"] = glfw.glfwGetWindowPos( main_window) session_window_size = glfw.glfwGetWindowSize(main_window) if 0 not in session_window_size: session_settings["window_size"] = session_window_size session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) glfw.glfwTerminate() except Exception: import traceback trace = traceback.format_exc() logger.error("Process Capture crashed with trace:\n{}".format(trace)) finally: # shut down eye processes: stop_eye_process(0) stop_eye_process(1) logger.info("Process shutting down.") ipc_pub.notify({"subject": "world_process.stopped"}) sleep(1.0)
def world(timebase, eyes_are_alive, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version): """Reads world video and runs plugins. Creates a window, gl context. Grabs images from a capture. Maps pupil to gaze data Can run various plug-ins. Reacts to notifications: ``set_detection_mapping_mode`` ``eye_process.started`` ``start_plugin`` Emits notifications: ``eye_process.should_start`` ``eye_process.should_stop`` ``set_detection_mapping_mode`` ``world_process.started`` ``world_process.stopped`` ``recording.should_stop``: Emits on camera failure ``launcher_process.should_stop`` Emits data: ``gaze``: Gaze data from current gaze mapping plugin.`` ``*``: any other plugin generated data in the events that it not [dt,pupil,gaze]. """ # We defer the imports because of multiprocessing. # Otherwise the world process each process also loads the other imports. # This is not harmful but unnecessary. # general imports import logging # networking import zmq import zmq_tools # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify',)) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.INFO) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) # display import glfw from pyglui import ui, graph, cygl, __version__ as pyglui_version assert pyglui_version >= '1.7' from pyglui.cygl.utils import Named_Texture import gl_utils # monitoring import psutil # helpers/utils from version_utils import VersionFormat from file_methods import Persistent_Dict from methods import normalize, denormalize, delta_t, get_system_info, timer from uvc import get_time_monotonic logger.info('Application Version: {}'.format(version)) logger.info('System Info: {}'.format(get_system_info())) import audio # trigger pupil detector cpp build: import pupil_detectors del pupil_detectors # Plug-ins from plugin import Plugin, Plugin_List, import_runtime_plugins from calibration_routines import calibration_plugins, gaze_mapping_plugins, Calibration_Plugin from fixation_detector import Fixation_Detector from recorder import Recorder from display_recent_gaze import Display_Recent_Gaze from time_sync import Time_Sync from pupil_remote import Pupil_Remote from pupil_groups import Pupil_Groups from surface_tracker import Surface_Tracker from log_display import Log_Display from annotations import Annotation_Capture from log_history import Log_History from frame_publisher import Frame_Publisher from blink_detection import Blink_Detection from video_capture import source_classes, manager_classes, Base_Manager from pupil_data_relay import Pupil_Data_Relay from remote_recorder import Remote_Recorder from audio_capture import Audio_Capture from accuracy_visualizer import Accuracy_Visualizer # UI Platform tweaks if platform.system() == 'Linux': scroll_factor = 10.0 window_position_default = (0, 0) elif platform.system() == 'Windows': scroll_factor = 10.0 window_position_default = (8, 31) else: scroll_factor = 1.0 window_position_default = (0, 0) # g_pool holds variables for this process they are accesible to all plugins g_pool = Global_Container() g_pool.app = 'capture' g_pool.process = 'world' g_pool.user_dir = user_dir g_pool.version = version g_pool.timebase = timebase g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.eyes_are_alive = eyes_are_alive def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic # manage plugins runtime_plugins = import_runtime_plugins(os.path.join(g_pool.user_dir, 'plugins')) calibration_plugins += [p for p in runtime_plugins if issubclass(p, Calibration_Plugin)] runtime_plugins = [p for p in runtime_plugins if not issubclass(p, Calibration_Plugin)] manager_classes += [p for p in runtime_plugins if issubclass(p, Base_Manager)] runtime_plugins = [p for p in runtime_plugins if not issubclass(p, Base_Manager)] user_launchable_plugins = [Audio_Capture, Pupil_Groups, Frame_Publisher, Pupil_Remote, Time_Sync, Surface_Tracker, Annotation_Capture, Log_History, Fixation_Detector, Blink_Detection, Remote_Recorder, Accuracy_Visualizer] + runtime_plugins system_plugins = [Log_Display, Display_Recent_Gaze, Recorder, Pupil_Data_Relay] plugin_by_index = (system_plugins + user_launchable_plugins + calibration_plugins + gaze_mapping_plugins + manager_classes + source_classes) name_by_index = [p.__name__ for p in plugin_by_index] plugin_by_name = dict(zip(name_by_index, plugin_by_index)) default_capture_settings = { 'preferred_names': ["Pupil Cam1 ID2", "Logitech Camera", "(046d:081d)", "C510", "B525", "C525", "C615", "C920", "C930e"], 'frame_size': (1280, 720), 'frame_rate': 30 } default_plugins = [("UVC_Source", default_capture_settings), ('Pupil_Data_Relay', {}), ('UVC_Manager', {}), ('Log_Display', {}), ('Dummy_Gaze_Mapper', {}), ('Display_Recent_Gaze', {}), ('Screen_Marker_Calibration', {}), ('Recorder', {}), ('Pupil_Remote', {})] # Callback functions def on_resize(window, w, h): if gl_utils.is_window_visible(window): hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() for g in g_pool.graphs: g.scale = hdpi_factor g.adjust_window_size(w, h) gl_utils.adjust_gl_view(w, h) for p in g_pool.plugins: p.on_window_resize(window, w, h) def on_iconify(window, iconified): g_pool.iconified = iconified def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): hdpi_factor = float(glfw.glfwGetFramebufferSize( window)[0] / glfw.glfwGetWindowSize(window)[0]) x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, count, paths): paths = [paths[x].decode('utf-8') for x in range(count)] for p in g_pool.plugins: p.on_drop(paths) tick = delta_t() def get_dt(): return next(tick) # load session persistent settings session_settings = Persistent_Dict(os.path.join(g_pool.user_dir, 'user_settings_world')) if VersionFormat(session_settings.get("version", '0.0')) != g_pool.version: logger.info("Session setting are from a different version of this app. I will not use those.") session_settings.clear() g_pool.iconified = False g_pool.detection_mapping_mode = session_settings.get('detection_mapping_mode', '3d') g_pool.active_calibration_plugin = None g_pool.active_gaze_mapping_plugin = None g_pool.capture_manager = None audio.audio_mode = session_settings.get('audio_mode', audio.default_audio_mode) def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def launch_eye_process(eye_id, delay=0): n = {'subject': 'eye_process.should_start.{}'.format(eye_id), 'eye_id': eye_id, 'delay': delay} ipc_pub.notify(n) def stop_eye_process(eye_id): n = {'subject': 'eye_process.should_stop.{}'.format(eye_id), 'eye_id': eye_id,'delay':0.2} ipc_pub.notify(n) def start_stop_eye(eye_id, make_alive): if make_alive: launch_eye_process(eye_id) else: stop_eye_process(eye_id) def set_detection_mapping_mode(new_mode): n = {'subject': 'set_detection_mapping_mode', 'mode': new_mode} ipc_pub.notify(n) def handle_notifications(n): subject = n['subject'] if subject == 'set_detection_mapping_mode': if n['mode'] == '2d': if ("Vector_Gaze_Mapper" in g_pool.active_gaze_mapping_plugin.class_name): logger.warning("The gaze mapper is not supported in 2d mode. Please recalibrate.") g_pool.plugins.add(plugin_by_name['Dummy_Gaze_Mapper']) g_pool.detection_mapping_mode = n['mode'] elif subject == 'start_plugin': g_pool.plugins.add(plugin_by_name[n['name']], args=n.get('args', {})) elif subject == 'stop_plugin': for p in g_pool.plugins: if p.class_name == n['name']: p.alive = False g_pool.plugins.clean() elif subject == 'eye_process.started': n = {'subject': 'set_detection_mapping_mode', 'mode': g_pool.detection_mapping_mode} ipc_pub.notify(n) elif subject.startswith('meta.should_doc'): ipc_pub.notify({'subject': 'meta.doc', 'actor': g_pool.app, 'doc': world.__doc__}) for p in g_pool.plugins: if (p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify): ipc_pub.notify({'subject': 'meta.doc', 'actor': p.class_name, 'doc': p.on_notify.__doc__}) # window and gl setup glfw.glfwInit() width, height = session_settings.get('window_size', (1280, 720)) main_window = glfw.glfwCreateWindow(width, height, "Pupil Capture - World") window_pos = session_settings.get('window_position', window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window def set_scale(new_scale): g_pool.gui_user_scale = new_scale on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) def reset_restart(): logger.warning("Resetting all settings and restarting Capture.") glfw.glfwSetWindowShouldClose(main_window, True) ipc_pub.notify({'subject': 'reset_restart_process.should_start'}) # setup GUI g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get('gui_scale', 1.) g_pool.sidebar = ui.Scrolling_Menu("Settings", pos=(-350, 0), size=(0, 0), header_pos='left') general_settings = ui.Growing_Menu('General') general_settings.append(ui.Button('Reset to default settings',reset_restart)) general_settings.append(ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2], label='Interface size')) general_settings.append(ui.Button('Reset window size', lambda: glfw.glfwSetWindowSize(main_window,g_pool.capture.frame_size[0],g_pool.capture.frame_size[1])) ) general_settings.append(ui.Selector('audio_mode', audio, selection=audio.audio_modes)) general_settings.append(ui.Selector('detection_mapping_mode', g_pool, label='detection & mapping mode', setter=set_detection_mapping_mode, selection=['2d','3d'] )) general_settings.append(ui.Switch('eye0_process', label='Detect eye 0', setter=lambda alive: start_stop_eye(0,alive), getter=lambda: eyes_are_alive[0].value )) general_settings.append(ui.Switch('eye1_process', label='Detect eye 1', setter=lambda alive: start_stop_eye(1,alive), getter=lambda: eyes_are_alive[1].value )) selector_label = "Select to load" labels = [p.__name__.replace('_', ' ') for p in user_launchable_plugins] user_launchable_plugins.insert(0, selector_label) labels.insert(0, selector_label) general_settings.append(ui.Selector('Open plugin', selection=user_launchable_plugins, labels=labels, setter=open_plugin, getter=lambda: selector_label)) general_settings.append(ui.Info_Text('Capture Version: {}'.format(g_pool.version))) g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100)) g_pool.capture_source_menu = ui.Growing_Menu('Capture Source') g_pool.capture_source_menu.collapsed = True g_pool.calibration_menu = ui.Growing_Menu('Calibration') g_pool.calibration_menu.collapsed = True g_pool.capture_selector_menu = ui.Growing_Menu('Capture Selection') g_pool.sidebar.append(general_settings) g_pool.sidebar.append(g_pool.capture_selector_menu) g_pool.sidebar.append(g_pool.capture_source_menu) g_pool.sidebar.append(g_pool.calibration_menu) g_pool.gui.append(g_pool.sidebar) g_pool.gui.append(g_pool.quickbar) # plugins that are loaded based on user settings from previous session g_pool.plugins = Plugin_List(g_pool, plugin_by_name, session_settings.get('loaded_plugins', default_plugins)) #We add the calibration menu selector, after a calibration has been added: g_pool.calibration_menu.insert(0,ui.Selector( 'active_calibration_plugin', getter=lambda: g_pool.active_calibration_plugin.__class__, selection = calibration_plugins, labels = [p.__name__.replace('_',' ') for p in calibration_plugins], setter= open_plugin,label='Method' )) #We add the capture selection menu, after a manager has been added: g_pool.capture_selector_menu.insert(0,ui.Selector( 'capture_manager', setter = open_plugin, getter = lambda: g_pool.capture_manager.__class__, selection = manager_classes, labels = [b.gui_name for b in manager_classes], label = 'Manager' )) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetWindowIconifyCallback(main_window, on_iconify) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() # now the we have aproper window we can load the last gui configuration g_pool.gui.configuration = session_settings.get('ui_config', {}) # create a timer to control window update frequency window_update_timer = timer(1 / 60) def window_should_update(): return next(window_update_timer) # set up performace graphs: pid = os.getpid() ps = psutil.Process(pid) ts = g_pool.get_timestamp() cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 130) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" pupil0_graph = graph.Bar_Graph(max_val=1.0) pupil0_graph.pos = (260, 130) pupil0_graph.update_rate = 5 pupil0_graph.label = "id0 conf: %0.2f" pupil1_graph = graph.Bar_Graph(max_val=1.0) pupil1_graph.pos = (380, 130) pupil1_graph.update_rate = 5 pupil1_graph.label = "id1 conf: %0.2f" pupil_graphs = pupil0_graph, pupil1_graph g_pool.graphs = [cpu_graph, fps_graph, pupil0_graph, pupil1_graph] # trigger setup of window and gl sizes on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) if session_settings.get('eye1_process_alive', False): launch_eye_process(1, delay=0.6) if session_settings.get('eye0_process_alive', True): launch_eye_process(0, delay=0.3) ipc_pub.notify({'subject': 'world_process.started'}) logger.warning('Process started.') # Event loop while not glfw.glfwWindowShouldClose(main_window): # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) #a dictionary that allows plugins to post and read events events = {} # report time between now and the last loop interation events['dt'] = get_dt() # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() # update performace graphs if 'frame' in events: t = events["frame"].timestamp dt, ts = t-ts, t try: fps_graph.add(1./dt) except ZeroDivisionError: pass for p in events["pupil_positions"]: pupil_graphs[p['id']].add(p['confidence']) cpu_graph.update() # send new events to ipc: del events['pupil_positions'] # already on the wire del events['gaze_positions'] # sent earlier if 'frame' in events: del events['frame'] # send explicity with frame publisher if 'depth_frame' in events: del events['depth_frame'] if 'audio_packets' in events: del events['audio_packets'] del events['dt'] # no need to send this for topic, data in events.items(): assert(isinstance(data, (list, tuple))) for d in data: ipc_pub.send(topic, d) glfw.glfwMakeContextCurrent(main_window) # render visual feedback from loaded plugins if window_should_update() and gl_utils.is_window_visible(main_window): for p in g_pool.plugins: p.gl_display() for g in g_pool.graphs: g.draw() unused_elements = g_pool.gui.update() for button, action, mods in unused_elements.buttons: pos = glfw.glfwGetCursorPos(main_window) pos = normalize(pos, glfw.glfwGetWindowSize(main_window)) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_click(pos, button, action) for key, scancode, action, mods in unused_elements.keys: for p in g_pool.plugins: p.on_key(key, scancode, action, mods) for char_ in unused_elements.chars: for p in g_pool.plugins: p.on_char(char_) glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() glfw.glfwRestoreWindow(main_window) # need to do this for windows os session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['gui_scale'] = g_pool.gui_user_scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos(main_window) session_settings['version'] = str(g_pool.version) session_settings['eye0_process_alive'] = eyes_are_alive[0].value session_settings['eye1_process_alive'] = eyes_are_alive[1].value session_settings['detection_mapping_mode'] = g_pool.detection_mapping_mode session_settings['audio_mode'] = audio.audio_mode session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) glfw.glfwTerminate() # shut down eye processes: stop_eye_process(0) stop_eye_process(1) logger.info("Process shutting down.") ipc_pub.notify({'subject': 'world_process.stopped'})
class Surface_Tracker(Plugin): """docstring """ def __init__(self,g_pool,mode="Show Markers and Surfaces",min_marker_perimeter = 100,invert_image=False): super(Surface_Tracker, self).__init__(g_pool) self.order = .2 # all markers that are detected in the most recent frame self.markers = [] self.camera_calibration = load_camera_calibration(self.g_pool) self.load_surface_definitions_from_file() # edit surfaces self.edit_surfaces = [] self.edit_surf_verts = [] self.marker_edit_surface = None #plugin state self.mode = mode self.running = True self.robust_detection = 1 self.aperture = 11 self.min_marker_perimeter = min_marker_perimeter self.locate_3d = False self.invert_image = invert_image self.img_shape = None self.menu = None self.button = None self.add_button = None def load_surface_definitions_from_file(self): # all registered surfaces self.surface_definitions = Persistent_Dict(os.path.join(self.g_pool.user_dir,'surface_definitions') ) self.surfaces = [Reference_Surface(saved_definition=d) for d in self.surface_definitions.get('realtime_square_marker_surfaces',[]) if isinstance(d,dict)] def on_click(self,pos,button,action): if self.mode == 'Show Markers and Surfaces': if action == GLFW_PRESS: for s in self.surfaces: toggle = s.get_mode_toggle(pos,self.img_shape) if toggle == 'surface_mode': if s in self.edit_surfaces: self.edit_surfaces.remove(s) else: self.edit_surfaces.append(s) elif toggle == 'marker_mode': if self.marker_edit_surface == s: self.marker_edit_surface = None else: self.marker_edit_surface = s if action == GLFW_RELEASE: self.edit_surf_verts = [] elif action == GLFW_PRESS: surf_verts = ((0.,0.),(1.,0.),(1.,1.),(0.,1.)) x,y = pos for s in self.edit_surfaces: if s.detected and s.defined: for (vx,vy),i in zip(s.ref_surface_to_img(np.array(surf_verts)),range(4)): vx,vy = denormalize((vx,vy),(self.img_shape[1],self.img_shape[0]),flip_y=True) if sqrt((x-vx)**2 + (y-vy)**2) <15: #img pixels self.edit_surf_verts.append((s,i)) return if self.marker_edit_surface: for m in self.markers: if m['perimeter']>=self.min_marker_perimeter: vx,vy = m['centroid'] if sqrt((x-vx)**2 + (y-vy)**2) <15: if self.marker_edit_surface.markers.has_key(m['id']): self.marker_edit_surface.remove_marker(m) else: self.marker_edit_surface.add_marker(m,self.markers,self.camera_calibration,self.min_marker_perimeter) def add_surface(self,_): self.surfaces.append(Reference_Surface()) self.update_gui_markers() def remove_surface(self,i): remove_surface = self.surfaces[i] if remove_surface == self.marker_edit_surface: self.marker_edit_surface = None if remove_surface in self.edit_surfaces: self.edit_surfaces.remove(remove_surface) self.surfaces[i].cleanup() del self.surfaces[i] self.update_gui_markers() def init_gui(self): self.menu = ui.Growing_Menu('Surface Tracker') self.g_pool.sidebar.append(self.menu) self.button = ui.Thumb('running',self,label='Track',hotkey='t') self.button.on_color[:] = (.1,.2,1.,.8) self.g_pool.quickbar.append(self.button) self.add_button = ui.Thumb('add_surface',setter=self.add_surface,getter=lambda:False,label='Add surface',hotkey='a') self.g_pool.quickbar.append(self.add_button) self.update_gui_markers() def deinit_gui(self): if self.menu: self.g_pool.sidebar.remove(self.menu) self.menu= None if self.button: self.g_pool.quickbar.remove(self.button) self.button = None if self.add_button: self.g_pool.quickbar.remove(self.add_button) self.add_button = None def update_gui_markers(self): def close(): self.alive = False self.menu.elements[:] = [] self.menu.append(ui.Button('Close',close)) self.menu.append(ui.Info_Text('This plugin detects and tracks fiducial markers visible in the scene. You can define surfaces using 1 or more marker visible within the world view by clicking *add surface*. You can edit defined surfaces by selecting *Surface edit mode*.')) self.menu.append(ui.Switch('robust_detection',self,label='Robust detection')) self.menu.append(ui.Switch('invert_image',self,label='Use inverted markers')) self.menu.append(ui.Slider('min_marker_perimeter',self,step=1,min=10,max=500)) self.menu.append(ui.Switch('locate_3d',self,label='3D localization')) self.menu.append(ui.Selector('mode',self,label="Mode",selection=['Show Markers and Surfaces','Show marker IDs'] )) self.menu.append(ui.Button("Add surface", lambda:self.add_surface('_'),)) for s in self.surfaces: idx = self.surfaces.index(s) s_menu = ui.Growing_Menu("Surface %s"%idx) s_menu.collapsed=True s_menu.append(ui.Text_Input('name',s)) s_menu.append(ui.Text_Input('x',s.real_world_size,label='X size')) s_menu.append(ui.Text_Input('y',s.real_world_size,label='Y size')) s_menu.append(ui.Button('Open Debug Window',s.open_close_window)) #closure to encapsulate idx def make_remove_s(i): return lambda: self.remove_surface(i) remove_s = make_remove_s(idx) s_menu.append(ui.Button('remove',remove_s)) self.menu.append(s_menu) def update(self,frame,events): self.img_shape = frame.height,frame.width,3 if self.running: gray = frame.gray if self.robust_detection: self.markers = detect_markers_robust(gray, grid_size = 5, prev_markers=self.markers, min_marker_perimeter=self.min_marker_perimeter, aperture=self.aperture, visualize=0, true_detect_every_frame=3, invert_image=self.invert_image) else: self.markers = detect_markers(gray, grid_size = 5, min_marker_perimeter=self.min_marker_perimeter, aperture=self.aperture, visualize=0, invert_image=self.invert_image) if self.mode == "Show marker IDs": draw_markers(frame.gray,self.markers) events['surface'] = [] # locate surfaces for s in self.surfaces: s.locate(self.markers,self.camera_calibration,self.min_marker_perimeter, self.locate_3d) if s.detected: events['surface'].append({'name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen.tolist(),'m_from_screen':s.m_from_screen.tolist(), 'timestamp':frame.timestamp,'camera_pose_3d':s.camera_pose_3d.tolist()}) if self.running: self.button.status_text = '%s/%s'%(len([s for s in self.surfaces if s.detected]),len(self.surfaces)) else: self.button.status_text = 'tracking paused' if self.mode == 'Show Markers and Surfaces': # edit surfaces by user if self.edit_surf_verts: window = glfwGetCurrentContext() pos = glfwGetCursorPos(window) pos = normalize(pos,glfwGetWindowSize(window),flip_y=True) for s,v_idx in self.edit_surf_verts: if s.detected: new_pos = s.img_to_ref_surface(np.array(pos)) s.move_vertex(v_idx,new_pos) #map recent gaze onto detected surfaces used for pupil server for s in self.surfaces: if s.detected: s.gaze_on_srf = [] for p in events.get('gaze_positions',[]): gp_on_s = tuple(s.img_to_ref_surface(np.array(p['norm_pos']))) p['realtime gaze on ' + s.name] = gp_on_s s.gaze_on_srf.append(gp_on_s) def get_init_dict(self): return {'mode':self.mode,'min_marker_perimeter':self.min_marker_perimeter} def gl_display(self): """ Display marker and surface info inside world screen """ if self.mode == "Show Markers and Surfaces": for m in self.markers: hat = np.array([[[0,0],[0,1],[.5,1.3],[1,1],[1,0],[0,0]]],dtype=np.float32) hat = cv2.perspectiveTransform(hat,m_marker_to_screen(m)) if m['perimeter']>=self.min_marker_perimeter: draw_polyline(hat.reshape((6,2)),color=RGBA(0.1,1.,1.,.5)) draw_polyline(hat.reshape((6,2)),color=RGBA(0.1,1.,1.,.3),line_type=GL_POLYGON) else: draw_polyline(hat.reshape((6,2)),color=RGBA(0.1,1.,1.,.5)) for s in self.surfaces: if s not in self.edit_surfaces and s is not self.marker_edit_surface: s.gl_draw_frame(self.img_shape) for s in self.edit_surfaces: s.gl_draw_frame(self.img_shape,highlight=True,surface_mode=True) s.gl_draw_corners() if self.marker_edit_surface: inc = [] exc = [] for m in self.markers: if m['perimeter']>=self.min_marker_perimeter: if self.marker_edit_surface.markers.has_key(m['id']): inc.append(m['centroid']) else: exc.append(m['centroid']) draw_points(exc,size=20,color=RGBA(1.,0.5,0.5,.8)) draw_points(inc,size=20,color=RGBA(0.5,1.,0.5,.8)) self.marker_edit_surface.gl_draw_frame(self.img_shape,color=(0.0,0.9,0.6,1.0),highlight=True,marker_mode=True) for s in self.surfaces: if self.locate_3d: s.gl_display_in_window_3d(self.g_pool.image_tex,self.camera_calibration) else: s.gl_display_in_window(self.g_pool.image_tex) def cleanup(self): """ called when the plugin gets terminated. This happens either voluntarily or forced. if you have a GUI or glfw window destroy it here. """ self.surface_definitions["realtime_square_marker_surfaces"] = [rs.save_to_dict() for rs in self.surfaces if rs.defined] self.surface_definitions.close() for s in self.surfaces: s.cleanup() self.deinit_gui()
def eye(timebase, is_alive_flag, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version, eye_id, overwrite_cap_settings=None): """reads eye video and detects the pupil. Creates a window, gl context. Grabs images from a capture. Streams Pupil coordinates. Reacts to notifications: ``set_detection_mapping_mode``: Sets detection method ``eye_process.should_stop``: Stops the eye process ``recording.started``: Starts recording eye video ``recording.stopped``: Stops recording eye video ``frame_publishing.started``: Starts frame publishing ``frame_publishing.stopped``: Stops frame publishing Emits notifications: ``eye_process.started``: Eye process started ``eye_process.stopped``: Eye process stopped Emits data: ``pupil.<eye id>``: Pupil data for eye with id ``<eye id>`` ``frame.eye.<eye id>``: Eye frames with id ``<eye id>`` """ # We deferr the imports becasue of multiprocessing. # Otherwise the world process each process also loads the other imports. import zmq import zmq_tools zmq_ctx = zmq.Context() ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",)) with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id): # logging setup import logging logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.INFO) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) # general imports import numpy as np import cv2 # display import glfw from pyglui import ui, graph, cygl from pyglui.cygl.utils import draw_points, RGBA, draw_polyline from pyglui.cygl.utils import Named_Texture from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen from gl_utils import make_coord_system_pixel_based from gl_utils import make_coord_system_norm_based from gl_utils import is_window_visible from ui_roi import UIRoi # monitoring import psutil # helpers/utils from uvc import get_time_monotonic from file_methods import Persistent_Dict from version_utils import VersionFormat from methods import normalize, denormalize, timer from av_writer import JPEG_Writer, AV_Writer from ndsi import H264Writer from video_capture import source_classes from video_capture import manager_classes # Pupil detectors from pupil_detectors import Detector_2D, Detector_3D pupil_detectors = {Detector_2D.__name__: Detector_2D, Detector_3D.__name__: Detector_3D} # UI Platform tweaks if platform.system() == 'Linux': scroll_factor = 10.0 window_position_default = (600, 300 * eye_id) elif platform.system() == 'Windows': scroll_factor = 10.0 window_position_default = (600,31+ 300 * eye_id) else: scroll_factor = 1.0 window_position_default = (600, 300 * eye_id) # g_pool holds variables for this process g_pool = Global_Container() # make some constants avaiable g_pool.user_dir = user_dir g_pool.version = version g_pool.app = 'capture' g_pool.process = 'eye{}'.format(eye_id) g_pool.timebase = timebase g_pool.ipc_pub = ipc_socket def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic # Callback functions def on_resize(window, w, h): if is_window_visible(window): active_window = glfw.glfwGetCurrentContext() glfw.glfwMakeContextCurrent(window) hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() for g in g_pool.graphs: g.scale = hdpi_factor g.adjust_window_size(w, h) adjust_gl_view(w, h) glfw.glfwMakeContextCurrent(active_window) def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_iconify(window, iconified): g_pool.iconified = iconified def on_window_mouse_button(window, button, action, mods): if g_pool.display_mode == 'roi': if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt: g_pool.u_r.active_edit_pt = False # if the roi interacts we dont want # the gui to interact as well return elif action == glfw.GLFW_PRESS: pos = glfw.glfwGetCursorPos(window) pos = normalize(pos, glfw.glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] # Position in img pixels pos = denormalize(pos,g_pool.capture.frame_size) # Position in img pixels if g_pool.u_r.mouse_over_edit_pt(pos, g_pool.u_r.handle_size + 40,g_pool.u_r.handle_size + 40): # if the roi interacts we dont want # the gui to interact as well return g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): hdpi_factor = glfw.glfwGetFramebufferSize( window)[0] / glfw.glfwGetWindowSize(window)[0] g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor) if g_pool.u_r.active_edit_pt: pos = normalize((x, y), glfw.glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1-pos[0],1-pos[1] pos = denormalize(pos,g_pool.capture.frame_size ) g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx,pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, count, paths): paths = [paths[x].decode('utf-8') for x in range(count)] g_pool.capture_manager.on_drop(paths) g_pool.capture.on_drop(paths) # load session persistent settings session_settings = Persistent_Dict(os.path.join(g_pool.user_dir, 'user_settings_eye{}'.format(eye_id))) if VersionFormat(session_settings.get("version", '0.0')) != g_pool.version: logger.info("Session setting are from a different version of this app. I will not use those.") session_settings.clear() g_pool.iconified = False g_pool.capture = None g_pool.capture_manager = None g_pool.flip = session_settings.get('flip', False) g_pool.display_mode = session_settings.get( 'display_mode', 'camera_image') g_pool.display_mode_info_text = {'camera_image': "Raw eye camera image. This uses the least amount of CPU power", 'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.", 'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below."} capture_manager_settings = session_settings.get( 'capture_manager_settings', ('UVC_Manager',{})) manager_class_name, manager_settings = capture_manager_settings manager_class_by_name = {c.__name__:c for c in manager_classes} g_pool.capture_manager = manager_class_by_name[manager_class_name](g_pool,**manager_settings) if eye_id == 0: cap_src = ["Pupil Cam1 ID0","HD-6000","Integrated Camera","HD USB Camera","USB 2.0 Camera"] else: cap_src = ["Pupil Cam1 ID1","HD-6000","Integrated Camera"] # Initialize capture default_settings = ('UVC_Source',{ 'preferred_names' : cap_src, 'frame_size': (640,480), 'frame_rate': 90 }) capture_source_settings = overwrite_cap_settings or session_settings.get('capture_settings', default_settings) source_class_name, source_settings = capture_source_settings source_class_by_name = {c.__name__:c for c in source_classes} g_pool.capture = source_class_by_name[source_class_name](g_pool,**source_settings) assert g_pool.capture g_pool.u_r = UIRoi((g_pool.capture.frame_size[1],g_pool.capture.frame_size[0])) roi_user_settings = session_settings.get('roi') if roi_user_settings and tuple(roi_user_settings[-1]) == g_pool.u_r.get()[-1]: g_pool.u_r.set(roi_user_settings) pupil_detector_settings = session_settings.get( 'pupil_detector_settings', None) last_pupil_detector = pupil_detectors[session_settings.get( 'last_pupil_detector', Detector_2D.__name__)] g_pool.pupil_detector = last_pupil_detector( g_pool, pupil_detector_settings) def set_display_mode_info(val): g_pool.display_mode = val g_pool.display_mode_info.text = g_pool.display_mode_info_text[val] def set_detector(new_detector): g_pool.pupil_detector.cleanup() g_pool.pupil_detector = new_detector(g_pool) g_pool.pupil_detector.init_gui(g_pool.sidebar) # Initialize glfw glfw.glfwInit() title = "Pupil Capture - eye {}".format(eye_id) width, height = session_settings.get( 'window_size', g_pool.capture.frame_size) main_window = glfw.glfwCreateWindow(width, height, title, None, None) window_pos = session_settings.get( 'window_position', window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() # UI callback functions def set_scale(new_scale): g_pool.gui_user_scale = new_scale on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() g_pool.image_tex.update_from_ndarray(np.ones((1,1),dtype=np.uint8)+125) # setup GUI g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get('gui_scale', 1.) g_pool.sidebar = ui.Scrolling_Menu("Settings", pos=(-300, 0), size=(0, 0), header_pos='left') general_settings = ui.Growing_Menu('General') general_settings.append(ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2], label='Interface Size')) general_settings.append(ui.Button('Reset window size',lambda: glfw.glfwSetWindowSize(main_window,*g_pool.capture.frame_size)) ) general_settings.append(ui.Switch('flip',g_pool,label='Flip image display')) general_settings.append(ui.Selector('display_mode', g_pool, setter=set_display_mode_info, selection=['camera_image','roi','algorithm'], labels=['Camera Image', 'ROI', 'Algorithm'], label="Mode") ) g_pool.display_mode_info = ui.Info_Text(g_pool.display_mode_info_text[g_pool.display_mode]) general_settings.append(g_pool.display_mode_info) g_pool.gui.append(g_pool.sidebar) detector_selector = ui.Selector('pupil_detector', getter=lambda: g_pool.pupil_detector.__class__, setter=set_detector, selection=[ Detector_2D, Detector_3D], labels=['C++ 2d detector', 'C++ 3d detector'], label="Detection method") general_settings.append(detector_selector) g_pool.capture_selector_menu = ui.Growing_Menu('Capture Selection') g_pool.capture_source_menu = ui.Growing_Menu('Capture Source') g_pool.capture_source_menu.collapsed = True g_pool.capture.init_gui() g_pool.sidebar.append(general_settings) g_pool.sidebar.append(g_pool.capture_selector_menu) g_pool.sidebar.append(g_pool.capture_source_menu) g_pool.pupil_detector.init_gui(g_pool.sidebar) g_pool.capture_manager.init_gui() g_pool.writer = None def replace_source(source_class_name,source_settings): g_pool.capture.cleanup() g_pool.capture = source_class_by_name[source_class_name](g_pool,**source_settings) g_pool.capture.init_gui() if g_pool.writer: logger.info("Done recording.") g_pool.writer.release() g_pool.writer = None g_pool.replace_source = replace_source # for ndsi capture def replace_manager(manager_class): g_pool.capture_manager.cleanup() g_pool.capture_manager = manager_class(g_pool) g_pool.capture_manager.init_gui() #We add the capture selection menu, after a manager has been added: g_pool.capture_selector_menu.insert(0,ui.Selector( 'capture_manager',g_pool, setter = replace_manager, getter = lambda: g_pool.capture_manager.__class__, selection = manager_classes, labels = [b.gui_name for b in manager_classes], label = 'Manager' )) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetWindowIconifyCallback(main_window, on_iconify) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) # load last gui configuration g_pool.gui.configuration = session_settings.get('ui_config', {}) # set up performance graphs pid = os.getpid() ps = psutil.Process(pid) ts = g_pool.get_timestamp() cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 130) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" g_pool.graphs = [cpu_graph, fps_graph] # set the last saved window size on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) should_publish_frames = False frame_publish_format = 'jpeg' # create a timer to control window update frequency window_update_timer = timer(1 / 60) def window_should_update(): return next(window_update_timer) logger.warning('Process started.') frame = None # Event loop while not glfw.glfwWindowShouldClose(main_window): if notify_sub.new_data: t, notification = notify_sub.recv() subject = notification['subject'] if subject.startswith('eye_process.should_stop'): if notification['eye_id'] == eye_id: break elif subject == 'set_detection_mapping_mode': if notification['mode'] == '3d': if not isinstance(g_pool.pupil_detector, Detector_3D): set_detector(Detector_3D) detector_selector.read_only = True else: if not isinstance(g_pool.pupil_detector, Detector_2D): set_detector(Detector_2D) detector_selector.read_only = False elif subject == 'recording.started': if notification['record_eye'] and g_pool.capture.online: record_path = notification['rec_path'] raw_mode = notification['compression'] logger.info("Will save eye video to: {}".format(record_path)) video_path = os.path.join(record_path, "eye{}.mp4".format(eye_id)) if raw_mode and frame and g_pool.capture.jpeg_support: g_pool.writer = JPEG_Writer(video_path, g_pool.capture.frame_rate) elif hasattr(g_pool.capture._recent_frame, 'h264_buffer'): g_pool.writer = H264Writer(video_path, g_pool.capture.frame_size[0], g_pool.capture.frame_size[1], g_pool.capture.frame_rate) else: g_pool.writer = AV_Writer(video_path, g_pool.capture.frame_rate) elif subject == 'recording.stopped': if g_pool.writer: logger.info("Done recording.") g_pool.writer.release() g_pool.writer = None elif subject.startswith('meta.should_doc'): ipc_socket.notify({ 'subject': 'meta.doc', 'actor': 'eye{}'.format(eye_id), 'doc': eye.__doc__ }) elif subject.startswith('frame_publishing.started'): should_publish_frames = True frame_publish_format = notification.get('format', 'jpeg') elif subject.startswith('frame_publishing.stopped'): should_publish_frames = False frame_publish_format = 'jpeg' elif subject.startswith('start_eye_capture') and notification['target'] == g_pool.process: replace_source(notification['name'],notification['args']) g_pool.capture.on_notify(notification) # Get an image from the grabber event = {} g_pool.capture.recent_events(event) frame = event.get('frame') g_pool.capture_manager.recent_events(event) if frame: f_width, f_height = g_pool.capture.frame_size if (g_pool.u_r.array_shape[0], g_pool.u_r.array_shape[1]) != (f_height, f_width): g_pool.pupil_detector.on_resolution_change((g_pool.u_r.array_shape[1], g_pool.u_r.array_shape[0]), g_pool.capture.frame_size) g_pool.u_r = UIRoi((f_height, f_width)) if should_publish_frames: try: if frame_publish_format == "jpeg": data = frame.jpeg_buffer elif frame_publish_format == "yuv": data = frame.yuv_buffer elif frame_publish_format == "bgr": data = frame.bgr elif frame_publish_format == "gray": data = frame.gray else: raise AttributeError() except AttributeError: pass else: pupil_socket.send('frame.eye.%s'%eye_id,{ 'width': frame.width, 'height': frame.height, 'index': frame.index, 'timestamp': frame.timestamp, 'format': frame_publish_format, '__raw_data__': [data] }) t = frame.timestamp dt, ts = t - ts, t try: fps_graph.add(1./dt) except ZeroDivisionError: pass if g_pool.writer: g_pool.writer.write_video_frame(frame) # pupil ellipse detection result = g_pool.pupil_detector.detect(frame, g_pool.u_r, g_pool.display_mode == 'algorithm') result['id'] = eye_id # stream the result pupil_socket.send('pupil.%s'%eye_id,result) cpu_graph.update() # GL drawing if window_should_update(): if is_window_visible(main_window): glfw.glfwMakeContextCurrent(main_window) clear_gl_screen() if frame: # switch to work in normalized coordinate space if g_pool.display_mode == 'algorithm': g_pool.image_tex.update_from_ndarray(frame.img) elif g_pool.display_mode in ('camera_image', 'roi'): g_pool.image_tex.update_from_ndarray(frame.gray) else: pass make_coord_system_norm_based(g_pool.flip) g_pool.image_tex.draw() f_width, f_height = g_pool.capture.frame_size make_coord_system_pixel_based((f_height, f_width, 3), g_pool.flip) if frame: if result['method'] == '3d c++': eye_ball = result['projected_sphere'] try: pts = cv2.ellipse2Poly( (int(eye_ball['center'][0]), int(eye_ball['center'][1])), (int(eye_ball['axes'][0] / 2), int(eye_ball['axes'][1] / 2)), int(eye_ball['angle']), 0, 360, 8) except ValueError as e: pass else: draw_polyline(pts, 2, RGBA(0., .9, .1, result['model_confidence'])) if result['confidence'] > 0: if 'ellipse' in result: pts = cv2.ellipse2Poly( (int(result['ellipse']['center'][0]), int(result['ellipse']['center'][1])), (int(result['ellipse']['axes'][0] / 2), int(result['ellipse']['axes'][1] / 2)), int(result['ellipse']['angle']), 0, 360, 15) confidence = result['confidence'] * 0.7 draw_polyline(pts, 1, RGBA(1., 0, 0, confidence)) draw_points([result['ellipse']['center']], size=20, color=RGBA(1., 0., 0., confidence), sharpness=1.) # render graphs fps_graph.draw() cpu_graph.draw() # render GUI g_pool.gui.update() # render the ROI g_pool.u_r.draw(g_pool.gui.scale) if g_pool.display_mode == 'roi': g_pool.u_r.draw_points(g_pool.gui.scale) # update screen glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() g_pool.pupil_detector.visualize() # detector decides if we visualize or not # END while running # in case eye recording was still runnnig: Save&close if g_pool.writer: logger.info("Done recording eye.") g_pool.writer = None glfw.glfwRestoreWindow(main_window) # need to do this for windows os # save session persistent settings session_settings['gui_scale'] = g_pool.gui_user_scale session_settings['roi'] = g_pool.u_r.get() session_settings['flip'] = g_pool.flip session_settings['display_mode'] = g_pool.display_mode session_settings['ui_config'] = g_pool.gui.configuration session_settings['capture_settings'] = g_pool.capture.class_name, g_pool.capture.get_init_dict() session_settings['capture_manager_settings'] = g_pool.capture_manager.class_name, g_pool.capture_manager.get_init_dict() session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos(main_window) session_settings['version'] = str(g_pool.version) session_settings['last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__ session_settings['pupil_detector_settings'] = g_pool.pupil_detector.get_settings() session_settings.close() g_pool.capture.deinit_gui() g_pool.pupil_detector.cleanup() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) glfw.glfwTerminate() g_pool.capture_manager.cleanup() g_pool.capture.cleanup() logger.info("Process shutting down.")
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_version): # general imports from time import sleep import logging import errno from glob import glob from time import time # networking import zmq import zmq_tools import numpy as np # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify', )) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.INFO) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) try: # imports from file_methods import Persistent_Dict, load_object # display import glfw # check versions for our own depedencies as they are fast-changing from pyglui import __version__ as pyglui_version from pyglui import ui, cygl from pyglui.cygl.utils import Named_Texture, RGBA import gl_utils # capture from video_capture import File_Source, EndofVideoFileError # helpers/utils from version_utils import VersionFormat from methods import normalize, denormalize, delta_t, get_system_info from player_methods import correlate_data, is_pupil_rec_dir, load_meta_info # Plug-ins from plugin import Plugin, Plugin_List, import_runtime_plugins from plugin_manager import Plugin_Manager from vis_circle import Vis_Circle from vis_cross import Vis_Cross from vis_polyline import Vis_Polyline from vis_light_points import Vis_Light_Points from vis_watermark import Vis_Watermark from vis_fixation import Vis_Fixation from vis_scan_path import Vis_Scan_Path from vis_eye_video_overlay import Vis_Eye_Video_Overlay from seek_control import Seek_Control from video_export_launcher import Video_Export_Launcher from offline_surface_tracker import Offline_Surface_Tracker # from marker_auto_trim_marks import Marker_Auto_Trim_Marks from fixation_detector import Offline_Fixation_Detector from batch_exporter import Batch_Exporter, Batch_Export from log_display import Log_Display from annotations import Annotation_Player from raw_data_exporter import Raw_Data_Exporter from log_history import Log_History from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection from gaze_producers import Gaze_From_Recording, Offline_Calibration from system_graphs import System_Graphs from system_timelines import System_Timelines from blink_detection import Offline_Blink_Detection assert VersionFormat(pyglui_version) >= VersionFormat( '1.17'), 'pyglui out of date, please upgrade to newest version' runtime_plugins = import_runtime_plugins( os.path.join(user_dir, 'plugins')) system_plugins = [ Log_Display, Seek_Control, Plugin_Manager, System_Graphs, Batch_Export, System_Timelines ] user_plugins = [ Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross, Vis_Watermark, Vis_Eye_Video_Overlay, Vis_Scan_Path, Offline_Fixation_Detector, Offline_Blink_Detection, Batch_Exporter, Video_Export_Launcher, Offline_Surface_Tracker, Raw_Data_Exporter, Annotation_Player, Log_History, Pupil_From_Recording, Offline_Pupil_Detection, Gaze_From_Recording, Offline_Calibration ] + runtime_plugins plugins = system_plugins + user_plugins # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal hdpi_factor hdpi_factor = float( glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor window_size = w, h g_pool.camera_render_size = w - int( icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(*window_size) g_pool.gui.collect_menus() for p in g_pool.plugins: p.on_window_resize(window, *g_pool.camera_render_size) def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) pos = x, y pos = normalize(pos, g_pool.camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_pos(pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, count, paths): for x in range(count): new_rec_dir = paths[x].decode('utf-8') if is_pupil_rec_dir(new_rec_dir): logger.debug( "Starting new session with '{}'".format(new_rec_dir)) ipc_pub.notify({ "subject": "player_drop_process.should_start", "rec_dir": new_rec_dir }) glfw.glfwSetWindowShouldClose(window, True) else: logger.error("'{}' is not a valid pupil recording".format( new_rec_dir)) tick = delta_t() def get_dt(): return next(tick) video_path = [ f for f in glob(os.path.join(rec_dir, "world.*")) if os.path.splitext(f)[1] in ('.mp4', '.mkv', '.avi', '.h264', '.mjpeg') ][0] pupil_data_path = os.path.join(rec_dir, "pupil_data") meta_info = load_meta_info(rec_dir) # log info about Pupil Platform and Platform in player.log logger.info('Application Version: {}'.format(app_version)) logger.info('System Info: {}'.format(get_system_info())) icon_bar_width = 50 window_size = None hdpi_factor = 1.0 # create container for globally scoped vars g_pool = Global_Container() g_pool.app = 'player' g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.plugin_by_name = {p.__name__: p for p in plugins} g_pool.camera_render_size = None # sets itself to g_pool.capture File_Source(g_pool, video_path) # load session persistent settings session_settings = Persistent_Dict( os.path.join(user_dir, "user_settings_player")) if VersionFormat(session_settings.get("version", '0.0')) != app_version: logger.info( "Session setting are a different version of this app. I will not use those." ) session_settings.clear() g_pool.capture.playback_speed = session_settings.get( 'playback_speed', 1.) width, height = session_settings.get('window_size', g_pool.capture.frame_size) window_pos = session_settings.get('window_position', window_position_default) glfw.glfwInit() main_window = glfw.glfwCreateWindow( width, height, "Pupil Player: " + meta_info["Recording Name"] + " - " + rec_dir.split(os.path.sep)[-1], None, None) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window def set_scale(new_scale): g_pool.gui_user_scale = new_scale window_size = ( g_pool.camera_render_size[0] + int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor), glfw.glfwGetFramebufferSize(main_window)[1]) logger.warning(icon_bar_width * g_pool.gui_user_scale * hdpi_factor) glfw.glfwSetWindowSize(main_window, *window_size) # load pupil_positions, gaze_positions g_pool.pupil_data = load_object(pupil_data_path) g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular' g_pool.version = app_version g_pool.timestamps = g_pool.capture.timestamps g_pool.get_timestamp = lambda: 0. g_pool.new_seek = True g_pool.user_dir = user_dir g_pool.rec_dir = rec_dir g_pool.meta_info = meta_info g_pool.min_data_confidence = session_settings.get( 'min_data_confidence', 0.6) g_pool.pupil_positions = [] g_pool.gaze_positions = [] g_pool.fixations = [] g_pool.notifications_by_frame = correlate_data( g_pool.pupil_data['notifications'], g_pool.timestamps) g_pool.pupil_positions_by_frame = [[] for x in g_pool.timestamps ] # populated by producer` g_pool.gaze_positions_by_frame = [[] for x in g_pool.timestamps ] # populated by producer g_pool.fixations_by_frame = [ [] for x in g_pool.timestamps ] # populated by the fixation detector plugin def set_data_confidence(new_confidence): g_pool.min_data_confidence = new_confidence notification = {'subject': 'min_data_confidence_changed'} notification['_notify_time_'] = time() + .8 g_pool.ipc_pub.notify(notification) def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def purge_plugins(): for p in g_pool.plugins: if p.__class__ in user_plugins: p.alive = False g_pool.plugins.clean() def do_export(_): export_range = g_pool.seek_control.trim_left, g_pool.seek_control.trim_right export_dir = os.path.join(g_pool.rec_dir, 'exports', '{}-{}'.format(*export_range)) try: os.makedirs(export_dir) except OSError as e: if e.errno != errno.EEXIST: logger.error("Could not create export dir") raise e else: overwrite_warning = "Previous export for range [{}-{}] already exists - overwriting." logger.warning(overwrite_warning.format(*export_range)) else: logger.info('Created export dir at "{}"'.format(export_dir)) notification = { 'subject': 'should_export', 'range': export_range, 'export_dir': export_dir } g_pool.ipc_pub.notify(notification) def reset_restart(): logger.warning("Resetting all settings and restarting Player.") glfw.glfwSetWindowShouldClose(main_window, True) ipc_pub.notify({'subject': 'clear_settings_process.should_start'}) ipc_pub.notify({ 'subject': 'player_process.should_start', 'rec_dir': rec_dir, 'delay': 2. }) def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is open the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get('gui_scale', 1.) g_pool.menubar = ui.Scrolling_Menu("Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos='left') g_pool.iconbar = ui.Scrolling_Menu("Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos='hidden') g_pool.timelines = ui.Container((0, 0), (0, 0), (0, 0)) g_pool.timelines.horizontal_constraint = g_pool.menubar g_pool.user_timelines = ui.Timeline_Menu('User Timelines', pos=(0., -150.), size=(0., 0.), header_pos='headline') g_pool.user_timelines.color = RGBA(a=0.) g_pool.user_timelines.collapsed = True # add container that constaints itself to the seekbar height vert_constr = ui.Container((0, 0), (0, -50.), (0, 0)) vert_constr.append(g_pool.user_timelines) g_pool.timelines.append(vert_constr) general_settings = ui.Growing_Menu('General', header_pos='headline') general_settings.append( ui.Button( 'Reset window size', lambda: glfw.glfwSetWindowSize( main_window, g_pool.capture.frame_size[0], g_pool.capture. frame_size[1]))) general_settings.append( ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2] + list(np.arange(1.5, 5.1, .5)), label='Interface Size')) general_settings.append( ui.Info_Text('Player Version: {}'.format(g_pool.version))) general_settings.append( ui.Info_Text('Capture Version: {}'.format( meta_info['Capture Software Version']))) general_settings.append( ui.Info_Text('Data Format Version: {}'.format( meta_info['Data Format Version']))) general_settings.append( ui.Slider('min_data_confidence', g_pool, setter=set_data_confidence, step=.05, min=0.0, max=1.0, label='Confidence threshold')) general_settings.append( ui.Button('Restart with default settings', reset_restart)) g_pool.menubar.append(general_settings) icon = ui.Icon('collapsed', general_settings, label=chr(0xe8b8), on_val=False, off_val=True, setter=toggle_general_settings, label_font='pupil_icons') icon.tooltip = 'General Settings' g_pool.iconbar.append(icon) user_plugin_separator = ui.Separator() user_plugin_separator.order = 0.35 g_pool.iconbar.append(user_plugin_separator) g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (100, -100)) g_pool.export_button = ui.Thumb('export', label=chr(0xe2c5), getter=lambda: False, setter=do_export, hotkey='e', label_font='pupil_icons') g_pool.quickbar.extend([g_pool.export_button]) g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.timelines) g_pool.gui.append(g_pool.iconbar) g_pool.gui.append(g_pool.quickbar) # we always load these plugins default_plugins = [('Plugin_Manager', {}), ('Seek_Control', {}), ('Log_Display', {}), ('Raw_Data_Exporter', {}), ('Vis_Polyline', {}), ('Vis_Circle', {}), ('System_Graphs', {}), ('System_Timelines', {}), ('Video_Export_Launcher', {}), ('Pupil_From_Recording', {}), ('Gaze_From_Recording', {})] g_pool.plugins = Plugin_List( g_pool, session_settings.get('loaded_plugins', default_plugins)) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) toggle_general_settings(True) g_pool.gui.configuration = session_settings.get('ui_config', {}) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() # trigger on_resize on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) def handle_notifications(n): subject = n['subject'] if subject == 'start_plugin': g_pool.plugins.add(g_pool.plugin_by_name[n['name']], args=n.get('args', {})) elif subject.startswith('meta.should_doc'): ipc_pub.notify({ 'subject': 'meta.doc', 'actor': g_pool.app, 'doc': player.__doc__ }) for p in g_pool.plugins: if (p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify): ipc_pub.notify({ 'subject': 'meta.doc', 'actor': p.class_name, 'doc': p.on_notify.__doc__ }) while not glfw.glfwWindowShouldClose(main_window): # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) # grab new frame if g_pool.capture.play or g_pool.new_seek: g_pool.new_seek = False try: new_frame = g_pool.capture.get_frame() except EndofVideoFileError: # end of video logic: pause at last frame. g_pool.capture.play = False logger.warning("end of video") frame = new_frame.copy() events = {} events['frame'] = frame # report time between now and the last loop interation events['dt'] = get_dt() # pupil and gaze positions are added by their respective producer plugins events['pupil_positions'] = [] events['gaze_positions'] = [] # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() glfw.glfwMakeContextCurrent(main_window) # render visual feedback from loaded plugins if gl_utils.is_window_visible(main_window): gl_utils.glViewport(0, 0, *g_pool.camera_render_size) g_pool.capture._recent_frame = frame g_pool.capture.gl_display() for p in g_pool.plugins: p.gl_display() gl_utils.glViewport(0, 0, *window_size) try: clipboard = glfw.glfwGetClipboardString( main_window).decode() except AttributeError: # clipbaord is None, might happen on startup clipboard = '' g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard and user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.glfwSetClipboardString(main_window, user_input.clipboard.encode()) for b in user_input.buttons: button, action, mods = b x, y = glfw.glfwGetCursorPos(main_window) pos = x * hdpi_factor, y * hdpi_factor pos = normalize(pos, g_pool.camera_render_size) pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_click(pos, button, action) for key, scancode, action, mods in user_input.keys: for p in g_pool.plugins: p.on_key(key, scancode, action, mods) for char_ in user_input.chars: for p in g_pool.plugins: p.on_char(char_) glfw.glfwSwapBuffers(main_window) # present frames at appropriate speed g_pool.capture.wait(frame) glfw.glfwPollEvents() session_settings['playback_speed'] = g_pool.capture.playback_speed session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['min_data_confidence'] = g_pool.min_data_confidence session_settings['gui_scale'] = g_pool.gui_user_scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos( main_window) session_settings['version'] = str(g_pool.version) session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.capture.cleanup() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) except: import traceback trace = traceback.format_exc() logger.error('Process Player crashed with trace:\n{}'.format(trace)) finally: logger.info("Process shutting down.") ipc_pub.notify({'subject': 'player_process.stopped'}) sleep(1.0)
class Marker_Detector(Plugin): """docstring """ def __init__(self,g_pool,menu_conf={},mode="Show markers and frames"): super(Marker_Detector, self).__init__(g_pool) self.order = .2 # all markers that are detected in the most recent frame self.markers = [] #load camera intrinsics try: K = np.load(os.path.join(self.g_pool.user_dir,'camera_matrix.npy')) dist_coef = np.load(os.path.join(self.g_pool.user_dir,"dist_coefs.npy")) img_size = np.load(os.path.join(self.g_pool.user_dir,"camera_resolution.npy")) self.camera_intrinsics = K, dist_coefs, img_size except: self.camera_intrinsics = None # all registered surfaces self.surface_definitions = Persistent_Dict(os.path.join(g_pool.user_dir,'surface_definitions') ) self.surfaces = [Reference_Surface(saved_definition=d) for d in self.surface_definitions.get('realtime_square_marker_surfaces',[]) if isinstance(d,dict)] # edit surfaces self.edit_surfaces = [] #plugin state self.mode = mode self.running = True self.robust_detection = 1 self.aperture = 11 self.min_marker_perimeter = 80 self.locate_3d = False #debug vars self.draw_markers = 0 self.show_surface_idx = 0 self.img_shape = None self.menu= None self.menu_conf= menu_conf self.button= None self.add_button = None def close(self): self.alive = False def on_click(self,pos,button,action): if self.mode == "Surface edit mode": if self.edit_surfaces: if action == GLFW_RELEASE: self.edit_surfaces = [] # no surfaces verts in edit mode, lets see if the cursor is close to one: else: if action == GLFW_PRESS: surf_verts = ((0.,0.),(1.,0.),(1.,1.),(0.,1.)) x,y = pos for s in self.surfaces: if s.detected and s.defined: for (vx,vy),i in zip(s.ref_surface_to_img(np.array(surf_verts)),range(4)): vx,vy = denormalize((vx,vy),(self.img_shape[1],self.img_shape[0]),flip_y=True) if sqrt((x-vx)**2 + (y-vy)**2) <15: #img pixels self.edit_surfaces.append((s,i)) def advance(self): pass def add_surface(self,_): self.surfaces.append(Reference_Surface()) self.update_gui_markers() def remove_surface(self,i): self.surfaces[i].cleanup() del self.surfaces[i] self.update_gui_markers() def init_gui(self): self.menu = ui.Growing_Menu('Marker Detector') self.menu.configuration = self.menu_conf self.g_pool.sidebar.append(self.menu) self.button = ui.Thumb('running',self,label='Track',hotkey='t') self.button.on_color[:] = (.1,.2,1.,.8) self.g_pool.quickbar.append(self.button) self.add_button = ui.Thumb('add_surface',setter=self.add_surface,getter=lambda:False,label='Add surface',hotkey='a') self.g_pool.quickbar.append(self.add_button) self.update_gui_markers() def deinit_gui(self): if self.menu: self.g_pool.sidebar.remove(self.menu) self.menu_conf= self.menu.configuration self.menu= None if self.button: self.g_pool.quickbar.remove(self.button) self.button = None if self.add_button: self.g_pool.quickbar.remove(self.add_button) self.add_button = None def update_gui_markers(self): self.menu.elements[:] = [] self.menu.append(ui.Info_Text('This plugin detects and tracks fiducial markers visible in the scene. You can define surfaces using 1 or more marker visible within the world view by clicking *add surface*. You can edit defined surfaces by selecting *Surface edit mode*.')) self.menu.append(ui.Button('Close',self.close)) self.menu.append(ui.Switch('robust_detection',self,label='Robust detection')) self.menu.append(ui.Switch('locate_3d',self,label='3D localization')) self.menu.append(ui.Selector('mode',self,label="Mode",selection=['Show markers and frames','Show marker IDs', 'Surface edit mode'] )) self.menu.append(ui.Button("Add surface", lambda:self.add_surface('_'),)) # disable locate_3d if camera intrinsics don't exist if self.camera_intrinsics is None: self.menu.elements[3].read_only = True for s in self.surfaces: idx = self.surfaces.index(s) s_menu = ui.Growing_Menu("Surface %s"%idx) s_menu.collapsed=True s_menu.append(ui.Text_Input('name',s,label='Name')) # self._bar.add_var("%s_markers"%i,create_string_buffer(512), getter=s.atb_marker_status,group=str(i),label='found/registered markers' ) s_menu.append(ui.Text_Input('x',s.real_world_size,'x_scale')) s_menu.append(ui.Text_Input('y',s.real_world_size,'y_scale')) s_menu.append(ui.Button('Open debug window',s.open_close_window)) #closure to encapsulate idx def make_remove_s(i): return lambda: self.remove_surface(i) remove_s = make_remove_s(idx) s_menu.append(ui.Button('Remove',remove_s)) self.menu.append(s_menu) def update(self,frame,events): self.img_shape = frame.height,frame.width,3 if self.running: gray = frame.gray if self.robust_detection: self.markers = detect_markers_robust(gray, grid_size = 5, prev_markers=self.markers, min_marker_perimeter=self.min_marker_perimeter, aperture=self.aperture, visualize=0, true_detect_every_frame=3) else: self.markers = detect_markers(gray, grid_size = 5, min_marker_perimeter=self.min_marker_perimeter, aperture=self.aperture, visualize=0) if self.mode == "Show marker IDs": draw_markers(frame.img,self.markers) # locate surfaces for s in self.surfaces: s.locate(self.markers, self.locate_3d,self.camera_intrinsics) # if s.detected: # events.append({'type':'marker_ref_surface','name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp}) if self.running: self.button.status_text = '%s/%s'%(len([s for s in self.surfaces if s.detected]),len(self.surfaces)) else: self.button.status_text = 'tracking paused' # edit surfaces by user if self.mode == "Surface edit mode": window = glfwGetCurrentContext() pos = glfwGetCursorPos(window) pos = normalize(pos,glfwGetWindowSize(window),flip_y=True) for s,v_idx in self.edit_surfaces: if s.detected: new_pos = s.img_to_ref_surface(np.array(pos)) s.move_vertex(v_idx,new_pos) #map recent gaze onto detected surfaces used for pupil server for s in self.surfaces: if s.detected: s.gaze_on_srf = [] for p in events.get('gaze',[]): gp_on_s = tuple(s.img_to_ref_surface(np.array(p['norm_pos']))) p['realtime gaze on ' + s.name] = gp_on_s s.gaze_on_srf.append(gp_on_s) def get_init_dict(self): if self.menu: d = {'menu_conf':self.menu.configuration,'mode':self.mode} else: d = {'menu_conf':self.menu_conf,'mode':self.mode} return d def gl_display(self): """ Display marker and surface info inside world screen """ if self.mode == "Show markers and frames": for m in self.markers: hat = np.array([[[0,0],[0,1],[.5,1.3],[1,1],[1,0],[0,0]]],dtype=np.float32) hat = cv2.perspectiveTransform(hat,m_marker_to_screen(m)) draw_gl_polyline(hat.reshape((6,2)),(0.1,1.,1.,.5)) for s in self.surfaces: s.gl_draw_frame(self.img_shape) for s in self.surfaces: if self.locate_3d: s.gl_display_in_window_3d(self.g_pool.image_tex,self.camera_intrinsics) else: s.gl_display_in_window(self.g_pool.image_tex) if self.mode == "Surface edit mode": for s in self.surfaces: s.gl_draw_frame(self.img_shape) s.gl_draw_corners() def cleanup(self): """ called when the plugin gets terminated. This happens either voluntarily or forced. if you have a GUI or glfw window destroy it here. """ self.surface_definitions["realtime_square_marker_surfaces"] = [rs.save_to_dict() for rs in self.surfaces if rs.defined] self.surface_definitions.close() for s in self.surfaces: s.cleanup() self.deinit_gui()
def player_drop(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_version): # general imports import logging # networking import zmq import zmq_tools from time import sleep # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.INFO) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) try: import glfw import gl_utils from OpenGL.GL import glClearColor from version_utils import VersionFormat from file_methods import Persistent_Dict from pyglui.pyfontstash import fontstash from pyglui.ui import get_roboto_font_path from player_methods import is_pupil_rec_dir, update_recording_to_recent def on_drop(window, count, paths): nonlocal rec_dir rec_dir = paths[0].decode('utf-8') if rec_dir: if not is_pupil_rec_dir(rec_dir): rec_dir = None # load session persistent settings session_settings = Persistent_Dict( os.path.join(user_dir, "user_settings_player")) if VersionFormat(session_settings.get("version", '0.0')) != app_version: logger.info( "Session setting are from a different version of this app. I will not use those." ) session_settings.clear() w, h = session_settings.get('window_size', (1280, 720)) window_pos = session_settings.get('window_position', window_position_default) glfw.glfwInit() glfw.glfwWindowHint(glfw.GLFW_RESIZABLE, 0) window = glfw.glfwCreateWindow(w, h, 'Pupil Player') glfw.glfwWindowHint(glfw.GLFW_RESIZABLE, 1) glfw.glfwMakeContextCurrent(window) glfw.glfwSetWindowPos(window, window_pos[0], window_pos[1]) glfw.glfwSetDropCallback(window, on_drop) glfont = fontstash.Context() glfont.add_font('roboto', get_roboto_font_path()) glfont.set_align_string(v_align="center", h_align="middle") glfont.set_color_float((0.2, 0.2, 0.2, 0.9)) gl_utils.basic_gl_setup() glClearColor(0.5, .5, 0.5, 0.0) text = 'Drop a recording directory onto this window.' tip = '(Tip: You can drop a recording directory onto the app icon.)' # text = "Please supply a Pupil recording directory as first arg when calling Pupil Player." while not glfw.glfwWindowShouldClose(window): fb_size = glfw.glfwGetFramebufferSize(window) hdpi_factor = float(fb_size[0] / glfw.glfwGetWindowSize(window)[0]) gl_utils.adjust_gl_view(*fb_size) if rec_dir: if is_pupil_rec_dir(rec_dir): logger.info( "Starting new session with '{}'".format(rec_dir)) text = "Updating recording format." tip = "This may take a while!" else: logger.error( "'{}' is not a valid pupil recording".format(rec_dir)) tip = "Oops! That was not a valid recording." rec_dir = None gl_utils.clear_gl_screen() glfont.set_blur(10.5) glfont.set_color_float((0.0, 0.0, 0.0, 1.)) glfont.set_size(w / 25. * hdpi_factor) glfont.draw_text(w / 2 * hdpi_factor, .3 * h * hdpi_factor, text) glfont.set_size(w / 30. * hdpi_factor) glfont.draw_text(w / 2 * hdpi_factor, .4 * h * hdpi_factor, tip) glfont.set_blur(0.96) glfont.set_color_float((1., 1., 1., 1.)) glfont.set_size(w / 25. * hdpi_factor) glfont.draw_text(w / 2 * hdpi_factor, .3 * h * hdpi_factor, text) glfont.set_size(w / 30. * hdpi_factor) glfont.draw_text(w / 2 * hdpi_factor, .4 * h * hdpi_factor, tip) glfw.glfwSwapBuffers(window) if rec_dir: update_recording_to_recent(rec_dir) glfw.glfwSetWindowShouldClose(window, True) glfw.glfwPollEvents() session_settings['window_position'] = glfw.glfwGetWindowPos(window) session_settings.close() glfw.glfwDestroyWindow(window) if rec_dir: ipc_pub.notify({ "subject": "player_process.should_start", "rec_dir": rec_dir }) except: import traceback trace = traceback.format_exc() logger.error( 'Process player_drop crashed with trace:\n{}'.format(trace)) finally: sleep(1.0)
class Offline_Surface_Tracker(Surface_Tracker, Analysis_Plugin_Base): """ Special version of surface tracker for use with videofile source. It uses a seperate process to search all frames in the world video file for markers. - self.cache is a list containing marker positions for each frame. - self.surfaces[i].cache is a list containing surface positions for each frame Both caches are build up over time. The marker cache is also session persistent. See marker_tracker.py for more info on this marker tracker. """ def __init__(self, g_pool, mode="Show Markers and Surfaces", min_marker_perimeter=100, invert_image=False, robust_detection=True): super().__init__( g_pool, mode, min_marker_perimeter, invert_image, robust_detection, ) self.order = .2 self.marker_cache_version = 2 self.min_marker_perimeter_cacher = 20 #find even super small markers. The surface locater will filter using min_marker_perimeter self.timeline_line_height = 16 self.load_marker_cache() self.init_marker_cacher() for s in self.surfaces: s.init_cache(self.cache, self.min_marker_perimeter, self.min_id_confidence) self.recalculate() def load_marker_cache(self): #check if marker cache is available from last session self.persistent_cache = Persistent_Dict( os.path.join(self.g_pool.rec_dir, 'square_marker_cache')) version = self.persistent_cache.get('version', 0) cache = self.persistent_cache.get('marker_cache', None) if cache is None: self.cache = Cache_List([False for _ in self.g_pool.timestamps]) self.persistent_cache['version'] = self.marker_cache_version self.persistent_cache['inverted_markers'] = self.invert_image elif version != self.marker_cache_version: self.persistent_cache['version'] = self.marker_cache_version self.invert_image = self.persistent_cache.get( 'inverted_markers', False) self.cache = Cache_List([False for _ in self.g_pool.timestamps]) logger.debug( "Marker cache version missmatch. Rebuilding marker cache.") else: self.cache = Cache_List(cache) #we overwrite the inverted_image setting from init with the one save in the marker cache. self.invert_image = self.persistent_cache.get( 'inverted_markers', False) logger.debug( "Loaded marker cache {} / {} frames had been searched before". format( len(self.cache) - self.cache.count(False), len(self.cache))) def clear_marker_cache(self): self.cache = Cache_List([False for _ in self.g_pool.timestamps]) self.persistent_cache['version'] = self.marker_cache_version def load_surface_definitions_from_file(self): self.surface_definitions = Persistent_Dict( os.path.join(self.g_pool.rec_dir, 'surface_definitions')) if self.surface_definitions.get('offline_square_marker_surfaces', []) != []: logger.debug( "Found ref surfaces defined or copied in previous session.") self.surfaces = [ Offline_Reference_Surface(self.g_pool, saved_definition=d) for d in self.surface_definitions.get( 'offline_square_marker_surfaces', []) ] elif self.surface_definitions.get('realtime_square_marker_surfaces', []) != []: logger.debug( "Did not find ref surfaces def created or used by the user in player from earlier session. Loading surfaces defined during capture." ) self.surfaces = [ Offline_Reference_Surface(self.g_pool, saved_definition=d) for d in self.surface_definitions.get( 'realtime_square_marker_surfaces', []) ] else: logger.debug("No surface defs found. Please define using GUI.") self.surfaces = [] def init_ui(self): self.add_menu() self.menu.label = 'Offline Surface Tracker' self.add_button = ui.Thumb('add_surface', setter=lambda x: self.add_surface(), getter=lambda: False, label='A', hotkey='a') self.g_pool.quickbar.append(self.add_button) self.glfont = fontstash.Context() self.glfont.add_font('opensans', ui.get_opensans_font_path()) self.glfont.set_color_float((1., 1., 1., .8)) self.glfont.set_align_string(v_align='right', h_align='top') self.timeline = ui.Timeline( 'Surface Tracker', self.gl_display_cache_bars, self.draw_labels, self.timeline_line_height * (len(self.surfaces) + 1)) self.g_pool.user_timelines.append(self.timeline) self.update_gui_markers() def deinit_ui(self): self.g_pool.user_timelines.remove(self.timeline) self.timeline = None self.glfont = None self.remove_menu() if self.add_button: self.g_pool.quickbar.remove(self.add_button) self.add_button = None def update_gui_markers(self): def set_min_marker_perimeter(val): self.min_marker_perimeter = val self.notify_all({ 'subject': 'min_marker_perimeter_changed', 'delay': 1 }) def set_invert_image(val): self.invert_image = val self.invalidate_marker_cache() self.invalidate_surface_caches() self.menu.elements[:] = [] self.menu.append( ui.Switch('invert_image', self, setter=set_invert_image, label='Use inverted markers')) self.menu.append( ui.Slider('min_marker_perimeter', self, min=20, max=500, step=1, setter=set_min_marker_perimeter)) self.menu.append( ui.Info_Text( 'The offline surface tracker will look for markers in the entire video. By default it uses surfaces defined in capture. You can change and add more surfaces here.' )) self.menu.append( ui.Info_Text( "Press the export button or type 'e' to start the export.")) self.menu.append( ui.Selector('mode', self, label='Mode', selection=[ "Show Markers and Surfaces", "Show marker IDs", "Show Heatmaps", "Show Metrics" ])) self.menu.append( ui.Info_Text( 'To see heatmap or surface metrics visualizations, click (re)-calculate gaze distributions. Set "X size" and "Y size" for each surface to see heatmap visualizations.' )) self.menu.append( ui.Button("(Re)-calculate gaze distributions", self.recalculate)) self.menu.append(ui.Button("Add surface", lambda: self.add_surface())) for s in self.surfaces: idx = self.surfaces.index(s) s_menu = ui.Growing_Menu("Surface {}".format(idx)) s_menu.collapsed = True s_menu.append(ui.Text_Input('name', s)) s_menu.append(ui.Text_Input('x', s.real_world_size, label='X size')) s_menu.append(ui.Text_Input('y', s.real_world_size, label='Y size')) s_menu.append(ui.Button('Open Debug Window', s.open_close_window)) #closure to encapsulate idx def make_remove_s(i): return lambda: self.remove_surface(i) remove_s = make_remove_s(idx) s_menu.append(ui.Button('remove', remove_s)) self.menu.append(s_menu) def on_notify(self, notification): if notification['subject'] == 'gaze_positions_changed': logger.info('Gaze postions changed. Recalculating.') self.recalculate() if notification['subject'] == 'min_data_confidence_changed': logger.info('Min_data_confidence changed. Recalculating.') self.recalculate() elif notification['subject'] == 'surfaces_changed': logger.info('Surfaces changed. Recalculating.') self.recalculate() elif notification['subject'] == 'min_marker_perimeter_changed': logger.info( 'Min marker perimeter adjusted. Re-detecting surfaces.') self.invalidate_surface_caches() elif notification['subject'] == "should_export": self.save_surface_statsics_to_file(notification['range'], notification['export_dir']) def add_surface(self): self.surfaces.append(Offline_Reference_Surface(self.g_pool)) self.timeline.height += self.timeline_line_height self.update_gui_markers() def remove_surface(self, i): super().remove_surface(i) self.timeline.height -= self.timeline_line_height def recalculate(self): in_mark = self.g_pool.seek_control.trim_left out_mark = self.g_pool.seek_control.trim_right section = slice(in_mark, out_mark) # calc heatmaps for s in self.surfaces: if s.defined: s.generate_heatmap(section) # calc distirbution accross all surfaces. results = [] for s in self.surfaces: gaze_on_srf = s.gaze_on_srf_in_section(section) results.append(len(gaze_on_srf)) self.metrics_gazecount = len(gaze_on_srf) if results == []: logger.warning("No surfaces defined.") return max_res = max(results) results = np.array(results, dtype=np.float32) if not max_res: logger.warning("No gaze on any surface for this section!") else: results *= 255. / max_res results = np.uint8(results) results_c_maps = cv2.applyColorMap(results, cv2.COLORMAP_JET) for s, c_map in zip(self.surfaces, results_c_maps): heatmap = np.ones((1, 1, 4), dtype=np.uint8) * 125 heatmap[:, :, :3] = c_map s.metrics_texture = Named_Texture() s.metrics_texture.update_from_ndarray(heatmap) def invalidate_surface_caches(self): for s in self.surfaces: s.cache = None def recent_events(self, events): frame = events.get('frame') if not frame: return self.img_shape = frame.img.shape self.update_marker_cache() # self.markers = [m for m in self.cache[frame.index] if m['perimeter'>=self.min_marker_perimeter] self.markers = self.cache[frame.index] if self.markers is False: self.markers = [] # tell precacher that it better have every thing from here on analyzed self.seek_marker_cacher(frame.index) events['surfaces'] = [] # locate surfaces for s in self.surfaces: if not s.locate_from_cache(frame.index): s.locate(self.markers, self.min_marker_perimeter, self.min_id_confidence) if s.detected: events['surfaces'].append({ 'name': s.name, 'uid': s.uid, 'm_to_screen': s.m_to_screen.tolist(), 'm_from_screen': s.m_from_screen.tolist(), 'gaze_on_srf': s.gaze_on_srf, 'timestamp': frame.timestamp, 'camera_pose_3d': s.camera_pose_3d.tolist() if s.camera_pose_3d is not None else None }) if self.mode == "Show marker IDs": draw_markers(frame.img, self.markers) elif self.mode == "Show Markers and Surfaces": # edit surfaces by user if self.edit_surf_verts: window = glfwGetCurrentContext() pos = glfwGetCursorPos(window) pos = normalize(pos, self.g_pool.camera_render_size, flip_y=True) for s, v_idx in self.edit_surf_verts: if s.detected: new_pos = s.img_to_ref_surface(np.array(pos)) s.move_vertex(v_idx, new_pos) else: # update srf with no or invald cache: for s in self.surfaces: if s.cache == None: s.init_cache(self.cache, self.min_marker_perimeter, self.min_id_confidence) self.notify_all({ 'subject': 'surfaces_changed', 'delay': 1 }) # allow surfaces to open/close windows for s in self.surfaces: if s.window_should_close: s.close_window() if s.window_should_open: s.open_window() def invalidate_marker_cache(self): self.close_marker_cacher() self.clear_marker_cache() self.init_marker_cacher() def init_marker_cacher(self): from marker_detector_cacher import fill_cache visited_list = [False if x is False else True for x in self.cache] video_file_path = self.g_pool.capture.source_path self.cache_queue = mp.Queue() self.cacher_seek_idx = mp.Value('i', 0) self.cacher_run = mp.Value(c_bool, True) self.cacher = mp.Process( target=fill_cache, args=(visited_list, video_file_path, self.cache_queue, self.cacher_seek_idx, self.cacher_run, self.min_marker_perimeter_cacher, self.invert_image)) self.cacher.start() def update_marker_cache(self): while not self.cache_queue.empty(): idx, c_m = self.cache_queue.get() self.cache.update(idx, c_m) for s in self.surfaces: s.update_cache(self.cache, min_marker_perimeter=self.min_marker_perimeter, min_id_confidence=self.min_id_confidence, idx=idx) if self.cacher_run.value is False: self.recalculate() if self.timeline: self.timeline.refresh() def seek_marker_cacher(self, idx): self.cacher_seek_idx.value = idx def close_marker_cacher(self): self.update_marker_cache() self.cacher_run.value = False self.cacher.join(1.0) if self.cacher.is_alive(): logger.error("Marker cacher unresponsive - terminating.") self.cacher.terminate() def gl_display(self): """ Display marker and surface info inside world screen """ super().gl_display() if self.mode == "Show Metrics": #todo: draw a backdrop to represent the gaze that is not on any surface for s in self.surfaces: #draw a quad on surface with false color of value. s.gl_display_metrics() def gl_display_cache_bars(self, width, height, scale): """ """ with gl_utils.Coord_System(0, self.cache.length - 1, height, 0): # Lines for areas that have been cached cached_ranges = [] for r in self.cache.visited_ranges: # [[0,1],[3,4]] cached_ranges += (r[0], 0), (r[1], 0 ) # [(0,0),(1,0),(3,0),(4,0)] glTranslatef(0, scale * self.timeline_line_height / 2, 0) color = RGBA(.8, .6, .2, .8) draw_polyline(cached_ranges, color=color, line_type=GL_LINES, thickness=scale * 4) # Lines where surfaces have been found in video cached_surfaces = [] for s in self.surfaces: found_at = [] if s.cache is not None: for r in s.cache.positive_ranges: # [[0,1],[3,4]] found_at += (r[0], 0), (r[1], 0 ) # [(0,0),(1,0),(3,0),(4,0)] cached_surfaces.append(found_at) color = RGBA(0, .7, .3, .8) for s in cached_surfaces: glTranslatef(0, scale * self.timeline_line_height, 0) draw_polyline(s, color=color, line_type=GL_LINES, thickness=scale * 2) def draw_labels(self, width, height, scale): self.glfont.set_size(self.timeline_line_height * .8 * scale) self.glfont.draw_text(width, 0, 'Marker Cache') for idx, s in enumerate(self.surfaces): glTranslatef(0, self.timeline_line_height * scale, 0) self.glfont.draw_text(width, 0, s.name) def save_surface_statsics_to_file(self, export_range, export_dir): """ between in and out mark report: gaze distribution: - total gazepoints - gaze points on surface x - gaze points not on any surface report: surface visisbility - total frames - surface x visible framecount surface events: frame_no, ts, surface "name", "id" enter/exit for each surface: fixations_on_name.csv gaze_on_name_id.csv positions_of_name_id.csv """ metrics_dir = os.path.join(export_dir, 'surfaces') section = slice(*export_range) in_mark = section.start out_mark = section.stop logger.info("exporting metrics to {}".format(metrics_dir)) if os.path.isdir(metrics_dir): logger.info("Will overwrite previous export for this section") else: try: os.mkdir(metrics_dir) except: logger.warning( "Could not make metrics dir {}".format(metrics_dir)) return with open(os.path.join(metrics_dir, 'surface_visibility.csv'), 'w', encoding='utf-8', newline='') as csvfile: csv_writer = csv.writer(csvfile, delimiter=',') # surface visibility report frame_count = len(self.g_pool.timestamps[section]) csv_writer.writerow(('frame_count', frame_count)) csv_writer.writerow(('')) csv_writer.writerow(('surface_name', 'visible_frame_count')) for s in self.surfaces: if s.cache == None: logger.warning( "The surface is not cached. Please wait for the cacher to collect data." ) return visible_count = s.visible_count_in_section(section) csv_writer.writerow((s.name, visible_count)) logger.info("Created 'surface_visibility.csv' file") with open(os.path.join(metrics_dir, 'surface_gaze_distribution.csv'), 'w', encoding='utf-8', newline='') as csvfile: csv_writer = csv.writer(csvfile, delimiter=',') # gaze distribution report gaze_in_section = list( chain(*self.g_pool.gaze_positions_by_frame[section])) not_on_any_srf = set([gp['timestamp'] for gp in gaze_in_section]) csv_writer.writerow( ('total_gaze_point_count', len(gaze_in_section))) csv_writer.writerow(('')) csv_writer.writerow(('surface_name', 'gaze_count')) for s in self.surfaces: gaze_on_srf = s.gaze_on_srf_in_section(section) gaze_on_srf = set( [gp['base_data']['timestamp'] for gp in gaze_on_srf]) not_on_any_srf -= gaze_on_srf csv_writer.writerow((s.name, len(gaze_on_srf))) csv_writer.writerow(('not_on_any_surface', len(not_on_any_srf))) logger.info("Created 'surface_gaze_distribution.csv' file") with open(os.path.join(metrics_dir, 'surface_events.csv'), 'w', encoding='utf-8', newline='') as csvfile: csv_writer = csv.writer(csvfile, delimiter=',') # surface events report csv_writer.writerow(('frame_number', 'timestamp', 'surface_name', 'surface_uid', 'event_type')) events = [] for s in self.surfaces: for enter_frame_id, exit_frame_id in s.cache.positive_ranges: events.append({ 'frame_id': enter_frame_id, 'srf_name': s.name, 'srf_uid': s.uid, 'event': 'enter' }) events.append({ 'frame_id': exit_frame_id, 'srf_name': s.name, 'srf_uid': s.uid, 'event': 'exit' }) events.sort(key=lambda x: x['frame_id']) for e in events: csv_writer.writerow( (e['frame_id'], self.g_pool.timestamps[e['frame_id']], e['srf_name'], e['srf_uid'], e['event'])) logger.info("Created 'surface_events.csv' file") for s in self.surfaces: # per surface names: surface_name = '_' + s.name.replace('/', '') + '_' + s.uid #save surface_positions as csv with open(os.path.join(metrics_dir, 'srf_positons' + surface_name + '.csv'), 'w', encoding='utf-8', newline='') as csvfile: csv_writer = csv.writer(csvfile, delimiter=',') csv_writer.writerow(('frame_idx', 'timestamp', 'm_to_screen', 'm_from_screen', 'detected_markers')) for idx, ts, ref_srf_data in zip( range(len(self.g_pool.timestamps)), self.g_pool.timestamps, s.cache): if in_mark <= idx <= out_mark: if ref_srf_data is not None and ref_srf_data is not False: csv_writer.writerow( (idx, ts, ref_srf_data['m_to_screen'], ref_srf_data['m_from_screen'], ref_srf_data['detected_markers'])) # save gaze on srf as csv. with open(os.path.join( metrics_dir, 'gaze_positions_on_surface' + surface_name + '.csv'), 'w', encoding='utf-8', newline='') as csvfile: csv_writer = csv.writer(csvfile, delimiter=',') csv_writer.writerow( ('world_timestamp', 'world_frame_idx', 'gaze_timestamp', 'x_norm', 'y_norm', 'x_scaled', 'y_scaled', 'on_srf')) for idx, ts, ref_srf_data in zip( range(len(self.g_pool.timestamps)), self.g_pool.timestamps, s.cache): if in_mark <= idx <= out_mark: if ref_srf_data is not None and ref_srf_data is not False: for gp in s.gaze_on_srf_by_frame_idx( idx, ref_srf_data['m_from_screen']): csv_writer.writerow( (ts, idx, gp['base_data']['timestamp'], gp['norm_pos'][0], gp['norm_pos'][1], gp['norm_pos'][0] * s.real_world_size['x'], gp['norm_pos'][1] * s.real_world_size['y'], gp['on_srf'])) # save fixation on srf as csv. with open(os.path.join( metrics_dir, 'fixations_on_surface' + surface_name + '.csv'), 'w', encoding='utf-8', newline='') as csvfile: csv_writer = csv.writer(csvfile, delimiter=',') csv_writer.writerow( ('id', 'start_timestamp', 'duration', 'start_frame', 'end_frame', 'norm_pos_x', 'norm_pos_y', 'x_scaled', 'y_scaled', 'on_srf')) fixations_on_surface = [] for idx, ref_srf_data in zip( range(len(self.g_pool.timestamps)), s.cache): if in_mark <= idx <= out_mark: if ref_srf_data is not None and ref_srf_data is not False: for f in s.fixations_on_srf_by_frame_idx( idx, ref_srf_data['m_from_screen']): fixations_on_surface.append(f) removed_duplicates = dict([ (f['base_data']['id'], f) for f in fixations_on_surface ]).values() for f_on_s in removed_duplicates: f = f_on_s['base_data'] f_x, f_y = f_on_s['norm_pos'] f_on_srf = f_on_s['on_srf'] csv_writer.writerow( (f['id'], f['timestamp'], f['duration'], f['start_frame_index'], f['end_frame_index'], f_x, f_y, f_x * s.real_world_size['x'], f_y * s.real_world_size['y'], f_on_srf)) logger.info( "Saved surface positon gaze and fixation data for '{}' with uid:'{}'" .format(s.name, s.uid)) if s.heatmap is not None: logger.info("Saved Heatmap as .png file.") cv2.imwrite( os.path.join(metrics_dir, 'heatmap' + surface_name + '.png'), s.heatmap) logger.info("Done exporting reference surface data.") # if s.detected and self.img is not None: # #let save out the current surface image found in video # #here we get the verts of the surface quad in norm_coords # mapped_space_one = np.array(((0,0),(1,0),(1,1),(0,1)),dtype=np.float32).reshape(-1,1,2) # screen_space = cv2.perspectiveTransform(mapped_space_one,s.m_to_screen).reshape(-1,2) # #now we convert to image pixel coods # screen_space[:,1] = 1-screen_space[:,1] # screen_space[:,1] *= self.img.shape[0] # screen_space[:,0] *= self.img.shape[1] # s_0,s_1 = s.real_world_size # #no we need to flip vertically again by setting the mapped_space verts accordingly. # mapped_space_scaled = np.array(((0,s_1),(s_0,s_1),(s_0,0),(0,0)),dtype=np.float32) # M = cv2.getPerspectiveTransform(screen_space,mapped_space_scaled) # #here we do the actual perspactive transform of the image. # srf_in_video = cv2.warpPerspective(self.img,M, (int(s.real_world_size['x']),int(s.real_world_size['y'])) ) # cv2.imwrite(os.path.join(metrics_dir,'surface'+surface_name+'.png'),srf_in_video) # logger.info("Saved current image as .png file.") # else: # logger.info("'%s' is not currently visible. Seek to appropriate frame and repeat this command."%s.name) def cleanup(self): """ called when the plugin gets terminated. This happens either voluntarily or forced. if you have a GUI or glfw window destroy it here. """ self.surface_definitions["offline_square_marker_surfaces"] = [ rs.save_to_dict() for rs in self.surfaces if rs.defined ] self.surface_definitions.close() self.close_marker_cacher() self.persistent_cache['inverted_markers'] = self.invert_image self.persistent_cache["marker_cache"] = self.cache.to_list() self.persistent_cache.close() for s in self.surfaces: s.close_window()
class Offline_Marker_Detector(Plugin): """ Special version of marker detector for use with videofile source. It uses a seperate process to search all frames in the world.avi file for markers. - self.cache is a list containing marker positions for each frame. - self.surfaces[i].cache is a list containing surface positions for each frame Both caches are build up over time. The marker cache is also session persistent. See marker_tracker.py for more info on this marker tracker. """ def __init__(self, g_pool, mode="Show Markers and Frames"): super(Offline_Marker_Detector, self).__init__(g_pool) self.order = .2 # all markers that are detected in the most recent frame self.markers = [] # all registered surfaces if g_pool.app == 'capture': raise Exception('For Player only.') #in player we load from the rec_dir: but we have a couple options: self.surface_definitions = Persistent_Dict( os.path.join(g_pool.rec_dir, 'surface_definitions')) if self.surface_definitions.get('offline_square_marker_surfaces', []) != []: logger.debug( "Found ref surfaces defined or copied in previous session.") self.surfaces = [ Offline_Reference_Surface(self.g_pool, saved_definition=d) for d in self.surface_definitions.get( 'offline_square_marker_surfaces', []) if isinstance(d, dict) ] elif self.surface_definitions.get('realtime_square_marker_surfaces', []) != []: logger.debug( "Did not find ref surfaces def created or used by the user in player from earlier session. Loading surfaces defined during capture." ) self.surfaces = [ Offline_Reference_Surface(self.g_pool, saved_definition=d) for d in self.surface_definitions.get( 'realtime_square_marker_surfaces', []) if isinstance(d, dict) ] else: logger.debug("No surface defs found. Please define using GUI.") self.surfaces = [] # ui mode settings self.mode = mode # edit surfaces self.edit_surfaces = [] #check if marker cache is available from last session self.persistent_cache = Persistent_Dict( os.path.join(g_pool.rec_dir, 'square_marker_cache')) self.cache = Cache_List( self.persistent_cache.get('marker_cache', [False for _ in g_pool.timestamps])) logger.debug( "Loaded marker cache %s / %s frames had been searched before" % (len(self.cache) - self.cache.count(False), len(self.cache))) self.init_marker_cacher() #debug vars self.show_surface_idx = c_int(0) self.img_shape = None self.img = None def init_gui(self): self.menu = ui.Scrolling_Menu('Offline Marker Tracker') self.g_pool.gui.append(self.menu) self.add_button = ui.Thumb('add_surface', setter=self.add_surface, getter=lambda: False, label='Add Surface', hotkey='a') self.g_pool.quickbar.append(self.add_button) self.update_gui_markers() self.on_window_resize(glfwGetCurrentContext(), *glfwGetWindowSize(glfwGetCurrentContext())) def deinit_gui(self): if self.menu: self.g_pool.gui.remove(self.menu) self.menu = None if self.add_button: self.g_pool.quickbar.remove(self.add_button) self.add_button = None def update_gui_markers(self): pass self.menu.elements[:] = [] self.menu.append( ui.Info_Text( 'The offline marker tracker will look for markers in the entire video. By default it uses surfaces defined in capture. You can change and add more surfaces here.' )) self.menu.append(ui.Button('Close', self.close)) self.menu.append( ui.Selector('mode', self, label='Mode', selection=[ "Show Markers and Frames", "Show marker IDs", "Surface edit mode", "Show Heatmaps", "Show Metrics" ])) self.menu.append( ui.Info_Text( 'To see heatmap or surface metrics visualizations, click (re)-calculate gaze distributions. Set "X size" and "Y size" for each surface to see heatmap visualizations.' )) self.menu.append( ui.Button("(Re)-calculate gaze distributions", self.recalculate)) self.menu.append( ui.Button("Export gaze and surface data", self.save_surface_statsics_to_file)) self.menu.append( ui.Button("Add surface", lambda: self.add_surface('_'))) for s in self.surfaces: idx = self.surfaces.index(s) s_menu = ui.Growing_Menu("Surface %s" % idx) s_menu.collapsed = True s_menu.append(ui.Text_Input('name', s)) s_menu.append(ui.Text_Input('x', s.real_world_size, label='X size')) s_menu.append(ui.Text_Input('y', s.real_world_size, label='Y size')) s_menu.append(ui.Button('Open Debug Window', s.open_close_window)) #closure to encapsulate idx def make_remove_s(i): return lambda: self.remove_surface(i) remove_s = make_remove_s(idx) s_menu.append(ui.Button('remove', remove_s)) self.menu.append(s_menu) def close(self): self.alive = False def on_window_resize(self, window, w, h): self.win_size = w, h def on_click(self, pos, button, action): if self.mode == "Surface edit mode": if self.edit_surfaces: if action == GLFW_RELEASE: self.edit_surfaces = [] # no surfaces verts in edit mode, lets see if the curser is close to one: else: if action == GLFW_PRESS: surf_verts = ((0., 0.), (1., 0.), (1., 1.), (0., 1.)) x, y = pos for s in self.surfaces: if s.detected and s.defined: for (vx, vy), i in zip( s.ref_surface_to_img(np.array(surf_verts)), range(4)): vx, vy = denormalize( (vx, vy), (self.img_shape[1], self.img_shape[0]), flip_y=True) if sqrt((x - vx)**2 + (y - vy)**2) < 15: #img pixels self.edit_surfaces.append((s, i)) def advance(self): pass def add_surface(self, _): self.surfaces.append(Offline_Reference_Surface(self.g_pool)) self.update_gui_markers() def remove_surface(self, i): self.surfaces[i].cleanup() del self.surfaces[i] self.update_gui_markers() def recalculate(self): in_mark = self.g_pool.trim_marks.in_mark out_mark = self.g_pool.trim_marks.out_mark section = slice(in_mark, out_mark) # calc heatmaps for s in self.surfaces: if s.defined: s.generate_heatmap(section) # calc distirbution accross all surfaces. results = [] for s in self.surfaces: gaze_on_srf = s.gaze_on_srf_in_section(section) results.append(len(gaze_on_srf)) self.metrics_gazecount = len(gaze_on_srf) if results == []: logger.warning("No surfaces defined.") return max_res = max(results) results = np.array(results, dtype=np.float32) if not max_res: logger.warning("No gaze on any surface for this section!") else: results *= 255. / max_res results = np.uint8(results) results_c_maps = cv2.applyColorMap(results, cv2.COLORMAP_JET) for s, c_map in zip(self.surfaces, results_c_maps): heatmap = np.ones((1, 1, 4), dtype=np.uint8) * 125 heatmap[:, :, :3] = c_map s.metrics_texture = create_named_texture(heatmap.shape) update_named_texture(s.metrics_texture, heatmap) def update(self, frame, events): self.img = frame.img self.img_shape = frame.img.shape self.update_marker_cache() self.markers = self.cache[frame.index] if self.markers == False: self.markers = [] self.seek_marker_cacher( frame.index ) # tell precacher that it better have every thing from here on analyzed # locate surfaces for s in self.surfaces: if not s.locate_from_cache(frame.index): s.locate(self.markers) if s.detected: pass # events.append({'type':'marker_ref_surface','name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp,'gaze_on_srf':s.gaze_on_srf}) if self.mode == "Show marker IDs": draw_markers(frame.img, self.markers) # edit surfaces by user if self.mode == "Surface edit mode": window = glfwGetCurrentContext() pos = glfwGetCursorPos(window) pos = normalize(pos, glfwGetWindowSize(window), flip_y=True) for s, v_idx in self.edit_surfaces: if s.detected: new_pos = s.img_to_ref_surface(np.array(pos)) s.move_vertex(v_idx, new_pos) s.cache = None self.heatmap = None else: # update srf with no or invald cache: for s in self.surfaces: if s.cache == None: s.init_cache(self.cache) #allow surfaces to open/close windows for s in self.surfaces: if s.window_should_close: s.close_window() if s.window_should_open: s.open_window() def init_marker_cacher(self): forking_enable(0) #for MacOs only from marker_detector_cacher import fill_cache visited_list = [False if x == False else True for x in self.cache] video_file_path = os.path.join(self.g_pool.rec_dir, 'world.mkv') if not os.path.isfile(video_file_path): video_file_path = os.path.join(self.g_pool.rec_dir, 'world.avi') self.cache_queue = Queue() self.cacher_seek_idx = Value('i', 0) self.cacher_run = Value(c_bool, True) self.cacher = Process(target=fill_cache, args=(visited_list, video_file_path, self.cache_queue, self.cacher_seek_idx, self.cacher_run)) self.cacher.start() def update_marker_cache(self): while not self.cache_queue.empty(): idx, c_m = self.cache_queue.get() self.cache.update(idx, c_m) for s in self.surfaces: s.update_cache(self.cache, idx=idx) def seek_marker_cacher(self, idx): self.cacher_seek_idx.value = idx def close_marker_cacher(self): self.update_marker_cache() self.cacher_run.value = False self.cacher.join() def gl_display(self): """ Display marker and surface info inside world screen """ self.gl_display_cache_bars() for s in self.surfaces: s.gl_display_in_window(self.g_pool.image_tex) if self.mode == "Show Markers and Frames": for m in self.markers: hat = np.array([[[0, 0], [0, 1], [1, 1], [1, 0], [0, 0]]], dtype=np.float32) hat = cv2.perspectiveTransform(hat, m_marker_to_screen(m)) draw_polyline(hat.reshape((5, 2)), color=RGBA(0.1, 1., 1., .3), line_type=GL_POLYGON) draw_polyline(hat.reshape((5, 2)), color=RGBA(0.1, 1., 1., .6)) for s in self.surfaces: s.gl_draw_frame(self.img_shape) if self.mode == "Surface edit mode": for s in self.surfaces: s.gl_draw_frame(self.img_shape) s.gl_draw_corners() if self.mode == "Show Heatmaps": for s in self.surfaces: s.gl_display_heatmap() if self.mode == "Show Metrics": #todo: draw a backdrop to represent the gaze that is not on any surface for s in self.surfaces: #draw a quad on surface with false color of value. s.gl_display_metrics() def gl_display_cache_bars(self): """ """ padding = 20. # Lines for areas that have been cached cached_ranges = [] for r in self.cache.visited_ranges: # [[0,1],[3,4]] cached_ranges += (r[0], 0), (r[1], 0) #[(0,0),(1,0),(3,0),(4,0)] # Lines where surfaces have been found in video cached_surfaces = [] for s in self.surfaces: found_at = [] if s.cache is not None: for r in s.cache.positive_ranges: # [[0,1],[3,4]] found_at += (r[0], 0), (r[1], 0 ) #[(0,0),(1,0),(3,0),(4,0)] cached_surfaces.append(found_at) glMatrixMode(GL_PROJECTION) glPushMatrix() glLoadIdentity() width, height = self.win_size h_pad = padding * (self.cache.length - 2) / float(width) v_pad = padding * 1. / (height - 2) glOrtho( -h_pad, (self.cache.length - 1) + h_pad, -v_pad, 1 + v_pad, -1, 1 ) # ranging from 0 to cache_len-1 (horizontal) and 0 to 1 (vertical) glMatrixMode(GL_MODELVIEW) glPushMatrix() glLoadIdentity() color = RGBA(8., .6, .2, 8.) draw_polyline(cached_ranges, color=color, line_type=GL_LINES, thickness=4) color = RGBA(0., .7, .3, 8.) for s in cached_surfaces: glTranslatef(0, .02, 0) draw_polyline(s, color=color, line_type=GL_LINES, thickness=2) glMatrixMode(GL_PROJECTION) glPopMatrix() glMatrixMode(GL_MODELVIEW) glPopMatrix() def save_surface_statsics_to_file(self): in_mark = self.g_pool.trim_marks.in_mark out_mark = self.g_pool.trim_marks.out_mark """ between in and out mark report: gaze distribution: - total gazepoints - gaze points on surface x - gaze points not on any surface report: surface visisbility - total frames - surface x visible framecount surface events: frame_no, ts, surface "name", "id" enter/exit for each surface: fixations_on_name.csv gaze_on_name_id.csv positions_of_name_id.csv """ section = slice(in_mark, out_mark) metrics_dir = os.path.join(self.g_pool.rec_dir, "metrics_%s-%s" % (in_mark, out_mark)) logger.info("exporting metrics to %s" % metrics_dir) if os.path.isdir(metrics_dir): logger.info("Will overwrite previous export for this section") else: try: os.mkdir(metrics_dir) except: logger.warning("Could not make metrics dir %s" % metrics_dir) return with open(os.path.join(metrics_dir, 'surface_visibility.csv'), 'wb') as csvfile: csv_writer = csv.writer(csvfile, delimiter='\t', quotechar='|', quoting=csv.QUOTE_MINIMAL) # surface visibility report frame_count = len(self.g_pool.timestamps[section]) csv_writer.writerow(('frame_count', frame_count)) csv_writer.writerow(('')) csv_writer.writerow(('surface_name', 'visible_frame_count')) for s in self.surfaces: if s.cache == None: logger.warning( "The surface is not cached. Please wait for the cacher to collect data." ) return visible_count = s.visible_count_in_section(section) csv_writer.writerow((s.name, visible_count)) logger.info("Created 'surface_visibility.csv' file") with open(os.path.join(metrics_dir, 'surface_gaze_distribution.csv'), 'wb') as csvfile: csv_writer = csv.writer(csvfile, delimiter='\t', quotechar='|', quoting=csv.QUOTE_MINIMAL) # gaze distribution report gaze_in_section = list( chain(*self.g_pool.gaze_positions_by_frame[section])) not_on_any_srf = set([gp['timestamp'] for gp in gaze_in_section]) csv_writer.writerow( ('total_gaze_point_count', len(gaze_in_section))) csv_writer.writerow(('')) csv_writer.writerow(('surface_name', 'gaze_count')) for s in self.surfaces: gaze_on_srf = s.gaze_on_srf_in_section(section) gaze_on_srf = set( [gp['base']["timestamp"] for gp in gaze_on_srf]) not_on_any_srf -= gaze_on_srf csv_writer.writerow((s.name, len(gaze_on_srf))) csv_writer.writerow(('not_on_any_surface', len(not_on_any_srf))) logger.info("Created 'surface_gaze_distribution.csv' file") with open(os.path.join(metrics_dir, 'surface_events.csv'), 'wb') as csvfile: csv_writer = csv.writer(csvfile, delimiter='\t', quotechar='|', quoting=csv.QUOTE_MINIMAL) # surface events report csv_writer.writerow(('frame_number', 'timestamp', 'surface_name', 'surface_uid', 'event_type')) events = [] for s in self.surfaces: for enter_frame_id, exit_frame_id in s.cache.positive_ranges: events.append({ 'frame_id': enter_frame_id, 'srf_name': s.name, 'srf_uid': s.uid, 'event': 'enter' }) events.append({ 'frame_id': exit_frame_id, 'srf_name': s.name, 'srf_uid': s.uid, 'event': 'exit' }) events.sort(key=lambda x: x['frame_id']) for e in events: csv_writer.writerow( (e['frame_id'], self.g_pool.timestamps[e['frame_id']], e['srf_name'], e['srf_uid'], e['event'])) logger.info("Created 'surface_events.csv' file") for s in self.surfaces: # per surface names: surface_name = '_' + s.name.replace('/', '') + '_' + s.uid # save surface_positions as pickle file save_object( s.cache.to_list(), os.path.join(metrics_dir, 'srf_positions' + surface_name)) #save surface_positions as csv with open( os.path.join(metrics_dir, 'srf_positons' + surface_name + '.csv'), 'wb') as csvfile: csv_writer = csv.writer(csvfile, delimiter='\t', quotechar='|', quoting=csv.QUOTE_MINIMAL) csv_writer.writerow(('frame_idx', 'timestamp', 'm_to_screen', 'm_from_screen', 'detected_markers')) for idx, ts, ref_srf_data in zip( range(len(self.g_pool.timestamps)), self.g_pool.timestamps, s.cache): if in_mark <= idx <= out_mark: if ref_srf_data is not None and ref_srf_data is not False: csv_writer.writerow( (idx, ts, ref_srf_data['m_to_screen'], ref_srf_data['m_from_screen'], ref_srf_data['detected_markers'])) # save gaze on srf as csv. with open( os.path.join( metrics_dir, 'gaze_positions_on_surface' + surface_name + '.csv'), 'wb') as csvfile: csv_writer = csv.writer(csvfile, delimiter='\t', quotechar='|', quoting=csv.QUOTE_MINIMAL) csv_writer.writerow( ('world_timestamp', 'world_frame_idx', 'gaze_timestamp', 'x_norm', 'y_norm', 'x_scaled', 'y_scaled', 'on_srf')) for idx, ts, ref_srf_data in zip( range(len(self.g_pool.timestamps)), self.g_pool.timestamps, s.cache): if in_mark <= idx <= out_mark: if ref_srf_data is not None and ref_srf_data is not False: for gp in s.gaze_on_srf_by_frame_idx( idx, ref_srf_data['m_from_screen']): csv_writer.writerow( (ts, idx, gp['base']['timestamp'], gp['norm_pos'][0], gp['norm_pos'][1], gp['norm_pos'][0] * s.real_world_size['x'], gp['norm_pos'][1] * s.real_world_size['y'], gp['on_srf'])) # # save fixation on srf as csv. with open( os.path.join( metrics_dir, 'fixations_on_surface' + surface_name + '.csv'), 'wb') as csvfile: csv_writer = csv.writer(csvfile, delimiter='\t', quotechar='|', quoting=csv.QUOTE_MINIMAL) csv_writer.writerow( ('id', 'start_timestamp', 'duration', 'start_frame', 'end_frame', 'norm_pos_x', 'norm_pos_y', 'x_scaled', 'y_scaled', 'on_srf')) fixations_on_surface = [] for idx, ref_srf_data in zip( range(len(self.g_pool.timestamps)), s.cache): if in_mark <= idx <= out_mark: if ref_srf_data is not None and ref_srf_data is not False: for f in s.fixations_on_srf_by_frame_idx( idx, ref_srf_data['m_from_screen']): fixations_on_surface.append(f) removed_dublicates = dict([ (f['base']['id'], f) for f in fixations_on_surface ]).values() for f_on_s in removed_dublicates: f = f_on_s['base'] f_x, f_y = f_on_s['norm_pos'] f_on_srf = f_on_s['on_srf'] csv_writer.writerow( (f['id'], f['timestamp'], f['duration'], f['start_frame_index'], f['end_frame_index'], f_x, f_y, f_x * s.real_world_size['x'], f_y * s.real_world_size['y'], f_on_srf)) logger.info( "Saved surface positon gaze and fixation data for '%s' with uid:'%s'" % (s.name, s.uid)) if s.heatmap is not None: logger.info("Saved Heatmap as .png file.") cv2.imwrite( os.path.join(metrics_dir, 'heatmap' + surface_name + '.png'), s.heatmap) logger.info("Done exporting reference surface data.") # if s.detected and self.img is not None: # #let save out the current surface image found in video # #here we get the verts of the surface quad in norm_coords # mapped_space_one = np.array(((0,0),(1,0),(1,1),(0,1)),dtype=np.float32).reshape(-1,1,2) # screen_space = cv2.perspectiveTransform(mapped_space_one,s.m_to_screen).reshape(-1,2) # #now we convert to image pixel coods # screen_space[:,1] = 1-screen_space[:,1] # screen_space[:,1] *= self.img.shape[0] # screen_space[:,0] *= self.img.shape[1] # s_0,s_1 = s.real_world_size # #no we need to flip vertically again by setting the mapped_space verts accordingly. # mapped_space_scaled = np.array(((0,s_1),(s_0,s_1),(s_0,0),(0,0)),dtype=np.float32) # M = cv2.getPerspectiveTransform(screen_space,mapped_space_scaled) # #here we do the actual perspactive transform of the image. # srf_in_video = cv2.warpPerspective(self.img,M, (int(s.real_world_size['x']),int(s.real_world_size['y'])) ) # cv2.imwrite(os.path.join(metrics_dir,'surface'+surface_name+'.png'),srf_in_video) # logger.info("Saved current image as .png file.") # else: # logger.info("'%s' is not currently visible. Seek to appropriate frame and repeat this command."%s.name) def get_init_dict(self): return {'mode': self.mode} def cleanup(self): """ called when the plugin gets terminated. This happens either voluntarily or forced. if you have a GUI or glfw window destroy it here. """ self.surface_definitions["offline_square_marker_surfaces"] = [ rs.save_to_dict() for rs in self.surfaces if rs.defined ] self.surface_definitions.close() self.close_marker_cacher() self.persistent_cache["marker_cache"] = self.cache.to_list() self.persistent_cache.close() for s in self.surfaces: s.close_window() self.deinit_gui()
class Offline_Surface_Tracker(Surface_Tracker): """ Special version of surface tracker for use with videofile source. It uses a seperate process to search all frames in the world video file for markers. - self.cache is a list containing marker positions for each frame. - self.surfaces[i].cache is a list containing surface positions for each frame Both caches are build up over time. The marker cache is also session persistent. See marker_tracker.py for more info on this marker tracker. """ def __init__(self,g_pool,mode="Show Markers and Surfaces",min_marker_perimeter = 100,invert_image=False,robust_detection=True): super(Offline_Surface_Tracker, self).__init__(g_pool,mode,min_marker_perimeter,robust_detection) self.order = .2 if g_pool.app == 'capture': raise Exception('For Player only.') self.marker_cache_version = 2 self.min_marker_perimeter_cacher = 20 #find even super small markers. The surface locater will filter using min_marker_perimeter #check if marker cache is available from last session self.persistent_cache = Persistent_Dict(os.path.join(g_pool.rec_dir,'square_marker_cache')) version = self.persistent_cache.get('version',0) cache = self.persistent_cache.get('marker_cache',None) if cache is None: self.cache = Cache_List([False for _ in g_pool.timestamps]) self.persistent_cache['version'] = self.marker_cache_version elif version != self.marker_cache_version: self.persistent_cache['version'] = self.marker_cache_version self.cache = Cache_List([False for _ in g_pool.timestamps]) logger.debug("Marker cache version missmatch. Rebuilding marker cache.") else: self.cache = Cache_List(cache) logger.debug("Loaded marker cache {} / {} frames had been searched before".format(len(self.cache)-self.cache.count(False),len(self.cache)) ) self.init_marker_cacher() for s in self.surfaces: s.init_cache(self.cache,self.camera_calibration,self.min_marker_perimeter,self.min_id_confidence) self.recalculate() def load_surface_definitions_from_file(self): self.surface_definitions = Persistent_Dict(os.path.join(self.g_pool.rec_dir,'surface_definitions')) if self.surface_definitions.get('offline_square_marker_surfaces',[]) != []: logger.debug("Found ref surfaces defined or copied in previous session.") self.surfaces = [Offline_Reference_Surface(self.g_pool,saved_definition=d) for d in self.surface_definitions.get('offline_square_marker_surfaces',[]) if isinstance(d,dict)] elif self.surface_definitions.get('realtime_square_marker_surfaces',[]) != []: logger.debug("Did not find ref surfaces def created or used by the user in player from earlier session. Loading surfaces defined during capture.") self.surfaces = [Offline_Reference_Surface(self.g_pool,saved_definition=d) for d in self.surface_definitions.get('realtime_square_marker_surfaces',[]) if isinstance(d,dict)] else: logger.debug("No surface defs found. Please define using GUI.") self.surfaces = [] def init_gui(self): self.menu = ui.Scrolling_Menu('Offline Surface Tracker') self.g_pool.gui.append(self.menu) self.add_button = ui.Thumb('add_surface',setter=lambda x: self.add_surface(),getter=lambda:False,label='A',hotkey='a') self.g_pool.quickbar.append(self.add_button) self.update_gui_markers() self.on_window_resize(glfwGetCurrentContext(),*glfwGetWindowSize(glfwGetCurrentContext())) def deinit_gui(self): if self.menu: self.g_pool.gui.remove(self.menu) self.menu= None if self.add_button: self.g_pool.quickbar.remove(self.add_button) self.add_button = None def update_gui_markers(self): def close(): self.alive=False def set_min_marker_perimeter(val): self.min_marker_perimeter = val self.notify_all({'subject':'min_marker_perimeter_changed','delay':1}) self.menu.elements[:] = [] self.menu.append(ui.Button('Close',close)) self.menu.append(ui.Slider('min_marker_perimeter',self,min=20,max=500,step=1,setter=set_min_marker_perimeter)) self.menu.append(ui.Info_Text('The offline surface tracker will look for markers in the entire video. By default it uses surfaces defined in capture. You can change and add more surfaces here.')) self.menu.append(ui.Info_Text("Press the export button or type 'e' to start the export.")) self.menu.append(ui.Selector('mode',self,label='Mode',selection=["Show Markers and Surfaces","Show marker IDs","Show Heatmaps","Show Metrics"] )) self.menu.append(ui.Info_Text('To see heatmap or surface metrics visualizations, click (re)-calculate gaze distributions. Set "X size" and "Y size" for each surface to see heatmap visualizations.')) self.menu.append(ui.Button("(Re)-calculate gaze distributions", self.recalculate)) self.menu.append(ui.Button("Add surface", lambda:self.add_surface())) for s in self.surfaces: idx = self.surfaces.index(s) s_menu = ui.Growing_Menu("Surface {}".format(idx)) s_menu.collapsed=True s_menu.append(ui.Text_Input('name',s)) s_menu.append(ui.Text_Input('x',s.real_world_size,label='X size')) s_menu.append(ui.Text_Input('y',s.real_world_size,label='Y size')) s_menu.append(ui.Button('Open Debug Window',s.open_close_window)) #closure to encapsulate idx def make_remove_s(i): return lambda: self.remove_surface(i) remove_s = make_remove_s(idx) s_menu.append(ui.Button('remove',remove_s)) self.menu.append(s_menu) def on_notify(self,notification): if notification['subject'] == 'gaze_positions_changed': logger.info('Gaze postions changed. Recalculating.') self.recalculate() if notification['subject'] == 'min_data_confidence_changed': logger.info('Min_data_confidence changed. Recalculating.') self.recalculate() elif notification['subject'] == 'surfaces_changed': logger.info('Surfaces changed. Recalculating.') self.recalculate() elif notification['subject'] == 'min_marker_perimeter_changed': logger.info('Min marker perimeter adjusted. Re-detecting surfaces.') self.invalidate_surface_caches() elif notification['subject'] is "should_export": self.save_surface_statsics_to_file(notification['range'],notification['export_dir']) def on_window_resize(self,window,w,h): self.win_size = w,h def add_surface(self): self.surfaces.append(Offline_Reference_Surface(self.g_pool)) self.update_gui_markers() def recalculate(self): in_mark = self.g_pool.trim_marks.in_mark out_mark = self.g_pool.trim_marks.out_mark section = slice(in_mark,out_mark) # calc heatmaps for s in self.surfaces: if s.defined: s.generate_heatmap(section) # calc distirbution accross all surfaces. results = [] for s in self.surfaces: gaze_on_srf = s.gaze_on_srf_in_section(section) results.append(len(gaze_on_srf)) self.metrics_gazecount = len(gaze_on_srf) if results == []: logger.warning("No surfaces defined.") return max_res = max(results) results = np.array(results,dtype=np.float32) if not max_res: logger.warning("No gaze on any surface for this section!") else: results *= 255./max_res results = np.uint8(results) results_c_maps = cv2.applyColorMap(results, cv2.COLORMAP_JET) for s,c_map in zip(self.surfaces,results_c_maps): heatmap = np.ones((1,1,4),dtype=np.uint8)*125 heatmap[:,:,:3] = c_map s.metrics_texture = Named_Texture() s.metrics_texture.update_from_ndarray(heatmap) def invalidate_surface_caches(self): for s in self.surfaces: s.cache = None def update(self,frame,events): self.img_shape = frame.img.shape self.update_marker_cache() # self.markers = [m for m in self.cache[frame.index] if m['perimeter'>=self.min_marker_perimeter] self.markers = self.cache[frame.index] if self.markers == False: self.markers = [] self.seek_marker_cacher(frame.index) # tell precacher that it better have every thing from here on analyzed events['surfaces'] = [] # locate surfaces for s in self.surfaces: if not s.locate_from_cache(frame.index): s.locate(self.markers,self.camera_calibration,self.min_marker_perimeter,self.min_id_confidence) if s.detected: events['surfaces'].append({'name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp}) if self.mode == "Show marker IDs": draw_markers(frame.img,self.markers) elif self.mode == "Show Markers and Surfaces": # edit surfaces by user if self.edit_surf_verts: window = glfwGetCurrentContext() pos = glfwGetCursorPos(window) pos = normalize(pos,glfwGetWindowSize(window),flip_y=True) for s,v_idx in self.edit_surf_verts: if s.detected: new_pos = s.img_to_ref_surface(np.array(pos)) s.move_vertex(v_idx,new_pos) else: # update srf with no or invald cache: for s in self.surfaces: if s.cache == None and s not in [s for s,i in self.edit_surf_verts]: s.init_cache(self.cache,self.camera_calibration,self.min_marker_perimeter,self.min_id_confidence) self.notify_all({'subject':'surfaces_changed','delay':1}) #allow surfaces to open/close windows for s in self.surfaces: if s.window_should_close: s.close_window() if s.window_should_open: s.open_window() def init_marker_cacher(self): if system() == 'Darwin': forking_enable(0) from marker_detector_cacher import fill_cache visited_list = [False if x == False else True for x in self.cache] video_file_path = self.g_pool.capture.source_path timestamps = self.g_pool.capture.timestamps self.cache_queue = Queue() self.cacher_seek_idx = Value('i',0) self.cacher_run = Value(c_bool,True) self.cacher = Process(target=fill_cache, args=(visited_list,video_file_path,timestamps,self.cache_queue,self.cacher_seek_idx,self.cacher_run,self.min_marker_perimeter_cacher)) self.cacher.start() def update_marker_cache(self): while not self.cache_queue.empty(): idx,c_m = self.cache_queue.get() self.cache.update(idx,c_m) for s in self.surfaces: s.update_cache(self.cache,camera_calibration=self.camera_calibration,min_marker_perimeter=self.min_marker_perimeter,min_id_confidence=self.min_id_confidence,idx=idx) if self.cacher_run.value == False: self.recalculate() def seek_marker_cacher(self,idx): self.cacher_seek_idx.value = idx def close_marker_cacher(self): self.update_marker_cache() self.cacher_run.value = False self.cacher.join() def gl_display(self): """ Display marker and surface info inside world screen """ self.gl_display_cache_bars() super(Offline_Surface_Tracker,self).gl_display() if self.mode == "Show Heatmaps": for s in self.surfaces: s.gl_display_heatmap() if self.mode == "Show Metrics": #todo: draw a backdrop to represent the gaze that is not on any surface for s in self.surfaces: #draw a quad on surface with false color of value. s.gl_display_metrics() def gl_display_cache_bars(self): """ """ padding = 30. # Lines for areas that have been cached cached_ranges = [] for r in self.cache.visited_ranges: # [[0,1],[3,4]] cached_ranges += (r[0],0),(r[1],0) #[(0,0),(1,0),(3,0),(4,0)] # Lines where surfaces have been found in video cached_surfaces = [] for s in self.surfaces: found_at = [] if s.cache is not None: for r in s.cache.positive_ranges: # [[0,1],[3,4]] found_at += (r[0],0),(r[1],0) #[(0,0),(1,0),(3,0),(4,0)] cached_surfaces.append(found_at) glMatrixMode(GL_PROJECTION) glPushMatrix() glLoadIdentity() width,height = self.win_size h_pad = padding * (self.cache.length-2)/float(width) v_pad = padding* 1./(height-2) glOrtho(-h_pad, (self.cache.length-1)+h_pad, -v_pad, 1+v_pad,-1,1) # ranging from 0 to cache_len-1 (horizontal) and 0 to 1 (vertical) glMatrixMode(GL_MODELVIEW) glPushMatrix() glLoadIdentity() color = RGBA(.8,.6,.2,.8) draw_polyline(cached_ranges,color=color,line_type=GL_LINES,thickness=4) color = RGBA(0,.7,.3,.8) for s in cached_surfaces: glTranslatef(0,.02,0) draw_polyline(s,color=color,line_type=GL_LINES,thickness=2) glMatrixMode(GL_PROJECTION) glPopMatrix() glMatrixMode(GL_MODELVIEW) glPopMatrix() def save_surface_statsics_to_file(self,export_range,export_dir): """ between in and out mark report: gaze distribution: - total gazepoints - gaze points on surface x - gaze points not on any surface report: surface visisbility - total frames - surface x visible framecount surface events: frame_no, ts, surface "name", "id" enter/exit for each surface: fixations_on_name.csv gaze_on_name_id.csv positions_of_name_id.csv """ metrics_dir = os.path.join(export_dir,'surfaces') section = export_range in_mark = export_range.start out_mark = export_range.stop logger.info("exporting metrics to {}".format(metrics_dir)) if os.path.isdir(metrics_dir): logger.info("Will overwrite previous export for this section") else: try: os.mkdir(metrics_dir) except: logger.warning("Could not make metrics dir {}".format(metrics_dir)) return with open(os.path.join(metrics_dir,'surface_visibility.csv'),'w',encoding='utf-8',newline='') as csvfile: csv_writer = csv.writer(csvfile, delimiter=',') # surface visibility report frame_count = len(self.g_pool.timestamps[section]) csv_writer.writerow(('frame_count',frame_count)) csv_writer.writerow(('')) csv_writer.writerow(('surface_name','visible_frame_count')) for s in self.surfaces: if s.cache == None: logger.warning("The surface is not cached. Please wait for the cacher to collect data.") return visible_count = s.visible_count_in_section(section) csv_writer.writerow( (s.name, visible_count) ) logger.info("Created 'surface_visibility.csv' file") with open(os.path.join(metrics_dir,'surface_gaze_distribution.csv'),'w',encoding='utf-8',newline='') as csvfile: csv_writer = csv.writer(csvfile, delimiter=',') # gaze distribution report gaze_in_section = list(chain(*self.g_pool.gaze_positions_by_frame[section])) not_on_any_srf = set([gp['timestamp'] for gp in gaze_in_section]) csv_writer.writerow(('total_gaze_point_count',len(gaze_in_section))) csv_writer.writerow(('')) csv_writer.writerow(('surface_name','gaze_count')) for s in self.surfaces: gaze_on_srf = s.gaze_on_srf_in_section(section) gaze_on_srf = set([gp['base_data']['timestamp'] for gp in gaze_on_srf]) not_on_any_srf -= gaze_on_srf csv_writer.writerow( (s.name, len(gaze_on_srf)) ) csv_writer.writerow(('not_on_any_surface', len(not_on_any_srf) ) ) logger.info("Created 'surface_gaze_distribution.csv' file") with open(os.path.join(metrics_dir,'surface_events.csv'),'w',encoding='utf-8',newline='') as csvfile: csv_writer = csv.writer(csvfile, delimiter=',') # surface events report csv_writer.writerow(('frame_number','timestamp','surface_name','surface_uid','event_type')) events = [] for s in self.surfaces: for enter_frame_id,exit_frame_id in s.cache.positive_ranges: events.append({'frame_id':enter_frame_id,'srf_name':s.name,'srf_uid':s.uid,'event':'enter'}) events.append({'frame_id':exit_frame_id,'srf_name':s.name,'srf_uid':s.uid,'event':'exit'}) events.sort(key=lambda x: x['frame_id']) for e in events: csv_writer.writerow( ( e['frame_id'],self.g_pool.timestamps[e['frame_id']],e['srf_name'],e['srf_uid'],e['event'] ) ) logger.info("Created 'surface_events.csv' file") for s in self.surfaces: # per surface names: surface_name = '_'+s.name.replace('/','')+'_'+s.uid # save surface_positions as pickle file save_object(s.cache.to_list(),os.path.join(metrics_dir,'srf_positions'+surface_name)) #save surface_positions as csv with open(os.path.join(metrics_dir,'srf_positons'+surface_name+'.csv'),'w',encoding='utf-8',newline='') as csvfile: csv_writer =csv.writer(csvfile, delimiter=',') csv_writer.writerow(('frame_idx','timestamp','m_to_screen','m_from_screen','detected_markers')) for idx,ts,ref_srf_data in zip(range(len(self.g_pool.timestamps)),self.g_pool.timestamps,s.cache): if in_mark <= idx <= out_mark: if ref_srf_data is not None and ref_srf_data is not False: csv_writer.writerow( (idx,ts,ref_srf_data['m_to_screen'],ref_srf_data['m_from_screen'],ref_srf_data['detected_markers']) ) # save gaze on srf as csv. with open(os.path.join(metrics_dir,'gaze_positions_on_surface'+surface_name+'.csv'),'w',encoding='utf-8',newline='') as csvfile: csv_writer = csv.writer(csvfile, delimiter=',') csv_writer.writerow(('world_timestamp','world_frame_idx','gaze_timestamp','x_norm','y_norm','x_scaled','y_scaled','on_srf')) for idx,ts,ref_srf_data in zip(range(len(self.g_pool.timestamps)),self.g_pool.timestamps,s.cache): if in_mark <= idx <= out_mark: if ref_srf_data is not None and ref_srf_data is not False: for gp in s.gaze_on_srf_by_frame_idx(idx,ref_srf_data['m_from_screen']): csv_writer.writerow( (ts,idx,gp['base_data']['timestamp'],gp['norm_pos'][0],gp['norm_pos'][1],gp['norm_pos'][0]*s.real_world_size['x'],gp['norm_pos'][1]*s.real_world_size['y'],gp['on_srf']) ) # save fixation on srf as csv. with open(os.path.join(metrics_dir,'fixations_on_surface'+surface_name+'.csv'),'w',encoding='utf-8',newline='') as csvfile: csv_writer = csv.writer(csvfile, delimiter=',') csv_writer.writerow(('id','start_timestamp','duration','start_frame','end_frame','norm_pos_x','norm_pos_y','x_scaled','y_scaled','on_srf')) fixations_on_surface = [] for idx,ref_srf_data in zip(range(len(self.g_pool.timestamps)),s.cache): if in_mark <= idx <= out_mark: if ref_srf_data is not None and ref_srf_data is not False: for f in s.fixations_on_srf_by_frame_idx(idx,ref_srf_data['m_from_screen']): fixations_on_surface.append(f) removed_dublicates = dict([(f['base_data']['id'],f) for f in fixations_on_surface]).values() for f_on_s in removed_dublicates: f = f_on_s['base_data'] f_x,f_y = f_on_s['norm_pos'] f_on_srf = f_on_s['on_srf'] csv_writer.writerow( (f['id'],f['timestamp'],f['duration'],f['start_frame_index'],f['end_frame_index'],f_x,f_y,f_x*s.real_world_size['x'],f_y*s.real_world_size['y'],f_on_srf) ) logger.info("Saved surface positon gaze and fixation data for '{}' with uid:'{}'".format(s.name,s.uid)) if s.heatmap is not None: logger.info("Saved Heatmap as .png file.") cv2.imwrite(os.path.join(metrics_dir,'heatmap'+surface_name+'.png'),s.heatmap) logger.info("Done exporting reference surface data.") # if s.detected and self.img is not None: # #let save out the current surface image found in video # #here we get the verts of the surface quad in norm_coords # mapped_space_one = np.array(((0,0),(1,0),(1,1),(0,1)),dtype=np.float32).reshape(-1,1,2) # screen_space = cv2.perspectiveTransform(mapped_space_one,s.m_to_screen).reshape(-1,2) # #now we convert to image pixel coods # screen_space[:,1] = 1-screen_space[:,1] # screen_space[:,1] *= self.img.shape[0] # screen_space[:,0] *= self.img.shape[1] # s_0,s_1 = s.real_world_size # #no we need to flip vertically again by setting the mapped_space verts accordingly. # mapped_space_scaled = np.array(((0,s_1),(s_0,s_1),(s_0,0),(0,0)),dtype=np.float32) # M = cv2.getPerspectiveTransform(screen_space,mapped_space_scaled) # #here we do the actual perspactive transform of the image. # srf_in_video = cv2.warpPerspective(self.img,M, (int(s.real_world_size['x']),int(s.real_world_size['y'])) ) # cv2.imwrite(os.path.join(metrics_dir,'surface'+surface_name+'.png'),srf_in_video) # logger.info("Saved current image as .png file.") # else: # logger.info("'%s' is not currently visible. Seek to appropriate frame and repeat this command."%s.name) def cleanup(self): """ called when the plugin gets terminated. This happens either voluntarily or forced. if you have a GUI or glfw window destroy it here. """ self.surface_definitions["offline_square_marker_surfaces"] = [rs.save_to_dict() for rs in self.surfaces if rs.defined] self.surface_definitions.close() self.close_marker_cacher() self.persistent_cache["marker_cache"] = self.cache.to_list() self.persistent_cache.close() for s in self.surfaces: s.close_window() self.deinit_gui()
def world(g_pool,cap_src,cap_size): """world Creates a window, gl context. Grabs images from a capture. Receives Pupil coordinates from eye process[es] Can run various plug-ins. """ #manage plugins runtime_plugins = import_runtime_plugins(os.path.join(g_pool.user_dir,'plugins')) user_launchable_plugins = [Show_Calibration,Pupil_Server,Pupil_Sync,Marker_Detector]+runtime_plugins system_plugins = [Log_Display,Display_Recent_Gaze,Recorder] plugin_by_index = system_plugins+user_launchable_plugins+calibration_plugins+gaze_mapping_plugins name_by_index = [p.__name__ for p in plugin_by_index] plugin_by_name = dict(zip(name_by_index,plugin_by_index)) default_plugins = [('Log_Display',{}),('Dummy_Gaze_Mapper',{}),('Display_Recent_Gaze',{}), ('Screen_Marker_Calibration',{}),('Recorder',{})] # Callback functions def on_resize(window,w, h): if not g_pool.iconified: g_pool.gui.update_window(w,h) g_pool.gui.collect_menus() graph.adjust_size(w,h) adjust_gl_view(w,h) for p in g_pool.plugins: p.on_window_resize(window,w,h) def on_iconify(window,iconified): g_pool.iconified = iconified def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key,scancode,action,mods) def on_char(window,char): g_pool.gui.update_char(char) def on_button(window,button, action, mods): g_pool.gui.update_button(button,action,mods) pos = glfwGetCursorPos(window) pos = normalize(pos,glfwGetWindowSize(main_window)) pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels for p in g_pool.plugins: p.on_click(pos,button,action) def on_pos(window,x, y): hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0]) x,y = x*hdpi_factor,y*hdpi_factor g_pool.gui.update_mouse(x,y) def on_scroll(window,x,y): g_pool.gui.update_scroll(x,y*scroll_factor) def on_close(window): g_pool.quit.value = True logger.info('Process closing from window') tick = delta_t() def get_dt(): return next(tick) # load session persistent settings session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_world')) if session_settings.get("version",VersionFormat('0.0')) < g_pool.version: logger.info("Session setting are from older version of this app. I will not use those.") session_settings.clear() # Initialize capture cap = autoCreateCapture(cap_src, timebase=g_pool.timebase) default_settings = {'frame_size':cap_size,'frame_rate':24} previous_settings = session_settings.get('capture_settings',None) if previous_settings and previous_settings['name'] == cap.name: cap.settings = previous_settings else: cap.settings = default_settings # Test capture try: frame = cap.get_frame() except CameraCaptureError: logger.error("Could not retrieve image from capture") cap.close() return # any object we attach to the g_pool object *from now on* will only be visible to this process! # vars should be declared here to make them visible to the code reader. g_pool.iconified = False g_pool.capture = cap g_pool.pupil_confidence_threshold = session_settings.get('pupil_confidence_threshold',.6) g_pool.active_calibration_plugin = None def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def set_scale(new_scale): g_pool.gui.scale = new_scale g_pool.gui.collect_menus() #window and gl setup glfwInit() width,height = session_settings.get('window_size',(frame.width, frame.height)) main_window = glfwCreateWindow(width,height, "World") window_pos = session_settings.get('window_position',window_position_default) glfwSetWindowPos(main_window,window_pos[0],window_pos[1]) glfwMakeContextCurrent(main_window) cygl.utils.init() #setup GUI g_pool.gui = ui.UI() g_pool.gui.scale = session_settings.get('gui_scale',1) g_pool.sidebar = ui.Scrolling_Menu("Settings",pos=(-350,0),size=(0,0),header_pos='left') general_settings = ui.Growing_Menu('General') general_settings.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=1.,max=2.5,label='Interface size')) general_settings.append(ui.Button('Reset window size',lambda: glfwSetWindowSize(main_window,frame.width,frame.height)) ) general_settings.append(ui.Selector('Open plugin', selection = user_launchable_plugins, labels = [p.__name__.replace('_',' ') for p in user_launchable_plugins], setter= open_plugin, getter=lambda: "Select to load")) general_settings.append(ui.Slider('pupil_confidence_threshold', g_pool,step = .01,min=0.,max=1.,label='Minimum pupil confidence')) general_settings.append(ui.Info_Text('Capture Version: %s'%g_pool.version)) g_pool.sidebar.append(general_settings) g_pool.calibration_menu = ui.Growing_Menu('Calibration') g_pool.sidebar.append(g_pool.calibration_menu) g_pool.gui.append(g_pool.sidebar) g_pool.quickbar = ui.Stretching_Menu('Quick Bar',(0,100),(120,-100)) g_pool.gui.append(g_pool.quickbar) g_pool.capture.init_gui(g_pool.sidebar) #plugins that are loaded based on user settings from previous session g_pool.notifications = [] g_pool.delayed_notifications = {} g_pool.plugins = Plugin_List(g_pool,plugin_by_name,session_settings.get('loaded_plugins',default_plugins)) #We add the calibration menu selector, after a calibration has been added: g_pool.calibration_menu.insert(0,ui.Selector('active_calibration_plugin',getter=lambda: g_pool.active_calibration_plugin.__class__, selection = calibration_plugins, labels = [p.__name__.replace('_',' ') for p in calibration_plugins], setter= open_plugin,label='Method')) # Register callbacks main_window glfwSetFramebufferSizeCallback(main_window,on_resize) glfwSetWindowCloseCallback(main_window,on_close) glfwSetWindowIconifyCallback(main_window,on_iconify) glfwSetKeyCallback(main_window,on_key) glfwSetCharCallback(main_window,on_char) glfwSetMouseButtonCallback(main_window,on_button) glfwSetCursorPosCallback(main_window,on_pos) glfwSetScrollCallback(main_window,on_scroll) # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() g_pool.image_tex.update_from_frame(frame) # refresh speed settings glfwSwapInterval(0) #trigger setup of window and gl sizes on_resize(main_window, *glfwGetFramebufferSize(main_window)) #now the we have aproper window we can load the last gui configuration g_pool.gui.configuration = session_settings.get('ui_config',{}) #set up performace graphs: pid = os.getpid() ps = psutil.Process(pid) ts = frame.timestamp cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20,130) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140,130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" pupil_graph = graph.Bar_Graph(max_val=1.0) pupil_graph.pos = (260,130) pupil_graph.update_rate = 5 pupil_graph.label = "Confidence: %0.2f" # Event loop while not g_pool.quit.value: # Get an image from the grabber try: frame = cap.get_frame() except CameraCaptureError: logger.error("Capture from camera failed. Stopping.") break except EndofVideoFileError: logger.warning("Video file is done. Stopping") break #update performace graphs t = frame.timestamp dt,ts = t-ts,t try: fps_graph.add(1./dt) except ZeroDivisionError: pass cpu_graph.update() #a dictionary that allows plugins to post and read events events = {} #report time between now and the last loop interation events['dt'] = get_dt() #receive and map pupil positions recent_pupil_positions = [] while not g_pool.pupil_queue.empty(): p = g_pool.pupil_queue.get() recent_pupil_positions.append(p) pupil_graph.add(p['confidence']) events['pupil_positions'] = recent_pupil_positions # publish delayed notifiactions when their time has come. for n in g_pool.delayed_notifications.values(): if n['_notify_time_'] < time(): del n['_notify_time_'] del g_pool.delayed_notifications[n['subject']] g_pool.notifications.append(n) # notify each plugin if there are new notifications: while g_pool.notifications: n = g_pool.notifications.pop(0) for p in g_pool.plugins: p.on_notify(n) # allow each Plugin to do its work. for p in g_pool.plugins: p.update(frame,events) #check if a plugin need to be destroyed g_pool.plugins.clean() # render camera image glfwMakeContextCurrent(main_window) if g_pool.iconified: pass else: g_pool.image_tex.update_from_frame(frame) make_coord_system_norm_based() g_pool.image_tex.draw() make_coord_system_pixel_based((frame.height,frame.width,3)) # render visual feedback from loaded plugins for p in g_pool.plugins: p.gl_display() if not g_pool.iconified: graph.push_view() fps_graph.draw() cpu_graph.draw() pupil_graph.draw() graph.pop_view() g_pool.gui.update() glfwSwapBuffers(main_window) glfwPollEvents() glfwRestoreWindow(main_window) #need to do this for windows os session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['pupil_confidence_threshold'] = g_pool.pupil_confidence_threshold session_settings['gui_scale'] = g_pool.gui.scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['capture_settings'] = g_pool.capture.settings session_settings['window_size'] = glfwGetWindowSize(main_window) session_settings['window_position'] = glfwGetWindowPos(main_window) session_settings['version'] = g_pool.version session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfwDestroyWindow(main_window) glfwTerminate() cap.close() logger.debug("Process done")
def service(timebase, eyes_are_alive, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version): """Maps pupil to gaze data, can run various plug-ins. Reacts to notifications: ``set_detection_mapping_mode``: Sets detection method ``start_plugin``: Starts given plugin with the given arguments ``eye_process.started``: Sets the detection method eye process ``service_process.should_stop``: Stops the service process Emits notifications: ``eye_process.should_start`` ``eye_process.should_stop`` ``set_detection_mapping_mode`` ``service_process.started`` ``service_process.stopped`` ``launcher_process.should_stop`` Emits data: ``gaze``: Gaze data from current gaze mapping plugin.`` ``*``: any other plugin generated data in the events that it not [dt,pupil,gaze]. """ # We defer the imports because of multiprocessing. # Otherwise the service process each process also loads the other imports. # This is not harmful but unnecessary. # general imports from time import sleep import logging import zmq import zmq_tools # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) gaze_pub = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url) pupil_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('pupil',)) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify',)) poller = zmq.Poller() poller.register(pupil_sub.socket) poller.register(notify_sub.socket) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) def launch_eye_process(eye_id, delay=0): n = {'subject': 'eye_process.should_start', 'eye_id': eye_id, 'delay': delay} ipc_pub.notify(n) def stop_eye_process(eye_id): n = {'subject': 'eye_process.should_stop', 'eye_id': eye_id} ipc_pub.notify(n) try: # helpers/utils from file_methods import Persistent_Dict from methods import delta_t, get_system_info from version_utils import VersionFormat import audio from uvc import get_time_monotonic # trigger pupil detector cpp build: import pupil_detectors del pupil_detectors # Plug-ins from plugin import Plugin, Plugin_List, import_runtime_plugins from calibration_routines import calibration_plugins, gaze_mapping_plugins from pupil_remote import Pupil_Remote from pupil_groups import Pupil_Groups logger.info('Application Version: {}'.format(version)) logger.info('System Info: {}'.format(get_system_info())) # g_pool holds variables for this process they are accesible to all plugins g_pool = Global_Container() g_pool.app = 'service' g_pool.user_dir = user_dir g_pool.version = version g_pool.get_now = get_time_monotonic g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.eyes_are_alive = eyes_are_alive g_pool.timebase = timebase def get_timestamp(): return get_time_monotonic()-g_pool.timebase.value g_pool.get_timestamp = get_timestamp # manage plugins runtime_plugins = import_runtime_plugins(os.path.join(g_pool.user_dir, 'plugins')) user_launchable_plugins = [Pupil_Groups, Pupil_Remote]+runtime_plugins plugin_by_index = runtime_plugins+calibration_plugins+gaze_mapping_plugins+user_launchable_plugins name_by_index = [p.__name__ for p in plugin_by_index] plugin_by_name = dict(zip(name_by_index, plugin_by_index)) default_plugins = [('Dummy_Gaze_Mapper', {}), ('HMD_Calibration', {}), ('Pupil_Remote', {})] g_pool.plugin_by_name = plugin_by_name tick = delta_t() def get_dt(): return next(tick) # load session persistent settings session_settings = Persistent_Dict(os.path.join(g_pool.user_dir, 'user_settings_service')) if session_settings.get("version", VersionFormat('0.0')) < g_pool.version: logger.info("Session setting are from older version of this app. I will not use those.") session_settings.clear() g_pool.detection_mapping_mode = session_settings.get('detection_mapping_mode', '2d') g_pool.active_calibration_plugin = None g_pool.active_gaze_mapping_plugin = None audio.audio_mode = session_settings.get('audio_mode', audio.default_audio_mode) # plugins that are loaded based on user settings from previous session g_pool.plugins = Plugin_List(g_pool, session_settings.get('loaded_plugins', default_plugins)) def handle_notifications(n): subject = n['subject'] if subject == 'set_detection_mapping_mode': if n['mode'] == '2d': if "Vector_Gaze_Mapper" in g_pool.active_gaze_mapping_plugin.class_name: logger.warning("The gaze mapper is not supported in 2d mode. Please recalibrate.") g_pool.plugins.add(plugin_by_name['Dummy_Gaze_Mapper']) g_pool.detection_mapping_mode = n['mode'] elif subject == 'start_plugin': g_pool.plugins.add(plugin_by_name[n['name']], args=n.get('args', {})) elif subject == 'eye_process.started': n = {'subject': 'set_detection_mapping_mode', 'mode': g_pool.detection_mapping_mode} ipc_pub.notify(n) elif subject == 'service_process.should_stop': g_pool.service_should_run = False elif subject.startswith('meta.should_doc'): ipc_pub.notify({ 'subject': 'meta.doc', 'actor': g_pool.app, 'doc': service.__doc__}) for p in g_pool.plugins: if p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify: ipc_pub.notify({ 'subject': 'meta.doc', 'actor': p.class_name, 'doc': p.on_notify.__doc__}) if session_settings.get('eye1_process_alive', False): launch_eye_process(1, delay=0.3) if session_settings.get('eye0_process_alive', True): launch_eye_process(0, delay=0.0) ipc_pub.notify({'subject': 'service_process.started'}) logger.warning('Process started.') g_pool.service_should_run = True # Event loop while g_pool.service_should_run: socks = dict(poller.poll()) if pupil_sub.socket in socks: t, p = pupil_sub.recv() new_gaze_data = g_pool.active_gaze_mapping_plugin.on_pupil_datum(p) for g in new_gaze_data: gaze_pub.send('gaze', g) events = {} events['gaze_positions'] = new_gaze_data events['pupil_positions'] = [p] for plugin in g_pool.plugins: plugin.recent_events(events=events) if notify_sub.socket in socks: t, n = notify_sub.recv() handle_notifications(n) for plugin in g_pool.plugins: plugin.on_notify(n) # check if a plugin need to be destroyed g_pool.plugins.clean() session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['version'] = str(g_pool.version) session_settings['eye0_process_alive'] = eyes_are_alive[0].value session_settings['eye1_process_alive'] = eyes_are_alive[1].value session_settings['detection_mapping_mode'] = g_pool.detection_mapping_mode session_settings['audio_mode'] = audio.audio_mode session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() except: import traceback trace = traceback.format_exc() logger.error('Process Service crashed with trace:\n{}'.format(trace)) finally: # shut down eye processes: stop_eye_process(0) stop_eye_process(1) logger.info("Process shutting down.") ipc_pub.notify({'subject': 'service_process.stopped'}) # shut down launcher n = {'subject': 'launcher_process.should_stop'} ipc_pub.notify(n) sleep(1.0)
def world(pupil_queue,timebase,launcher_pipe,eye_pipes,eyes_are_alive,user_dir,version,cap_src): """world Creates a window, gl context. Grabs images from a capture. Receives Pupil coordinates from eye process[es] Can run various plug-ins. """ import logging # Set up root logger for this process before doing imports of logged modules. logger = logging.getLogger() logger.setLevel(logging.INFO) #silence noisy modules logging.getLogger("OpenGL").setLevel(logging.ERROR) # create formatter formatter = logging.Formatter('%(processName)s - [%(levelname)s] %(name)s : %(message)s') # create file handler which logs even debug messages fh = logging.FileHandler(os.path.join(user_dir,'capture.log'),mode='w') fh.setLevel(logger.level) fh.setFormatter(formatter) # create console handler with a higher log level ch = logging.StreamHandler() ch.setLevel(logger.level+10) ch.setFormatter(formatter) # add the handlers to the logger logger.addHandler(fh) logger.addHandler(ch) #setup thread to recv log recrods from other processes. def log_loop(logging): import zmq ctx = zmq.Context() sub = ctx.socket(zmq.SUB) sub.bind('tcp://127.0.0.1:502020') sub.setsockopt(zmq.SUBSCRIBE, "") while True: record = sub.recv_pyobj() logger = logging.getLogger(record.name) logger.handle(record) import threading log_thread = threading.Thread(target=log_loop, args=(logging,)) log_thread.setDaemon(True) log_thread.start() # create logger for the context of this function logger = logging.getLogger(__name__) # We defer the imports because of multiprocessing. # Otherwise the world process each process also loads the other imports. # This is not harmful but unnecessary. #general imports from time import time,sleep import numpy as np #display import glfw from pyglui import ui,graph,cygl from pyglui.cygl.utils import Named_Texture from gl_utils import basic_gl_setup,adjust_gl_view, clear_gl_screen,make_coord_system_pixel_based,make_coord_system_norm_based #check versions for our own depedencies as they are fast-changing from pyglui import __version__ as pyglui_version assert pyglui_version >= '0.8' #monitoring import psutil # helpers/utils from file_methods import Persistent_Dict from methods import normalize, denormalize, delta_t, get_system_info from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError from version_utils import VersionFormat import audio # Plug-ins from plugin import Plugin_List,import_runtime_plugins from calibration_routines import calibration_plugins, gaze_mapping_plugins from recorder import Recorder from show_calibration import Show_Calibration from display_recent_gaze import Display_Recent_Gaze from pupil_server import Pupil_Server from pupil_sync import Pupil_Sync from surface_tracker import Surface_Tracker from log_display import Log_Display from annotations import Annotation_Capture from pupil_remote import Pupil_Remote from log_history import Log_History from game_controller import GameController logger.info('Application Version: %s'%version) logger.info('System Info: %s'%get_system_info()) #UI Platform tweaks if platform.system() == 'Linux': scroll_factor = 10.0 window_position_default = (0,0) elif platform.system() == 'Windows': scroll_factor = 1.0 window_position_default = (8,31) else: scroll_factor = 1.0 window_position_default = (0,0) #g_pool holds variables for this process g_pool = Global_Container() # make some constants avaiable g_pool.user_dir = user_dir g_pool.version = version g_pool.app = 'capture' g_pool.pupil_queue = pupil_queue g_pool.timebase = timebase # g_pool.launcher_pipe = launcher_pipe g_pool.eye_pipes = eye_pipes g_pool.eyes_are_alive = eyes_are_alive #manage plugins runtime_plugins = import_runtime_plugins(os.path.join(g_pool.user_dir,'plugins')) user_launchable_plugins = [GameController, Show_Calibration,Pupil_Remote,Pupil_Server,Pupil_Sync,Surface_Tracker,Annotation_Capture,Log_History]+runtime_plugins system_plugins = [Log_Display,Display_Recent_Gaze,Recorder] plugin_by_index = system_plugins+user_launchable_plugins+calibration_plugins+gaze_mapping_plugins name_by_index = [p.__name__ for p in plugin_by_index] plugin_by_name = dict(zip(name_by_index,plugin_by_index)) default_plugins = [('Log_Display',{}),('Dummy_Gaze_Mapper',{}),('Display_Recent_Gaze',{}), ('Screen_Marker_Calibration',{}),('Recorder',{})] # Callback functions def on_resize(window,w, h): if not g_pool.iconified: g_pool.gui.update_window(w,h) g_pool.gui.collect_menus() graph.adjust_size(w,h) adjust_gl_view(w,h) for p in g_pool.plugins: p.on_window_resize(window,w,h) def on_iconify(window,iconified): g_pool.iconified = iconified def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key,scancode,action,mods) def on_char(window,char): g_pool.gui.update_char(char) def on_button(window,button, action, mods): g_pool.gui.update_button(button,action,mods) pos = glfw.glfwGetCursorPos(window) pos = normalize(pos,glfw.glfwGetWindowSize(main_window)) pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels for p in g_pool.plugins: p.on_click(pos,button,action) def on_pos(window,x, y): hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0]/glfw.glfwGetWindowSize(window)[0]) x,y = x*hdpi_factor,y*hdpi_factor g_pool.gui.update_mouse(x,y) def on_scroll(window,x,y): g_pool.gui.update_scroll(x,y*scroll_factor) tick = delta_t() def get_dt(): return next(tick) # load session persistent settings session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_world')) if session_settings.get("version",VersionFormat('0.0')) < g_pool.version: logger.info("Session setting are from older version of this app. I will not use those.") session_settings.clear() # Initialize capture cap = autoCreateCapture(cap_src, timebase=g_pool.timebase) default_settings = {'frame_size':(1280,720),'frame_rate':30} previous_settings = session_settings.get('capture_settings',None) if previous_settings and previous_settings['name'] == cap.name: cap.settings = previous_settings else: cap.settings = default_settings # Test capture try: frame = cap.get_frame() except CameraCaptureError: logger.error("Could not retrieve image from capture") cap.close() launcher_pipe.send("Exit") return g_pool.iconified = False g_pool.capture = cap g_pool.pupil_confidence_threshold = session_settings.get('pupil_confidence_threshold',.6) g_pool.detection_mapping_mode = session_settings.get('detection_mapping_mode','2d') g_pool.active_calibration_plugin = None audio.audio_mode = session_settings.get('audio_mode',audio.default_audio_mode) def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def set_scale(new_scale): g_pool.gui.scale = new_scale g_pool.gui.collect_menus() def launch_eye_process(eye_id,blocking=False): if eyes_are_alive[eye_id].value: logger.error("Eye%s process already running."%eye_id) return launcher_pipe.send(eye_id) eye_pipes[eye_id].send( ('Set_Detection_Mapping_Mode',g_pool.detection_mapping_mode) ) if blocking: #wait for ready message from eye to sequentialize startup eye_pipes[eye_id].send('Ping') eye_pipes[eye_id].recv() logger.warning('Eye %s process started.'%eye_id) def stop_eye_process(eye_id,blocking=False): if eyes_are_alive[eye_id].value: eye_pipes[eye_id].send('Exit') if blocking: while eyes_are_alive[eye_id].value: sleep(.1) def start_stop_eye(eye_id,make_alive): if make_alive: launch_eye_process(eye_id) else: stop_eye_process(eye_id) def set_detection_mapping_mode(new_mode): if new_mode == '2d': for p in g_pool.plugins: if "Vector_Gaze_Mapper" in p.class_name: logger.warning("The gaze mapper is not supported in 2d mode. Please recalibrate.") p.alive = False g_pool.plugins.clean() for alive, pipe in zip(g_pool.eyes_are_alive,g_pool.eye_pipes): if alive.value: pipe.send( ('Set_Detection_Mapping_Mode',new_mode) ) g_pool.detection_mapping_mode = new_mode #window and gl setup glfw.glfwInit() width,height = session_settings.get('window_size',(frame.width, frame.height)) main_window = glfw.glfwCreateWindow(width,height, "World") window_pos = session_settings.get('window_position',window_position_default) glfw.glfwSetWindowPos(main_window,window_pos[0],window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window #setup GUI g_pool.gui = ui.UI() g_pool.gui.scale = session_settings.get('gui_scale',1) g_pool.sidebar = ui.Scrolling_Menu("Settings",pos=(-350,0),size=(0,0),header_pos='left') general_settings = ui.Growing_Menu('General') general_settings.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=1.,max=2.5,label='Interface size')) general_settings.append(ui.Button('Reset window size',lambda: glfw.glfwSetWindowSize(main_window,frame.width,frame.height)) ) general_settings.append(ui.Selector('audio_mode',audio,selection=audio.audio_modes)) general_settings.append(ui.Selector('detection_mapping_mode',g_pool,label='detection & mapping mode',setter=set_detection_mapping_mode,selection=['2d','3d'])) general_settings.append(ui.Switch('eye0_process',label='Detect eye 0',setter=lambda alive: start_stop_eye(0,alive),getter=lambda: eyes_are_alive[0].value )) general_settings.append(ui.Switch('eye1_process',label='Detect eye 1',setter=lambda alive: start_stop_eye(1,alive),getter=lambda: eyes_are_alive[1].value )) general_settings.append(ui.Selector('Open plugin', selection = user_launchable_plugins, labels = [p.__name__.replace('_',' ') for p in user_launchable_plugins], setter= open_plugin, getter=lambda: "Select to load")) general_settings.append(ui.Slider('pupil_confidence_threshold', g_pool,step = .01,min=0.,max=1.,label='Minimum pupil confidence')) general_settings.append(ui.Info_Text('Capture Version: %s'%g_pool.version)) g_pool.sidebar.append(general_settings) g_pool.calibration_menu = ui.Growing_Menu('Calibration') g_pool.sidebar.append(g_pool.calibration_menu) g_pool.gui.append(g_pool.sidebar) g_pool.quickbar = ui.Stretching_Menu('Quick Bar',(0,100),(120,-100)) g_pool.gui.append(g_pool.quickbar) g_pool.capture.init_gui(g_pool.sidebar) #plugins that are loaded based on user settings from previous session g_pool.notifications = [] g_pool.delayed_notifications = {} g_pool.plugins = Plugin_List(g_pool,plugin_by_name,session_settings.get('loaded_plugins',default_plugins)) #We add the calibration menu selector, after a calibration has been added: g_pool.calibration_menu.insert(0,ui.Selector('active_calibration_plugin',getter=lambda: g_pool.active_calibration_plugin.__class__, selection = calibration_plugins, labels = [p.__name__.replace('_',' ') for p in calibration_plugins], setter= open_plugin,label='Method')) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window,on_resize) glfw.glfwSetWindowIconifyCallback(main_window,on_iconify) glfw.glfwSetKeyCallback(main_window,on_key) glfw.glfwSetCharCallback(main_window,on_char) glfw.glfwSetMouseButtonCallback(main_window,on_button) glfw.glfwSetCursorPosCallback(main_window,on_pos) glfw.glfwSetScrollCallback(main_window,on_scroll) # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() g_pool.image_tex.update_from_frame(frame) # refresh speed settings glfw.glfwSwapInterval(0) #trigger setup of window and gl sizes on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) #now the we have aproper window we can load the last gui configuration g_pool.gui.configuration = session_settings.get('ui_config',{}) #set up performace graphs: pid = os.getpid() ps = psutil.Process(pid) ts = frame.timestamp cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20,130) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140,130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" pupil_graph = graph.Bar_Graph(max_val=1.0) pupil_graph.pos = (260,130) pupil_graph.update_rate = 5 pupil_graph.label = "Confidence: %0.2f" if session_settings.get('eye1_process_alive',False): launch_eye_process(1,blocking=True) if session_settings.get('eye0_process_alive',True): launch_eye_process(0,blocking=False) # Event loop while not glfw.glfwWindowShouldClose(main_window): # Get an image from the grabber try: frame = g_pool.capture.get_frame() except CameraCaptureError: logger.error("Capture from camera failed. Starting Fake Capture.") settings = g_pool.capture.settings g_pool.capture.close() g_pool.capture = autoCreateCapture(None, timebase=g_pool.timebase) g_pool.capture.init_gui(g_pool.sidebar) g_pool.capture.settings = settings g_pool.notifications.append({'subject':'should_stop_recording'}) continue except EndofVideoFileError: logger.warning("Video file is done. Stopping") break #update performace graphs t = frame.timestamp dt,ts = t-ts,t try: fps_graph.add(1./dt) except ZeroDivisionError: pass cpu_graph.update() #a dictionary that allows plugins to post and read events events = {} #report time between now and the last loop interation events['dt'] = get_dt() #receive and map pupil positions recent_pupil_positions = [] while not g_pool.pupil_queue.empty(): p = g_pool.pupil_queue.get() recent_pupil_positions.append(p) pupil_graph.add(p['confidence']) events['pupil_positions'] = recent_pupil_positions # publish delayed notifiactions when their time has come. for n in g_pool.delayed_notifications.values(): if n['_notify_time_'] < time(): del n['_notify_time_'] del g_pool.delayed_notifications[n['subject']] g_pool.notifications.append(n) # notify each plugin if there are new notifications: while g_pool.notifications: n = g_pool.notifications.pop(0) for p in g_pool.plugins: p.on_notify(n) # allow each Plugin to do its work. for p in g_pool.plugins: p.update(frame,events) #check if a plugin need to be destroyed g_pool.plugins.clean() # render camera image glfw.glfwMakeContextCurrent(main_window) if g_pool.iconified: pass else: g_pool.image_tex.update_from_frame(frame) make_coord_system_norm_based() g_pool.image_tex.draw() make_coord_system_pixel_based((frame.height,frame.width,3)) # render visual feedback from loaded plugins for p in g_pool.plugins: p.gl_display() if not g_pool.iconified: graph.push_view() fps_graph.draw() cpu_graph.draw() pupil_graph.draw() graph.pop_view() g_pool.gui.update() glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() glfw.glfwRestoreWindow(main_window) #need to do this for windows os session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['pupil_confidence_threshold'] = g_pool.pupil_confidence_threshold session_settings['gui_scale'] = g_pool.gui.scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['capture_settings'] = g_pool.capture.settings session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos(main_window) session_settings['version'] = g_pool.version session_settings['eye0_process_alive'] = eyes_are_alive[0].value session_settings['eye1_process_alive'] = eyes_are_alive[1].value session_settings['detection_mapping_mode'] = g_pool.detection_mapping_mode session_settings['audio_mode'] = audio.audio_mode session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) glfw.glfwTerminate() g_pool.capture.close() #shut down eye processes: stop_eye_process(0,blocking = True) stop_eye_process(1,blocking = True) #shut down laucher launcher_pipe.send("Exit") logger.info("Process Shutting down.")
def show_no_rec_window(): from pyglui.pyfontstash import fontstash from pyglui.ui import get_roboto_font_path def on_drop(window,count,paths): for x in range(count): new_rec_dir = paths[x] if is_pupil_rec_dir(new_rec_dir): logger.debug("Starting new session with '%s'"%new_rec_dir) global rec_dir rec_dir = new_rec_dir glfwSetWindowShouldClose(window,True) else: logger.error("'%s' is not a valid pupil recording"%new_rec_dir) # load session persistent settings session_settings = Persistent_Dict(os.path.join(user_dir,"user_settings")) if session_settings.get("version",VersionFormat('0.0')) < get_version(version_file): logger.info("Session setting are from older version of this app. I will not use those.") session_settings.clear() w,h = session_settings.get('window_size',(1280,720)) window_pos = session_settings.get('window_position',(0,0)) glfwWindowHint(GLFW_RESIZABLE,0) window = glfwCreateWindow(w, h,'Pupil Player') glfwWindowHint(GLFW_RESIZABLE,1) glfwMakeContextCurrent(window) glfwSetWindowPos(window,window_pos[0],window_pos[1]) glfwSetDropCallback(window,on_drop) adjust_gl_view(w,h) glfont = fontstash.Context() glfont.add_font('roboto',get_roboto_font_path()) glfont.set_align_string(v_align="center",h_align="middle") glfont.set_color_float((0.2,0.2,0.2,0.9)) basic_gl_setup() glClearColor(0.5,.5,0.5,0.0) text = 'Drop a recording directory onto this window.' tip = '(Tip: You can drop a recording directory onto the app icon.)' # text = "Please supply a Pupil recording directory as first arg when calling Pupil Player." while not glfwWindowShouldClose(window): clear_gl_screen() glfont.set_blur(10.5) glfont.set_color_float((0.0,0.0,0.0,1.)) glfont.set_size(w/25.) glfont.draw_text(w/2,.3*h,text) glfont.set_size(w/30.) glfont.draw_text(w/2,.4*h,tip) glfont.set_blur(0.96) glfont.set_color_float((1.,1.,1.,1.)) glfont.set_size(w/25.) glfont.draw_text(w/2,.3*h,text) glfont.set_size(w/30.) glfont.draw_text(w/2,.4*h,tip) glfwSwapBuffers(window) glfwPollEvents() session_settings['window_position'] = glfwGetWindowPos(window) session_settings['version'] = get_version(version_file) session_settings.close() del glfont glfwDestroyWindow(window)
def world(timebase, eyes_are_alive, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version): """Reads world video and runs plugins. Creates a window, gl context. Grabs images from a capture. Maps pupil to gaze data Can run various plug-ins. Reacts to notifications: ``set_detection_mapping_mode`` ``eye_process.started`` ``start_plugin`` Emits notifications: ``eye_process.should_start`` ``eye_process.should_stop`` ``set_detection_mapping_mode`` ``world_process.started`` ``world_process.stopped`` ``recording.should_stop``: Emits on camera failure ``launcher_process.should_stop`` Emits data: ``gaze``: Gaze data from current gaze mapping plugin.`` ``*``: any other plugin generated data in the events that it not [dt,pupil,gaze]. """ # We defer the imports because of multiprocessing. # Otherwise the world process each process also loads the other imports. # This is not harmful but unnecessary. # general imports from time import sleep import logging # networking import zmq import zmq_tools # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify', )) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] # logger.setLevel(logging.DEBUG) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) def launch_eye_process(eye_id, delay=0): n = { 'subject': 'eye_process.should_start.{}'.format(eye_id), 'eye_id': eye_id, 'delay': delay } ipc_pub.notify(n) def stop_eye_process(eye_id): n = { 'subject': 'eye_process.should_stop.{}'.format(eye_id), 'eye_id': eye_id, 'delay': 0.2 } ipc_pub.notify(n) def start_stop_eye(eye_id, make_alive): if make_alive: launch_eye_process(eye_id) else: stop_eye_process(eye_id) def set_detection_mapping_mode(new_mode): n = {'subject': 'set_detection_mapping_mode', 'mode': new_mode} ipc_pub.notify(n) try: # display import glfw from version_utils import VersionFormat from pyglui import ui, cygl, __version__ as pyglui_version assert VersionFormat(pyglui_version) >= VersionFormat( '1.9'), 'pyglui out of date, please upgrade to newest version' from pyglui.cygl.utils import Named_Texture import gl_utils # helpers/utils from file_methods import Persistent_Dict from methods import normalize, denormalize, delta_t, get_system_info, timer from uvc import get_time_monotonic logger.info('Application Version: {}'.format(version)) logger.info('System Info: {}'.format(get_system_info())) import audio # trigger pupil detector cpp build: import pupil_detectors del pupil_detectors # Plug-ins from plugin import Plugin, System_Plugin_Base, Plugin_List, import_runtime_plugins from plugin_manager import Plugin_Manager from calibration_routines import calibration_plugins, gaze_mapping_plugins, Calibration_Plugin, Gaze_Mapping_Plugin from fixation_detector import Fixation_Detector from recorder import Recorder from display_recent_gaze import Display_Recent_Gaze from time_sync import Time_Sync from pupil_remote import Pupil_Remote from pupil_groups import Pupil_Groups from surface_tracker import Surface_Tracker from log_display import Log_Display from annotations import Annotation_Capture from log_history import Log_History from frame_publisher import Frame_Publisher from blink_detection import Blink_Detection from video_capture import source_classes, manager_classes, Base_Manager, Base_Source from pupil_data_relay import Pupil_Data_Relay from remote_recorder import Remote_Recorder from audio_capture import Audio_Capture from accuracy_visualizer import Accuracy_Visualizer # from saccade_detector import Saccade_Detector from system_graphs import System_Graphs from camera_intrinsics_estimation import Camera_Intrinsics_Estimation from hololens_relay import Hololens_Relay # UI Platform tweaks if platform.system() == 'Linux': scroll_factor = 10.0 window_position_default = (30, 30) elif platform.system() == 'Windows': scroll_factor = 10.0 window_position_default = (8, 31) else: scroll_factor = 1.0 window_position_default = (0, 0) icon_bar_width = 50 window_size = None camera_render_size = None hdpi_factor = 1.0 # g_pool holds variables for this process they are accesible to all plugins g_pool = Global_Container() g_pool.app = 'capture' g_pool.process = 'world' g_pool.user_dir = user_dir g_pool.version = version g_pool.timebase = timebase g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.eyes_are_alive = eyes_are_alive def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic # manage plugins runtime_plugins = import_runtime_plugins( os.path.join(g_pool.user_dir, 'plugins')) user_plugins = [ Audio_Capture, Pupil_Groups, Frame_Publisher, Pupil_Remote, Time_Sync, Surface_Tracker, Annotation_Capture, Log_History, Fixation_Detector, Blink_Detection, Remote_Recorder, Accuracy_Visualizer, Camera_Intrinsics_Estimation, Hololens_Relay ] system_plugins = [ Log_Display, Display_Recent_Gaze, Recorder, Pupil_Data_Relay, Plugin_Manager, System_Graphs ] + manager_classes + source_classes plugins = system_plugins + user_plugins + runtime_plugins + calibration_plugins + gaze_mapping_plugins user_plugins += [ p for p in runtime_plugins if not isinstance(p, (Base_Manager, Base_Source, System_Plugin_Base, Calibration_Plugin, Gaze_Mapping_Plugin)) ] g_pool.plugin_by_name = {p.__name__: p for p in plugins} default_capture_settings = { 'preferred_names': [ "Pupil Cam1 ID2", "Logitech Camera", "(046d:081d)", "C510", "B525", "C525", "C615", "C920", "C930e" ], 'frame_size': (1280, 720), 'frame_rate': 30 } default_plugins = [("UVC_Source", default_capture_settings), ('Pupil_Data_Relay', {}), ('UVC_Manager', {}), ('Log_Display', {}), ('Dummy_Gaze_Mapper', {}), ('Display_Recent_Gaze', {}), ('Screen_Marker_Calibration', {}), ('Recorder', {}), ('Pupil_Remote', {}), ('Plugin_Manager', {}), ('System_Graphs', {})] # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal camera_render_size nonlocal hdpi_factor hdpi_factor = float( glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor window_size = w, h camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(*window_size) g_pool.gui.collect_menus() for p in g_pool.plugins: p.on_window_resize(window, *camera_render_size) def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) pos = x, y pos = normalize(pos, camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_pos(pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, count, paths): paths = [paths[x].decode('utf-8') for x in range(count)] for p in g_pool.plugins: p.on_drop(paths) tick = delta_t() def get_dt(): return next(tick) # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, 'user_settings_world')) if VersionFormat(session_settings.get("version", '0.0')) != g_pool.version: logger.info( "Session setting are from a different version of this app. I will not use those." ) session_settings.clear() g_pool.detection_mapping_mode = session_settings.get( 'detection_mapping_mode', '3d') g_pool.active_calibration_plugin = None g_pool.active_gaze_mapping_plugin = None g_pool.capture = None audio.audio_mode = session_settings.get('audio_mode', audio.default_audio_mode) def handle_notifications(n): subject = n['subject'] if subject == 'set_detection_mapping_mode': if n['mode'] == '2d': if ("Vector_Gaze_Mapper" in g_pool.active_gaze_mapping_plugin.class_name): logger.warning( "The gaze mapper is not supported in 2d mode. Please recalibrate." ) g_pool.plugins.add( g_pool.plugin_by_name['Dummy_Gaze_Mapper']) g_pool.detection_mapping_mode = n['mode'] elif subject == 'start_plugin': g_pool.plugins.add(g_pool.plugin_by_name[n['name']], args=n.get('args', {})) elif subject == 'stop_plugin': for p in g_pool.plugins: if p.class_name == n['name']: p.alive = False g_pool.plugins.clean() elif subject == 'eye_process.started': n = { 'subject': 'set_detection_mapping_mode', 'mode': g_pool.detection_mapping_mode } ipc_pub.notify(n) elif subject.startswith('meta.should_doc'): ipc_pub.notify({ 'subject': 'meta.doc', 'actor': g_pool.app, 'doc': world.__doc__ }) for p in g_pool.plugins: if (p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify): ipc_pub.notify({ 'subject': 'meta.doc', 'actor': p.class_name, 'doc': p.on_notify.__doc__ }) # window and gl setup glfw.glfwInit() width, height = session_settings.get('window_size', (1280 + icon_bar_width, 720)) main_window = glfw.glfwCreateWindow(width, height, "Pupil Capture - World") window_pos = session_settings.get('window_position', window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window def set_scale(new_scale): g_pool.gui_user_scale = new_scale window_size = camera_render_size[0] + \ int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor), \ glfw.glfwGetFramebufferSize(main_window)[1] logger.warning(icon_bar_width * g_pool.gui_user_scale * hdpi_factor) glfw.glfwSetWindowSize(main_window, *window_size) def reset_restart(): logger.warning("Resetting all settings and restarting Capture.") glfw.glfwSetWindowShouldClose(main_window, True) ipc_pub.notify({'subject': 'clear_settings_process.should_start'}) ipc_pub.notify({ 'subject': 'world_process.should_start', 'delay': 2. }) def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is open the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed # setup GUI g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get('gui_scale', 1.) g_pool.menubar = ui.Scrolling_Menu("Settings", pos=(-400, 0), size=(-icon_bar_width, 0), header_pos='left') g_pool.iconbar = ui.Scrolling_Menu("Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos='hidden') g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100)) g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.iconbar) g_pool.gui.append(g_pool.quickbar) general_settings = ui.Growing_Menu('General', header_pos='headline') general_settings.append( ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.6, .8, 1., 1.2, 1.4], label='Interface size')) def set_window_size(): f_width, f_height = g_pool.capture.frame_size f_width += int(icon_bar_width * g_pool.gui.scale) glfw.glfwSetWindowSize(main_window, f_width, f_height) general_settings.append(ui.Button('Reset window size', set_window_size)) general_settings.append( ui.Selector('audio_mode', audio, selection=audio.audio_modes)) general_settings.append( ui.Selector('detection_mapping_mode', g_pool, label='detection & mapping mode', setter=set_detection_mapping_mode, selection=['2d', '3d'])) general_settings.append( ui.Switch('eye0_process', label='Detect eye 0', setter=lambda alive: start_stop_eye(0, alive), getter=lambda: eyes_are_alive[0].value)) general_settings.append( ui.Switch('eye1_process', label='Detect eye 1', setter=lambda alive: start_stop_eye(1, alive), getter=lambda: eyes_are_alive[1].value)) general_settings.append( ui.Info_Text('Capture Version: {}'.format(g_pool.version))) general_settings.append( ui.Button('Restart with default settings', reset_restart)) g_pool.menubar.append(general_settings) icon = ui.Icon('collapsed', general_settings, label=chr(0xe8b8), on_val=False, off_val=True, setter=toggle_general_settings, label_font='pupil_icons') icon.tooltip = 'General Settings' g_pool.iconbar.append(icon) user_plugin_separator = ui.Separator() user_plugin_separator.order = 0.35 g_pool.iconbar.append(user_plugin_separator) # plugins that are loaded based on user settings from previous session g_pool.plugins = Plugin_List( g_pool, session_settings.get('loaded_plugins', default_plugins)) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() toggle_general_settings(False) # now the we have aproper window we can load the last gui configuration g_pool.gui.configuration = session_settings.get('ui_config', {}) # create a timer to control window update frequency window_update_timer = timer(1 / 60) def window_should_update(): return next(window_update_timer) # trigger setup of window and gl sizes on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) if session_settings.get('eye1_process_alive', False): launch_eye_process(1, delay=0.6) if session_settings.get('eye0_process_alive', True): launch_eye_process(0, delay=0.3) ipc_pub.notify({'subject': 'world_process.started'}) logger.warning('Process started.') # Event loop while not glfw.glfwWindowShouldClose(main_window): # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) #a dictionary that allows plugins to post and read events events = {} # report time between now and the last loop interation events['dt'] = get_dt() # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() # send new events to ipc: del events['pupil_positions'] # already on the wire del events['gaze_positions'] # sent earlier if 'frame' in events: del events['frame'] # send explicity with frame publisher if 'depth_frame' in events: del events['depth_frame'] if 'audio_packets' in events: del events['audio_packets'] del events['dt'] # no need to send this for topic, data in events.items(): assert (isinstance(data, (list, tuple))) for d in data: ipc_pub.send(topic, d) glfw.glfwMakeContextCurrent(main_window) # render visual feedback from loaded plugins if window_should_update() and gl_utils.is_window_visible( main_window): gl_utils.glViewport(0, 0, *camera_render_size) for p in g_pool.plugins: p.gl_display() gl_utils.glViewport(0, 0, *window_size) unused_elements = g_pool.gui.update() for button, action, mods in unused_elements.buttons: x, y = glfw.glfwGetCursorPos(main_window) pos = x * hdpi_factor, y * hdpi_factor pos = normalize(pos, camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_click(pos, button, action) for key, scancode, action, mods in unused_elements.keys: for p in g_pool.plugins: p.on_key(key, scancode, action, mods) for char_ in unused_elements.chars: for p in g_pool.plugins: p.on_char(char_) glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() glfw.glfwRestoreWindow(main_window) # need to do this for windows os session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['gui_scale'] = g_pool.gui_user_scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos( main_window) session_settings['version'] = str(g_pool.version) session_settings['eye0_process_alive'] = eyes_are_alive[0].value session_settings['eye1_process_alive'] = eyes_are_alive[1].value session_settings[ 'detection_mapping_mode'] = g_pool.detection_mapping_mode session_settings['audio_mode'] = audio.audio_mode session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) glfw.glfwTerminate() except: import traceback trace = traceback.format_exc() logger.error('Process Capture crashed with trace:\n{}'.format(trace)) finally: # shut down eye processes: stop_eye_process(0) stop_eye_process(1) logger.info("Process shutting down.") ipc_pub.notify({'subject': 'world_process.stopped'}) sleep(1.0)
def eye(pupil_queue, timebase, pipe_to_world, is_alive_flag, user_dir, version, eye_id, cap_src): """ Creates a window, gl context. Grabs images from a capture. Streams Pupil coordinates into g_pool.pupil_queue """ is_alive = Is_Alive_Manager(is_alive_flag) with is_alive: import logging # Set up root logger for this process before doing imports of logged modules. logger = logging.getLogger() logger.setLevel(logging.INFO) # remove inherited handlers logger.handlers = [] # create file handler which logs even debug messages fh = logging.FileHandler(os.path.join(user_dir, 'eye%s.log' % eye_id), mode='w') # fh.setLevel(logging.DEBUG) # create console handler with a higher log level ch = logging.StreamHandler() ch.setLevel(logger.level + 10) # create formatter and add it to the handlers formatter = logging.Formatter( 'Eye' + str(eye_id) + ' Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s') fh.setFormatter(formatter) formatter = logging.Formatter( 'EYE' + str(eye_id) + ' Process [%(levelname)s] %(name)s : %(message)s') ch.setFormatter(formatter) # add the handlers to the logger logger.addHandler(fh) logger.addHandler(ch) #silence noisy modules logging.getLogger("OpenGL").setLevel(logging.ERROR) logging.getLogger("libav").setLevel(logging.ERROR) # create logger for the context of this function logger = logging.getLogger(__name__) # We deferr the imports becasue of multiprocessing. # Otherwise the world process each process also loads the other imports. #general imports import numpy as np import cv2 #display import glfw from pyglui import ui, graph, cygl from pyglui.cygl.utils import draw_points, RGBA, draw_polyline, Named_Texture from OpenGL.GL import GL_LINE_LOOP from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen, make_coord_system_pixel_based, make_coord_system_norm_based from ui_roi import UIRoi #monitoring import psutil # helpers/utils from file_methods import Persistent_Dict from version_utils import VersionFormat from methods import normalize, denormalize, Roi, timer from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError from av_writer import JPEG_Writer, AV_Writer # Pupil detectors from pupil_detectors import Canny_Detector, Detector_2D, Detector_3D pupil_detectors = { Canny_Detector.__name__: Canny_Detector, Detector_2D.__name__: Detector_2D, Detector_3D.__name__: Detector_3D } #UI Platform tweaks if platform.system() == 'Linux': scroll_factor = 10.0 window_position_default = (600, 300 * eye_id) elif platform.system() == 'Windows': scroll_factor = 1.0 window_position_default = (600, 31 + 300 * eye_id) else: scroll_factor = 1.0 window_position_default = (600, 300 * eye_id) #g_pool holds variables for this process g_pool = Global_Container() # make some constants avaiable g_pool.user_dir = user_dir g_pool.version = version g_pool.app = 'capture' g_pool.pupil_queue = pupil_queue g_pool.timebase = timebase # Callback functions def on_resize(window, w, h): if not g_pool.iconified: active_window = glfw.glfwGetCurrentContext() glfw.glfwMakeContextCurrent(window) g_pool.gui.update_window(w, h) graph.adjust_size(w, h) adjust_gl_view(w, h) glfw.glfwMakeContextCurrent(active_window) def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_char(window, char): g_pool.gui.update_char(char) def on_iconify(window, iconified): g_pool.iconified = iconified def on_button(window, button, action, mods): if g_pool.display_mode == 'roi': if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt: g_pool.u_r.active_edit_pt = False return # if the roi interacts we dont what the gui to interact as well elif action == glfw.GLFW_PRESS: pos = glfw.glfwGetCursorPos(window) pos = normalize(pos, glfw.glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] pos = denormalize( pos, (frame.width, frame.height)) # Position in img pixels if g_pool.u_r.mouse_over_edit_pt( pos, g_pool.u_r.handle_size + 40, g_pool.u_r.handle_size + 40): return # if the roi interacts we dont what the gui to interact as well g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): hdpi_factor = float( glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor) if g_pool.u_r.active_edit_pt: pos = normalize((x, y), glfw.glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] pos = denormalize(pos, (frame.width, frame.height)) g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx, pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, 'user_settings_eye%s' % eye_id)) if session_settings.get("version", VersionFormat('0.0')) < g_pool.version: logger.info( "Session setting are from older version of this app. I will not use those." ) session_settings.clear() # Initialize capture cap = autoCreateCapture(cap_src, timebase=g_pool.timebase) default_settings = {'frame_size': (640, 480), 'frame_rate': 60} previous_settings = session_settings.get('capture_settings', None) if previous_settings and previous_settings['name'] == cap.name: cap.settings = previous_settings else: cap.settings = default_settings # Test capture try: frame = cap.get_frame() except CameraCaptureError: logger.error("Could not retrieve image from capture") cap.close() return #signal world that we are ready to go # pipe_to_world.send('eye%s process ready'%eye_id) # any object we attach to the g_pool object *from now on* will only be visible to this process! # vars should be declared here to make them visible to the code reader. g_pool.iconified = False g_pool.capture = cap g_pool.flip = session_settings.get('flip', False) g_pool.display_mode = session_settings.get('display_mode', 'camera_image') g_pool.display_mode_info_text = { 'camera_image': "Raw eye camera image. This uses the least amount of CPU power", 'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be a small as possible but big enough to capture to pupil in its movements", 'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters with in the Pupil Detection menu below." } g_pool.u_r = UIRoi(frame.img.shape) g_pool.u_r.set(session_settings.get('roi', g_pool.u_r.get())) def on_frame_size_change(new_size): g_pool.u_r = UIRoi((new_size[1], new_size[0])) cap.on_frame_size_change = on_frame_size_change writer = None pupil_detector_settings = session_settings.get( 'pupil_detector_settings', None) last_pupil_detector = pupil_detectors[session_settings.get( 'last_pupil_detector', Detector_2D.__name__)] g_pool.pupil_detector = last_pupil_detector(g_pool, pupil_detector_settings) # UI callback functions def set_scale(new_scale): g_pool.gui.scale = new_scale g_pool.gui.collect_menus() def set_display_mode_info(val): g_pool.display_mode = val g_pool.display_mode_info.text = g_pool.display_mode_info_text[val] def set_detector(new_detector): g_pool.pupil_detector.cleanup() g_pool.pupil_detector = new_detector(g_pool) g_pool.pupil_detector.init_gui(g_pool.sidebar) # Initialize glfw glfw.glfwInit() title = "eye %s" % eye_id width, height = session_settings.get('window_size', (frame.width, frame.height)) main_window = glfw.glfwCreateWindow(width, height, title, None, None) window_pos = session_settings.get('window_position', window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() g_pool.image_tex.update_from_frame(frame) glfw.glfwSwapInterval(0) #setup GUI g_pool.gui = ui.UI() g_pool.gui.scale = session_settings.get('gui_scale', 1) g_pool.sidebar = ui.Scrolling_Menu("Settings", pos=(-300, 0), size=(0, 0), header_pos='left') general_settings = ui.Growing_Menu('General') general_settings.append( ui.Slider('scale', g_pool.gui, setter=set_scale, step=.05, min=1., max=2.5, label='Interface Size')) general_settings.append( ui.Button( 'Reset window size', lambda: glfw.glfwSetWindowSize( main_window, frame.width, frame.height))) general_settings.append( ui.Switch('flip', g_pool, label='Flip image display')) general_settings.append( ui.Selector('display_mode', g_pool, setter=set_display_mode_info, selection=['camera_image', 'roi', 'algorithm'], labels=['Camera Image', 'ROI', 'Algorithm'], label="Mode")) g_pool.display_mode_info = ui.Info_Text( g_pool.display_mode_info_text[g_pool.display_mode]) general_settings.append(g_pool.display_mode_info) g_pool.sidebar.append(general_settings) g_pool.gui.append(g_pool.sidebar) detector_selector = ui.Selector( 'pupil_detector', getter=lambda: g_pool.pupil_detector.__class__, setter=set_detector, selection=[Canny_Detector, Detector_2D, Detector_3D], labels=[ 'Python 2D detector', 'C++ 2d detector', 'C++ 3d detector' ], label="Detection method") general_settings.append(detector_selector) # let detector add its GUI g_pool.pupil_detector.init_gui(g_pool.sidebar) # let the camera add its GUI g_pool.capture.init_gui(g_pool.sidebar) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetWindowIconifyCallback(main_window, on_iconify) glfw.glfwSetKeyCallback(main_window, on_key) glfw.glfwSetCharCallback(main_window, on_char) glfw.glfwSetMouseButtonCallback(main_window, on_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) #set the last saved window size on_resize(main_window, *glfw.glfwGetWindowSize(main_window)) # load last gui configuration g_pool.gui.configuration = session_settings.get('ui_config', {}) #set up performance graphs pid = os.getpid() ps = psutil.Process(pid) ts = frame.timestamp cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 130) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" #create a timer to control window update frequency window_update_timer = timer(1 / 60.) def window_should_update(): return next(window_update_timer) # Event loop while not glfw.glfwWindowShouldClose(main_window): if pipe_to_world.poll(): cmd = pipe_to_world.recv() if cmd == 'Exit': break elif cmd == "Ping": pipe_to_world.send("Pong") command = None else: command, payload = cmd if command == 'Set_Detection_Mapping_Mode': if payload == '3d': if not isinstance(g_pool.pupil_detector, Detector_3D): set_detector(Detector_3D) detector_selector.read_only = True else: set_detector(Detector_2D) detector_selector.read_only = False else: command = None # Get an image from the grabber try: frame = cap.get_frame() except CameraCaptureError: logger.error("Capture from Camera Failed. Stopping.") break except EndofVideoFileError: logger.warning("Video File is done. Stopping") cap.seek_to_frame(0) frame = cap.get_frame() #update performace graphs t = frame.timestamp dt, ts = t - ts, t try: fps_graph.add(1. / dt) except ZeroDivisionError: pass cpu_graph.update() ### RECORDING of Eye Video (on demand) ### # Setup variables and lists for recording if 'Rec_Start' == command: record_path, raw_mode = payload logger.info("Will save eye video to: %s" % record_path) timestamps_path = os.path.join(record_path, "eye%s_timestamps.npy" % eye_id) if raw_mode and frame.jpeg_buffer: video_path = os.path.join(record_path, "eye%s.mp4" % eye_id) writer = JPEG_Writer(video_path, cap.frame_rate) else: video_path = os.path.join(record_path, "eye%s.mp4" % eye_id) writer = AV_Writer(video_path, cap.frame_rate) timestamps = [] elif 'Rec_Stop' == command: logger.info("Done recording.") writer.release() writer = None np.save(timestamps_path, np.asarray(timestamps)) del timestamps if writer: writer.write_video_frame(frame) timestamps.append(frame.timestamp) # pupil ellipse detection result = g_pool.pupil_detector.detect( frame, g_pool.u_r, g_pool.display_mode == 'algorithm') result['id'] = eye_id # stream the result g_pool.pupil_queue.put(result) # GL drawing if window_should_update(): if not g_pool.iconified: glfw.glfwMakeContextCurrent(main_window) clear_gl_screen() # switch to work in normalized coordinate space if g_pool.display_mode == 'algorithm': g_pool.image_tex.update_from_ndarray(frame.img) elif g_pool.display_mode in ('camera_image', 'roi'): g_pool.image_tex.update_from_ndarray(frame.gray) else: pass make_coord_system_norm_based(g_pool.flip) g_pool.image_tex.draw() # switch to work in pixel space make_coord_system_pixel_based( (frame.height, frame.width, 3), g_pool.flip) if result['confidence'] > 0: if result.has_key('ellipse'): pts = cv2.ellipse2Poly( (int(result['ellipse']['center'][0]), int(result['ellipse']['center'][1])), (int(result['ellipse']['axes'][0] / 2), int(result['ellipse']['axes'][1] / 2)), int(result['ellipse']['angle']), 0, 360, 15) draw_polyline(pts, 1, RGBA(1., 0, 0, .5)) draw_points([result['ellipse']['center']], size=20, color=RGBA(1., 0., 0., .5), sharpness=1.) # render graphs graph.push_view() fps_graph.draw() cpu_graph.draw() graph.pop_view() # render GUI g_pool.gui.update() #render the ROI if g_pool.display_mode == 'roi': g_pool.u_r.draw(g_pool.gui.scale) #update screen glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() g_pool.pupil_detector.visualize( ) #detector decides if we visualize or not # END while running # in case eye recording was still runnnig: Save&close if writer: logger.info("Done recording eye.") writer = None np.save(timestamps_path, np.asarray(timestamps)) glfw.glfwRestoreWindow(main_window) #need to do this for windows os # save session persistent settings session_settings['gui_scale'] = g_pool.gui.scale session_settings['roi'] = g_pool.u_r.get() session_settings['flip'] = g_pool.flip session_settings['display_mode'] = g_pool.display_mode session_settings['ui_config'] = g_pool.gui.configuration session_settings['capture_settings'] = g_pool.capture.settings session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos( main_window) session_settings['version'] = g_pool.version session_settings[ 'last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__ session_settings[ 'pupil_detector_settings'] = g_pool.pupil_detector.get_settings() session_settings.close() g_pool.pupil_detector.cleanup() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) glfw.glfwTerminate() cap.close() logger.debug("Process done")
def main(): # Callback functions def on_resize(window, w, h): active_window = glfwGetCurrentContext() glfwMakeContextCurrent(window) adjust_gl_view(w, h, window) norm_size = normalize((w, h), glfwGetWindowSize(window)) fb_size = denormalize(norm_size, glfwGetFramebufferSize(window)) atb.TwWindowSize(*map(int, fb_size)) glfwMakeContextCurrent(active_window) for p in g.plugins: p.on_window_resize(window, w, h) def on_key(window, key, scancode, action, mods): if not atb.TwEventKeyboardGLFW(key, action): if action == GLFW_PRESS: pass def on_char(window, char): if not atb.TwEventCharGLFW(char, 1): pass def on_button(window, button, action, mods): if not atb.TwEventMouseButtonGLFW(button, action): pos = glfwGetCursorPos(window) pos = normalize(pos, glfwGetWindowSize(main_window)) pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels for p in g.plugins: p.on_click(pos, button, action) def on_pos(window, x, y): norm_pos = normalize((x, y), glfwGetWindowSize(window)) fb_x, fb_y = denormalize(norm_pos, glfwGetFramebufferSize(window)) if atb.TwMouseMotion(int(fb_x), int(fb_y)): pass def on_scroll(window, x, y): if not atb.TwMouseWheel(int(x)): pass def on_close(window): glfwSetWindowShouldClose(main_window, True) logger.debug('Process closing from window') try: rec_dir = sys.argv[1] except: #for dev, supply hardcoded dir: rec_dir = '/Users/mkassner/Desktop/Marker_Tracking_Demo_Recording/' if os.path.isdir(rec_dir): logger.debug("Dev option: Using hadcoded data dir.") else: if getattr(sys, 'frozen', False): logger.warning( "You did not supply a data directory when you called this script! \ \nPlease drag a Pupil recoding directory onto the launch icon." ) else: logger.warning( "You did not supply a data directory when you called this script! \ \nPlease supply a Pupil recoding directory as first arg when calling Pupil Player." ) return if not is_pupil_rec_dir(rec_dir): logger.error( "You did not supply a dir with the required files inside.") return #backwards compatibility fn. patch_meta_info(rec_dir) #parse and load data folder info video_path = rec_dir + "/world.avi" timestamps_path = rec_dir + "/timestamps.npy" gaze_positions_path = rec_dir + "/gaze_positions.npy" meta_info_path = rec_dir + "/info.csv" #parse info.csv file with open(meta_info_path) as info: meta_info = dict( ((line.strip().split('\t')) for line in info.readlines())) rec_version = meta_info["Capture Software Version"] rec_version_float = int( filter(type(rec_version).isdigit, rec_version)[:3]) / 100. #(get major,minor,fix of version) logger.debug("Recording version: %s , %s" % (rec_version, rec_version_float)) #load gaze information gaze_list = np.load(gaze_positions_path) timestamps = np.load(timestamps_path) #correlate data positions_by_frame = correlate_gaze(gaze_list, timestamps) # load session persistent settings session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings")) def load(var_name, default): return session_settings.get(var_name, default) def save(var_name, var): session_settings[var_name] = var # Initialize capture cap = autoCreateCapture(video_path, timestamps=timestamps_path) if isinstance(cap, FakeCapture): logger.error("could not start capture.") return width, height = cap.get_size() # Initialize glfw glfwInit() main_window = glfwCreateWindow( width, height, "Pupil Player: " + meta_info["Recording Name"] + " - " + rec_dir.split(os.path.sep)[-1], None, None) glfwMakeContextCurrent(main_window) # Register callbacks main_window glfwSetWindowSizeCallback(main_window, on_resize) glfwSetWindowCloseCallback(main_window, on_close) glfwSetKeyCallback(main_window, on_key) glfwSetCharCallback(main_window, on_char) glfwSetMouseButtonCallback(main_window, on_button) glfwSetCursorPosCallback(main_window, on_pos) glfwSetScrollCallback(main_window, on_scroll) # create container for globally scoped varfs (within world) g = Temp() g.plugins = [] g.play = False g.new_seek = True g.user_dir = user_dir g.rec_dir = rec_dir g.app = 'player' g.timestamps = timestamps g.positions_by_frame = positions_by_frame # helpers called by the main atb bar def update_fps(): old_time, bar.timestamp = bar.timestamp, time() dt = bar.timestamp - old_time if dt: bar.fps.value += .1 * (1. / dt - bar.fps.value) def set_window_size(mode, data): width, height = cap.get_size() ratio = (1, .75, .5, .25)[mode] w, h = int(width * ratio), int(height * ratio) glfwSetWindowSize(main_window, w, h) data.value = mode # update the bar.value def get_from_data(data): """ helper for atb getter and setter use """ return data.value def get_play(): return g.play def set_play(value): g.play = value def next_frame(): try: cap.seek_to_frame(cap.get_frame_index()) except FileSeekError: pass g.new_seek = True def prev_frame(): try: cap.seek_to_frame(cap.get_frame_index() - 2) except FileSeekError: pass g.new_seek = True def open_plugin(selection, data): if plugin_by_index[selection] not in additive_plugins: for p in g.plugins: if isinstance(p, plugin_by_index[selection]): return g.plugins = [p for p in g.plugins if p.alive] logger.debug('Open Plugin: %s' % name_by_index[selection]) new_plugin = plugin_by_index[selection](g) g.plugins.append(new_plugin) g.plugins.sort(key=lambda p: p.order) if hasattr(new_plugin, 'init_gui'): new_plugin.init_gui() # save the value for atb bar data.value = selection def get_from_data(data): """ helper for atb getter and setter use """ return data.value atb.init() # add main controls ATB bar bar = atb.Bar(name="Controls", label="Controls", help="Scene controls", color=(50, 50, 50), alpha=100, valueswidth=150, text='light', position=(10, 10), refresh=.1, size=(300, 160)) bar.next_atb_pos = (10, 220) bar.fps = c_float(0.0) bar.timestamp = time() bar.window_size = c_int(load("window_size", 0)) window_size_enum = atb.enum("Display Size", { "Full": 0, "Medium": 1, "Half": 2, "Mini": 3 }) bar.version = create_string_buffer(version, 512) bar.recording_version = create_string_buffer(rec_version, 512) bar.add_var("fps", bar.fps, step=1., readonly=True) bar._fps = c_float(cap.get_fps()) bar.add_var("recoding fps", bar._fps, readonly=True) bar.add_var("display size", vtype=window_size_enum, setter=set_window_size, getter=get_from_data, data=bar.window_size) bar.add_var("play", vtype=c_bool, getter=get_play, setter=set_play, key="space") bar.add_button('step next', next_frame, key='right') bar.add_button('step prev', prev_frame, key='left') bar.add_var("frame index", vtype=c_int, getter=lambda: cap.get_frame_index() - 1) bar.plugin_to_load = c_int(0) plugin_type_enum = atb.enum("Plug In", index_by_name) bar.add_var("plugin", setter=open_plugin, getter=get_from_data, data=bar.plugin_to_load, vtype=plugin_type_enum) bar.add_var( "version of recording", bar.recording_version, readonly=True, help="version of the capture software used to make this recording") bar.add_var("version of player", bar.version, readonly=True, help="version of the Pupil Player") bar.add_button("exit", on_close, data=main_window, key="esc") #set the last saved window size set_window_size(bar.window_size.value, bar.window_size) on_resize(main_window, *glfwGetWindowSize(main_window)) glfwSetWindowPos(main_window, 0, 0) #we always load these plugins g.plugins.append( Export_Launcher(g, data_dir=rec_dir, frame_count=len(timestamps))) g.plugins.append(Seek_Bar(g, capture=cap)) g.trim_marks = Trim_Marks(g, capture=cap) g.plugins.append(g.trim_marks) #these are loaded based on user settings for initializer in load('plugins', []): name, args = initializer logger.debug("Loading plugin: %s with settings %s" % (name, args)) try: p = plugin_by_name[name](g, **args) g.plugins.append(p) except: logger.warning("Plugin '%s' failed to load from settings file." % name) if load('plugins', "_") == "_": #lets load some default if we dont have presets g.plugins.append(Scan_Path(g)) g.plugins.append(Vis_Polyline(g)) g.plugins.append(Vis_Circle(g)) # g.plugins.append(Vis_Light_Points(g)) #sort by exec order g.plugins.sort(key=lambda p: p.order) #init gui for p in g.plugins: if hasattr(p, 'init_gui'): p.init_gui() # gl_state settings basic_gl_setup() g.image_tex = create_named_texture((height, width, 3)) while not glfwWindowShouldClose(main_window): update_fps() #grab new frame if g.play or g.new_seek: try: new_frame = cap.get_frame() except EndofVideoFileError: #end of video logic: pause at last frame. g.play = False if g.new_seek: display_time = new_frame.timestamp g.new_seek = False frame = new_frame.copy() #new positons and events we make a deepcopy just like the image is a copy. current_pupil_positions = deepcopy(positions_by_frame[frame.index]) events = [] # allow each Plugin to do its work. for p in g.plugins: p.update(frame, current_pupil_positions, events) #check if a plugin need to be destroyed g.plugins = [p for p in g.plugins if p.alive] # render camera image glfwMakeContextCurrent(main_window) make_coord_system_norm_based() draw_named_texture(g.image_tex, frame.img) make_coord_system_pixel_based(frame.img.shape) # render visual feedback from loaded plugins for p in g.plugins: p.gl_display() #present frames at appropriate speed wait_time = frame.timestamp - display_time display_time = frame.timestamp try: spent_time = time() - timestamp sleep(wait_time - spent_time) except: pass timestamp = time() atb.draw() glfwSwapBuffers(main_window) glfwPollEvents() plugin_save = [] for p in g.plugins: try: p_initializer = p.get_class_name(), p.get_init_dict() plugin_save.append(p_initializer) except AttributeError: #not all plugins need to be savable, they will not have the init dict. # any object without a get_init_dict method will throw this exception. pass # de-init all running plugins for p in g.plugins: p.alive = False #reading p.alive actually runs plug-in cleanup _ = p.alive save('plugins', plugin_save) save('window_size', bar.window_size.value) session_settings.close() cap.close() bar.destroy() glfwDestroyWindow(main_window) glfwTerminate() logger.debug("Process done")
def player_drop(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_version, debug): # general imports import logging # networking import zmq import zmq_tools from time import sleep # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.INFO) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) try: import glfw import gl_utils from OpenGL.GL import glClearColor from version_utils import VersionFormat from file_methods import Persistent_Dict from pyglui.pyfontstash import fontstash from pyglui.ui import get_roboto_font_path import player_methods as pm from pupil_recording import ( assert_valid_recording_type, InvalidRecordingException, ) from pupil_recording.update import update_recording process_was_interrupted = False def interrupt_handler(sig, frame): import traceback trace = traceback.format_stack(f=frame) logger.debug(f"Caught signal {sig} in:\n" + "".join(trace)) nonlocal process_was_interrupted process_was_interrupted = True signal.signal(signal.SIGINT, interrupt_handler) def on_drop(window, count, paths): nonlocal rec_dir rec_dir = paths[0].decode("utf-8") if rec_dir: try: assert_valid_recording_type(rec_dir) except InvalidRecordingException as err: logger.error(str(err)) rec_dir = None # load session persistent settings session_settings = Persistent_Dict( os.path.join(user_dir, "user_settings_player")) if VersionFormat(session_settings.get("version", "0.0")) != app_version: logger.info( "Session setting are from a different version of this app. I will not use those." ) session_settings.clear() w, h = session_settings.get("window_size", (1280, 720)) window_pos = session_settings.get("window_position", window_position_default) glfw.glfwInit() glfw.glfwWindowHint(glfw.GLFW_RESIZABLE, 0) window = glfw.glfwCreateWindow(w, h, "Pupil Player") glfw.glfwWindowHint(glfw.GLFW_RESIZABLE, 1) glfw.glfwMakeContextCurrent(window) glfw.glfwSetWindowPos(window, window_pos[0], window_pos[1]) glfw.glfwSetDropCallback(window, on_drop) glfont = fontstash.Context() glfont.add_font("roboto", get_roboto_font_path()) glfont.set_align_string(v_align="center", h_align="middle") glfont.set_color_float((0.2, 0.2, 0.2, 0.9)) gl_utils.basic_gl_setup() glClearColor(0.5, 0.5, 0.5, 0.0) text = "Drop a recording directory onto this window." tip = "(Tip: You can drop a recording directory onto the app icon.)" # text = "Please supply a Pupil recording directory as first arg when calling Pupil Player." def display_string(string, font_size, center_y): x = w / 2 * hdpi_factor y = center_y * hdpi_factor glfont.set_size(font_size * hdpi_factor) glfont.set_blur(10.5) glfont.set_color_float((0.0, 0.0, 0.0, 1.0)) glfont.draw_text(x, y, string) glfont.set_blur(0.96) glfont.set_color_float((1.0, 1.0, 1.0, 1.0)) glfont.draw_text(x, y, string) while not glfw.glfwWindowShouldClose( window) and not process_was_interrupted: fb_size = glfw.glfwGetFramebufferSize(window) hdpi_factor = glfw.getHDPIFactor(window) gl_utils.adjust_gl_view(*fb_size) if rec_dir: try: assert_valid_recording_type(rec_dir) logger.info( "Starting new session with '{}'".format(rec_dir)) text = "Updating recording format." tip = "This may take a while!" except InvalidRecordingException as err: logger.error(str(err)) if err.recovery: text = err.reason tip = err.recovery else: text = "Invalid recording" tip = err.reason rec_dir = None gl_utils.clear_gl_screen() display_string(text, font_size=51, center_y=216) for idx, line in enumerate(tip.split("\n")): tip_font_size = 42 center_y = 288 + tip_font_size * idx * 1.2 display_string(line, font_size=tip_font_size, center_y=center_y) glfw.glfwSwapBuffers(window) if rec_dir: try: update_recording(rec_dir) except AssertionError as err: logger.error(str(err)) tip = "Oops! There was an error updating the recording." rec_dir = None except InvalidRecordingException as err: logger.error(str(err)) if err.recovery: text = err.reason tip = err.recovery else: text = "Invalid recording" tip = err.reason rec_dir = None else: glfw.glfwSetWindowShouldClose(window, True) glfw.glfwPollEvents() session_settings["window_position"] = glfw.glfwGetWindowPos(window) session_settings.close() glfw.glfwDestroyWindow(window) if rec_dir: ipc_pub.notify({ "subject": "player_process.should_start", "rec_dir": rec_dir }) except Exception: import traceback trace = traceback.format_exc() logger.error( "Process player_drop crashed with trace:\n{}".format(trace)) finally: sleep(1.0)
def service(timebase,eyes_are_alive,ipc_pub_url,ipc_sub_url,ipc_push_url,user_dir,version): """Maps pupil to gaze data, can run various plug-ins. Reacts to notifications: ``set_detection_mapping_mode``: Sets detection method ``start_plugin``: Starts given plugin with the given arguments ``eye_process.started``: Sets the detection method eye process ``service_process.should_stop``: Stops the service process Emits notifications: ``eye_process.should_start`` ``eye_process.should_stop`` ``set_detection_mapping_mode`` ``service_process.started`` ``service_process.stopped`` ``launcher_process.should_stop`` Emits data: ``gaze``: Gaze data from current gaze mapping plugin.`` ``*``: any other plugin generated data in the events that it not [dt,pupil,gaze]. """ # We defer the imports because of multiprocessing. # Otherwise the service process each process also loads the other imports. # This is not harmful but unnecessary. #general imports import numpy as np import logging import zmq import zmq_tools #zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx,ipc_push_url) gaze_pub = zmq_tools.Msg_Streamer(zmq_ctx,ipc_pub_url) pupil_sub = zmq_tools.Msg_Receiver(zmq_ctx,ipc_sub_url,topics=('pupil',)) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,ipc_sub_url,topics=('notify',)) poller = zmq.Poller() poller.register(pupil_sub.socket) poller.register(notify_sub.socket) #log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx,ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) #monitoring import psutil # helpers/utils from file_methods import Persistent_Dict from methods import normalize, denormalize, delta_t, get_system_info from version_utils import VersionFormat import audio from uvc import get_time_monotonic #trigger pupil detector cpp build: import pupil_detectors del pupil_detectors # Plug-ins from plugin import Plugin,Plugin_List,import_runtime_plugins from calibration_routines import calibration_plugins, gaze_mapping_plugins from pupil_sync import Pupil_Sync from pupil_remote import Pupil_Remote from pupil_groups import Pupil_Groups logger.info('Application Version: %s'%version) logger.info('System Info: %s'%get_system_info()) #g_pool holds variables for this process they are accesible to all plugins g_pool = Global_Container() g_pool.app = 'service' g_pool.user_dir = user_dir g_pool.version = version g_pool.get_now = get_time_monotonic g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.eyes_are_alive = eyes_are_alive g_pool.timebase = timebase def get_timestamp(): return get_time_monotonic()-g_pool.timebase.value g_pool.get_timestamp = get_timestamp #manage plugins runtime_plugins = import_runtime_plugins(os.path.join(g_pool.user_dir,'plugins')) user_launchable_plugins = [Pupil_Groups,Pupil_Remote,Pupil_Sync]+runtime_plugins plugin_by_index = runtime_plugins+calibration_plugins+gaze_mapping_plugins+user_launchable_plugins name_by_index = [p.__name__ for p in plugin_by_index] plugin_by_name = dict(zip(name_by_index,plugin_by_index)) default_plugins = [('Dummy_Gaze_Mapper',{}),('HMD_Calibration',{}),('Pupil_Remote',{})] tick = delta_t() def get_dt(): return next(tick) # load session persistent settings session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_service')) if session_settings.get("version",VersionFormat('0.0')) < g_pool.version: logger.info("Session setting are from older version of this app. I will not use those.") session_settings.clear() g_pool.detection_mapping_mode = session_settings.get('detection_mapping_mode','2d') g_pool.active_calibration_plugin = None g_pool.active_gaze_mapping_plugin = None audio.audio_mode = session_settings.get('audio_mode',audio.default_audio_mode) #plugins that are loaded based on user settings from previous session g_pool.plugins = Plugin_List(g_pool,plugin_by_name,session_settings.get('loaded_plugins',default_plugins)) def launch_eye_process(eye_id,delay=0): n = {'subject':'eye_process.should_start','eye_id':eye_id,'delay':delay} ipc_pub.notify(n) def stop_eye_process(eye_id): n = {'subject':'eye_process.should_stop','eye_id':eye_id} ipc_pub.notify(n) def handle_notifications(n): subject = n['subject'] if subject == 'set_detection_mapping_mode': if n['mode'] == '2d': if "Vector_Gaze_Mapper" in g_pool.active_gaze_mapping_plugin.class_name: logger.warning("The gaze mapper is not supported in 2d mode. Please recalibrate.") g_pool.plugins.add(plugin_by_name['Dummy_Gaze_Mapper']) g_pool.detection_mapping_mode = n['mode'] elif subject == 'start_plugin': g_pool.plugins.add(plugin_by_name[n['name']],args=n.get('args',{}) ) elif subject == 'eye_process.started': n = {'subject':'set_detection_mapping_mode','mode':g_pool.detection_mapping_mode} ipc_pub.notify(n) elif subject == 'service_process.should_stop': g_pool.service_should_run = False elif subject.startswith('meta.should_doc'): ipc_pub.notify({ 'subject':'meta.doc', 'actor':g_pool.app, 'doc':service.__doc__}) for p in g_pool.plugins: if p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify: ipc_pub.notify({ 'subject':'meta.doc', 'actor': p.class_name, 'doc':p.on_notify.__doc__}) if session_settings.get('eye1_process_alive',False): launch_eye_process(1,delay=0.3) if session_settings.get('eye0_process_alive',True): launch_eye_process(0,delay=0.0) ipc_pub.notify({'subject':'service_process.started'}) logger.warning('Process started.') g_pool.service_should_run = True # Event loop while g_pool.service_should_run: socks = dict(poller.poll()) if pupil_sub.socket in socks: t,p = pupil_sub.recv() new_gaze_data = g_pool.active_gaze_mapping_plugin.on_pupil_datum(p) for g in new_gaze_data: gaze_pub.send('gaze',g) #simulate the update loop. events = {} events['gaze_positions'] = new_gaze_data events['pupil_positions'] = [p,] for plugin in g_pool.plugins: plugin.update(frame=None,events=events) if notify_sub.socket in socks: t,n = notify_sub.recv() handle_notifications(n) for plugin in g_pool.plugins: plugin.on_notify(n) #check if a plugin need to be destroyed g_pool.plugins.clean() session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['version'] = g_pool.version session_settings['eye0_process_alive'] = eyes_are_alive[0].value session_settings['eye1_process_alive'] = eyes_are_alive[1].value session_settings['detection_mapping_mode'] = g_pool.detection_mapping_mode session_settings['audio_mode'] = audio.audio_mode session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() #shut down eye processes: stop_eye_process(0) stop_eye_process(1) logger.info("Process shutting down.") ipc_pub.notify({'subject':'service_process.stopped'}) #shut down launcher n = {'subject':'launcher_process.should_stop'} ipc_pub.notify(n)
def load_surface_definitions_from_file(self): # all registered surfaces self.surface_definitions = Persistent_Dict(os.path.join(self.g_pool.user_dir,'surface_definitions') ) self.surfaces = [Reference_Surface(saved_definition=d) for d in self.surface_definitions.get('realtime_square_marker_surfaces',[]) if isinstance(d,dict)]
def world(g_pool, cap_src, cap_size): """world Creates a window, gl context. Grabs images from a capture. Receives Pupil coordinates from g_pool.pupil_queue Can run various plug-ins. """ # Callback functions def on_resize(window, w, h): active_window = glfwGetCurrentContext() glfwMakeContextCurrent(window) norm_size = normalize((w, h), glfwGetWindowSize(window)) fb_size = denormalize(norm_size, glfwGetFramebufferSize(window)) atb.TwWindowSize(*map(int, fb_size)) adjust_gl_view(w, h, window) glfwMakeContextCurrent(active_window) for p in g_pool.plugins: p.on_window_resize(window, w, h) def on_iconify(window, iconfied): if not isinstance(cap, FakeCapture): g_pool.update_textures.value = not iconfied def on_key(window, key, scancode, action, mods): if not atb.TwEventKeyboardGLFW(key, action): if action == GLFW_PRESS: if key == GLFW_KEY_ESCAPE: on_close(window) def on_char(window, char): if not atb.TwEventCharGLFW(char, 1): pass def on_button(window, button, action, mods): if not atb.TwEventMouseButtonGLFW(button, action): pos = glfwGetCursorPos(window) pos = normalize(pos, glfwGetWindowSize(world_window)) pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels for p in g_pool.plugins: p.on_click(pos, button, action) def on_pos(window, x, y): norm_pos = normalize((x, y), glfwGetWindowSize(window)) fb_x, fb_y = denormalize(norm_pos, glfwGetFramebufferSize(window)) if atb.TwMouseMotion(int(fb_x), int(fb_y)): pass def on_scroll(window, x, y): if not atb.TwMouseWheel(int(x)): pass def on_close(window): g_pool.quit.value = True logger.info('Process closing from window') # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, 'user_settings_world')) def load(var_name, default): return session_settings.get(var_name, default) def save(var_name, var): session_settings[var_name] = var # Initialize capture cap = autoCreateCapture(cap_src, cap_size, 24, timebase=g_pool.timebase) # Get an image from the grabber try: frame = cap.get_frame() except CameraCaptureError: logger.error("Could not retrieve image from capture") cap.close() return height, width = frame.img.shape[:2] # load last calibration data try: pt_cloud = np.load(os.path.join(g_pool.user_dir, 'cal_pt_cloud.npy')) logger.debug("Using calibration found in %s" % g_pool.user_dir) map_pupil = calibrate.get_map_from_cloud(pt_cloud, (width, height)) except: logger.debug("No calibration found.") def map_pupil(vector): """ 1 to 1 mapping """ return vector # any object we attach to the g_pool object *from now on* will only be visible to this process! # vars should be declared here to make them visible to the code reader. g_pool.plugins = [] g_pool.map_pupil = map_pupil g_pool.update_textures = c_bool(1) if isinstance(cap, FakeCapture): g_pool.update_textures.value = False g_pool.capture = cap g_pool.rec_name = recorder.get_auto_name() # helpers called by the main atb bar def update_fps(): old_time, bar.timestamp = bar.timestamp, time() dt = bar.timestamp - old_time if dt: bar.fps.value += .05 * (1. / dt - bar.fps.value) def set_window_size(mode, data): height, width = frame.img.shape[:2] ratio = (1, .75, .5, .25)[mode] w, h = int(width * ratio), int(height * ratio) glfwSetWindowSize(world_window, w, h) data.value = mode # update the bar.value def get_from_data(data): """ helper for atb getter and setter use """ return data.value def set_rec_name(val): if not val.value: g_pool.rec_name = recorder.get_auto_name() else: g_pool.rec_name = val.value def get_rec_name(): return create_string_buffer(g_pool.rec_name, 512) def open_calibration(selection, data): # prepare destruction of current ref_detector... and remove it for p in g_pool.plugins: if isinstance(p, calibration_routines.detector_by_index): p.alive = False g_pool.plugins = [p for p in g_pool.plugins if p.alive] new_ref_detector = calibration_routines.detector_by_index[selection]( g_pool, atb_pos=bar.next_atb_pos) g_pool.plugins.append(new_ref_detector) g_pool.plugins.sort(key=lambda p: p.order) # save the value for atb bar data.value = selection def toggle_record_video(): for p in g_pool.plugins: if isinstance(p, recorder.Recorder): p.alive = False return new_plugin = recorder.Recorder(g_pool, g_pool.rec_name, bar.fps.value, frame.img.shape, bar.record_eye.value, g_pool.eye_tx, bar.audio.value) g_pool.plugins.append(new_plugin) g_pool.plugins.sort(key=lambda p: p.order) def toggle_show_calib_result(): for p in g_pool.plugins: if isinstance(p, Show_Calibration): p.alive = False return new_plugin = Show_Calibration(g_pool, frame.img.shape) g_pool.plugins.append(new_plugin) g_pool.plugins.sort(key=lambda p: p.order) def toggle_server(): for p in g_pool.plugins: if isinstance(p, Pupil_Server): p.alive = False return new_plugin = Pupil_Server(g_pool, (10, 300)) g_pool.plugins.append(new_plugin) g_pool.plugins.sort(key=lambda p: p.order) def toggle_remote(): for p in g_pool.plugins: if isinstance(p, Pupil_Remote): p.alive = False return new_plugin = Pupil_Remote(g_pool, (10, 360), on_char) g_pool.plugins.append(new_plugin) g_pool.plugins.sort(key=lambda p: p.order) def toggle_ar(): for p in g_pool.plugins: if isinstance(p, Marker_Detector): p.alive = False return new_plugin = Marker_Detector(g_pool, (10, 400)) g_pool.plugins.append(new_plugin) g_pool.plugins.sort(key=lambda p: p.order) def reset_timebase(): #the last frame from worldcam will be t0 g_pool.timebase.value = g_pool.capure.get_now() logger.info( "New timebase set to %s all timestamps will count from here now." % g_pool.timebase.value) atb.init() # add main controls ATB bar bar = atb.Bar(name="World", label="Controls", help="Scene controls", color=(50, 50, 50), alpha=100, valueswidth=150, text='light', position=(10, 10), refresh=.3, size=(300, 200)) bar.next_atb_pos = (10, 220) bar.fps = c_float(0.0) bar.timestamp = time() bar.calibration_type = c_int(load("calibration_type", 0)) bar.record_eye = c_bool(load("record_eye", 0)) bar.audio = c_int(load("audio", -1)) bar.window_size = c_int(load("window_size", 0)) window_size_enum = atb.enum("Display Size", { "Full": 0, "Medium": 1, "Half": 2, "Mini": 3 }) calibrate_type_enum = atb.enum("Calibration Method", calibration_routines.index_by_name) audio_enum = atb.enum("Audio Input", dict(Audio_Input_List())) bar.version = create_string_buffer(g_pool.version, 512) bar.add_var( "fps", bar.fps, step=1., readonly=True, help= "Refresh speed of this process. Especially during recording it should not drop below the camera set frame rate." ) bar.add_var( "display size", vtype=window_size_enum, setter=set_window_size, getter=get_from_data, data=bar.window_size, help="Resize the world window. This has no effect on the actual image." ) bar.add_var("calibration method", setter=open_calibration, getter=get_from_data, data=bar.calibration_type, vtype=calibrate_type_enum, group="Calibration", help="Please choose your desired calibration method.") bar.add_button("show calibration result", toggle_show_calib_result, group="Calibration", help="Click to show calibration result.") bar.add_var("session name", create_string_buffer(512), getter=get_rec_name, setter=set_rec_name, group="Recording", help="Give your recording session a custom name.") bar.add_button("record", toggle_record_video, key="r", group="Recording", help="Start/Stop Recording") bar.add_var("record eye", bar.record_eye, group="Recording", help="check to save raw video of eye") bar.add_var("record audio", bar.audio, vtype=audio_enum, group="Recording", help="Select from audio recording options.") bar.add_button( "start/stop marker tracking", toggle_ar, key="x", help="find markers in scene to map gaze onto referace surfaces") bar.add_button( "start/stop server", toggle_server, key="s", help= "the server broadcasts pupil and gaze positions locally or via network" ) bar.add_button("start/stop remote", toggle_remote, key="w", help="remote allows seding commad to pupil via network") bar.add_button( "set timebase to now", reset_timebase, help="this button allows the timestamps to count from now on.", key="t") bar.add_var( "update screen", g_pool.update_textures, help= "if you dont need to see the camera image updated, you can turn this of to reduce CPU load." ) bar.add_separator("Sep1") bar.add_var("version", bar.version, readonly=True) bar.add_var("exit", g_pool.quit) # add uvc camera controls ATB bar cap.create_atb_bar(pos=(320, 10)) # Initialize glfw glfwInit() world_window = glfwCreateWindow(width, height, "World", None, None) glfwMakeContextCurrent(world_window) # Register callbacks world_window glfwSetWindowSizeCallback(world_window, on_resize) glfwSetWindowCloseCallback(world_window, on_close) glfwSetWindowIconifyCallback(world_window, on_iconify) glfwSetKeyCallback(world_window, on_key) glfwSetCharCallback(world_window, on_char) glfwSetMouseButtonCallback(world_window, on_button) glfwSetCursorPosCallback(world_window, on_pos) glfwSetScrollCallback(world_window, on_scroll) #set the last saved window size set_window_size(bar.window_size.value, bar.window_size) on_resize(world_window, *glfwGetWindowSize(world_window)) glfwSetWindowPos(world_window, 0, 0) # gl_state settings basic_gl_setup() g_pool.image_tex = create_named_texture(frame.img) # refresh speed settings glfwSwapInterval(0) #load calibration plugin open_calibration(bar.calibration_type.value, bar.calibration_type) #load gaze_display plugin g_pool.plugins.append(Display_Recent_Gaze(g_pool)) # Event loop while not g_pool.quit.value: # Get an image from the grabber try: frame = cap.get_frame() except CameraCaptureError: logger.error("Capture from Camera Failed. Stopping.") break except EndofVideoFileError: logger.warning("Video File is done. Stopping") break update_fps() #a container that allows plugins to post and read events events = [] #receive and map pupil positions recent_pupil_positions = [] while not g_pool.pupil_queue.empty(): p = g_pool.pupil_queue.get() if p['norm_pupil'] is None: p['norm_gaze'] = None else: p['norm_gaze'] = g_pool.map_pupil(p['norm_pupil']) recent_pupil_positions.append(p) # allow each Plugin to do its work. for p in g_pool.plugins: p.update(frame, recent_pupil_positions, events) #check if a plugin need to be destroyed g_pool.plugins = [p for p in g_pool.plugins if p.alive] # render camera image glfwMakeContextCurrent(world_window) make_coord_system_norm_based() if g_pool.update_textures.value: draw_named_texture(g_pool.image_tex, frame.img) else: draw_named_texture(g_pool.image_tex) make_coord_system_pixel_based(frame.img.shape) # render visual feedback from loaded plugins for p in g_pool.plugins: p.gl_display() atb.draw() glfwSwapBuffers(world_window) glfwPollEvents() # de-init all running plugins for p in g_pool.plugins: p.alive = False #reading p.alive actually runs plug-in cleanup _ = p.alive save('window_size', bar.window_size.value) save('calibration_type', bar.calibration_type.value) save('record_eye', bar.record_eye.value) save('audio', bar.audio.value) session_settings.close() cap.close() atb.terminate() glfwDestroyWindow(world_window) glfwTerminate() logger.debug("Process done")
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_version): # general imports from time import sleep import logging from glob import glob from time import time, strftime, localtime # networking import zmq import zmq_tools import numpy as np # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",)) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.NOTSET) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) try: from background_helper import IPC_Logging_Task_Proxy IPC_Logging_Task_Proxy.push_url = ipc_push_url from tasklib.background.patches import IPCLoggingPatch IPCLoggingPatch.ipc_push_url = ipc_push_url # imports from file_methods import Persistent_Dict, next_export_sub_dir # display import glfw # check versions for our own depedencies as they are fast-changing from pyglui import __version__ as pyglui_version from pyglui import ui, cygl from pyglui.cygl.utils import Named_Texture, RGBA import gl_utils # capture from video_capture import File_Source # helpers/utils from version_utils import VersionFormat from methods import normalize, denormalize, delta_t, get_system_info import player_methods as pm from csv_utils import write_key_value_file # Plug-ins from plugin import Plugin, Plugin_List, import_runtime_plugins from plugin_manager import Plugin_Manager from vis_circle import Vis_Circle from vis_cross import Vis_Cross from vis_polyline import Vis_Polyline from vis_light_points import Vis_Light_Points from vis_watermark import Vis_Watermark from vis_fixation import Vis_Fixation # from vis_scan_path import Vis_Scan_Path from seek_control import Seek_Control from surface_tracker import Surface_Tracker_Offline # from marker_auto_trim_marks import Marker_Auto_Trim_Marks from fixation_detector import Offline_Fixation_Detector from eye_movement import Offline_Eye_Movement_Detector from log_display import Log_Display from annotations import Annotation_Player from raw_data_exporter import Raw_Data_Exporter from log_history import Log_History from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection from gaze_producer.gaze_from_recording import GazeFromRecording from gaze_producer.gaze_from_offline_calibration import ( GazeFromOfflineCalibration, ) from system_graphs import System_Graphs from system_timelines import System_Timelines from blink_detection import Offline_Blink_Detection from audio_playback import Audio_Playback from video_export.plugins.imotions_exporter import iMotions_Exporter from video_export.plugins.eye_video_exporter import Eye_Video_Exporter from video_export.plugins.world_video_exporter import World_Video_Exporter from video_capture import File_Source from video_overlay.plugins import Video_Overlay, Eye_Overlay assert VersionFormat(pyglui_version) >= VersionFormat( "1.23" ), "pyglui out of date, please upgrade to newest version" runtime_plugins = import_runtime_plugins(os.path.join(user_dir, "plugins")) system_plugins = [ Log_Display, Seek_Control, Plugin_Manager, System_Graphs, System_Timelines, Audio_Playback, ] user_plugins = [ Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross, Vis_Watermark, Eye_Overlay, Video_Overlay, # Vis_Scan_Path, Offline_Fixation_Detector, Offline_Eye_Movement_Detector, Offline_Blink_Detection, Surface_Tracker_Offline, Raw_Data_Exporter, Annotation_Player, Log_History, Pupil_From_Recording, Offline_Pupil_Detection, GazeFromRecording, GazeFromOfflineCalibration, World_Video_Exporter, iMotions_Exporter, Eye_Video_Exporter, ] + runtime_plugins if platform.system() != "Windows": # Head pose tracking is currently not available on Windows from head_pose_tracker.offline_head_pose_tracker import ( Offline_Head_Pose_Tracker, ) user_plugins.append(Offline_Head_Pose_Tracker) plugins = system_plugins + user_plugins # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal hdpi_factor if w == 0 or h == 0: return hdpi_factor = glfw.getHDPIFactor(window) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor window_size = w, h g_pool.camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(*window_size) g_pool.gui.collect_menus() for p in g_pool.plugins: p.on_window_resize(window, *g_pool.camera_render_size) def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) pos = x, y pos = normalize(pos, g_pool.camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_pos(pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, count, paths): paths = [paths[x].decode("utf-8") for x in range(count)] for path in paths: if pm.is_pupil_rec_dir(path): _restart_with_recording(path) return # call `on_drop` callbacks until a plugin indicates # that it has consumed the event (by returning True) any(p.on_drop(paths) for p in g_pool.plugins) def _restart_with_recording(rec_dir): logger.debug("Starting new session with '{}'".format(rec_dir)) ipc_pub.notify( {"subject": "player_drop_process.should_start", "rec_dir": rec_dir} ) glfw.glfwSetWindowShouldClose(g_pool.main_window, True) tick = delta_t() def get_dt(): return next(tick) meta_info = pm.load_meta_info(rec_dir) # log info about Pupil Platform and Platform in player.log logger.info("Application Version: {}".format(app_version)) logger.info("System Info: {}".format(get_system_info())) icon_bar_width = 50 window_size = None hdpi_factor = 1.0 # create container for globally scoped vars g_pool = SimpleNamespace() g_pool.app = "player" g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.plugin_by_name = {p.__name__: p for p in plugins} g_pool.camera_render_size = None valid_ext = (".mp4", ".mkv", ".avi", ".h264", ".mjpeg", ".fake") video_path = [ f for f in glob(os.path.join(rec_dir, "world.*")) if os.path.splitext(f)[1] in valid_ext ][0] File_Source( g_pool, timing="external", source_path=video_path, buffered_decoding=True, fill_gaps=True, ) # load session persistent settings session_settings = Persistent_Dict( os.path.join(user_dir, "user_settings_player") ) if VersionFormat(session_settings.get("version", "0.0")) != app_version: logger.info( "Session setting are a different version of this app. I will not use those." ) session_settings.clear() width, height = g_pool.capture.frame_size width += icon_bar_width width, height = session_settings.get("window_size", (width, height)) window_pos = session_settings.get("window_position", window_position_default) window_name = "Pupil Player: {} - {}".format( meta_info["Recording Name"], os.path.split(rec_dir)[-1] ) glfw.glfwInit() main_window = glfw.glfwCreateWindow(width, height, window_name, None, None) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window def set_scale(new_scale): g_pool.gui_user_scale = new_scale window_size = ( g_pool.camera_render_size[0] + int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor), glfw.glfwGetFramebufferSize(main_window)[1], ) logger.warning(icon_bar_width * g_pool.gui_user_scale * hdpi_factor) glfw.glfwSetWindowSize(main_window, *window_size) # load pupil_positions, gaze_positions g_pool.binocular = meta_info.get("Eye Mode", "monocular") == "binocular" g_pool.version = app_version g_pool.timestamps = g_pool.capture.timestamps g_pool.get_timestamp = lambda: 0.0 g_pool.user_dir = user_dir g_pool.rec_dir = rec_dir g_pool.meta_info = meta_info g_pool.min_data_confidence = session_settings.get("min_data_confidence", MIN_DATA_CONFIDENCE_DEFAULT) g_pool.min_calibration_confidence = session_settings.get( "min_calibration_confidence", MIN_CALIBRATION_CONFIDENCE_DEFAULT ) # populated by producers g_pool.pupil_positions = pm.Bisector() g_pool.pupil_positions_by_id = (pm.Bisector(), pm.Bisector()) g_pool.gaze_positions = pm.Bisector() g_pool.fixations = pm.Affiliator() g_pool.eye_movements = pm.Affiliator() def set_data_confidence(new_confidence): g_pool.min_data_confidence = new_confidence notification = {"subject": "min_data_confidence_changed"} notification["_notify_time_"] = time() + 0.8 g_pool.ipc_pub.notify(notification) def do_export(_): left_idx = g_pool.seek_control.trim_left right_idx = g_pool.seek_control.trim_right export_range = left_idx, right_idx + 1 # exclusive range.stop export_ts_window = pm.exact_window(g_pool.timestamps, (left_idx, right_idx)) export_dir = os.path.join(g_pool.rec_dir, "exports") export_dir = next_export_sub_dir(export_dir) os.makedirs(export_dir) logger.info('Created export dir at "{}"'.format(export_dir)) export_info = { "Player Software Version": str(g_pool.version), "Data Format Version": meta_info["Data Format Version"], "Export Date": strftime("%d.%m.%Y", localtime()), "Export Time": strftime("%H:%M:%S", localtime()), "Frame Index Range:": g_pool.seek_control.get_frame_index_trim_range_string(), "Relative Time Range": g_pool.seek_control.get_rel_time_trim_range_string(), "Absolute Time Range": g_pool.seek_control.get_abs_time_trim_range_string(), } with open(os.path.join(export_dir, "export_info.csv"), "w") as csv: write_key_value_file(csv, export_info) notification = { "subject": "should_export", "range": export_range, "ts_window": export_ts_window, "export_dir": export_dir, } g_pool.ipc_pub.notify(notification) def reset_restart(): logger.warning("Resetting all settings and restarting Player.") glfw.glfwSetWindowShouldClose(main_window, True) ipc_pub.notify({"subject": "clear_settings_process.should_start"}) ipc_pub.notify( { "subject": "player_process.should_start", "rec_dir": rec_dir, "delay": 2.0, } ) def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is open the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0) g_pool.menubar = ui.Scrolling_Menu( "Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos="left" ) g_pool.iconbar = ui.Scrolling_Menu( "Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden" ) g_pool.timelines = ui.Container((0, 0), (0, 0), (0, 0)) g_pool.timelines.horizontal_constraint = g_pool.menubar g_pool.user_timelines = ui.Timeline_Menu( "User Timelines", pos=(0.0, -150.0), size=(0.0, 0.0), header_pos="headline" ) g_pool.user_timelines.color = RGBA(a=0.0) g_pool.user_timelines.collapsed = True # add container that constaints itself to the seekbar height vert_constr = ui.Container((0, 0), (0, -50.0), (0, 0)) vert_constr.append(g_pool.user_timelines) g_pool.timelines.append(vert_constr) def set_window_size(): f_width, f_height = g_pool.capture.frame_size f_width += int(icon_bar_width * g_pool.gui.scale) glfw.glfwSetWindowSize(main_window, f_width, f_height) general_settings = ui.Growing_Menu("General", header_pos="headline") general_settings.append(ui.Button("Reset window size", set_window_size)) general_settings.append( ui.Selector( "gui_user_scale", g_pool, setter=set_scale, selection=[0.8, 0.9, 1.0, 1.1, 1.2] + list(np.arange(1.5, 5.1, 0.5)), label="Interface Size", ) ) general_settings.append( ui.Info_Text("Player Version: {}".format(g_pool.version)) ) general_settings.append( ui.Info_Text( "Capture Version: {}".format(meta_info["Capture Software Version"]) ) ) general_settings.append( ui.Info_Text( "Data Format Version: {}".format(meta_info["Data Format Version"]) ) ) general_settings.append( ui.Info_Text( "High level data, e.g. fixations, or visualizations only consider gaze data that has an equal or higher confidence than the minimum data confidence." ) ) general_settings.append( ui.Slider( "min_data_confidence", g_pool, setter=set_data_confidence, step=0.05, min=0.0, max=1.0, label="Minimum data confidence", ) ) general_settings.append( ui.Button("Restart with default settings", reset_restart) ) g_pool.menubar.append(general_settings) icon = ui.Icon( "collapsed", general_settings, label=chr(0xE8B8), on_val=False, off_val=True, setter=toggle_general_settings, label_font="pupil_icons", ) icon.tooltip = "General Settings" g_pool.iconbar.append(icon) user_plugin_separator = ui.Separator() user_plugin_separator.order = 0.35 g_pool.iconbar.append(user_plugin_separator) g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100), (100, -100)) g_pool.export_button = ui.Thumb( "export", label=chr(0xE2C5), getter=lambda: False, setter=do_export, hotkey="e", label_font="pupil_icons", ) g_pool.quickbar.extend([g_pool.export_button]) g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.timelines) g_pool.gui.append(g_pool.iconbar) g_pool.gui.append(g_pool.quickbar) # we always load these plugins default_plugins = [ ("Plugin_Manager", {}), ("Seek_Control", {}), ("Log_Display", {}), ("Raw_Data_Exporter", {}), ("Vis_Polyline", {}), ("Vis_Circle", {}), ("System_Graphs", {}), ("System_Timelines", {}), ("World_Video_Exporter", {}), ("Pupil_From_Recording", {}), ("GazeFromRecording", {}), ("Audio_Playback", {}), ] g_pool.plugins = Plugin_List( g_pool, session_settings.get("loaded_plugins", default_plugins) ) # Manually add g_pool.capture to the plugin list g_pool.plugins._plugins.append(g_pool.capture) g_pool.plugins._plugins.sort(key=lambda p: p.order) g_pool.capture.init_ui() general_settings.insert( -1, ui.Text_Input( "rel_time_trim_section", getter=g_pool.seek_control.get_rel_time_trim_range_string, setter=g_pool.seek_control.set_rel_time_trim_range_string, label="Relative time range to export", ), ) general_settings.insert( -1, ui.Text_Input( "frame_idx_trim_section", getter=g_pool.seek_control.get_frame_index_trim_range_string, setter=g_pool.seek_control.set_frame_index_trim_range_string, label="Frame index range to export", ), ) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) toggle_general_settings(True) g_pool.gui.configuration = session_settings.get("ui_config", {}) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() # trigger on_resize on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) def handle_notifications(n): subject = n["subject"] if subject == "start_plugin": g_pool.plugins.add( g_pool.plugin_by_name[n["name"]], args=n.get("args", {}) ) elif subject.startswith("meta.should_doc"): ipc_pub.notify( {"subject": "meta.doc", "actor": g_pool.app, "doc": player.__doc__} ) for p in g_pool.plugins: if ( p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify ): ipc_pub.notify( { "subject": "meta.doc", "actor": p.class_name, "doc": p.on_notify.__doc__, } ) while not glfw.glfwWindowShouldClose(main_window): # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) events = {} # report time between now and the last loop interation events["dt"] = get_dt() # pupil and gaze positions are added by their respective producer plugins events["pupil"] = [] events["gaze"] = [] # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() glfw.glfwMakeContextCurrent(main_window) # render visual feedback from loaded plugins if gl_utils.is_window_visible(main_window): gl_utils.glViewport(0, 0, *g_pool.camera_render_size) g_pool.capture.gl_display() for p in g_pool.plugins: p.gl_display() gl_utils.glViewport(0, 0, *window_size) try: clipboard = glfw.glfwGetClipboardString(main_window).decode() except AttributeError: # clipbaord is None, might happen on startup clipboard = "" g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard and user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.glfwSetClipboardString( main_window, user_input.clipboard.encode() ) for b in user_input.buttons: button, action, mods = b x, y = glfw.glfwGetCursorPos(main_window) pos = x * hdpi_factor, y * hdpi_factor pos = normalize(pos, g_pool.camera_render_size) pos = denormalize(pos, g_pool.capture.frame_size) # call `on_click` callbacks until a plugin indicates # that it has consumed the event (by returning True) any(p.on_click(pos, button, action) for p in g_pool.plugins) for key, scancode, action, mods in user_input.keys: # call `on_key` callbacks until a plugin indicates # that it has consumed the event (by returning True) any(p.on_key(key, scancode, action, mods) for p in g_pool.plugins) for char_ in user_input.chars: # call `char_` callbacks until a plugin indicates # that it has consumed the event (by returning True) any(p.on_char(char_) for p in g_pool.plugins) # present frames at appropriate speed g_pool.seek_control.wait(events["frame"].timestamp) glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() session_settings["loaded_plugins"] = g_pool.plugins.get_initializers() session_settings["min_data_confidence"] = g_pool.min_data_confidence session_settings[ "min_calibration_confidence" ] = g_pool.min_calibration_confidence session_settings["gui_scale"] = g_pool.gui_user_scale session_settings["ui_config"] = g_pool.gui.configuration session_settings["window_position"] = glfw.glfwGetWindowPos(main_window) session_settings["version"] = str(g_pool.version) session_window_size = glfw.glfwGetWindowSize(main_window) if 0 not in session_window_size: session_settings["window_size"] = session_window_size session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) except: import traceback trace = traceback.format_exc() logger.error("Process Player crashed with trace:\n{}".format(trace)) finally: logger.info("Process shutting down.") ipc_pub.notify({"subject": "player_process.stopped"}) sleep(1.0)
def world( timebase, eye_procs_alive, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version, preferred_remote_port, hide_ui, debug, ): """Reads world video and runs plugins. Creates a window, gl context. Grabs images from a capture. Maps pupil to gaze data Can run various plug-ins. Reacts to notifications: ``eye_process.started`` ``start_plugin`` ``should_stop`` Emits notifications: ``eye_process.should_start`` ``eye_process.should_stop`` ``world_process.started`` ``world_process.stopped`` ``recording.should_stop``: Emits on camera failure ``launcher_process.should_stop`` Emits data: ``gaze``: Gaze data from current gaze mapping plugin.`` ``*``: any other plugin generated data in the events that it not [dt,pupil,gaze]. """ # We defer the imports because of multiprocessing. # Otherwise the world process each process also loads the other imports. # This is not harmful but unnecessary. # general imports from time import sleep import logging # networking import zmq import zmq_tools # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",)) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.NOTSET) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) def launch_eye_process(eye_id, delay=0): n = { "subject": "eye_process.should_start.{}".format(eye_id), "eye_id": eye_id, "delay": delay, } ipc_pub.notify(n) def stop_eye_process(eye_id): n = { "subject": "eye_process.should_stop.{}".format(eye_id), "eye_id": eye_id, "delay": 0.2, } ipc_pub.notify(n) def start_stop_eye(eye_id, make_alive): if make_alive: launch_eye_process(eye_id) else: stop_eye_process(eye_id) def detection_enabled_getter() -> bool: return g_pool.pupil_detection_enabled def detection_enabled_setter(is_on: bool): g_pool.pupil_detection_enabled = is_on n = {"subject": "set_pupil_detection_enabled", "value": is_on} ipc_pub.notify(n) try: from background_helper import IPC_Logging_Task_Proxy IPC_Logging_Task_Proxy.push_url = ipc_push_url from tasklib.background.patches import IPCLoggingPatch IPCLoggingPatch.ipc_push_url = ipc_push_url from OpenGL.GL import GL_COLOR_BUFFER_BIT # display import glfw glfw.ERROR_REPORTING = "raise" from version_utils import parse_version from pyglui import ui, cygl, __version__ as pyglui_version assert parse_version(pyglui_version) >= parse_version( "1.27" ), "pyglui out of date, please upgrade to newest version" from pyglui.cygl.utils import Named_Texture import gl_utils # helpers/utils from file_methods import Persistent_Dict from methods import normalize, denormalize, delta_t, get_system_info, timer from uvc import get_time_monotonic logger.info("Application Version: {}".format(version)) logger.info("System Info: {}".format(get_system_info())) logger.debug(f"Debug flag: {debug}") import audio # Plug-ins from plugin import ( Plugin, System_Plugin_Base, Plugin_List, import_runtime_plugins, ) from plugin_manager import Plugin_Manager from calibration_choreography import ( available_calibration_choreography_plugins, CalibrationChoreographyPlugin, patch_loaded_plugins_with_choreography_plugin, ) available_choreography_plugins = available_calibration_choreography_plugins() from gaze_mapping import registered_gazer_classes from gaze_mapping.gazer_base import GazerBase from pupil_detector_plugins.detector_base_plugin import PupilDetectorPlugin from fixation_detector import Fixation_Detector from recorder import Recorder from display_recent_gaze import Display_Recent_Gaze from time_sync import Time_Sync from network_api import NetworkApiPlugin from pupil_groups import Pupil_Groups from surface_tracker import Surface_Tracker_Online from log_display import Log_Display from annotations import Annotation_Capture from log_history import Log_History from blink_detection import Blink_Detection from video_capture import ( source_classes, manager_classes, Base_Manager, Base_Source, ) from pupil_data_relay import Pupil_Data_Relay from remote_recorder import Remote_Recorder from accuracy_visualizer import Accuracy_Visualizer from system_graphs import System_Graphs from camera_intrinsics_estimation import Camera_Intrinsics_Estimation from hololens_relay import Hololens_Relay from head_pose_tracker.online_head_pose_tracker import Online_Head_Pose_Tracker # UI Platform tweaks if platform.system() == "Linux": scroll_factor = 10.0 window_position_default = (30, 30) elif platform.system() == "Windows": scroll_factor = 10.0 window_position_default = (8, 90) else: scroll_factor = 1.0 window_position_default = (0, 0) process_was_interrupted = False def interrupt_handler(sig, frame): import traceback trace = traceback.format_stack(f=frame) logger.debug(f"Caught signal {sig} in:\n" + "".join(trace)) nonlocal process_was_interrupted process_was_interrupted = True signal.signal(signal.SIGINT, interrupt_handler) icon_bar_width = 50 window_size = None camera_render_size = None content_scale = 1.0 # g_pool holds variables for this process they are accessible to all plugins g_pool = SimpleNamespace() g_pool.debug = debug g_pool.app = "capture" g_pool.process = "world" g_pool.user_dir = user_dir g_pool.version = version g_pool.timebase = timebase g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.eye_procs_alive = eye_procs_alive g_pool.preferred_remote_port = preferred_remote_port def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic # manage plugins runtime_plugins = import_runtime_plugins( os.path.join(g_pool.user_dir, "plugins") ) runtime_plugins = [ p for p in runtime_plugins if not issubclass(p, PupilDetectorPlugin) ] user_plugins = [ Pupil_Groups, NetworkApiPlugin, Time_Sync, Surface_Tracker_Online, Annotation_Capture, Log_History, Fixation_Detector, Blink_Detection, Remote_Recorder, Accuracy_Visualizer, Camera_Intrinsics_Estimation, Hololens_Relay, Online_Head_Pose_Tracker, ] system_plugins = ( [ Log_Display, Display_Recent_Gaze, Recorder, Pupil_Data_Relay, Plugin_Manager, System_Graphs, ] + manager_classes + source_classes ) plugins = ( system_plugins + user_plugins + runtime_plugins + available_choreography_plugins + registered_gazer_classes() ) user_plugins += [ p for p in runtime_plugins if not isinstance( p, ( Base_Manager, Base_Source, System_Plugin_Base, CalibrationChoreographyPlugin, GazerBase, ), ) ] g_pool.plugin_by_name = {p.__name__: p for p in plugins} default_capture_name = "UVC_Source" default_capture_settings = { "preferred_names": [ "Pupil Cam1 ID2", "Logitech Camera", "(046d:081d)", "C510", "B525", "C525", "C615", "C920", "C930e", ], "frame_size": (1280, 720), "frame_rate": 30, } default_plugins = [ (default_capture_name, default_capture_settings), ("Pupil_Data_Relay", {}), ("UVC_Manager", {}), ("NDSI_Manager", {}), ("HMD_Streaming_Manager", {}), ("File_Manager", {}), ("Log_Display", {}), ("Dummy_Gaze_Mapper", {}), ("Display_Recent_Gaze", {}), # Calibration choreography plugin is added below by calling # patch_loaded_plugins_with_choreography_plugin ("Recorder", {}), ("NetworkApiPlugin", {}), ("Fixation_Detector", {}), ("Blink_Detection", {}), ("Accuracy_Visualizer", {}), ("Plugin_Manager", {}), ("System_Graphs", {}), ] def consume_events_and_render_buffer(): gl_utils.glViewport(0, 0, *camera_render_size) for p in g_pool.plugins: p.gl_display() gl_utils.glViewport(0, 0, *window_size) try: clipboard = glfw.get_clipboard_string(main_window).decode() except (AttributeError, glfw.GLFWError): # clipboard is None, might happen on startup clipboard = "" g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.set_clipboard_string(main_window, user_input.clipboard) for button, action, mods in user_input.buttons: x, y = glfw.get_cursor_pos(main_window) pos = gl_utils.window_coordinate_to_framebuffer_coordinate( main_window, x, y, cached_scale=None ) pos = normalize(pos, camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for plugin in g_pool.plugins: if plugin.on_click(pos, button, action): break for key, scancode, action, mods in user_input.keys: for plugin in g_pool.plugins: if plugin.on_key(key, scancode, action, mods): break for char_ in user_input.chars: for plugin in g_pool.plugins: if plugin.on_char(char_): break glfw.swap_buffers(main_window) # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal camera_render_size nonlocal content_scale if w == 0 or h == 0: return # Always clear buffers on resize to make sure that there are no overlapping # artifacts from previous frames. gl_utils.glClear(GL_COLOR_BUFFER_BIT) gl_utils.glClearColor(0, 0, 0, 1) content_scale = gl_utils.get_content_scale(window) framebuffer_scale = gl_utils.get_framebuffer_scale(window) g_pool.gui.scale = content_scale window_size = w, h camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(*window_size) g_pool.gui.collect_menus() for p in g_pool.plugins: p.on_window_resize(window, *camera_render_size) # Minimum window size required, otherwise parts of the UI can cause openGL # issues with permanent effects. Depends on the content scale, which can # potentially be dynamically modified, so we re-adjust the size limits every # time here. min_size = int(2 * icon_bar_width * g_pool.gui.scale / framebuffer_scale) glfw.set_window_size_limits( window, min_size, min_size, glfw.DONT_CARE, glfw.DONT_CARE, ) # Needed, to update the window buffer while resizing consume_events_and_render_buffer() def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x, y = gl_utils.window_coordinate_to_framebuffer_coordinate( window, x, y, cached_scale=None ) g_pool.gui.update_mouse(x, y) pos = x, y pos = normalize(pos, camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_pos(pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, paths): for plugin in g_pool.plugins: if plugin.on_drop(paths): break tick = delta_t() def get_dt(): return next(tick) # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, "user_settings_world") ) if parse_version(session_settings.get("version", "0.0")) != g_pool.version: logger.info( "Session setting are from a different version of this app. I will not use those." ) session_settings.clear() g_pool.min_data_confidence = 0.6 g_pool.min_calibration_confidence = session_settings.get( "min_calibration_confidence", 0.8 ) g_pool.pupil_detection_enabled = session_settings.get( "pupil_detection_enabled", True ) g_pool.active_gaze_mapping_plugin = None g_pool.capture = None audio.set_audio_mode( session_settings.get("audio_mode", audio.get_default_audio_mode()) ) def handle_notifications(noti): subject = noti["subject"] if subject == "set_pupil_detection_enabled": g_pool.pupil_detection_enabled = noti["value"] elif subject == "start_plugin": try: g_pool.plugins.add( g_pool.plugin_by_name[noti["name"]], args=noti.get("args", {}) ) except KeyError as err: logger.error(f"Attempt to load unknown plugin: {err}") elif subject == "stop_plugin": for p in g_pool.plugins: if p.class_name == noti["name"]: p.alive = False g_pool.plugins.clean() elif subject == "eye_process.started": noti = { "subject": "set_pupil_detection_enabled", "value": g_pool.pupil_detection_enabled, } ipc_pub.notify(noti) elif subject == "set_min_calibration_confidence": g_pool.min_calibration_confidence = noti["value"] elif subject.startswith("meta.should_doc"): ipc_pub.notify( {"subject": "meta.doc", "actor": g_pool.app, "doc": world.__doc__} ) for p in g_pool.plugins: if ( p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify ): ipc_pub.notify( { "subject": "meta.doc", "actor": p.class_name, "doc": p.on_notify.__doc__, } ) elif subject == "world_process.adapt_window_size": set_window_size() elif subject == "world_process.should_stop": glfw.set_window_should_close(main_window, True) width, height = session_settings.get( "window_size", (1280 + icon_bar_width, 720) ) # window and gl setup glfw.init() glfw.window_hint(glfw.SCALE_TO_MONITOR, glfw.TRUE) if hide_ui: glfw.window_hint(glfw.VISIBLE, 0) # hide window main_window = glfw.create_window( width, height, "Pupil Capture - World", None, None ) window_position_manager = gl_utils.WindowPositionManager() window_pos = window_position_manager.new_window_position( window=main_window, default_position=window_position_default, previous_position=session_settings.get("window_position", None), ) glfw.set_window_pos(main_window, window_pos[0], window_pos[1]) glfw.make_context_current(main_window) cygl.utils.init() g_pool.main_window = main_window def reset_restart(): logger.warning("Resetting all settings and restarting Capture.") glfw.set_window_should_close(main_window, True) ipc_pub.notify({"subject": "clear_settings_process.should_start"}) ipc_pub.notify({"subject": "world_process.should_start", "delay": 2.0}) def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is opened, the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed # setup GUI g_pool.gui = ui.UI() g_pool.menubar = ui.Scrolling_Menu( "Settings", pos=(-400, 0), size=(-icon_bar_width, 0), header_pos="left" ) g_pool.iconbar = ui.Scrolling_Menu( "Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden" ) g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100), (120, -100)) g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.iconbar) g_pool.gui.append(g_pool.quickbar) general_settings = ui.Growing_Menu("General", header_pos="headline") def set_window_size(): # Get current capture frame size f_width, f_height = g_pool.capture.frame_size # Get current display scale factor content_scale = gl_utils.get_content_scale(main_window) framebuffer_scale = gl_utils.get_framebuffer_scale(main_window) display_scale_factor = content_scale / framebuffer_scale # Scale the capture frame size by display scale factor f_width *= display_scale_factor f_height *= display_scale_factor # Increas the width to account for the added scaled icon bar width f_width += icon_bar_width * display_scale_factor # Set the newly calculated size (scaled capture frame size + scaled icon bar width) glfw.set_window_size(main_window, int(f_width), int(f_height)) general_settings.append(ui.Button("Reset window size", set_window_size)) general_settings.append( ui.Selector( "Audio mode", None, getter=audio.get_audio_mode, setter=audio.set_audio_mode, selection=audio.get_audio_mode_list(), ) ) general_settings.append( ui.Switch( "pupil_detection_enabled", label="Pupil detection", getter=detection_enabled_getter, setter=detection_enabled_setter, ) ) general_settings.append( ui.Switch( "eye0_process", label="Detect eye 0", setter=lambda alive: start_stop_eye(0, alive), getter=lambda: eye_procs_alive[0].value, ) ) general_settings.append( ui.Switch( "eye1_process", label="Detect eye 1", setter=lambda alive: start_stop_eye(1, alive), getter=lambda: eye_procs_alive[1].value, ) ) general_settings.append( ui.Info_Text("Capture Version: {}".format(g_pool.version)) ) general_settings.append( ui.Button("Restart with default settings", reset_restart) ) g_pool.menubar.append(general_settings) icon = ui.Icon( "collapsed", general_settings, label=chr(0xE8B8), on_val=False, off_val=True, setter=toggle_general_settings, label_font="pupil_icons", ) icon.tooltip = "General Settings" g_pool.iconbar.append(icon) user_plugin_separator = ui.Separator() user_plugin_separator.order = 0.35 g_pool.iconbar.append(user_plugin_separator) loaded_plugins = session_settings.get("loaded_plugins", default_plugins) # Resolve the active calibration choreography plugin loaded_plugins = patch_loaded_plugins_with_choreography_plugin( loaded_plugins, app=g_pool.app ) session_settings["loaded_plugins"] = loaded_plugins # plugins that are loaded based on user settings from previous session g_pool.plugins = Plugin_List(g_pool, loaded_plugins) if not g_pool.capture: # Make sure we always have a capture running. Important if there was no # capture stored in session settings. g_pool.plugins.add( g_pool.plugin_by_name[default_capture_name], default_capture_settings ) # Register callbacks main_window glfw.set_framebuffer_size_callback(main_window, on_resize) glfw.set_key_callback(main_window, on_window_key) glfw.set_char_callback(main_window, on_window_char) glfw.set_mouse_button_callback(main_window, on_window_mouse_button) glfw.set_cursor_pos_callback(main_window, on_pos) glfw.set_scroll_callback(main_window, on_scroll) glfw.set_drop_callback(main_window, on_drop) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() toggle_general_settings(True) # now that we have a proper window we can load the last gui configuration g_pool.gui.configuration = session_settings.get("ui_config", {}) # If previously selected plugin was not loaded this time, we will have an # expanded menubar without any menu selected. We need to ensure the menubar is # collapsed in this case. if all(submenu.collapsed for submenu in g_pool.menubar.elements): g_pool.menubar.collapsed = True # create a timer to control window update frequency window_update_timer = timer(1 / 60) def window_should_update(): return next(window_update_timer) # trigger setup of window and gl sizes on_resize(main_window, *glfw.get_framebuffer_size(main_window)) if session_settings.get("eye1_process_alive", True): launch_eye_process(1, delay=0.6) if session_settings.get("eye0_process_alive", True): launch_eye_process(0, delay=0.3) ipc_pub.notify({"subject": "world_process.started"}) logger.warning("Process started.") if platform.system() == "Darwin": # On macOS, calls to glfw.swap_buffers() deliberately take longer in case of # occluded windows, based on the swap interval value. This causes an FPS drop # and leads to problems when recording. To side-step this behaviour, the swap # interval is set to zero. # # Read more about window occlusion on macOS here: # https://developer.apple.com/library/archive/documentation/Performance/Conceptual/power_efficiency_guidelines_osx/WorkWhenVisible.html glfw.swap_interval(0) # Event loop while not glfw.window_should_close(main_window) and not process_was_interrupted: # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) # a dictionary that allows plugins to post and read events events = {} # report time between now and the last loop interation events["dt"] = get_dt() # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() # "blacklisted" events that were already sent del events["pupil"] del events["gaze"] # delete if exists. More expensive than del, so only use it when key might not exist events.pop("annotation", None) # send new events to ipc: if "frame" in events: del events["frame"] # send explicitly with frame publisher if "depth_frame" in events: del events["depth_frame"] if "audio_packets" in events: del events["audio_packets"] del events["dt"] # no need to send this for data in events.values(): assert isinstance(data, (list, tuple)) for d in data: ipc_pub.send(d) glfw.make_context_current(main_window) # render visual feedback from loaded plugins glfw.poll_events() if window_should_update() and gl_utils.is_window_visible(main_window): gl_utils.glViewport(0, 0, *camera_render_size) for p in g_pool.plugins: p.gl_display() gl_utils.glViewport(0, 0, *window_size) try: clipboard = glfw.get_clipboard_string(main_window).decode() except (AttributeError, glfw.GLFWError): # clipboard is None, might happen on startup clipboard = "" g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.set_clipboard_string(main_window, user_input.clipboard) for button, action, mods in user_input.buttons: x, y = glfw.get_cursor_pos(main_window) pos = gl_utils.window_coordinate_to_framebuffer_coordinate( main_window, x, y, cached_scale=None ) pos = normalize(pos, camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for plugin in g_pool.plugins: if plugin.on_click(pos, button, action): break for key, scancode, action, mods in user_input.keys: for plugin in g_pool.plugins: if plugin.on_key(key, scancode, action, mods): break for char_ in user_input.chars: for plugin in g_pool.plugins: if plugin.on_char(char_): break glfw.swap_buffers(main_window) session_settings["loaded_plugins"] = g_pool.plugins.get_initializers() session_settings["ui_config"] = g_pool.gui.configuration session_settings["version"] = str(g_pool.version) session_settings["eye0_process_alive"] = eye_procs_alive[0].value session_settings["eye1_process_alive"] = eye_procs_alive[1].value session_settings[ "min_calibration_confidence" ] = g_pool.min_calibration_confidence session_settings["pupil_detection_enabled"] = g_pool.pupil_detection_enabled session_settings["audio_mode"] = audio.get_audio_mode() if not hide_ui: glfw.restore_window(main_window) # need to do this for windows os session_settings["window_position"] = glfw.get_window_pos(main_window) session_window_size = glfw.get_window_size(main_window) if 0 not in session_window_size: f_width, f_height = session_window_size if platform.system() in ("Windows", "Linux"): f_width, f_height = ( f_width / content_scale, f_height / content_scale, ) session_settings["window_size"] = int(f_width), int(f_height) session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfw.destroy_window(main_window) glfw.terminate() except Exception: import traceback trace = traceback.format_exc() logger.error("Process Capture crashed with trace:\n{}".format(trace)) finally: # shut down eye processes: stop_eye_process(0) stop_eye_process(1) logger.info("Process shutting down.") ipc_pub.notify({"subject": "world_process.stopped"}) sleep(1.0)
def eye(pupil_queue, timebase, pipe_to_world, is_alive_flag, user_dir, version, eye_id, cap_src): """ Creates a window, gl context. Grabs images from a capture. Streams Pupil coordinates into g_pool.pupil_queue """ is_alive = Is_Alive_Manager(is_alive_flag) with is_alive: import logging # Set up root logger for this process before doing imports of logged modules. logger = logging.getLogger() logger.setLevel(logging.INFO) # remove inherited handlers logger.handlers = [] # create file handler which logs even debug messages fh = logging.FileHandler(os.path.join(user_dir,'eye%s.log'%eye_id),mode='w') # fh.setLevel(logging.DEBUG) # create console handler with a higher log level ch = logging.StreamHandler() ch.setLevel(logger.level+10) # create formatter and add it to the handlers formatter = logging.Formatter('Eye'+str(eye_id)+' Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s') fh.setFormatter(formatter) formatter = logging.Formatter('EYE'+str(eye_id)+' Process [%(levelname)s] %(name)s : %(message)s') ch.setFormatter(formatter) # add the handlers to the logger logger.addHandler(fh) logger.addHandler(ch) #silence noisy modules logging.getLogger("OpenGL").setLevel(logging.ERROR) # create logger for the context of this function logger = logging.getLogger(__name__) # We deferr the imports becasue of multiprocessing. # Otherwise the world process each process also loads the other imports. #general imports import numpy as np import cv2 #display import glfw from pyglui import ui,graph,cygl from pyglui.cygl.utils import draw_points, RGBA, draw_polyline, Named_Texture, Sphere import OpenGL.GL as gl from gl_utils import basic_gl_setup,adjust_gl_view, clear_gl_screen ,make_coord_system_pixel_based,make_coord_system_norm_based, make_coord_system_eye_camera_based from ui_roi import UIRoi #monitoring import psutil import math # helpers/utils from file_methods import Persistent_Dict from version_utils import VersionFormat from methods import normalize, denormalize, Roi, timer from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError from av_writer import JPEG_Writer,AV_Writer # Pupil detectors from pupil_detectors import Detector_2D, Detector_3D pupil_detectors = {Detector_2D.__name__:Detector_2D,Detector_3D.__name__:Detector_3D} #UI Platform tweaks if platform.system() == 'Linux': scroll_factor = 10.0 window_position_default = (600,300*eye_id) elif platform.system() == 'Windows': scroll_factor = 1.0 window_position_default = (600,31+300*eye_id) else: scroll_factor = 1.0 window_position_default = (600,300*eye_id) #g_pool holds variables for this process g_pool = Global_Container() # make some constants avaiable g_pool.user_dir = user_dir g_pool.version = version g_pool.app = 'capture' g_pool.pupil_queue = pupil_queue g_pool.timebase = timebase # Callback functions def on_resize(window,w, h): if not g_pool.iconified: active_window = glfw.glfwGetCurrentContext() glfw.glfwMakeContextCurrent(window) g_pool.gui.update_window(w,h) graph.adjust_size(w,h) adjust_gl_view(w,h) glfw.glfwMakeContextCurrent(active_window) def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key,scancode,action,mods) def on_char(window,char): g_pool.gui.update_char(char) def on_iconify(window,iconified): g_pool.iconified = iconified def on_button(window,button, action, mods): if g_pool.display_mode == 'roi': if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt: g_pool.u_r.active_edit_pt = False return # if the roi interacts we dont what the gui to interact as well elif action == glfw.GLFW_PRESS: pos = glfw.glfwGetCursorPos(window) pos = normalize(pos,glfw.glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1-pos[0],1-pos[1] pos = denormalize(pos,(frame.width,frame.height)) # Position in img pixels if g_pool.u_r.mouse_over_edit_pt(pos,g_pool.u_r.handle_size+40,g_pool.u_r.handle_size+40): return # if the roi interacts we dont what the gui to interact as well g_pool.gui.update_button(button,action,mods) def on_pos(window,x, y): hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0]/glfw.glfwGetWindowSize(window)[0]) g_pool.gui.update_mouse(x*hdpi_factor,y*hdpi_factor) if g_pool.u_r.active_edit_pt: pos = normalize((x,y),glfw.glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1-pos[0],1-pos[1] pos = denormalize(pos,(frame.width,frame.height) ) g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx,pos) def on_scroll(window,x,y): g_pool.gui.update_scroll(x,y*scroll_factor) # load session persistent settings session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_eye%s'%eye_id)) if session_settings.get("version",VersionFormat('0.0')) < g_pool.version: logger.info("Session setting are from older version of this app. I will not use those.") session_settings.clear() # Initialize capture cap = autoCreateCapture(cap_src, timebase=g_pool.timebase) default_settings = {'frame_size':(640,480),'frame_rate':60} previous_settings = session_settings.get('capture_settings',None) if previous_settings and previous_settings['name'] == cap.name: cap.settings = previous_settings else: cap.settings = default_settings # Test capture try: frame = cap.get_frame() except CameraCaptureError: logger.error("Could not retrieve image from capture") cap.close() return #signal world that we are ready to go # pipe_to_world.send('eye%s process ready'%eye_id) # any object we attach to the g_pool object *from now on* will only be visible to this process! # vars should be declared here to make them visible to the code reader. g_pool.iconified = False g_pool.capture = cap g_pool.flip = session_settings.get('flip',False) g_pool.display_mode = session_settings.get('display_mode','camera_image') g_pool.display_mode_info_text = {'camera_image': "Raw eye camera image. This uses the least amount of CPU power", 'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.", 'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below."} g_pool.u_r = UIRoi(frame.img.shape) g_pool.u_r.set(session_settings.get('roi',g_pool.u_r.get())) def on_frame_size_change(new_size): g_pool.u_r = UIRoi((new_size[1],new_size[0])) cap.on_frame_size_change = on_frame_size_change writer = None pupil_detector_settings = session_settings.get('pupil_detector_settings',None) last_pupil_detector = pupil_detectors[session_settings.get('last_pupil_detector',Detector_2D.__name__)] g_pool.pupil_detector = last_pupil_detector(g_pool,pupil_detector_settings) # UI callback functions def set_scale(new_scale): g_pool.gui.scale = new_scale g_pool.gui.collect_menus() def set_display_mode_info(val): g_pool.display_mode = val g_pool.display_mode_info.text = g_pool.display_mode_info_text[val] def set_detector(new_detector): g_pool.pupil_detector.cleanup() g_pool.pupil_detector = new_detector(g_pool) g_pool.pupil_detector.init_gui(g_pool.sidebar) # Initialize glfw glfw.glfwInit() title = "eye %s"%eye_id width,height = session_settings.get('window_size',(frame.width, frame.height)) main_window = glfw.glfwCreateWindow(width,height, title, None, None) window_pos = session_settings.get('window_position',window_position_default) glfw.glfwSetWindowPos(main_window,window_pos[0],window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() g_pool.image_tex.update_from_frame(frame) glfw.glfwSwapInterval(0) sphere = Sphere(20) #setup GUI g_pool.gui = ui.UI() g_pool.gui.scale = session_settings.get('gui_scale',1) g_pool.sidebar = ui.Scrolling_Menu("Settings",pos=(-300,0),size=(0,0),header_pos='left') general_settings = ui.Growing_Menu('General') general_settings.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=1.,max=2.5,label='Interface Size')) general_settings.append(ui.Button('Reset window size',lambda: glfw.glfwSetWindowSize(main_window,frame.width,frame.height)) ) general_settings.append(ui.Switch('flip',g_pool,label='Flip image display')) general_settings.append(ui.Selector('display_mode',g_pool,setter=set_display_mode_info,selection=['camera_image','roi','algorithm'], labels=['Camera Image', 'ROI', 'Algorithm'], label="Mode") ) g_pool.display_mode_info = ui.Info_Text(g_pool.display_mode_info_text[g_pool.display_mode]) general_settings.append(g_pool.display_mode_info) g_pool.sidebar.append(general_settings) g_pool.gui.append(g_pool.sidebar) detector_selector = ui.Selector('pupil_detector',getter = lambda: g_pool.pupil_detector.__class__ ,setter=set_detector,selection=[Detector_2D, Detector_3D],labels=['C++ 2d detector', 'C++ 3d detector'], label="Detection method") general_settings.append(detector_selector) # let detector add its GUI g_pool.pupil_detector.init_gui(g_pool.sidebar) # let the camera add its GUI g_pool.capture.init_gui(g_pool.sidebar) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window,on_resize) glfw.glfwSetWindowIconifyCallback(main_window,on_iconify) glfw.glfwSetKeyCallback(main_window,on_key) glfw.glfwSetCharCallback(main_window,on_char) glfw.glfwSetMouseButtonCallback(main_window,on_button) glfw.glfwSetCursorPosCallback(main_window,on_pos) glfw.glfwSetScrollCallback(main_window,on_scroll) #set the last saved window size on_resize(main_window, *glfw.glfwGetWindowSize(main_window)) # load last gui configuration g_pool.gui.configuration = session_settings.get('ui_config',{}) #set up performance graphs pid = os.getpid() ps = psutil.Process(pid) ts = frame.timestamp cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20,130) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140,130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" #create a timer to control window update frequency window_update_timer = timer(1/60.) def window_should_update(): return next(window_update_timer) # Event loop while not glfw.glfwWindowShouldClose(main_window): if pipe_to_world.poll(): cmd = pipe_to_world.recv() if cmd == 'Exit': break elif cmd == "Ping": pipe_to_world.send("Pong") command = None else: command,payload = cmd if command == 'Set_Detection_Mapping_Mode': if payload == '3d': if not isinstance(g_pool.pupil_detector,Detector_3D): set_detector(Detector_3D) detector_selector.read_only = True else: set_detector(Detector_2D) detector_selector.read_only = False else: command = None # Get an image from the grabber try: frame = cap.get_frame() except CameraCaptureError: logger.error("Capture from Camera Failed. Stopping.") break except EndofVideoFileError: logger.warning("Video File is done. Stopping") cap.seek_to_frame(0) frame = cap.get_frame() #update performace graphs t = frame.timestamp dt,ts = t-ts,t try: fps_graph.add(1./dt) except ZeroDivisionError: pass cpu_graph.update() ### RECORDING of Eye Video (on demand) ### # Setup variables and lists for recording if 'Rec_Start' == command: record_path,raw_mode = payload logger.info("Will save eye video to: %s"%record_path) timestamps_path = os.path.join(record_path, "eye%s_timestamps.npy"%eye_id) if raw_mode and frame.jpeg_buffer: video_path = os.path.join(record_path, "eye%s.mp4"%eye_id) writer = JPEG_Writer(video_path,cap.frame_rate) else: video_path = os.path.join(record_path, "eye%s.mp4"%eye_id) writer = AV_Writer(video_path,cap.frame_rate) timestamps = [] elif 'Rec_Stop' == command: logger.info("Done recording.") writer.release() writer = None np.save(timestamps_path,np.asarray(timestamps)) del timestamps if writer: writer.write_video_frame(frame) timestamps.append(frame.timestamp) # pupil ellipse detection result = g_pool.pupil_detector.detect(frame, g_pool.u_r, g_pool.display_mode == 'algorithm') result['id'] = eye_id # stream the result g_pool.pupil_queue.put(result) # GL drawing if window_should_update(): if not g_pool.iconified: glfw.glfwMakeContextCurrent(main_window) clear_gl_screen() # switch to work in normalized coordinate space if g_pool.display_mode == 'algorithm': g_pool.image_tex.update_from_ndarray(frame.img) elif g_pool.display_mode in ('camera_image','roi'): g_pool.image_tex.update_from_ndarray(frame.gray) else: pass make_coord_system_norm_based(g_pool.flip) g_pool.image_tex.draw() window_size = glfw.glfwGetWindowSize(main_window) make_coord_system_pixel_based((frame.height,frame.width,3),g_pool.flip) if result['method'] == '3d c++': eye_ball = result['projected_sphere'] try: pts = cv2.ellipse2Poly( (int(eye_ball['center'][0]),int(eye_ball['center'][1])), (int(eye_ball['axes'][0]/2),int(eye_ball['axes'][1]/2)), int(eye_ball['angle']),0,360,8) except ValueError as e: pass else: draw_polyline(pts,2,RGBA(0.,.9,.1,result['model_confidence']) ) if result['confidence'] >0: if result.has_key('ellipse'): pts = cv2.ellipse2Poly( (int(result['ellipse']['center'][0]),int(result['ellipse']['center'][1])), (int(result['ellipse']['axes'][0]/2),int(result['ellipse']['axes'][1]/2)), int(result['ellipse']['angle']),0,360,15) confidence = result['confidence'] * 0.7 #scale it a little draw_polyline(pts,1,RGBA(1.,0,0,confidence)) draw_points([result['ellipse']['center']],size=20,color=RGBA(1.,0.,0.,confidence),sharpness=1.) # render graphs graph.push_view() fps_graph.draw() cpu_graph.draw() graph.pop_view() # render GUI g_pool.gui.update() #render the ROI if g_pool.display_mode == 'roi': g_pool.u_r.draw(g_pool.gui.scale) #update screen glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() g_pool.pupil_detector.visualize() #detector decides if we visualize or not # END while running # in case eye recording was still runnnig: Save&close if writer: logger.info("Done recording eye.") writer = None np.save(timestamps_path,np.asarray(timestamps)) glfw.glfwRestoreWindow(main_window) #need to do this for windows os # save session persistent settings session_settings['gui_scale'] = g_pool.gui.scale session_settings['roi'] = g_pool.u_r.get() session_settings['flip'] = g_pool.flip session_settings['display_mode'] = g_pool.display_mode session_settings['ui_config'] = g_pool.gui.configuration session_settings['capture_settings'] = g_pool.capture.settings session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos(main_window) session_settings['version'] = g_pool.version session_settings['last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__ session_settings['pupil_detector_settings'] = g_pool.pupil_detector.get_settings() session_settings.close() g_pool.pupil_detector.cleanup() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) glfw.glfwTerminate() cap.close() logger.debug("Process done")
def service( timebase, eye_procs_alive, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version, preferred_remote_port, hide_ui, debug, ): """Maps pupil to gaze data, can run various plug-ins. Reacts to notifications: ``start_plugin``: Starts given plugin with the given arguments ``eye_process.started``: Sets the detection method eye process ``service_process.should_stop``: Stops the service process Emits notifications: ``eye_process.should_start`` ``eye_process.should_stop`` ``service_process.started`` ``service_process.stopped`` ``launcher_process.should_stop`` Emits data: ``gaze``: Gaze data from current gaze mapping plugin.`` ``*``: any other plugin generated data in the events that it not [dt,pupil,gaze]. """ # We defer the imports because of multiprocessing. # Otherwise the service process each process also loads the other imports. # This is not harmful but unnecessary. # general imports from time import sleep import logging import zmq import zmq_tools # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) gaze_pub = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url) pupil_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("pupil",)) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",)) poller = zmq.Poller() poller.register(pupil_sub.socket) poller.register(notify_sub.socket) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) def launch_eye_process(eye_id, delay=0): n = {"subject": "eye_process.should_start", "eye_id": eye_id, "delay": delay} ipc_pub.notify(n) def stop_eye_process(eye_id): n = {"subject": "eye_process.should_stop", "eye_id": eye_id} ipc_pub.notify(n) try: # helpers/utils from file_methods import Persistent_Dict from methods import delta_t, get_system_info from version_utils import parse_version import audio from uvc import get_time_monotonic # Plug-ins from plugin import Plugin, Plugin_List, import_runtime_plugins from calibration_choreography import ( available_calibration_choreography_plugins, patch_loaded_plugins_with_choreography_plugin, ) from gaze_mapping import registered_gazer_classes from network_api import NetworkApiPlugin from pupil_groups import Pupil_Groups from blink_detection import Blink_Detection from fixation_detector import Fixation_Detector from service_ui import Service_UI from background_helper import IPC_Logging_Task_Proxy IPC_Logging_Task_Proxy.push_url = ipc_push_url process_was_interrupted = False def interrupt_handler(sig, frame): import traceback trace = traceback.format_stack(f=frame) logger.debug(f"Caught signal {sig} in:\n" + "".join(trace)) nonlocal process_was_interrupted process_was_interrupted = True signal.signal(signal.SIGINT, interrupt_handler) logger.info("Application Version: {}".format(version)) logger.info("System Info: {}".format(get_system_info())) logger.debug(f"Debug flag: {debug}") # g_pool holds variables for this process they are accesible to all plugins g_pool = SimpleNamespace() g_pool.debug = debug g_pool.app = "service" g_pool.user_dir = user_dir g_pool.version = version g_pool.get_now = get_time_monotonic g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.eye_procs_alive = eye_procs_alive g_pool.timebase = timebase g_pool.preferred_remote_port = preferred_remote_port g_pool.hide_ui = hide_ui def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp # manage plugins runtime_plugins = import_runtime_plugins( os.path.join(g_pool.user_dir, "plugins") ) user_launchable_plugins = [ Service_UI, Pupil_Groups, NetworkApiPlugin, Blink_Detection, ] + runtime_plugins plugin_by_index = ( runtime_plugins + available_calibration_choreography_plugins() + registered_gazer_classes() + user_launchable_plugins ) name_by_index = [pupil_datum.__name__ for pupil_datum in plugin_by_index] plugin_by_name = dict(zip(name_by_index, plugin_by_index)) default_plugins = [ ("Service_UI", {}), # Calibration choreography plugin is added bellow by calling # patch_world_session_settings_with_choreography_plugin ("NetworkApiPlugin", {}), ("Blink_Detection", {}), ("Fixation_Detector", {}), ] g_pool.plugin_by_name = plugin_by_name tick = delta_t() def get_dt(): return next(tick) # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, "user_settings_service") ) if parse_version(session_settings.get("version", "0.0")) < g_pool.version: logger.info( "Session setting are from older version of this app. I will not use those." ) session_settings.clear() g_pool.min_calibration_confidence = session_settings.get( "min_calibration_confidence", 0.8 ) audio.set_audio_mode( session_settings.get("audio_mode", audio.get_default_audio_mode()) ) ipc_pub.notify({"subject": "service_process.started"}) logger.warning("Process started.") g_pool.service_should_run = True loaded_plugins = session_settings.get("loaded_plugins", default_plugins) # Resolve the active calibration choreography plugin loaded_plugins = patch_loaded_plugins_with_choreography_plugin( loaded_plugins, app=g_pool.app ) session_settings["loaded_plugins"] = loaded_plugins # plugins that are loaded based on user settings from previous session g_pool.plugins = Plugin_List(g_pool, loaded_plugins) # NOTE: The NetworkApiPlugin plugin fails to load when the port is already in use # and will set this variable to false. Then we should not even start the eye # processes. Otherwise we would have to wait for their initialization before # attempting cleanup in Service. if g_pool.service_should_run: if session_settings.get("eye1_process_alive", True): launch_eye_process(1, delay=0.3) if session_settings.get("eye0_process_alive", True): launch_eye_process(0, delay=0.0) def handle_notifications(n): subject = n["subject"] if subject == "start_plugin": try: g_pool.plugins.add( plugin_by_name[n["name"]], args=n.get("args", {}) ) except KeyError as err: logger.error(f"Attempt to load unknown plugin: {err}") elif subject == "service_process.should_stop": g_pool.service_should_run = False elif subject.startswith("meta.should_doc"): ipc_pub.notify( {"subject": "meta.doc", "actor": g_pool.app, "doc": service.__doc__} ) for p in g_pool.plugins: if ( p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify ): ipc_pub.notify( { "subject": "meta.doc", "actor": p.class_name, "doc": p.on_notify.__doc__, } ) # initiate ui update loop ipc_pub.notify( {"subject": "service_process.ui.should_update", "initial_delay": 1 / 40} ) g_pool.active_gaze_mapping_plugin = None # Event loop while g_pool.service_should_run and not process_was_interrupted: socks = dict(poller.poll()) if pupil_sub.socket in socks: topic, pupil_datum = pupil_sub.recv() events = {} events["pupil"] = [pupil_datum] if g_pool.active_gaze_mapping_plugin: new_gaze_data = g_pool.active_gaze_mapping_plugin.map_pupil_to_gaze( [pupil_datum] ) events["gaze"] = [] for gaze_datum in new_gaze_data: gaze_pub.send(gaze_datum) events["gaze"].append(gaze_datum) for plugin in g_pool.plugins: plugin.recent_events(events=events) if notify_sub.socket in socks: topic, n = notify_sub.recv() handle_notifications(n) for plugin in g_pool.plugins: plugin.on_notify(n) # check if a plugin need to be destroyed g_pool.plugins.clean() session_settings["loaded_plugins"] = g_pool.plugins.get_initializers() session_settings["version"] = str(g_pool.version) session_settings["eye0_process_alive"] = eye_procs_alive[0].value session_settings["eye1_process_alive"] = eye_procs_alive[1].value session_settings[ "min_calibration_confidence" ] = g_pool.min_calibration_confidence session_settings["audio_mode"] = audio.get_audio_mode() session_settings.close() # de-init all running plugins for pupil_datum in g_pool.plugins: pupil_datum.alive = False g_pool.plugins.clean() except Exception: import traceback trace = traceback.format_exc() logger.error("Process Service crashed with trace:\n{}".format(trace)) finally: # shut down eye processes: stop_eye_process(0) stop_eye_process(1) logger.info("Process shutting down.") ipc_pub.notify({"subject": "service_process.stopped"}) # shut down launcher n = {"subject": "launcher_process.should_stop"} ipc_pub.notify(n) sleep(1.0)
def world(timebase, eyes_are_alive, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version): """Reads world video and runs plugins. Creates a window, gl context. Grabs images from a capture. Maps pupil to gaze data Can run various plug-ins. Reacts to notifications: ``set_detection_mapping_mode`` ``eye_process.started`` ``start_plugin`` Emits notifications: ``eye_process.should_start`` ``eye_process.should_stop`` ``set_detection_mapping_mode`` ``world_process.started`` ``world_process.stopped`` ``recording.should_stop``: Emits on camera failure ``launcher_process.should_stop`` Emits data: ``gaze``: Gaze data from current gaze mapping plugin.`` ``*``: any other plugin generated data in the events that it not [dt,pupil,gaze]. """ # We defer the imports because of multiprocessing. # Otherwise the world process each process also loads the other imports. # This is not harmful but unnecessary. #general imports from time import time, sleep import numpy as np import logging #networking import zmq import zmq_tools #zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) gaze_pub = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url) pupil_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('pupil', )) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify', )) #log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) #display import glfw from pyglui import ui, graph, cygl, __version__ as pyglui_version assert pyglui_version >= '1.0' from pyglui.cygl.utils import Named_Texture from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen, make_coord_system_pixel_based, make_coord_system_norm_based, glFlush, is_window_visible #monitoring import psutil # helpers/utils from version_utils import VersionFormat from file_methods import Persistent_Dict from methods import normalize, denormalize, delta_t, get_system_info from uvc import get_time_monotonic logger.info('Application Version: %s' % version) logger.info('System Info: %s' % get_system_info()) # video sources from video_capture import InitialisationError, StreamError, Fake_Source, EndofVideoFileError, source_classes, manager_classes source_by_name = {src.class_name(): src for src in source_classes} import audio #trigger pupil detector cpp build: import pupil_detectors del pupil_detectors # Plug-ins from plugin import Plugin, Plugin_List, import_runtime_plugins from calibration_routines import calibration_plugins, gaze_mapping_plugins from fixation_detector import Fixation_Detector_3D from recorder import Recorder from show_calibration import Show_Calibration from display_recent_gaze import Display_Recent_Gaze from time_sync import Time_Sync from pupil_remote import Pupil_Remote from pupil_groups import Pupil_Groups from surface_tracker import Surface_Tracker from log_display import Log_Display from annotations import Annotation_Capture from log_history import Log_History from frame_publisher import Frame_Publisher #UI Platform tweaks if platform.system() == 'Linux': scroll_factor = 10.0 window_position_default = (0, 0) elif platform.system() == 'Windows': scroll_factor = 1.0 window_position_default = (8, 31) else: scroll_factor = 1.0 window_position_default = (0, 0) #g_pool holds variables for this process they are accesible to all plugins g_pool = Global_Container() g_pool.app = 'capture' g_pool.user_dir = user_dir g_pool.version = version g_pool.timebase = timebase g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.eyes_are_alive = eyes_are_alive def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic #manage plugins runtime_plugins = import_runtime_plugins( os.path.join(g_pool.user_dir, 'plugins')) user_launchable_plugins = [ Pupil_Groups, Frame_Publisher, Show_Calibration, Pupil_Remote, Time_Sync, Surface_Tracker, Annotation_Capture, Log_History, Fixation_Detector_3D ] + runtime_plugins system_plugins = [Log_Display, Display_Recent_Gaze, Recorder] plugin_by_index = system_plugins + user_launchable_plugins + calibration_plugins + gaze_mapping_plugins + manager_classes name_by_index = [p.__name__ for p in plugin_by_index] plugin_by_name = dict(zip(name_by_index, plugin_by_index)) default_plugins = [('UVC_Manager', {}), ('Log_Display', {}), ('Dummy_Gaze_Mapper', {}), ('Display_Recent_Gaze', {}), ('Screen_Marker_Calibration', {}), ('Recorder', {}), ('Pupil_Remote', {}), ('Fixation_Detector_3D', {})] # Callback functions def on_resize(window, w, h): if is_window_visible(window): g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() graph.adjust_size(w, h) adjust_gl_view(w, h) for p in g_pool.plugins: p.on_window_resize(window, w, h) def on_iconify(window, iconified): g_pool.iconified = iconified def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_char(window, char): g_pool.gui.update_char(char) def on_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) pos = glfw.glfwGetCursorPos(window) pos = normalize(pos, glfw.glfwGetWindowSize(main_window)) pos = denormalize( pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels for p in g_pool.plugins: p.on_click(pos, button, action) def on_pos(window, x, y): hdpi_factor = float( glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) tick = delta_t() def get_dt(): return next(tick) g_pool.on_frame_size_change = lambda new_size: None # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, 'user_settings_world')) if session_settings.get("version", VersionFormat('0.0')) < g_pool.version: logger.info( "Session setting are from older version of this app. I will not use those." ) session_settings.clear() # Initialize capture default_settings = { 'source_class_name': 'UVC_Source', 'preferred_names': [ "Pupil Cam1 ID2", "Logitech Camera", "(046d:081d)", "C510", "B525", "C525", "C615", "C920", "C930e" ], 'frame_size': (1280, 720), 'frame_rate': 30 } settings = session_settings.get('capture_settings', default_settings) try: cap = source_by_name[settings['source_class_name']](g_pool, **settings) except (KeyError, InitialisationError) as e: if isinstance(e, KeyError): logger.warning( 'Incompatible capture setting encountered. Falling back to fake source.' ) cap = Fake_Source(g_pool, **settings) g_pool.iconified = False g_pool.detection_mapping_mode = session_settings.get( 'detection_mapping_mode', '3d') g_pool.active_calibration_plugin = None g_pool.active_gaze_mapping_plugin = None g_pool.capture = cap g_pool.capture_manager = None audio.audio_mode = session_settings.get('audio_mode', audio.default_audio_mode) def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def set_scale(new_scale): g_pool.gui.scale = new_scale g_pool.gui.collect_menus() def launch_eye_process(eye_id, delay=0): n = { 'subject': 'eye_process.should_start.%s' % eye_id, 'eye_id': eye_id, 'delay': delay } ipc_pub.notify(n) def stop_eye_process(eye_id): n = {'subject': 'eye_process.should_stop', 'eye_id': eye_id} ipc_pub.notify(n) def start_stop_eye(eye_id, make_alive): if make_alive: launch_eye_process(eye_id) else: stop_eye_process(eye_id) def set_detection_mapping_mode(new_mode): n = {'subject': 'set_detection_mapping_mode', 'mode': new_mode} ipc_pub.notify(n) def handle_notifications(n): subject = n['subject'] if subject == 'set_detection_mapping_mode': if n['mode'] == '2d': if "Vector_Gaze_Mapper" in g_pool.active_gaze_mapping_plugin.class_name: logger.warning( "The gaze mapper is not supported in 2d mode. Please recalibrate." ) g_pool.plugins.add(plugin_by_name['Dummy_Gaze_Mapper']) g_pool.detection_mapping_mode = n['mode'] elif subject == 'start_plugin': g_pool.plugins.add(plugin_by_name[n['name']], args=n.get('args', {})) elif subject == 'eye_process.started': n = { 'subject': 'set_detection_mapping_mode', 'mode': g_pool.detection_mapping_mode } ipc_pub.notify(n) elif subject.startswith('meta.should_doc'): ipc_pub.notify({ 'subject': 'meta.doc', 'actor': g_pool.app, 'doc': world.__doc__ }) for p in g_pool.plugins: if p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify: ipc_pub.notify({ 'subject': 'meta.doc', 'actor': p.class_name, 'doc': p.on_notify.__doc__ }) #window and gl setup glfw.glfwInit() width, height = session_settings.get('window_size', cap.frame_size) main_window = glfw.glfwCreateWindow(width, height, "World") window_pos = session_settings.get('window_position', window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window #setup GUI g_pool.gui = ui.UI() g_pool.gui.scale = session_settings.get('gui_scale', 1) g_pool.sidebar = ui.Scrolling_Menu("Settings", pos=(-350, 0), size=(0, 0), header_pos='left') general_settings = ui.Growing_Menu('General') general_settings.append( ui.Slider('scale', g_pool.gui, setter=set_scale, step=.05, min=1., max=2.5, label='Interface size')) general_settings.append( ui.Button( 'Reset window size', lambda: glfw.glfwSetWindowSize( main_window, frame.width, frame.height))) general_settings.append( ui.Selector('audio_mode', audio, selection=audio.audio_modes)) general_settings.append( ui.Selector('detection_mapping_mode', g_pool, label='detection & mapping mode', setter=set_detection_mapping_mode, selection=['2d', '3d'])) general_settings.append( ui.Switch('eye0_process', label='Detect eye 0', setter=lambda alive: start_stop_eye(0, alive), getter=lambda: eyes_are_alive[0].value)) general_settings.append( ui.Switch('eye1_process', label='Detect eye 1', setter=lambda alive: start_stop_eye(1, alive), getter=lambda: eyes_are_alive[1].value)) selector_label = "Select to load" labels = [p.__name__.replace('_', ' ') for p in user_launchable_plugins] user_launchable_plugins.insert(0, selector_label) labels.insert(0, selector_label) general_settings.append( ui.Selector('Open plugin', selection=user_launchable_plugins, labels=labels, setter=open_plugin, getter=lambda: selector_label)) general_settings.append( ui.Info_Text('Capture Version: %s' % g_pool.version)) g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100)) g_pool.capture_source_menu = ui.Growing_Menu('Capture Source') g_pool.capture.init_gui() g_pool.calibration_menu = ui.Growing_Menu('Calibration') g_pool.capture_selector_menu = ui.Growing_Menu('Capture Selection') g_pool.sidebar.append(general_settings) g_pool.sidebar.append(g_pool.capture_selector_menu) g_pool.sidebar.append(g_pool.capture_source_menu) g_pool.sidebar.append(g_pool.calibration_menu) g_pool.gui.append(g_pool.sidebar) g_pool.gui.append(g_pool.quickbar) #plugins that are loaded based on user settings from previous session g_pool.plugins = Plugin_List( g_pool, plugin_by_name, session_settings.get('loaded_plugins', default_plugins)) #We add the calibration menu selector, after a calibration has been added: g_pool.calibration_menu.insert( 0, ui.Selector( 'active_calibration_plugin', getter=lambda: g_pool.active_calibration_plugin.__class__, selection=calibration_plugins, labels=[p.__name__.replace('_', ' ') for p in calibration_plugins], setter=open_plugin, label='Method')) #We add the capture selection menu, after a manager has been added: g_pool.capture_selector_menu.insert( 0, ui.Selector('capture_manager', setter=open_plugin, getter=lambda: g_pool.capture_manager.__class__, selection=manager_classes, labels=[b.gui_name for b in manager_classes], label='Manager')) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetWindowIconifyCallback(main_window, on_iconify) glfw.glfwSetKeyCallback(main_window, on_key) glfw.glfwSetCharCallback(main_window, on_char) glfw.glfwSetMouseButtonCallback(main_window, on_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() # refresh speed settings glfw.glfwSwapInterval(0) #trigger setup of window and gl sizes on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) #now the we have aproper window we can load the last gui configuration g_pool.gui.configuration = session_settings.get('ui_config', {}) #set up performace graphs: pid = os.getpid() ps = psutil.Process(pid) ts = g_pool.get_timestamp() cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 130) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" pupil0_graph = graph.Bar_Graph(max_val=1.0) pupil0_graph.pos = (260, 130) pupil0_graph.update_rate = 5 pupil0_graph.label = "id0 conf: %0.2f" pupil1_graph = graph.Bar_Graph(max_val=1.0) pupil1_graph.pos = (380, 130) pupil1_graph.update_rate = 5 pupil1_graph.label = "id1 conf: %0.2f" pupil_graphs = pupil0_graph, pupil1_graph if session_settings.get('eye1_process_alive', False): launch_eye_process(1, delay=0.6) if session_settings.get('eye0_process_alive', True): launch_eye_process(0, delay=0.3) ipc_pub.notify({'subject': 'world_process.started'}) logger.warning('Process started.') # Event loop while not glfw.glfwWindowShouldClose(main_window): # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) g_pool.capture.on_notify(n) for p in g_pool.plugins: p.on_notify(n) # Get an image from the grabber try: frame = g_pool.capture.get_frame() except StreamError as e: prev_settings = g_pool.capture.settings g_pool.capture.deinit_gui() g_pool.capture.cleanup() g_pool.capture = None prev_settings[ 'info_text'] = "'%s' disconnected." % prev_settings['name'] g_pool.capture = Fake_Source(g_pool, **prev_settings) g_pool.capture.init_gui() ipc_pub.notify({'subject': 'recording.should_stop'}) logger.error("Error getting frame. Falling back to Fake source.") logger.debug("Caught error: %s" % e) sleep(.2) continue except EndofVideoFileError: logger.warning("Video file is done. Rewinding") g_pool.capture.seek_to_frame(0) continue #update performace graphs t = frame.timestamp dt, ts = t - ts, t try: fps_graph.add(1. / dt) except ZeroDivisionError: pass cpu_graph.update() #a dictionary that allows plugins to post and read events events = {} #report time between now and the last loop interation events['dt'] = get_dt() recent_pupil_data = [] recent_gaze_data = [] while pupil_sub.new_data: t, p = pupil_sub.recv() pupil_graphs[p['id']].add(p['confidence']) recent_pupil_data.append(p) new_gaze_data = g_pool.active_gaze_mapping_plugin.on_pupil_datum(p) for g in new_gaze_data: gaze_pub.send('gaze', g) recent_gaze_data += new_gaze_data events['pupil_positions'] = recent_pupil_data events['gaze_positions'] = recent_gaze_data # allow each Plugin to do its work. for p in g_pool.plugins: p.update(frame, events) #check if a plugin need to be destroyed g_pool.plugins.clean() #send new events to ipc: del events['pupil_positions'] #already on the wire del events['gaze_positions'] #send earlier in this loop del events['dt'] #no need to send this for topic, data in events.iteritems(): assert (isinstance(data, (list, tuple))) for d in data: ipc_pub.send(topic, d) # render camera image glfw.glfwMakeContextCurrent(main_window) if is_window_visible(main_window): g_pool.image_tex.update_from_frame(frame) glFlush() make_coord_system_norm_based() g_pool.image_tex.draw() make_coord_system_pixel_based((frame.height, frame.width, 3)) # render visual feedback from loaded plugins if is_window_visible(main_window): g_pool.capture.gl_display() for p in g_pool.plugins: p.gl_display() graph.push_view() fps_graph.draw() cpu_graph.draw() pupil0_graph.draw() pupil1_graph.draw() graph.pop_view() g_pool.gui.update() glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() glfw.glfwRestoreWindow(main_window) #need to do this for windows os session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['gui_scale'] = g_pool.gui.scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['capture_settings'] = g_pool.capture.settings session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos(main_window) session_settings['version'] = g_pool.version session_settings['eye0_process_alive'] = eyes_are_alive[0].value session_settings['eye1_process_alive'] = eyes_are_alive[1].value session_settings['detection_mapping_mode'] = g_pool.detection_mapping_mode session_settings['audio_mode'] = audio.audio_mode session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) glfw.glfwTerminate() g_pool.capture.deinit_gui() g_pool.capture.cleanup() #shut down eye processes: stop_eye_process(0) stop_eye_process(1) logger.info("Process shutting down.") ipc_pub.notify({'subject': 'world_process.stopped'}) #shut down launcher n = {'subject': 'launcher_process.should_stop'} ipc_pub.notify(n) zmq_ctx.destroy()
def world(g_pool, cap_src, cap_size): """world Creates a window, gl context. Grabs images from a capture. Receives Pupil coordinates from eye process[es] Can run various plug-ins. """ # Callback functions def on_resize(window, w, h): active_window = glfwGetCurrentContext() glfwMakeContextCurrent(window) hdpi_factor = glfwGetFramebufferSize(window)[0] / glfwGetWindowSize( window)[0] w, h = w * hdpi_factor, h * hdpi_factor g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() graph.adjust_size(w, h) adjust_gl_view(w, h) for p in g_pool.plugins: p.on_window_resize(window, w, h) glfwMakeContextCurrent(active_window) def on_iconify(window, iconfied): if not isinstance(cap, FakeCapture): g_pool.update_textures = not iconfied def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_char(window, char): g_pool.gui.update_char(char) def on_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) pos = glfwGetCursorPos(window) pos = normalize(pos, glfwGetWindowSize(main_window)) pos = denormalize( pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels for p in g_pool.plugins: p.on_click(pos, button, action) def on_pos(window, x, y): hdpi_factor = float( glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0]) x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_close(window): g_pool.quit.value = True logger.info('Process closing from window') # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, 'user_settings_world')) # Initialize capture cap = autoCreateCapture(cap_src, cap_size, 24, timebase=g_pool.timebase) # Test capture try: frame = cap.get_frame() except CameraCaptureError: logger.error("Could not retrieve image from capture") cap.close() return # any object we attach to the g_pool object *from now on* will only be visible to this process! # vars should be declared here to make them visible to the code reader. g_pool.update_textures = 2 if isinstance(cap, FakeCapture): g_pool.update_textures = 0 g_pool.capture = cap g_pool.pupil_confidence_threshold = session_settings.get( 'pupil_confidence_threshold', .6) g_pool.active_calibration_plugin = None #UI callback functions def reset_timebase(): #the last frame from worldcam will be t0 g_pool.timebase.value = g_pool.capture.get_now() logger.info( "New timebase set to %s all timestamps will count from here now." % g_pool.timebase.value) def set_calibration_plugin(new_calibration): g_pool.active_calibration_plugin = new_calibration new_plugin = new_calibration(g_pool) g_pool.plugins.add(new_plugin) def open_plugin(plugin): if plugin == "Select to load": return logger.debug('Open Plugin: %s' % plugin) new_plugin = plugin(g_pool) g_pool.plugins.add(new_plugin) def set_scale(new_scale): g_pool.gui.scale = new_scale g_pool.gui.collect_menus() def get_scale(): return g_pool.gui.scale width, height = session_settings.get('window_size', (frame.width, frame.height)) window_pos = session_settings.get('window_position', (0, 0)) # not yet using this one. # Initialize glfw glfwInit() main_window = glfwCreateWindow(width, height, "World", None, None) glfwMakeContextCurrent(main_window) cygl.utils.init() # Register callbacks main_window glfwSetWindowSizeCallback(main_window, on_resize) glfwSetWindowCloseCallback(main_window, on_close) glfwSetWindowIconifyCallback(main_window, on_iconify) glfwSetKeyCallback(main_window, on_key) glfwSetCharCallback(main_window, on_char) glfwSetMouseButtonCallback(main_window, on_button) glfwSetCursorPosCallback(main_window, on_pos) glfwSetScrollCallback(main_window, on_scroll) # gl_state settings basic_gl_setup() g_pool.image_tex = create_named_texture(frame.img.shape) update_named_texture(g_pool.image_tex, frame.img) # refresh speed settings glfwSwapInterval(0) glfwSetWindowPos(main_window, 0, 0) #setup GUI g_pool.gui = ui.UI() g_pool.gui.scale = session_settings.get('gui_scale', 1) g_pool.sidebar = ui.Scrolling_Menu("Settings", pos=(-250, 0), size=(0, 0), header_pos='left') g_pool.sidebar.configuration = session_settings.get('side_bar_config', {}) general_settings = ui.Growing_Menu('General') general_settings.configuration = session_settings.get( 'general_menu_config', {}) general_settings.append( ui.Slider('scale', setter=set_scale, getter=get_scale, step=.05, min=1., max=2.5, label='Interface size')) general_settings.append( ui.Button( 'Reset window size', lambda: glfwSetWindowSize(main_window, frame.width, frame.height))) general_settings.append( ui.Selector('Open plugin', selection=user_launchable_plugins, labels=[ p.__name__.replace('_', ' ') for p in user_launchable_plugins ], setter=open_plugin, getter=lambda: "Select to load")) g_pool.sidebar.append(general_settings) advanced_settings = ui.Growing_Menu('Advanced') advanced_settings.configuration = session_settings.get( 'advanced_menu_config', {'collapsed': True}) advanced_settings.append( ui.Selector('update_textures', g_pool, label="Update display", selection=range(3), labels=('No update', 'Gray', 'Color'))) advanced_settings.append( ui.Slider('pupil_confidence_threshold', g_pool, step=.01, min=0., max=1., label='Minimum pupil confidence')) advanced_settings.append(ui.Button('Set timebase to 0', reset_timebase)) advanced_settings.append( ui.Info_Text('Capture Version: %s' % g_pool.version)) general_settings.append(advanced_settings) g_pool.calibration_menu = ui.Growing_Menu('Calibration') g_pool.calibration_menu.configuration = session_settings.get( 'calibration_menu_config', {}) g_pool.calibration_menu.append( ui.Selector( 'active_calibration_plugin', g_pool, selection=calibration_plugins, labels=[p.__name__.replace('_', ' ') for p in calibration_plugins], setter=set_calibration_plugin, label='Method')) g_pool.sidebar.append(g_pool.calibration_menu) g_pool.gui.append(g_pool.sidebar) g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100)) g_pool.gui.append(g_pool.quickbar) g_pool.gui.append( ui.Hot_Key("quit", setter=on_close, getter=lambda: True, label="X", hotkey=GLFW_KEY_ESCAPE)) g_pool.capture.init_gui(g_pool.sidebar) g_pool.capture.menu.configuration = session_settings.get( 'capture_menu_config', {}) #plugins that are loaded based on user settings from previous session g_pool.plugins = Plugin_List( g_pool, plugin_by_name, session_settings.get('loaded_plugins', default_plugins)) #only needed for the gui to show the loaded calibration type for p in g_pool.plugins: if p.base_class_name == 'Calibration_Plugin': g_pool.active_calibration_plugin = p.__class__ break on_resize(main_window, *glfwGetWindowSize(main_window)) #set up performace graphs: pid = os.getpid() ps = psutil.Process(pid) ts = frame.timestamp cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 130) cpu_graph.update_fn = ps.get_cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" pupil_graph = graph.Bar_Graph(max_val=1.0) pupil_graph.pos = (260, 130) pupil_graph.update_rate = 5 pupil_graph.label = "Confidence: %0.2f" # Event loop while not g_pool.quit.value: # Get an image from the grabber try: frame = cap.get_frame() except CameraCaptureError: logger.error("Capture from camera failed. Stopping.") break except EndofVideoFileError: logger.warning("Video file is done. Stopping") break #update performace graphs t = frame.timestamp dt, ts = t - ts, t fps_graph.add(1. / dt) cpu_graph.update() #a dictionary that allows plugins to post and read events events = {} #receive and map pupil positions recent_pupil_positions = [] while not g_pool.pupil_queue.empty(): p = g_pool.pupil_queue.get() recent_pupil_positions.append(p) pupil_graph.add(p['confidence']) events['pupil_positions'] = recent_pupil_positions # allow each Plugin to do its work. for p in g_pool.plugins: p.update(frame, events) #check if a plugin need to be destroyed g_pool.plugins.clean() # render camera image glfwMakeContextCurrent(main_window) if g_pool.update_textures == 2: update_named_texture(g_pool.image_tex, frame.img) elif g_pool.update_textures == 1: update_named_texture(g_pool.image_tex, frame.gray) make_coord_system_norm_based() draw_named_texture(g_pool.image_tex) make_coord_system_pixel_based((frame.height, frame.width, 3)) # render visual feedback from loaded plugins for p in g_pool.plugins: p.gl_display() graph.push_view() fps_graph.draw() cpu_graph.draw() pupil_graph.draw() graph.pop_view() g_pool.gui.update() glfwSwapBuffers(main_window) glfwPollEvents() session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings[ 'pupil_confidence_threshold'] = g_pool.pupil_confidence_threshold session_settings['gui_scale'] = g_pool.gui.scale session_settings['side_bar_config'] = g_pool.sidebar.configuration session_settings['capture_menu_config'] = g_pool.capture.menu.configuration session_settings['general_menu_config'] = general_settings.configuration session_settings['advanced_menu_config'] = advanced_settings.configuration session_settings[ 'calibration_menu_config'] = g_pool.calibration_menu.configuration session_settings['window_size'] = glfwGetWindowSize(main_window) session_settings['window_position'] = glfwGetWindowPos(main_window) session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() cap.close() glfwDestroyWindow(main_window) glfwTerminate() logger.debug("Process done")