def start(self): self.data = { 'pupil_positions': [], 'gaze_positions': [], 'notifications': [] } self.frame_count = 0 self.running = True self.menu.read_only = True self.start_time = time() session = os.path.join(self.rec_dir, self.session_name) try: os.makedirs(session) logger.debug( "Created new recordings session dir {}".format(session)) except: logger.debug( "Recordings session dir {} already exists, using it.".format( session)) if self.rec_path is None: # set up self incrementing folder within session folder counter = 0 while True: self.rec_path = os.path.join(session, "{:03d}/".format(counter)) try: os.mkdir(self.rec_path) logger.debug("Created new recording dir {}".format( self.rec_path)) break except: logger.debug( "We dont want to overwrite data, incrementing counter & trying to make new data folder" ) counter += 1 self.meta_info_path = os.path.join(self.rec_path, "info.csv") with open(self.meta_info_path, 'w', newline='') as csvfile: csv_utils.write_key_value_file( csvfile, { 'Recording Name': self.session_name, 'Start Date': strftime("%d.%m.%Y", localtime(self.start_time)), 'Start Time': strftime("%H:%M:%S", localtime(self.start_time)) }) self.video_path = os.path.join(self.rec_path, "world.mp4") if self.raw_jpeg and self.g_pool.capture.jpeg_support: self.writer = JPEG_Writer(self.video_path, self.g_pool.capture.frame_rate) elif hasattr(self.g_pool.capture._recent_frame, 'h264_buffer'): self.writer = H264Writer(self.video_path, self.g_pool.capture.frame_size[0], self.g_pool.capture.frame_size[1], int(self.g_pool.capture.frame_rate)) else: self.writer = AV_Writer(self.video_path, fps=self.g_pool.capture.frame_rate) try: cal_pt_path = os.path.join(self.g_pool.user_dir, "user_calibration_data") cal_data = load_object(cal_pt_path) notification = { 'subject': 'calibration.calibration_data', 'record': True } notification.update(cal_data) self.data['notifications'].append(notification) except: pass if self.show_info_menu: self.open_info_menu() logger.info("Started Recording.") self.notify_all({ 'subject': 'recording.started', 'rec_path': self.rec_path, 'session_name': self.session_name, 'record_eye': self.record_eye, 'compression': self.raw_jpeg })
def eye(timebase, is_alive_flag, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version, eye_id, overwrite_cap_settings=None): """reads eye video and detects the pupil. Creates a window, gl context. Grabs images from a capture. Streams Pupil coordinates. Reacts to notifications: ``set_detection_mapping_mode``: Sets detection method ``eye_process.should_stop``: Stops the eye process ``recording.started``: Starts recording eye video ``recording.stopped``: Stops recording eye video ``frame_publishing.started``: Starts frame publishing ``frame_publishing.stopped``: Stops frame publishing Emits notifications: ``eye_process.started``: Eye process started ``eye_process.stopped``: Eye process stopped Emits data: ``pupil.<eye id>``: Pupil data for eye with id ``<eye id>`` ``frame.eye.<eye id>``: Eye frames with id ``<eye id>`` """ # We deferr the imports becasue of multiprocessing. # Otherwise the world process each process also loads the other imports. import zmq import zmq_tools zmq_ctx = zmq.Context() ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",)) with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id): # logging setup import logging logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.INFO) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) # general imports import numpy as np import cv2 # display import glfw from pyglui import ui, graph, cygl from pyglui.cygl.utils import draw_points, RGBA, draw_polyline from pyglui.cygl.utils import Named_Texture from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen from gl_utils import make_coord_system_pixel_based from gl_utils import make_coord_system_norm_based from gl_utils import is_window_visible from ui_roi import UIRoi # monitoring import psutil # helpers/utils from uvc import get_time_monotonic from file_methods import Persistent_Dict from version_utils import VersionFormat from methods import normalize, denormalize, timer from av_writer import JPEG_Writer, AV_Writer from ndsi import H264Writer from video_capture import source_classes from video_capture import manager_classes # Pupil detectors from pupil_detectors import Detector_2D, Detector_3D pupil_detectors = {Detector_2D.__name__: Detector_2D, Detector_3D.__name__: Detector_3D} # UI Platform tweaks if platform.system() == 'Linux': scroll_factor = 10.0 window_position_default = (600, 300 * eye_id) elif platform.system() == 'Windows': scroll_factor = 10.0 window_position_default = (600,31+ 300 * eye_id) else: scroll_factor = 1.0 window_position_default = (600, 300 * eye_id) # g_pool holds variables for this process g_pool = Global_Container() # make some constants avaiable g_pool.user_dir = user_dir g_pool.version = version g_pool.app = 'capture' g_pool.process = 'eye{}'.format(eye_id) g_pool.timebase = timebase g_pool.ipc_pub = ipc_socket def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic # Callback functions def on_resize(window, w, h): if is_window_visible(window): active_window = glfw.glfwGetCurrentContext() glfw.glfwMakeContextCurrent(window) hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() for g in g_pool.graphs: g.scale = hdpi_factor g.adjust_window_size(w, h) adjust_gl_view(w, h) glfw.glfwMakeContextCurrent(active_window) def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_iconify(window, iconified): g_pool.iconified = iconified def on_window_mouse_button(window, button, action, mods): if g_pool.display_mode == 'roi': if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt: g_pool.u_r.active_edit_pt = False # if the roi interacts we dont want # the gui to interact as well return elif action == glfw.GLFW_PRESS: pos = glfw.glfwGetCursorPos(window) pos = normalize(pos, glfw.glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] # Position in img pixels pos = denormalize(pos,g_pool.capture.frame_size) # Position in img pixels if g_pool.u_r.mouse_over_edit_pt(pos, g_pool.u_r.handle_size + 40,g_pool.u_r.handle_size + 40): # if the roi interacts we dont want # the gui to interact as well return g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): hdpi_factor = glfw.glfwGetFramebufferSize( window)[0] / glfw.glfwGetWindowSize(window)[0] g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor) if g_pool.u_r.active_edit_pt: pos = normalize((x, y), glfw.glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1-pos[0],1-pos[1] pos = denormalize(pos,g_pool.capture.frame_size ) g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx,pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, count, paths): paths = [paths[x].decode('utf-8') for x in range(count)] g_pool.capture_manager.on_drop(paths) g_pool.capture.on_drop(paths) # load session persistent settings session_settings = Persistent_Dict(os.path.join(g_pool.user_dir, 'user_settings_eye{}'.format(eye_id))) if VersionFormat(session_settings.get("version", '0.0')) != g_pool.version: logger.info("Session setting are from a different version of this app. I will not use those.") session_settings.clear() g_pool.iconified = False g_pool.capture = None g_pool.capture_manager = None g_pool.flip = session_settings.get('flip', False) g_pool.display_mode = session_settings.get( 'display_mode', 'camera_image') g_pool.display_mode_info_text = {'camera_image': "Raw eye camera image. This uses the least amount of CPU power", 'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.", 'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below."} capture_manager_settings = session_settings.get( 'capture_manager_settings', ('UVC_Manager',{})) manager_class_name, manager_settings = capture_manager_settings manager_class_by_name = {c.__name__:c for c in manager_classes} g_pool.capture_manager = manager_class_by_name[manager_class_name](g_pool,**manager_settings) if eye_id == 0: cap_src = ["Pupil Cam1 ID0","HD-6000","Integrated Camera","HD USB Camera","USB 2.0 Camera"] else: cap_src = ["Pupil Cam1 ID1","HD-6000","Integrated Camera"] # Initialize capture default_settings = ('UVC_Source',{ 'preferred_names' : cap_src, 'frame_size': (640,480), 'frame_rate': 90 }) capture_source_settings = overwrite_cap_settings or session_settings.get('capture_settings', default_settings) source_class_name, source_settings = capture_source_settings source_class_by_name = {c.__name__:c for c in source_classes} g_pool.capture = source_class_by_name[source_class_name](g_pool,**source_settings) assert g_pool.capture g_pool.u_r = UIRoi((g_pool.capture.frame_size[1],g_pool.capture.frame_size[0])) roi_user_settings = session_settings.get('roi') if roi_user_settings and tuple(roi_user_settings[-1]) == g_pool.u_r.get()[-1]: g_pool.u_r.set(roi_user_settings) pupil_detector_settings = session_settings.get( 'pupil_detector_settings', None) last_pupil_detector = pupil_detectors[session_settings.get( 'last_pupil_detector', Detector_2D.__name__)] g_pool.pupil_detector = last_pupil_detector( g_pool, pupil_detector_settings) def set_display_mode_info(val): g_pool.display_mode = val g_pool.display_mode_info.text = g_pool.display_mode_info_text[val] def set_detector(new_detector): g_pool.pupil_detector.cleanup() g_pool.pupil_detector = new_detector(g_pool) g_pool.pupil_detector.init_gui(g_pool.sidebar) # Initialize glfw glfw.glfwInit() title = "Pupil Capture - eye {}".format(eye_id) width, height = session_settings.get( 'window_size', g_pool.capture.frame_size) main_window = glfw.glfwCreateWindow(width, height, title, None, None) window_pos = session_settings.get( 'window_position', window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() # UI callback functions def set_scale(new_scale): g_pool.gui_user_scale = new_scale on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() g_pool.image_tex.update_from_ndarray(np.ones((1,1),dtype=np.uint8)+125) # setup GUI g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get('gui_scale', 1.) g_pool.sidebar = ui.Scrolling_Menu("Settings", pos=(-300, 0), size=(0, 0), header_pos='left') general_settings = ui.Growing_Menu('General') general_settings.append(ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2], label='Interface Size')) general_settings.append(ui.Button('Reset window size',lambda: glfw.glfwSetWindowSize(main_window,*g_pool.capture.frame_size)) ) general_settings.append(ui.Switch('flip',g_pool,label='Flip image display')) general_settings.append(ui.Selector('display_mode', g_pool, setter=set_display_mode_info, selection=['camera_image','roi','algorithm'], labels=['Camera Image', 'ROI', 'Algorithm'], label="Mode") ) g_pool.display_mode_info = ui.Info_Text(g_pool.display_mode_info_text[g_pool.display_mode]) general_settings.append(g_pool.display_mode_info) g_pool.gui.append(g_pool.sidebar) detector_selector = ui.Selector('pupil_detector', getter=lambda: g_pool.pupil_detector.__class__, setter=set_detector, selection=[ Detector_2D, Detector_3D], labels=['C++ 2d detector', 'C++ 3d detector'], label="Detection method") general_settings.append(detector_selector) g_pool.capture_selector_menu = ui.Growing_Menu('Capture Selection') g_pool.capture_source_menu = ui.Growing_Menu('Capture Source') g_pool.capture_source_menu.collapsed = True g_pool.capture.init_gui() g_pool.sidebar.append(general_settings) g_pool.sidebar.append(g_pool.capture_selector_menu) g_pool.sidebar.append(g_pool.capture_source_menu) g_pool.pupil_detector.init_gui(g_pool.sidebar) g_pool.capture_manager.init_gui() g_pool.writer = None def replace_source(source_class_name,source_settings): g_pool.capture.cleanup() g_pool.capture = source_class_by_name[source_class_name](g_pool,**source_settings) g_pool.capture.init_gui() if g_pool.writer: logger.info("Done recording.") g_pool.writer.release() g_pool.writer = None g_pool.replace_source = replace_source # for ndsi capture def replace_manager(manager_class): g_pool.capture_manager.cleanup() g_pool.capture_manager = manager_class(g_pool) g_pool.capture_manager.init_gui() #We add the capture selection menu, after a manager has been added: g_pool.capture_selector_menu.insert(0,ui.Selector( 'capture_manager',g_pool, setter = replace_manager, getter = lambda: g_pool.capture_manager.__class__, selection = manager_classes, labels = [b.gui_name for b in manager_classes], label = 'Manager' )) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetWindowIconifyCallback(main_window, on_iconify) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) # load last gui configuration g_pool.gui.configuration = session_settings.get('ui_config', {}) # set up performance graphs pid = os.getpid() ps = psutil.Process(pid) ts = g_pool.get_timestamp() cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 130) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" g_pool.graphs = [cpu_graph, fps_graph] # set the last saved window size on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) should_publish_frames = False frame_publish_format = 'jpeg' # create a timer to control window update frequency window_update_timer = timer(1 / 60) def window_should_update(): return next(window_update_timer) logger.warning('Process started.') frame = None # Event loop while not glfw.glfwWindowShouldClose(main_window): if notify_sub.new_data: t, notification = notify_sub.recv() subject = notification['subject'] if subject.startswith('eye_process.should_stop'): if notification['eye_id'] == eye_id: break elif subject == 'set_detection_mapping_mode': if notification['mode'] == '3d': if not isinstance(g_pool.pupil_detector, Detector_3D): set_detector(Detector_3D) detector_selector.read_only = True else: if not isinstance(g_pool.pupil_detector, Detector_2D): set_detector(Detector_2D) detector_selector.read_only = False elif subject == 'recording.started': if notification['record_eye'] and g_pool.capture.online: record_path = notification['rec_path'] raw_mode = notification['compression'] logger.info("Will save eye video to: {}".format(record_path)) video_path = os.path.join(record_path, "eye{}.mp4".format(eye_id)) if raw_mode and frame and g_pool.capture.jpeg_support: g_pool.writer = JPEG_Writer(video_path, g_pool.capture.frame_rate) elif hasattr(g_pool.capture._recent_frame, 'h264_buffer'): g_pool.writer = H264Writer(video_path, g_pool.capture.frame_size[0], g_pool.capture.frame_size[1], g_pool.capture.frame_rate) else: g_pool.writer = AV_Writer(video_path, g_pool.capture.frame_rate) elif subject == 'recording.stopped': if g_pool.writer: logger.info("Done recording.") g_pool.writer.release() g_pool.writer = None elif subject.startswith('meta.should_doc'): ipc_socket.notify({ 'subject': 'meta.doc', 'actor': 'eye{}'.format(eye_id), 'doc': eye.__doc__ }) elif subject.startswith('frame_publishing.started'): should_publish_frames = True frame_publish_format = notification.get('format', 'jpeg') elif subject.startswith('frame_publishing.stopped'): should_publish_frames = False frame_publish_format = 'jpeg' elif subject.startswith('start_eye_capture') and notification['target'] == g_pool.process: replace_source(notification['name'],notification['args']) g_pool.capture.on_notify(notification) # Get an image from the grabber event = {} g_pool.capture.recent_events(event) frame = event.get('frame') g_pool.capture_manager.recent_events(event) if frame: f_width, f_height = g_pool.capture.frame_size if (g_pool.u_r.array_shape[0], g_pool.u_r.array_shape[1]) != (f_height, f_width): g_pool.pupil_detector.on_resolution_change((g_pool.u_r.array_shape[1], g_pool.u_r.array_shape[0]), g_pool.capture.frame_size) g_pool.u_r = UIRoi((f_height, f_width)) if should_publish_frames: try: if frame_publish_format == "jpeg": data = frame.jpeg_buffer elif frame_publish_format == "yuv": data = frame.yuv_buffer elif frame_publish_format == "bgr": data = frame.bgr elif frame_publish_format == "gray": data = frame.gray else: raise AttributeError() except AttributeError: pass else: pupil_socket.send('frame.eye.%s'%eye_id,{ 'width': frame.width, 'height': frame.height, 'index': frame.index, 'timestamp': frame.timestamp, 'format': frame_publish_format, '__raw_data__': [data] }) t = frame.timestamp dt, ts = t - ts, t try: fps_graph.add(1./dt) except ZeroDivisionError: pass if g_pool.writer: g_pool.writer.write_video_frame(frame) # pupil ellipse detection result = g_pool.pupil_detector.detect(frame, g_pool.u_r, g_pool.display_mode == 'algorithm') result['id'] = eye_id # stream the result pupil_socket.send('pupil.%s'%eye_id,result) cpu_graph.update() # GL drawing if window_should_update(): if is_window_visible(main_window): glfw.glfwMakeContextCurrent(main_window) clear_gl_screen() if frame: # switch to work in normalized coordinate space if g_pool.display_mode == 'algorithm': g_pool.image_tex.update_from_ndarray(frame.img) elif g_pool.display_mode in ('camera_image', 'roi'): g_pool.image_tex.update_from_ndarray(frame.gray) else: pass make_coord_system_norm_based(g_pool.flip) g_pool.image_tex.draw() f_width, f_height = g_pool.capture.frame_size make_coord_system_pixel_based((f_height, f_width, 3), g_pool.flip) if frame: if result['method'] == '3d c++': eye_ball = result['projected_sphere'] try: pts = cv2.ellipse2Poly( (int(eye_ball['center'][0]), int(eye_ball['center'][1])), (int(eye_ball['axes'][0] / 2), int(eye_ball['axes'][1] / 2)), int(eye_ball['angle']), 0, 360, 8) except ValueError as e: pass else: draw_polyline(pts, 2, RGBA(0., .9, .1, result['model_confidence'])) if result['confidence'] > 0: if 'ellipse' in result: pts = cv2.ellipse2Poly( (int(result['ellipse']['center'][0]), int(result['ellipse']['center'][1])), (int(result['ellipse']['axes'][0] / 2), int(result['ellipse']['axes'][1] / 2)), int(result['ellipse']['angle']), 0, 360, 15) confidence = result['confidence'] * 0.7 draw_polyline(pts, 1, RGBA(1., 0, 0, confidence)) draw_points([result['ellipse']['center']], size=20, color=RGBA(1., 0., 0., confidence), sharpness=1.) # render graphs fps_graph.draw() cpu_graph.draw() # render GUI g_pool.gui.update() # render the ROI g_pool.u_r.draw(g_pool.gui.scale) if g_pool.display_mode == 'roi': g_pool.u_r.draw_points(g_pool.gui.scale) # update screen glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() g_pool.pupil_detector.visualize() # detector decides if we visualize or not # END while running # in case eye recording was still runnnig: Save&close if g_pool.writer: logger.info("Done recording eye.") g_pool.writer = None glfw.glfwRestoreWindow(main_window) # need to do this for windows os # save session persistent settings session_settings['gui_scale'] = g_pool.gui_user_scale session_settings['roi'] = g_pool.u_r.get() session_settings['flip'] = g_pool.flip session_settings['display_mode'] = g_pool.display_mode session_settings['ui_config'] = g_pool.gui.configuration session_settings['capture_settings'] = g_pool.capture.class_name, g_pool.capture.get_init_dict() session_settings['capture_manager_settings'] = g_pool.capture_manager.class_name, g_pool.capture_manager.get_init_dict() session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos(main_window) session_settings['version'] = str(g_pool.version) session_settings['last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__ session_settings['pupil_detector_settings'] = g_pool.pupil_detector.get_settings() session_settings.close() g_pool.capture.deinit_gui() g_pool.pupil_detector.cleanup() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) glfw.glfwTerminate() g_pool.capture_manager.cleanup() g_pool.capture.cleanup() logger.info("Process shutting down.")
def start(self): self.start_time = time() start_time_synced = self.g_pool.get_timestamp() if isinstance(self.g_pool.capture, NDSI_Source): # If the user did not enable TimeSync, the timestamps will be way off and # the recording code will crash. We check the difference between the last # frame's time and the start_time_synced and if this does not match, we stop # the recording and show a warning instead. TIMESTAMP_ERROR_THRESHOLD = 5.0 frame = self.g_pool.capture._recent_frame if frame is None: logger.error( "Your connection does not seem to be stable enough for " "recording Pupil Mobile via WiFi. We recommend recording " "on the phone.") return if abs(frame.timestamp - start_time_synced) > TIMESTAMP_ERROR_THRESHOLD: logger.error( "Pupil Mobile stream is not in sync. Aborting recording." " Enable the Time Sync plugin and try again.") return session = os.path.join(self.rec_root_dir, self.session_name) try: os.makedirs(session, exist_ok=True) logger.debug( "Created new recordings session dir {}".format(session)) except OSError: logger.error( "Could not start recording. Session dir {} not writable.". format(session)) return self.pldata_writers = {} self.frame_count = 0 self.running = True self.menu.read_only = True recording_uuid = uuid.uuid4() # set up self incrementing folder within session folder counter = 0 while True: self.rec_path = os.path.join(session, "{:03d}/".format(counter)) try: os.mkdir(self.rec_path) logger.debug("Created new recording dir {}".format( self.rec_path)) break except FileExistsError: logger.debug( "We dont want to overwrite data, incrementing counter & trying to make new data folder" ) counter += 1 self.meta_info = RecordingInfoFile.create_empty_file(self.rec_path) self.meta_info.recording_software_name = ( RecordingInfoFile.RECORDING_SOFTWARE_NAME_PUPIL_CAPTURE) self.meta_info.recording_software_version = str(self.g_pool.version) self.meta_info.recording_name = self.session_name self.meta_info.start_time_synced_s = start_time_synced self.meta_info.start_time_system_s = self.start_time self.meta_info.recording_uuid = recording_uuid self.meta_info.system_info = get_system_info() self.video_path = os.path.join(self.rec_path, "world.mp4") if self.raw_jpeg and self.g_pool.capture.jpeg_support: self.writer = JPEG_Writer(self.video_path, start_time_synced) elif hasattr(self.g_pool.capture._recent_frame, "h264_buffer"): self.writer = H264Writer( self.video_path, self.g_pool.capture.frame_size[0], self.g_pool.capture.frame_size[1], int(self.g_pool.capture.frame_rate), ) else: self.writer = MPEG_Writer(self.video_path, start_time_synced) calibration_data_notification_classes = [ CalibrationSetupNotification, CalibrationResultNotification, ] writer = PLData_Writer(self.rec_path, "notify") for note_class in calibration_data_notification_classes: try: file_path = os.path.join(self.g_pool.user_dir, note_class.file_name()) note = note_class.from_dict(load_object(file_path)) note_dict = note.as_dict() note_dict["topic"] = "notify." + note_dict["subject"] writer.append(note_dict) except FileNotFoundError: continue self.pldata_writers["notify"] = writer if self.show_info_menu: self.open_info_menu() logger.info("Started Recording.") self.notify_all({ "subject": "recording.started", "rec_path": self.rec_path, "session_name": self.session_name, "record_eye": self.record_eye, "compression": self.raw_jpeg, "start_time_synced": float(start_time_synced), })
def eye( timebase, is_alive_flag, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version, eye_id, overwrite_cap_settings=None, ): """reads eye video and detects the pupil. Creates a window, gl context. Grabs images from a capture. Streams Pupil coordinates. Reacts to notifications: ``set_detection_mapping_mode``: Sets detection method ``eye_process.should_stop``: Stops the eye process ``recording.started``: Starts recording eye video ``recording.stopped``: Stops recording eye video ``frame_publishing.started``: Starts frame publishing ``frame_publishing.stopped``: Stops frame publishing Emits notifications: ``eye_process.started``: Eye process started ``eye_process.stopped``: Eye process stopped Emits data: ``pupil.<eye id>``: Pupil data for eye with id ``<eye id>`` ``frame.eye.<eye id>``: Eye frames with id ``<eye id>`` """ # We deferr the imports becasue of multiprocessing. # Otherwise the world process each process also loads the other imports. import zmq import zmq_tools zmq_ctx = zmq.Context() ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",)) # logging setup import logging logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.NOTSET) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) if is_alive_flag.value: # indicates eye process that this is a duplicated startup logger.warning("Aborting redundant eye process startup") return with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id, logger): # general imports import traceback import numpy as np import cv2 # display import glfw from pyglui import ui, graph, cygl from pyglui.cygl.utils import draw_points, RGBA, draw_polyline from pyglui.cygl.utils import Named_Texture from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen from gl_utils import make_coord_system_pixel_based from gl_utils import make_coord_system_norm_based from gl_utils import is_window_visible, glViewport from ui_roi import UIRoi # monitoring import psutil # helpers/utils from uvc import get_time_monotonic from file_methods import Persistent_Dict from version_utils import VersionFormat from methods import normalize, denormalize, timer from av_writer import JPEG_Writer, AV_Writer from ndsi import H264Writer from video_capture import source_classes from video_capture import manager_classes from background_helper import IPC_Logging_Task_Proxy IPC_Logging_Task_Proxy.push_url = ipc_push_url # Pupil detectors from pupil_detectors import Detector_2D, Detector_3D, Detector_Dummy pupil_detectors = { Detector_2D.__name__: Detector_2D, Detector_3D.__name__: Detector_3D, Detector_Dummy.__name__: Detector_Dummy, } # UI Platform tweaks if platform.system() == "Linux": scroll_factor = 10.0 window_position_default = (600, 300 * eye_id + 30) elif platform.system() == "Windows": scroll_factor = 10.0 window_position_default = (600, 90 + 300 * eye_id) else: scroll_factor = 1.0 window_position_default = (600, 300 * eye_id) icon_bar_width = 50 window_size = None camera_render_size = None hdpi_factor = 1.0 # g_pool holds variables for this process g_pool = SimpleNamespace() # make some constants avaiable g_pool.user_dir = user_dir g_pool.version = version g_pool.app = "capture" g_pool.process = "eye{}".format(eye_id) g_pool.timebase = timebase g_pool.ipc_pub = ipc_socket def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal camera_render_size nonlocal hdpi_factor active_window = glfw.glfwGetCurrentContext() glfw.glfwMakeContextCurrent(window) hdpi_factor = glfw.getHDPIFactor(window) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor window_size = w, h camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() for g in g_pool.graphs: g.scale = hdpi_factor g.adjust_window_size(w, h) adjust_gl_view(w, h) glfw.glfwMakeContextCurrent(active_window) def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_iconify(window, iconified): g_pool.iconified = iconified def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x *= hdpi_factor y *= hdpi_factor g_pool.gui.update_mouse(x, y) if g_pool.u_r.active_edit_pt: pos = normalize((x, y), camera_render_size) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] pos = denormalize(pos, g_pool.capture.frame_size) g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx, pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, count, paths): paths = [paths[x].decode("utf-8") for x in range(count)] plugins = (g_pool.capture_manager, g_pool.capture) # call `on_drop` callbacks until a plugin indicates # that it has consumed the event (by returning True) any(p.on_drop(paths) for p in plugins) # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, "user_settings_eye{}".format(eye_id)) ) if VersionFormat(session_settings.get("version", "0.0")) != g_pool.version: logger.info( "Session setting are from a different version of this app. I will not use those." ) session_settings.clear() g_pool.iconified = False g_pool.capture = None g_pool.capture_manager = None g_pool.flip = session_settings.get("flip", False) g_pool.display_mode = session_settings.get("display_mode", "camera_image") g_pool.display_mode_info_text = { "camera_image": "Raw eye camera image. This uses the least amount of CPU power", "roi": "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.", "algorithm": "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below.", } capture_manager_settings = session_settings.get( "capture_manager_settings", ("UVC_Manager", {}) ) manager_class_name, manager_settings = capture_manager_settings manager_class_by_name = {c.__name__: c for c in manager_classes} g_pool.capture_manager = manager_class_by_name[manager_class_name]( g_pool, **manager_settings ) if eye_id == 0: cap_src = ["Pupil Cam3 ID0", "Pupil Cam2 ID0", "Pupil Cam1 ID0", "HD-6000"] else: cap_src = ["Pupil Cam3 ID1", "Pupil Cam2 ID1", "Pupil Cam1 ID1"] # Initialize capture default_settings = ( "UVC_Source", {"preferred_names": cap_src, "frame_size": (320, 240), "frame_rate": 120}, ) capture_source_settings = overwrite_cap_settings or session_settings.get( "capture_settings", default_settings ) source_class_name, source_settings = capture_source_settings source_class_by_name = {c.__name__: c for c in source_classes} g_pool.capture = source_class_by_name[source_class_name]( g_pool, **source_settings ) assert g_pool.capture g_pool.u_r = UIRoi((g_pool.capture.frame_size[1], g_pool.capture.frame_size[0])) roi_user_settings = session_settings.get("roi") if roi_user_settings and tuple(roi_user_settings[-1]) == g_pool.u_r.get()[-1]: g_pool.u_r.set(roi_user_settings) pupil_detector_settings = session_settings.get("pupil_detector_settings", None) last_pupil_detector = pupil_detectors[ session_settings.get("last_pupil_detector", Detector_2D.__name__) ] g_pool.pupil_detector = last_pupil_detector(g_pool, pupil_detector_settings) def set_display_mode_info(val): g_pool.display_mode = val g_pool.display_mode_info.text = g_pool.display_mode_info_text[val] def set_detector(new_detector): g_pool.pupil_detector.deinit_ui() g_pool.pupil_detector.cleanup() g_pool.pupil_detector = new_detector(g_pool) g_pool.pupil_detector.init_ui() def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is open the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed # Initialize glfw glfw.glfwInit() title = "Pupil Capture - eye {}".format(eye_id) width, height = g_pool.capture.frame_size width *= 2 height *= 2 width += icon_bar_width width, height = session_settings.get("window_size", (width, height)) main_window = glfw.glfwCreateWindow(width, height, title, None, None) window_pos = session_settings.get("window_position", window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() # UI callback functions def set_scale(new_scale): g_pool.gui_user_scale = new_scale on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() g_pool.image_tex.update_from_ndarray(np.ones((1, 1), dtype=np.uint8) + 125) # setup GUI g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0) g_pool.menubar = ui.Scrolling_Menu( "Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos="left" ) g_pool.iconbar = ui.Scrolling_Menu( "Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden" ) g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.iconbar) general_settings = ui.Growing_Menu("General", header_pos="headline") general_settings.append( ui.Selector( "gui_user_scale", g_pool, setter=set_scale, selection=[0.8, 0.9, 1.0, 1.1, 1.2], label="Interface Size", ) ) def set_window_size(): f_width, f_height = g_pool.capture.frame_size f_width *= 2 f_height *= 2 f_width += int(icon_bar_width * g_pool.gui.scale) glfw.glfwSetWindowSize(main_window, f_width, f_height) def uroi_on_mouse_button(button, action, mods): if g_pool.display_mode == "roi": if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt: g_pool.u_r.active_edit_pt = False # if the roi interacts we dont want # the gui to interact as well return elif action == glfw.GLFW_PRESS: x, y = glfw.glfwGetCursorPos(main_window) # pos = normalize(pos, glfw.glfwGetWindowSize(main_window)) x *= hdpi_factor y *= hdpi_factor pos = normalize((x, y), camera_render_size) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] # Position in img pixels pos = denormalize( pos, g_pool.capture.frame_size ) # Position in img pixels if g_pool.u_r.mouse_over_edit_pt( pos, g_pool.u_r.handle_size, g_pool.u_r.handle_size ): # if the roi interacts we dont want # the gui to interact as well return general_settings.append(ui.Button("Reset window size", set_window_size)) general_settings.append(ui.Switch("flip", g_pool, label="Flip image display")) general_settings.append( ui.Selector( "display_mode", g_pool, setter=set_display_mode_info, selection=["camera_image", "roi", "algorithm"], labels=["Camera Image", "ROI", "Algorithm"], label="Mode", ) ) g_pool.display_mode_info = ui.Info_Text( g_pool.display_mode_info_text[g_pool.display_mode] ) general_settings.append(g_pool.display_mode_info) detector_selector = ui.Selector( "pupil_detector", getter=lambda: g_pool.pupil_detector.__class__, setter=set_detector, selection=[Detector_Dummy, Detector_2D, Detector_3D], labels=["disabled", "C++ 2d detector", "C++ 3d detector"], label="Detection method", ) general_settings.append(detector_selector) g_pool.menubar.append(general_settings) icon = ui.Icon( "collapsed", general_settings, label=chr(0xE8B8), on_val=False, off_val=True, setter=toggle_general_settings, label_font="pupil_icons", ) icon.tooltip = "General Settings" g_pool.iconbar.append(icon) toggle_general_settings(False) g_pool.pupil_detector.init_ui() g_pool.capture.init_ui() g_pool.capture_manager.init_ui() g_pool.writer = None def replace_source(source_class_name, source_settings): g_pool.capture.deinit_ui() g_pool.capture.cleanup() g_pool.capture = source_class_by_name[source_class_name]( g_pool, **source_settings ) g_pool.capture.init_ui() if g_pool.writer: logger.info("Done recording.") try: g_pool.writer.release() except RuntimeError: logger.error("No eye video recorded") g_pool.writer = None g_pool.replace_source = replace_source # for ndsi capture # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetWindowIconifyCallback(main_window, on_iconify) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) # load last gui configuration g_pool.gui.configuration = session_settings.get("ui_config", {}) # set up performance graphs pid = os.getpid() ps = psutil.Process(pid) ts = g_pool.get_timestamp() cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 50) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = "CPU %0.1f" fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 50) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" g_pool.graphs = [cpu_graph, fps_graph] # set the last saved window size on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) should_publish_frames = False frame_publish_format = "jpeg" frame_publish_format_recent_warning = False # create a timer to control window update frequency window_update_timer = timer(1 / 60) def window_should_update(): return next(window_update_timer) logger.warning("Process started.") frame = None # Event loop while not glfw.glfwWindowShouldClose(main_window): if notify_sub.new_data: t, notification = notify_sub.recv() subject = notification["subject"] if subject.startswith("eye_process.should_stop"): if notification["eye_id"] == eye_id: break elif subject == "set_detection_mapping_mode": if notification["mode"] == "3d": if not isinstance(g_pool.pupil_detector, Detector_3D): set_detector(Detector_3D) detector_selector.read_only = True elif notification["mode"] == "2d": if not isinstance(g_pool.pupil_detector, Detector_2D): set_detector(Detector_2D) detector_selector.read_only = False else: if not isinstance(g_pool.pupil_detector, Detector_Dummy): set_detector(Detector_Dummy) detector_selector.read_only = True elif subject == "recording.started": if notification["record_eye"] and g_pool.capture.online: record_path = notification["rec_path"] raw_mode = notification["compression"] logger.info("Will save eye video to: {}".format(record_path)) video_path = os.path.join( record_path, "eye{}.mp4".format(eye_id) ) if raw_mode and frame and g_pool.capture.jpeg_support: g_pool.writer = JPEG_Writer( video_path, g_pool.capture.frame_rate ) elif hasattr(g_pool.capture._recent_frame, "h264_buffer"): g_pool.writer = H264Writer( video_path, g_pool.capture.frame_size[0], g_pool.capture.frame_size[1], g_pool.capture.frame_rate, ) else: g_pool.writer = AV_Writer( video_path, g_pool.capture.frame_rate ) elif subject == "recording.stopped": if g_pool.writer: logger.info("Done recording.") try: g_pool.writer.release() except RuntimeError: logger.error("No eye video recorded") g_pool.writer = None elif subject.startswith("meta.should_doc"): ipc_socket.notify( { "subject": "meta.doc", "actor": "eye{}".format(eye_id), "doc": eye.__doc__, } ) elif subject.startswith("frame_publishing.started"): should_publish_frames = True frame_publish_format = notification.get("format", "jpeg") elif subject.startswith("frame_publishing.stopped"): should_publish_frames = False frame_publish_format = "jpeg" elif ( subject.startswith("start_eye_capture") and notification["target"] == g_pool.process ): replace_source(notification["name"], notification["args"]) elif notification["subject"].startswith("pupil_detector.set_property"): target_process = notification.get("target", g_pool.process) should_apply = target_process == g_pool.process if should_apply: try: property_name = notification["name"] property_value = notification["value"] if "2d" in notification["subject"]: g_pool.pupil_detector.set_2d_detector_property( property_name, property_value ) elif "3d" in notification["subject"]: if not isinstance(g_pool.pupil_detector, Detector_3D): raise ValueError( "3d properties are only available" " if 3d detector is active" ) g_pool.pupil_detector.set_3d_detector_property( property_name, property_value ) else: raise KeyError( "Notification subject does not " "specifiy detector type." ) logger.debug( "`{}` property set to {}".format( property_name, property_value ) ) except KeyError: logger.error("Malformed notification received") logger.debug(traceback.format_exc()) except (ValueError, TypeError): logger.error("Invalid property or value") logger.debug(traceback.format_exc()) elif notification["subject"].startswith( "pupil_detector.broadcast_properties" ): target_process = notification.get("target", g_pool.process) should_respond = target_process == g_pool.process if should_respond: props = g_pool.pupil_detector.get_detector_properties() properties_broadcast = { "subject": "pupil_detector.properties.{}".format(eye_id), **props, # add properties to broadcast } ipc_socket.notify(properties_broadcast) g_pool.capture.on_notify(notification) g_pool.capture_manager.on_notify(notification) # Get an image from the grabber event = {} g_pool.capture.recent_events(event) frame = event.get("frame") g_pool.capture_manager.recent_events(event) if frame: f_width, f_height = g_pool.capture.frame_size if (g_pool.u_r.array_shape[0], g_pool.u_r.array_shape[1]) != ( f_height, f_width, ): g_pool.pupil_detector.on_resolution_change( (g_pool.u_r.array_shape[1], g_pool.u_r.array_shape[0]), g_pool.capture.frame_size, ) g_pool.u_r = UIRoi((f_height, f_width)) if should_publish_frames: try: if frame_publish_format == "jpeg": data = frame.jpeg_buffer elif frame_publish_format == "yuv": data = frame.yuv_buffer elif frame_publish_format == "bgr": data = frame.bgr elif frame_publish_format == "gray": data = frame.gray assert data is not None except (AttributeError, AssertionError, NameError): if not frame_publish_format_recent_warning: frame_publish_format_recent_warning = True logger.warning( '{}s are not compatible with format "{}"'.format( type(frame), frame_publish_format ) ) else: frame_publish_format_recent_warning = False pupil_socket.send( { "topic": "frame.eye.{}".format(eye_id), "width": frame.width, "height": frame.height, "index": frame.index, "timestamp": frame.timestamp, "format": frame_publish_format, "__raw_data__": [data], } ) t = frame.timestamp dt, ts = t - ts, t try: fps_graph.add(1.0 / dt) except ZeroDivisionError: pass if g_pool.writer: g_pool.writer.write_video_frame(frame) # pupil ellipse detection result = g_pool.pupil_detector.detect( frame, g_pool.u_r, g_pool.display_mode == "algorithm" ) if result is not None: result["id"] = eye_id result["topic"] = "pupil.{}".format(eye_id) pupil_socket.send(result) cpu_graph.update() # GL drawing if window_should_update(): if is_window_visible(main_window): glfw.glfwMakeContextCurrent(main_window) clear_gl_screen() if frame: # switch to work in normalized coordinate space if g_pool.display_mode == "algorithm": g_pool.image_tex.update_from_ndarray(frame.img) elif g_pool.display_mode in ("camera_image", "roi"): g_pool.image_tex.update_from_ndarray(frame.gray) else: pass glViewport(0, 0, *camera_render_size) make_coord_system_norm_based(g_pool.flip) g_pool.image_tex.draw() f_width, f_height = g_pool.capture.frame_size make_coord_system_pixel_based((f_height, f_width, 3), g_pool.flip) if frame and result: if result["method"] == "3d c++": eye_ball = result["projected_sphere"] try: pts = cv2.ellipse2Poly( ( int(eye_ball["center"][0]), int(eye_ball["center"][1]), ), ( int(eye_ball["axes"][0] / 2), int(eye_ball["axes"][1] / 2), ), int(eye_ball["angle"]), 0, 360, 8, ) except ValueError as e: pass else: draw_polyline( pts, 2, RGBA(0.0, 0.9, 0.1, result["model_confidence"]), ) if result["confidence"] > 0: if "ellipse" in result: pts = cv2.ellipse2Poly( ( int(result["ellipse"]["center"][0]), int(result["ellipse"]["center"][1]), ), ( int(result["ellipse"]["axes"][0] / 2), int(result["ellipse"]["axes"][1] / 2), ), int(result["ellipse"]["angle"]), 0, 360, 15, ) confidence = result["confidence"] * 0.7 draw_polyline(pts, 1, RGBA(1.0, 0, 0, confidence)) draw_points( [result["ellipse"]["center"]], size=20, color=RGBA(1.0, 0.0, 0.0, confidence), sharpness=1.0, ) glViewport(0, 0, *camera_render_size) make_coord_system_pixel_based((f_height, f_width, 3), g_pool.flip) # render the ROI g_pool.u_r.draw(g_pool.gui.scale) if g_pool.display_mode == "roi": g_pool.u_r.draw_points(g_pool.gui.scale) glViewport(0, 0, *window_size) make_coord_system_pixel_based((*window_size[::-1], 3), g_pool.flip) # render graphs fps_graph.draw() cpu_graph.draw() # render GUI unused_elements = g_pool.gui.update() for butt in unused_elements.buttons: uroi_on_mouse_button(*butt) make_coord_system_pixel_based((*window_size[::-1], 3), g_pool.flip) g_pool.pupil_detector.visualize() # detector decides if we visualize or not # update screen glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() # END while running # in case eye recording was still runnnig: Save&close if g_pool.writer: logger.info("Done recording eye.") g_pool.writer = None glfw.glfwRestoreWindow(main_window) # need to do this for windows os # save session persistent settings session_settings["gui_scale"] = g_pool.gui_user_scale session_settings["roi"] = g_pool.u_r.get() session_settings["flip"] = g_pool.flip session_settings["display_mode"] = g_pool.display_mode session_settings["ui_config"] = g_pool.gui.configuration session_settings["capture_settings"] = ( g_pool.capture.class_name, g_pool.capture.get_init_dict(), ) session_settings["capture_manager_settings"] = ( g_pool.capture_manager.class_name, g_pool.capture_manager.get_init_dict(), ) session_settings["window_position"] = glfw.glfwGetWindowPos(main_window) session_settings["version"] = str(g_pool.version) session_settings[ "last_pupil_detector" ] = g_pool.pupil_detector.__class__.__name__ session_settings[ "pupil_detector_settings" ] = g_pool.pupil_detector.get_settings() session_window_size = glfw.glfwGetWindowSize(main_window) if 0 not in session_window_size: session_settings["window_size"] = session_window_size session_settings.close() g_pool.capture.deinit_ui() g_pool.capture_manager.deinit_ui() g_pool.pupil_detector.deinit_ui() g_pool.pupil_detector.cleanup() g_pool.capture_manager.cleanup() g_pool.capture.cleanup() glfw.glfwDestroyWindow(main_window) g_pool.gui.terminate() glfw.glfwTerminate() logger.info("Process shutting down.")
def start(self): session = os.path.join(self.rec_root_dir, self.session_name) try: os.makedirs(session, exist_ok=True) logger.debug("Created new recordings session dir {}".format(session)) except OSError: logger.error( "Could not start recording. Session dir {} not writable.".format( session ) ) return self.pldata_writers = {} self.frame_count = 0 self.running = True self.menu.read_only = True self.start_time = time() start_time_synced = self.g_pool.get_timestamp() recording_uuid = uuid.uuid4() # set up self incrementing folder within session folder counter = 0 while True: self.rec_path = os.path.join(session, "{:03d}/".format(counter)) try: os.mkdir(self.rec_path) logger.debug("Created new recording dir {}".format(self.rec_path)) break except: logger.debug( "We dont want to overwrite data, incrementing counter & trying to make new data folder" ) counter += 1 self.meta_info_path = os.path.join(self.rec_path, "info.csv") with open(self.meta_info_path, "w", newline="", encoding="utf-8") as csvfile: csv_utils.write_key_value_file( csvfile, { "Recording Name": self.session_name, "Start Date": strftime("%d.%m.%Y", localtime(self.start_time)), "Start Time": strftime("%H:%M:%S", localtime(self.start_time)), "Start Time (System)": self.start_time, "Start Time (Synced)": start_time_synced, "Recording UUID": recording_uuid, }, ) self.video_path = os.path.join(self.rec_path, "world.mp4") if self.raw_jpeg and self.g_pool.capture.jpeg_support: self.writer = JPEG_Writer(self.video_path, self.g_pool.capture.frame_rate) elif hasattr(self.g_pool.capture._recent_frame, "h264_buffer"): self.writer = H264Writer( self.video_path, self.g_pool.capture.frame_size[0], self.g_pool.capture.frame_size[1], int(self.g_pool.capture.frame_rate), ) else: self.writer = AV_Writer(self.video_path, fps=self.g_pool.capture.frame_rate) try: cal_pt_path = os.path.join(self.g_pool.user_dir, "user_calibration_data") cal_data = load_object(cal_pt_path) notification = {"subject": "calibration.calibration_data", "record": True} notification.update(cal_data) notification["topic"] = "notify." + notification["subject"] writer = PLData_Writer(self.rec_path, "notify") writer.append(notification) self.pldata_writers["notify"] = writer except FileNotFoundError: pass if self.show_info_menu: self.open_info_menu() logger.info("Started Recording.") self.notify_all( { "subject": "recording.started", "rec_path": self.rec_path, "session_name": self.session_name, "record_eye": self.record_eye, "compression": self.raw_jpeg, } )
def eye( timebase, is_alive_flag, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version, eye_id, overwrite_cap_settings=None, hide_ui=False, ): """reads eye video and detects the pupil. Creates a window, gl context. Grabs images from a capture. Streams Pupil coordinates. Reacts to notifications: ``set_detection_mapping_mode``: Sets detection method ``eye_process.should_stop``: Stops the eye process ``recording.started``: Starts recording eye video ``recording.stopped``: Stops recording eye video ``frame_publishing.started``: Starts frame publishing ``frame_publishing.stopped``: Stops frame publishing ``start_eye_plugin``: Start plugins in eye process Emits notifications: ``eye_process.started``: Eye process started ``eye_process.stopped``: Eye process stopped Emits data: ``pupil.<eye id>``: Pupil data for eye with id ``<eye id>`` ``frame.eye.<eye id>``: Eye frames with id ``<eye id>`` """ # We deferr the imports becasue of multiprocessing. # Otherwise the world process each process also loads the other imports. import zmq import zmq_tools zmq_ctx = zmq.Context() ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify", )) # logging setup import logging logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.NOTSET) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) if is_alive_flag.value: # indicates eye process that this is a duplicated startup logger.warning("Aborting redundant eye process startup") return with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id, logger): # general imports import traceback import numpy as np import cv2 # display import glfw from pyglui import ui, graph, cygl from pyglui.cygl.utils import draw_points, RGBA, draw_polyline from pyglui.cygl.utils import Named_Texture from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen from gl_utils import make_coord_system_pixel_based from gl_utils import make_coord_system_norm_based from gl_utils import is_window_visible, glViewport from ui_roi import UIRoi # monitoring import psutil # Plug-ins from plugin import Plugin_List # helpers/utils from uvc import get_time_monotonic from file_methods import Persistent_Dict from version_utils import VersionFormat from methods import normalize, denormalize, timer from av_writer import JPEG_Writer, MPEG_Writer from ndsi import H264Writer from video_capture import source_classes, manager_classes from background_helper import IPC_Logging_Task_Proxy from pupil_detector_plugins import available_detector_plugins from pupil_detector_plugins.manager import PupilDetectorManager IPC_Logging_Task_Proxy.push_url = ipc_push_url def interrupt_handler(sig, frame): import traceback trace = traceback.format_stack(f=frame) logger.debug(f"Caught signal {sig} in:\n" + "".join(trace)) # NOTE: Interrupt is handled in world/service/player which are responsible for # shutting down the eye process properly signal.signal(signal.SIGINT, interrupt_handler) # UI Platform tweaks if platform.system() == "Linux": scroll_factor = 10.0 window_position_default = (600, 300 * eye_id + 30) elif platform.system() == "Windows": scroll_factor = 10.0 window_position_default = (600, 90 + 300 * eye_id) else: scroll_factor = 1.0 window_position_default = (600, 300 * eye_id) icon_bar_width = 50 window_size = None camera_render_size = None hdpi_factor = 1.0 # g_pool holds variables for this process g_pool = SimpleNamespace() # make some constants avaiable g_pool.user_dir = user_dir g_pool.version = version g_pool.app = "capture" g_pool.eye_id = eye_id g_pool.process = f"eye{eye_id}" g_pool.timebase = timebase g_pool.ipc_pub = ipc_socket def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic default_detector_cls, available_detectors = available_detector_plugins( ) plugins = (manager_classes + source_classes + available_detectors + [PupilDetectorManager]) g_pool.plugin_by_name = {p.__name__: p for p in plugins} preferred_names = [ f"Pupil Cam3 ID{eye_id}", f"Pupil Cam2 ID{eye_id}", f"Pupil Cam1 ID{eye_id}", ] if eye_id == 0: preferred_names += ["HD-6000"] default_capture_settings = ( "UVC_Source", { "preferred_names": preferred_names, "frame_size": (320, 240), "frame_rate": 120, }, ) default_plugins = [ # TODO: extend with plugins default_capture_settings, ("UVC_Manager", {}), # Detector needs to be loaded first to set `g_pool.pupil_detector` (default_detector_cls.__name__, {}), ("PupilDetectorManager", {}), ] # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal camera_render_size nonlocal hdpi_factor active_window = glfw.glfwGetCurrentContext() glfw.glfwMakeContextCurrent(window) hdpi_factor = glfw.getHDPIFactor(window) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor window_size = w, h camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() for g in g_pool.graphs: g.scale = hdpi_factor g.adjust_window_size(w, h) adjust_gl_view(w, h) glfw.glfwMakeContextCurrent(active_window) def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_iconify(window, iconified): g_pool.iconified = iconified def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x *= hdpi_factor y *= hdpi_factor g_pool.gui.update_mouse(x, y) if g_pool.u_r.active_edit_pt: pos = normalize((x, y), camera_render_size) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] pos = denormalize(pos, g_pool.capture.frame_size) g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx, pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, count, paths): paths = [paths[x].decode("utf-8") for x in range(count)] for plugin in g_pool.plugins: if plugin.on_drop(paths): break # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, "user_settings_eye{}".format(eye_id))) if VersionFormat(session_settings.get("version", "0.0")) != g_pool.version: logger.info( "Session setting are from a different version of this app. I will not use those." ) session_settings.clear() g_pool.iconified = False g_pool.capture = None g_pool.flip = session_settings.get("flip", False) g_pool.display_mode = session_settings.get("display_mode", "camera_image") g_pool.display_mode_info_text = { "camera_image": "Raw eye camera image. This uses the least amount of CPU power", "roi": "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.", "algorithm": "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below.", } def set_display_mode_info(val): g_pool.display_mode = val g_pool.display_mode_info.text = g_pool.display_mode_info_text[val] def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is open the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed # Initialize glfw glfw.glfwInit() if hide_ui: glfw.glfwWindowHint(glfw.GLFW_VISIBLE, 0) # hide window title = "Pupil Capture - eye {}".format(eye_id) width, height = session_settings.get("window_size", (640 + icon_bar_width, 480)) main_window = glfw.glfwCreateWindow(width, height, title, None, None) window_pos = session_settings.get("window_position", window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() # UI callback functions def set_scale(new_scale): g_pool.gui_user_scale = new_scale on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() g_pool.image_tex.update_from_ndarray( np.ones((1, 1), dtype=np.uint8) + 125) # setup GUI g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0) g_pool.menubar = ui.Scrolling_Menu("Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos="left") g_pool.iconbar = ui.Scrolling_Menu("Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden") g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.iconbar) general_settings = ui.Growing_Menu("General", header_pos="headline") general_settings.append( ui.Selector( "gui_user_scale", g_pool, setter=set_scale, selection=[0.8, 0.9, 1.0, 1.1, 1.2], label="Interface Size", )) def set_window_size(): f_width, f_height = g_pool.capture.frame_size f_width *= 2 f_height *= 2 f_width += int(icon_bar_width * g_pool.gui.scale) glfw.glfwSetWindowSize(main_window, f_width, f_height) def uroi_on_mouse_button(button, action, mods): if g_pool.display_mode == "roi": if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt: g_pool.u_r.active_edit_pt = False # if the roi interacts we dont want # the gui to interact as well return elif action == glfw.GLFW_PRESS: x, y = glfw.glfwGetCursorPos(main_window) # pos = normalize(pos, glfw.glfwGetWindowSize(main_window)) x *= hdpi_factor y *= hdpi_factor pos = normalize((x, y), camera_render_size) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] # Position in img pixels pos = denormalize( pos, g_pool.capture.frame_size) # Position in img pixels if g_pool.u_r.mouse_over_edit_pt(pos, g_pool.u_r.handle_size, g_pool.u_r.handle_size): # if the roi interacts we dont want # the gui to interact as well return general_settings.append(ui.Button("Reset window size", set_window_size)) general_settings.append( ui.Switch("flip", g_pool, label="Flip image display")) general_settings.append( ui.Selector( "display_mode", g_pool, setter=set_display_mode_info, selection=["camera_image", "roi", "algorithm"], labels=["Camera Image", "ROI", "Algorithm"], label="Mode", )) g_pool.display_mode_info = ui.Info_Text( g_pool.display_mode_info_text[g_pool.display_mode]) general_settings.append(g_pool.display_mode_info) g_pool.menubar.append(general_settings) icon = ui.Icon( "collapsed", general_settings, label=chr(0xE8B8), on_val=False, off_val=True, setter=toggle_general_settings, label_font="pupil_icons", ) icon.tooltip = "General Settings" g_pool.iconbar.append(icon) toggle_general_settings(False) plugins_to_load = session_settings.get("loaded_plugins", default_plugins) if overwrite_cap_settings: # Ensure that overwrite_cap_settings takes preference over source plugins # with incorrect settings that were loaded from session settings. plugins_to_load.append(overwrite_cap_settings) g_pool.plugins = Plugin_List(g_pool, plugins_to_load) g_pool.writer = None g_pool.u_r = UIRoi( (g_pool.capture.frame_size[1], g_pool.capture.frame_size[0])) roi_user_settings = session_settings.get("roi") if roi_user_settings and tuple( roi_user_settings[-1]) == g_pool.u_r.get()[-1]: g_pool.u_r.set(roi_user_settings) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetWindowIconifyCallback(main_window, on_iconify) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) # load last gui configuration g_pool.gui.configuration = session_settings.get("ui_config", {}) # set up performance graphs pid = os.getpid() ps = psutil.Process(pid) ts = g_pool.get_timestamp() cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 50) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = "CPU %0.1f" fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 50) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" g_pool.graphs = [cpu_graph, fps_graph] # set the last saved window size on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) should_publish_frames = False frame_publish_format = "jpeg" frame_publish_format_recent_warning = False # create a timer to control window update frequency window_update_timer = timer(1 / 60) def window_should_update(): return next(window_update_timer) logger.warning("Process started.") frame = None # Event loop while not glfw.glfwWindowShouldClose(main_window): if notify_sub.new_data: t, notification = notify_sub.recv() subject = notification["subject"] if subject.startswith("eye_process.should_stop"): if notification["eye_id"] == eye_id: break elif subject == "recording.started": if notification["record_eye"] and g_pool.capture.online: record_path = notification["rec_path"] raw_mode = notification["compression"] start_time_synced = notification["start_time_synced"] logger.info( "Will save eye video to: {}".format(record_path)) video_path = os.path.join(record_path, "eye{}.mp4".format(eye_id)) if raw_mode and frame and g_pool.capture.jpeg_support: g_pool.writer = JPEG_Writer( video_path, start_time_synced) elif hasattr(g_pool.capture._recent_frame, "h264_buffer"): g_pool.writer = H264Writer( video_path, g_pool.capture.frame_size[0], g_pool.capture.frame_size[1], g_pool.capture.frame_rate, ) else: g_pool.writer = MPEG_Writer( video_path, start_time_synced) elif subject == "recording.stopped": if g_pool.writer: logger.info("Done recording.") try: g_pool.writer.release() except RuntimeError: logger.error("No eye video recorded") g_pool.writer = None elif subject.startswith("meta.should_doc"): ipc_socket.notify({ "subject": "meta.doc", "actor": "eye{}".format(eye_id), "doc": eye.__doc__, }) elif subject.startswith("frame_publishing.started"): should_publish_frames = True frame_publish_format = notification.get("format", "jpeg") elif subject.startswith("frame_publishing.stopped"): should_publish_frames = False frame_publish_format = "jpeg" elif (subject.startswith("start_eye_plugin") and notification["target"] == g_pool.process): try: g_pool.plugins.add( g_pool.plugin_by_name[notification["name"]], notification.get("args", {}), ) except KeyError as err: logger.error(f"Attempt to load unknown plugin: {err}") for plugin in g_pool.plugins: plugin.on_notify(notification) event = {} for plugin in g_pool.plugins: plugin.recent_events(event) frame = event.get("frame") if frame: f_width, f_height = g_pool.capture.frame_size if (g_pool.u_r.array_shape[0], g_pool.u_r.array_shape[1]) != ( f_height, f_width, ): g_pool.pupil_detector.on_resolution_change( (g_pool.u_r.array_shape[1], g_pool.u_r.array_shape[0]), g_pool.capture.frame_size, ) g_pool.u_r = UIRoi((f_height, f_width)) if should_publish_frames: try: if frame_publish_format == "jpeg": data = frame.jpeg_buffer elif frame_publish_format == "yuv": data = frame.yuv_buffer elif frame_publish_format == "bgr": data = frame.bgr elif frame_publish_format == "gray": data = frame.gray assert data is not None except (AttributeError, AssertionError, NameError): if not frame_publish_format_recent_warning: frame_publish_format_recent_warning = True logger.warning( '{}s are not compatible with format "{}"'. format(type(frame), frame_publish_format)) else: frame_publish_format_recent_warning = False pupil_socket.send({ "topic": "frame.eye.{}".format(eye_id), "width": frame.width, "height": frame.height, "index": frame.index, "timestamp": frame.timestamp, "format": frame_publish_format, "__raw_data__": [data], }) t = frame.timestamp dt, ts = t - ts, t try: fps_graph.add(1.0 / dt) except ZeroDivisionError: pass if g_pool.writer: g_pool.writer.write_video_frame(frame) result = event.get("pupil_detection_result", None) if result is not None: pupil_socket.send(result) cpu_graph.update() # GL drawing if window_should_update(): if is_window_visible(main_window): glfw.glfwMakeContextCurrent(main_window) clear_gl_screen() glViewport(0, 0, *camera_render_size) for p in g_pool.plugins: p.gl_display() glViewport(0, 0, *camera_render_size) # render the ROI g_pool.u_r.draw(g_pool.gui.scale) if g_pool.display_mode == "roi": g_pool.u_r.draw_points(g_pool.gui.scale) glViewport(0, 0, *window_size) # render graphs fps_graph.draw() cpu_graph.draw() # render GUI try: clipboard = glfw.glfwGetClipboardString( main_window).decode() except AttributeError: # clipboard is None, might happen on startup clipboard = "" g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.glfwSetClipboardString( main_window, user_input.clipboard.encode()) for button, action, mods in user_input.buttons: x, y = glfw.glfwGetCursorPos(main_window) pos = x * hdpi_factor, y * hdpi_factor pos = normalize(pos, camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for plugin in g_pool.plugins: if plugin.on_click(pos, button, action): break for key, scancode, action, mods in user_input.keys: for plugin in g_pool.plugins: if plugin.on_key(key, scancode, action, mods): break for char_ in user_input.chars: for plugin in g_pool.plugins: if plugin.on_char(char_): break # update screen glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() # END while running # in case eye recording was still runnnig: Save&close if g_pool.writer: logger.info("Done recording eye.") g_pool.writer.release() g_pool.writer = None session_settings["loaded_plugins"] = g_pool.plugins.get_initializers() # save session persistent settings session_settings["gui_scale"] = g_pool.gui_user_scale session_settings["roi"] = g_pool.u_r.get() session_settings["flip"] = g_pool.flip session_settings["display_mode"] = g_pool.display_mode session_settings["ui_config"] = g_pool.gui.configuration session_settings["version"] = str(g_pool.version) if not hide_ui: glfw.glfwRestoreWindow( main_window) # need to do this for windows os session_settings["window_position"] = glfw.glfwGetWindowPos( main_window) session_window_size = glfw.glfwGetWindowSize(main_window) if 0 not in session_window_size: session_settings["window_size"] = session_window_size session_settings.close() for plugin in g_pool.plugins: plugin.alive = False g_pool.plugins.clean() glfw.glfwDestroyWindow(main_window) g_pool.gui.terminate() glfw.glfwTerminate() logger.info("Process shutting down.")
def eye( timebase, is_alive_flag, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version, eye_id, overwrite_cap_settings=None, hide_ui=False, debug=False, pub_socket_hwm=None, parent_application="capture", ): """reads eye video and detects the pupil. Creates a window, gl context. Grabs images from a capture. Streams Pupil coordinates. Reacts to notifications: ``eye_process.should_stop``: Stops the eye process ``recording.started``: Starts recording eye video ``recording.stopped``: Stops recording eye video ``frame_publishing.started``: Starts frame publishing ``frame_publishing.stopped``: Stops frame publishing ``start_eye_plugin``: Start plugins in eye process Emits notifications: ``eye_process.started``: Eye process started ``eye_process.stopped``: Eye process stopped Emits data: ``pupil.<eye id>``: Pupil data for eye with id ``<eye id>`` ``frame.eye.<eye id>``: Eye frames with id ``<eye id>`` """ # We deferr the imports becasue of multiprocessing. # Otherwise the world process each process also loads the other imports. import zmq import zmq_tools zmq_ctx = zmq.Context() ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url, pub_socket_hwm) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",)) # logging setup import logging logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.NOTSET) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) if is_alive_flag.value: # indicates eye process that this is a duplicated startup logger.warning("Aborting redundant eye process startup") return with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id, logger): # general imports import traceback import numpy as np import cv2 from OpenGL.GL import GL_COLOR_BUFFER_BIT # display import glfw from gl_utils import GLFWErrorReporting GLFWErrorReporting.set_default() from pyglui import ui, graph, cygl from pyglui.cygl.utils import Named_Texture import gl_utils from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen from gl_utils import make_coord_system_pixel_based from gl_utils import make_coord_system_norm_based from gl_utils import is_window_visible, glViewport # monitoring import psutil # Plug-ins from plugin import Plugin_List # helpers/utils from uvc import get_time_monotonic from file_methods import Persistent_Dict from version_utils import parse_version from methods import normalize, denormalize, timer from av_writer import JPEG_Writer, MPEG_Writer, NonMonotonicTimestampError from ndsi import H264Writer from video_capture import source_classes, manager_classes from roi import Roi from background_helper import IPC_Logging_Task_Proxy from pupil_detector_plugins import available_detector_plugins, EVENT_KEY IPC_Logging_Task_Proxy.push_url = ipc_push_url def interrupt_handler(sig, frame): import traceback trace = traceback.format_stack(f=frame) logger.debug(f"Caught signal {sig} in:\n" + "".join(trace)) # NOTE: Interrupt is handled in world/service/player which are responsible for # shutting down the eye process properly signal.signal(signal.SIGINT, interrupt_handler) # UI Platform tweaks if platform.system() == "Linux": scroll_factor = 10.0 window_position_default = (600, 300 * eye_id + 30) elif platform.system() == "Windows": scroll_factor = 10.0 window_position_default = (600, 90 + 300 * eye_id) else: scroll_factor = 1.0 window_position_default = (600, 300 * eye_id) icon_bar_width = 50 window_size = None content_scale = 1.0 # g_pool holds variables for this process g_pool = SimpleNamespace() # make some constants avaiable g_pool.debug = debug g_pool.user_dir = user_dir g_pool.version = version g_pool.app = parent_application g_pool.eye_id = eye_id g_pool.process = f"eye{eye_id}" g_pool.timebase = timebase g_pool.camera_render_size = None g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_socket g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic def load_runtime_pupil_detection_plugins(): from plugin import import_runtime_plugins from pupil_detector_plugins.detector_base_plugin import PupilDetectorPlugin plugins_path = os.path.join(g_pool.user_dir, "plugins") for plugin in import_runtime_plugins(plugins_path): if not isinstance(plugin, type): continue if not issubclass(plugin, PupilDetectorPlugin): continue if plugin is PupilDetectorPlugin: continue yield plugin available_detectors = available_detector_plugins() runtime_detectors = list(load_runtime_pupil_detection_plugins()) plugins = ( manager_classes + source_classes + available_detectors + runtime_detectors + [Roi] ) g_pool.plugin_by_name = {p.__name__: p for p in plugins} preferred_names = [ f"Pupil Cam3 ID{eye_id}", f"Pupil Cam2 ID{eye_id}", f"Pupil Cam1 ID{eye_id}", ] if eye_id == 0: preferred_names += ["HD-6000"] default_capture_name = "UVC_Source" default_capture_settings = { "preferred_names": preferred_names, "frame_size": (192, 192), "frame_rate": 120, } default_plugins = [ # TODO: extend with plugins (default_capture_name, default_capture_settings), ("UVC_Manager", {}), *[(p.__name__, {}) for p in available_detectors], ("NDSI_Manager", {}), ("HMD_Streaming_Manager", {}), ("File_Manager", {}), ("Roi", {}), ] def consume_events_and_render_buffer(): glfw.make_context_current(main_window) clear_gl_screen() if all(c > 0 for c in g_pool.camera_render_size): glViewport(0, 0, *g_pool.camera_render_size) for p in g_pool.plugins: p.gl_display() glViewport(0, 0, *window_size) # render graphs fps_graph.draw() cpu_graph.draw() # render GUI try: clipboard = glfw.get_clipboard_string(main_window).decode() except (AttributeError, glfw.GLFWError): # clipboard is None, might happen on startup clipboard = "" g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.set_clipboard_string(main_window, user_input.clipboard) for button, action, mods in user_input.buttons: x, y = glfw.get_cursor_pos(main_window) pos = gl_utils.window_coordinate_to_framebuffer_coordinate( main_window, x, y, cached_scale=None ) pos = normalize(pos, g_pool.camera_render_size) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for plugin in g_pool.plugins: if plugin.on_click(pos, button, action): break for key, scancode, action, mods in user_input.keys: for plugin in g_pool.plugins: if plugin.on_key(key, scancode, action, mods): break for char_ in user_input.chars: for plugin in g_pool.plugins: if plugin.on_char(char_): break # update screen glfw.swap_buffers(main_window) # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal content_scale is_minimized = bool(glfw.get_window_attrib(window, glfw.ICONIFIED)) if is_minimized: return # Always clear buffers on resize to make sure that there are no overlapping # artifacts from previous frames. gl_utils.glClear(GL_COLOR_BUFFER_BIT) gl_utils.glClearColor(0, 0, 0, 1) active_window = glfw.get_current_context() glfw.make_context_current(window) content_scale = gl_utils.get_content_scale(window) framebuffer_scale = gl_utils.get_framebuffer_scale(window) g_pool.gui.scale = content_scale window_size = w, h g_pool.camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() for g in g_pool.graphs: g.scale = content_scale g.adjust_window_size(w, h) adjust_gl_view(w, h) glfw.make_context_current(active_window) # Minimum window size required, otherwise parts of the UI can cause openGL # issues with permanent effects. Depends on the content scale, which can # potentially be dynamically modified, so we re-adjust the size limits every # time here. min_size = int(2 * icon_bar_width * g_pool.gui.scale / framebuffer_scale) glfw.set_window_size_limits( window, min_size, min_size, glfw.DONT_CARE, glfw.DONT_CARE, ) # Needed, to update the window buffer while resizing consume_events_and_render_buffer() def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_iconify(window, iconified): g_pool.iconified = iconified def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x, y = gl_utils.window_coordinate_to_framebuffer_coordinate( window, x, y, cached_scale=None ) g_pool.gui.update_mouse(x, y) pos = x, y pos = normalize(pos, g_pool.camera_render_size) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_pos(pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, paths): for plugin in g_pool.plugins: if plugin.on_drop(paths): break # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, "user_settings_eye{}".format(eye_id)) ) if parse_version(session_settings.get("version", "0.0")) != g_pool.version: logger.info( "Session setting are from a different version of this app. I will not use those." ) session_settings.clear() camera_is_physically_flipped = eye_id == 0 g_pool.iconified = False g_pool.capture = None g_pool.flip = session_settings.get("flip", camera_is_physically_flipped) g_pool.display_mode = session_settings.get("display_mode", "camera_image") g_pool.display_mode_info_text = { "camera_image": "Raw eye camera image. This uses the least amount of CPU power", "roi": "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.", "algorithm": "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below.", } def set_display_mode_info(val): g_pool.display_mode = val g_pool.display_mode_info.text = g_pool.display_mode_info_text[val] def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is open the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed # Initialize glfw glfw.init() glfw.window_hint(glfw.SCALE_TO_MONITOR, glfw.TRUE) if hide_ui: glfw.window_hint(glfw.VISIBLE, 0) # hide window title = "Pupil Capture - eye {}".format(eye_id) # Pupil Cam1 uses 4:3 resolutions. Pupil Cam2 and Cam3 use 1:1 resolutions. # As all Pupil Core and VR/AR add-ons are shipped with Pupil Cam2 and Cam3 # cameras, we adjust the default eye window size to a 1:1 content aspect ratio. # The size of 500 was chosen s.t. the menu still fits. default_window_size = 500 + icon_bar_width, 500 width, height = session_settings.get("window_size", default_window_size) main_window = glfw.create_window(width, height, title, None, None) window_position_manager = gl_utils.WindowPositionManager() window_pos = window_position_manager.new_window_position( window=main_window, default_position=window_position_default, previous_position=session_settings.get("window_position", None), ) glfw.set_window_pos(main_window, window_pos[0], window_pos[1]) glfw.make_context_current(main_window) cygl.utils.init() # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() g_pool.image_tex.update_from_ndarray(np.ones((1, 1), dtype=np.uint8) + 125) # setup GUI g_pool.gui = ui.UI() g_pool.menubar = ui.Scrolling_Menu( "Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos="left" ) g_pool.iconbar = ui.Scrolling_Menu( "Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden" ) g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.iconbar) general_settings = ui.Growing_Menu("General", header_pos="headline") def set_window_size(): # Get current capture frame size f_width, f_height = g_pool.capture.frame_size # Eye camera resolutions are too small to be used as default window sizes. # We use double their size instead. frame_scale_factor = 2 f_width *= frame_scale_factor f_height *= frame_scale_factor # Get current display scale factor content_scale = gl_utils.get_content_scale(main_window) framebuffer_scale = gl_utils.get_framebuffer_scale(main_window) display_scale_factor = content_scale / framebuffer_scale # Scale the capture frame size by display scale factor f_width *= display_scale_factor f_height *= display_scale_factor # Increas the width to account for the added scaled icon bar width f_width += icon_bar_width * display_scale_factor # Set the newly calculated size (scaled capture frame size + scaled icon bar width) glfw.set_window_size(main_window, int(f_width), int(f_height)) general_settings.append(ui.Button("Reset window size", set_window_size)) general_settings.append(ui.Switch("flip", g_pool, label="Flip image display")) general_settings.append( ui.Selector( "display_mode", g_pool, setter=set_display_mode_info, selection=["camera_image", "roi", "algorithm"], labels=["Camera Image", "ROI", "Algorithm"], label="Mode", ) ) g_pool.display_mode_info = ui.Info_Text( g_pool.display_mode_info_text[g_pool.display_mode] ) general_settings.append(g_pool.display_mode_info) g_pool.menubar.append(general_settings) icon = ui.Icon( "collapsed", general_settings, label=chr(0xE8B8), on_val=False, off_val=True, setter=toggle_general_settings, label_font="pupil_icons", ) icon.tooltip = "General Settings" g_pool.iconbar.append(icon) plugins_to_load = session_settings.get("loaded_plugins", default_plugins) if overwrite_cap_settings: # Ensure that overwrite_cap_settings takes preference over source plugins # with incorrect settings that were loaded from session settings. plugins_to_load.append(overwrite_cap_settings) # Add runtime plugins to the list of plugins to load with default arguments, # if not already restored from session settings plugins_to_load_names = set(name for name, _ in plugins_to_load) for runtime_detector in runtime_detectors: runtime_name = runtime_detector.__name__ if runtime_name not in plugins_to_load_names: plugins_to_load.append((runtime_name, {})) g_pool.plugins = Plugin_List(g_pool, plugins_to_load) if not g_pool.capture: # Make sure we always have a capture running. Important if there was no # capture stored in session settings. g_pool.plugins.add( g_pool.plugin_by_name[default_capture_name], default_capture_settings ) toggle_general_settings(True) g_pool.writer = None g_pool.rec_path = None # Register callbacks main_window glfw.set_framebuffer_size_callback(main_window, on_resize) glfw.set_window_iconify_callback(main_window, on_iconify) glfw.set_key_callback(main_window, on_window_key) glfw.set_char_callback(main_window, on_window_char) glfw.set_mouse_button_callback(main_window, on_window_mouse_button) glfw.set_cursor_pos_callback(main_window, on_pos) glfw.set_scroll_callback(main_window, on_scroll) glfw.set_drop_callback(main_window, on_drop) # load last gui configuration g_pool.gui.configuration = session_settings.get("ui_config", {}) # If previously selected plugin was not loaded this time, we will have an # expanded menubar without any menu selected. We need to ensure the menubar is # collapsed in this case. if all(submenu.collapsed for submenu in g_pool.menubar.elements): g_pool.menubar.collapsed = True # set up performance graphs pid = os.getpid() ps = psutil.Process(pid) ts = g_pool.get_timestamp() cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 50) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = "CPU %0.1f" fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 50) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" g_pool.graphs = [cpu_graph, fps_graph] # set the last saved window size on_resize(main_window, *glfw.get_framebuffer_size(main_window)) should_publish_frames = False frame_publish_format = "jpeg" frame_publish_format_recent_warning = False # create a timer to control window update frequency window_update_timer = timer(1 / 60) def window_should_update(): return next(window_update_timer) logger.warning("Process started.") frame = None if platform.system() == "Darwin": # On macOS, calls to glfw.swap_buffers() deliberately take longer in case of # occluded windows, based on the swap interval value. This causes an FPS drop # and leads to problems when recording. To side-step this behaviour, the swap # interval is set to zero. # # Read more about window occlusion on macOS here: # https://developer.apple.com/library/archive/documentation/Performance/Conceptual/power_efficiency_guidelines_osx/WorkWhenVisible.html glfw.swap_interval(0) # Event loop window_should_close = False while not window_should_close: if notify_sub.new_data: t, notification = notify_sub.recv() subject = notification["subject"] if subject.startswith("eye_process.should_stop"): if notification["eye_id"] == eye_id: break elif subject == "recording.started": if notification["record_eye"] and g_pool.capture.online: g_pool.rec_path = notification["rec_path"] raw_mode = notification["compression"] start_time_synced = notification["start_time_synced"] logger.info(f"Will save eye video to: {g_pool.rec_path}") video_path = os.path.join( g_pool.rec_path, "eye{}.mp4".format(eye_id) ) if raw_mode and frame and g_pool.capture.jpeg_support: g_pool.writer = JPEG_Writer(video_path, start_time_synced) elif hasattr(g_pool.capture._recent_frame, "h264_buffer"): g_pool.writer = H264Writer( video_path, g_pool.capture.frame_size[0], g_pool.capture.frame_size[1], g_pool.capture.frame_rate, ) else: g_pool.writer = MPEG_Writer(video_path, start_time_synced) elif subject == "recording.stopped": if g_pool.writer: logger.info("Done recording.") try: g_pool.writer.release() except RuntimeError: logger.error("No eye video recorded") else: # TODO: wrap recording logic into plugin g_pool.capture.intrinsics.save( g_pool.rec_path, custom_name=f"eye{eye_id}" ) finally: g_pool.writer = None elif subject.startswith("meta.should_doc"): ipc_socket.notify( { "subject": "meta.doc", "actor": "eye{}".format(eye_id), "doc": eye.__doc__, } ) elif subject.startswith("frame_publishing.started"): should_publish_frames = True frame_publish_format = notification.get("format", "jpeg") elif subject.startswith("frame_publishing.stopped"): should_publish_frames = False frame_publish_format = "jpeg" elif ( subject.startswith("start_eye_plugin") and notification["target"] == g_pool.process ): try: g_pool.plugins.add( g_pool.plugin_by_name[notification["name"]], notification.get("args", {}), ) except KeyError as err: logger.error(f"Attempt to load unknown plugin: {err}") elif ( subject.startswith("stop_eye_plugin") and notification["target"] == g_pool.process ): try: plugin_to_stop = g_pool.plugin_by_name[notification["name"]] except KeyError as err: logger.error(f"Attempt to load unknown plugin: {err}") else: plugin_to_stop.alive = False g_pool.plugins.clean() for plugin in g_pool.plugins: plugin.on_notify(notification) event = {} for plugin in g_pool.plugins: plugin.recent_events(event) frame = event.get("frame") if frame: if should_publish_frames: try: if frame_publish_format == "jpeg": data = frame.jpeg_buffer elif frame_publish_format == "yuv": data = frame.yuv_buffer elif frame_publish_format == "bgr": data = frame.bgr elif frame_publish_format == "gray": data = frame.gray assert data is not None except (AttributeError, AssertionError, NameError): if not frame_publish_format_recent_warning: frame_publish_format_recent_warning = True logger.warning( '{}s are not compatible with format "{}"'.format( type(frame), frame_publish_format ) ) else: frame_publish_format_recent_warning = False pupil_socket.send( { "topic": "frame.eye.{}".format(eye_id), "width": frame.width, "height": frame.height, "index": frame.index, "timestamp": frame.timestamp, "format": frame_publish_format, "__raw_data__": [data], } ) t = frame.timestamp dt, ts = t - ts, t try: fps_graph.add(1.0 / dt) except ZeroDivisionError: pass if g_pool.writer: try: g_pool.writer.write_video_frame(frame) except NonMonotonicTimestampError as e: logger.error( "Recorder received non-monotonic timestamp!" " Stopping the recording!" ) logger.debug(str(e)) ipc_socket.notify({"subject": "recording.should_stop"}) ipc_socket.notify( {"subject": "recording.should_stop", "remote_notify": "all"} ) for result in event.get(EVENT_KEY, ()): pupil_socket.send(result) # GL drawing if window_should_update(): cpu_graph.update() if is_window_visible(main_window): consume_events_and_render_buffer() glfw.poll_events() window_should_close = glfw.window_should_close(main_window) # END while running # in case eye recording was still runnnig: Save&close if g_pool.writer: logger.info("Done recording eye.") g_pool.writer.release() g_pool.writer = None session_settings["loaded_plugins"] = g_pool.plugins.get_initializers() # save session persistent settings session_settings["flip"] = g_pool.flip session_settings["display_mode"] = g_pool.display_mode session_settings["ui_config"] = g_pool.gui.configuration session_settings["version"] = str(g_pool.version) if not hide_ui: glfw.restore_window(main_window) # need to do this for windows os session_settings["window_position"] = glfw.get_window_pos(main_window) session_window_size = glfw.get_window_size(main_window) if 0 not in session_window_size: f_width, f_height = session_window_size if platform.system() in ("Windows", "Linux"): # Store unscaled window size as the operating system will scale the # windows appropriately during launch on Windows and Linux. f_width, f_height = ( f_width / content_scale, f_height / content_scale, ) session_settings["window_size"] = int(f_width), int(f_height) session_settings.close() for plugin in g_pool.plugins: plugin.alive = False g_pool.plugins.clean() glfw.destroy_window(main_window) g_pool.gui.terminate() glfw.terminate() logger.info("Process shutting down.")