def export(should_terminate, frames_to_export, current_frame, rec_dir, user_dir, min_data_confidence, start_frame=None, end_frame=None, plugin_initializers=(), out_file_path=None): vis_plugins = sorted([ Vis_Circle, Vis_Cross, Vis_Polyline, Vis_Light_Points, Vis_Watermark, Scan_Path ], key=lambda x: x.__name__) analysis_plugins = sorted([ Manual_Gaze_Correction, Eye_Video_Overlay, Pupil_Angle_3D_Fixation_Detector, Gaze_Position_2D_Fixation_Detector ], key=lambda x: x.__name__) user_plugins = sorted(import_runtime_plugins( os.path.join(user_dir, 'plugins')), key=lambda x: x.__name__) available_plugins = vis_plugins + analysis_plugins + user_plugins name_by_index = [p.__name__ for p in available_plugins] index_by_name = dict(zip(name_by_index, range(len(name_by_index)))) plugin_by_name = dict(zip(name_by_index, available_plugins)) logger = logging.getLogger(__name__ + ' with pid: ' + str(os.getpid())) update_recording_to_recent(rec_dir) video_path = [ f for f in glob(os.path.join(rec_dir, "world.*")) if f[-3:] in ('mp4', 'mkv', 'avi') ][0] timestamps_path = os.path.join(rec_dir, "world_timestamps.npy") pupil_data_path = os.path.join(rec_dir, "pupil_data") meta_info = load_meta_info(rec_dir) rec_version = read_rec_version(meta_info) g_pool = Global_Container() g_pool.app = 'exporter' g_pool.min_data_confidence = min_data_confidence timestamps = np.load(timestamps_path) cap = File_Source(g_pool, video_path, timestamps=timestamps) #Out file path verification, we do this before but if one uses a seperate tool, this will kick in. if out_file_path is None: out_file_path = os.path.join(rec_dir, "world_viz.mp4") else: file_name = os.path.basename(out_file_path) dir_name = os.path.dirname(out_file_path) if not dir_name: dir_name = rec_dir if not file_name: file_name = 'world_viz.mp4' out_file_path = os.path.expanduser(os.path.join(dir_name, file_name)) if os.path.isfile(out_file_path): logger.warning("Video out file already exsists. I will overwrite!") os.remove(out_file_path) logger.debug("Saving Video to %s" % out_file_path) #Trim mark verification #make sure the trim marks (start frame, endframe) make sense: We define them like python list slices,thus we can test them like such. trimmed_timestamps = timestamps[start_frame:end_frame] if len(trimmed_timestamps) == 0: logger.warn( "Start and end frames are set such that no video will be exported." ) return False if start_frame == None: start_frame = 0 #these two vars are shared with the lauching process and give a job length and progress report. frames_to_export.value = len(trimmed_timestamps) current_frame.value = 0 logger.debug( "Will export from frame %s to frame %s. This means I will export %s frames." % (start_frame, start_frame + frames_to_export.value, frames_to_export.value)) #setup of writer writer = AV_Writer(out_file_path, fps=cap.frame_rate, use_timestamps=True) cap.seek_to_frame(start_frame) start_time = time() g_pool.capture = cap g_pool.rec_dir = rec_dir g_pool.user_dir = user_dir g_pool.rec_version = rec_version g_pool.timestamps = timestamps g_pool.delayed_notifications = {} g_pool.notifications = [] # load pupil_positions, gaze_positions pupil_data = load_object(pupil_data_path) pupil_list = pupil_data['pupil_positions'] gaze_list = pupil_data['gaze_positions'] g_pool.pupil_positions_by_frame = correlate_data(pupil_list, g_pool.timestamps) g_pool.gaze_positions_by_frame = correlate_data(gaze_list, g_pool.timestamps) g_pool.fixations_by_frame = [[] for x in g_pool.timestamps ] #populated by the fixation detector plugin #add plugins g_pool.plugins = Plugin_List(g_pool, plugin_by_name, plugin_initializers) while frames_to_export.value > current_frame.value: if should_terminate.value: logger.warning("User aborted export. Exported %s frames to %s." % (current_frame.value, out_file_path)) #explicit release of VideoWriter writer.close() writer = None return False try: frame = cap.get_frame_nowait() except EndofVideoFileError: break events = {} #new positons and events events['gaze_positions'] = g_pool.gaze_positions_by_frame[frame.index] events['pupil_positions'] = g_pool.pupil_positions_by_frame[ frame.index] # publish delayed notifiactions when their time has come. for n in g_pool.delayed_notifications.values(): if n['_notify_time_'] < time(): del n['_notify_time_'] del g_pool.delayed_notifications[n['subject']] g_pool.notifications.append(n) # notify each plugin if there are new notifactions: while g_pool.notifications: n = g_pool.notifications.pop(0) for p in g_pool.plugins: p.on_notify(n) # allow each Plugin to do its work. for p in g_pool.plugins: p.update(frame, events) writer.write_video_frame(frame) current_frame.value += 1 writer.close() writer = None duration = time() - start_time effective_fps = float(current_frame.value) / duration logger.info( "Export done: Exported %s frames to %s. This took %s seconds. Exporter ran at %s frames per second" % (current_frame.value, out_file_path, duration, effective_fps)) return True
def export( should_terminate, frames_to_export, current_frame, rec_dir, user_dir, min_data_confidence, start_frame=None, end_frame=None, plugin_initializers=(), out_file_path=None, ): vis_plugins = sorted( [Vis_Circle, Vis_Cross, Vis_Polyline, Vis_Light_Points, Vis_Watermark, Vis_Scan_Path, Vis_Eye_Video_Overlay], key=lambda x: x.__name__, ) analysis_plugins = sorted( [Manual_Gaze_Correction, Pupil_Angle_3D_Fixation_Detector, Gaze_Position_2D_Fixation_Detector], key=lambda x: x.__name__, ) user_plugins = sorted(import_runtime_plugins(os.path.join(user_dir, "plugins")), key=lambda x: x.__name__) available_plugins = vis_plugins + analysis_plugins + user_plugins name_by_index = [p.__name__ for p in available_plugins] plugin_by_name = dict(zip(name_by_index, available_plugins)) logger = logging.getLogger(__name__ + " with pid: " + str(os.getpid())) update_recording_to_recent(rec_dir) video_path = [f for f in glob(os.path.join(rec_dir, "world.*")) if f[-3:] in ("mp4", "mkv", "avi")][0] timestamps_path = os.path.join(rec_dir, "world_timestamps.npy") pupil_data_path = os.path.join(rec_dir, "pupil_data") meta_info = load_meta_info(rec_dir) rec_version = read_rec_version(meta_info) g_pool = Global_Container() g_pool.app = "exporter" g_pool.min_data_confidence = min_data_confidence timestamps = np.load(timestamps_path) cap = File_Source(g_pool, video_path, timestamps=timestamps) # Out file path verification, we do this before but if one uses a seperate tool, this will kick in. if out_file_path is None: out_file_path = os.path.join(rec_dir, "world_viz.mp4") else: file_name = os.path.basename(out_file_path) dir_name = os.path.dirname(out_file_path) if not dir_name: dir_name = rec_dir if not file_name: file_name = "world_viz.mp4" out_file_path = os.path.expanduser(os.path.join(dir_name, file_name)) if os.path.isfile(out_file_path): logger.warning("Video out file already exsists. I will overwrite!") os.remove(out_file_path) logger.debug("Saving Video to {}".format(out_file_path)) # Trim mark verification # make sure the trim marks (start frame, endframe) make sense: # We define them like python list slices, thus we can test them like such. trimmed_timestamps = timestamps[start_frame:end_frame] if len(trimmed_timestamps) == 0: logger.warn("Start and end frames are set such that no video will be exported.") return False if start_frame is None: start_frame = 0 # these two vars are shared with the lauching process and give a job length and progress report. frames_to_export.value = len(trimmed_timestamps) current_frame.value = 0 exp_info = "Will export from frame {} to frame {}. This means I will export {} frames." logger.debug(exp_info.format(start_frame, start_frame + frames_to_export.value, frames_to_export.value)) # setup of writer writer = AV_Writer(out_file_path, fps=cap.frame_rate, use_timestamps=True) cap.seek_to_frame(start_frame) start_time = time() g_pool.capture = cap g_pool.rec_dir = rec_dir g_pool.user_dir = user_dir g_pool.rec_version = rec_version g_pool.timestamps = timestamps g_pool.delayed_notifications = {} g_pool.notifications = [] # load pupil_positions, gaze_positions pupil_data = load_object(pupil_data_path) pupil_list = pupil_data["pupil_positions"] gaze_list = pupil_data["gaze_positions"] g_pool.pupil_positions_by_frame = correlate_data(pupil_list, g_pool.timestamps) g_pool.gaze_positions_by_frame = correlate_data(gaze_list, g_pool.timestamps) g_pool.fixations_by_frame = [[] for x in g_pool.timestamps] # populated by the fixation detector plugin # add plugins g_pool.plugins = Plugin_List(g_pool, plugin_by_name, plugin_initializers) while frames_to_export.value > current_frame.value: if should_terminate.value: logger.warning("User aborted export. Exported {} frames to {}.".format(current_frame.value, out_file_path)) # explicit release of VideoWriter writer.close() writer = None return False try: frame = cap.get_frame_nowait() except EndofVideoFileError: break events = {} # new positons and events events["gaze_positions"] = g_pool.gaze_positions_by_frame[frame.index] events["pupil_positions"] = g_pool.pupil_positions_by_frame[frame.index] # publish delayed notifiactions when their time has come. for n in list(g_pool.delayed_notifications.values()): if n["_notify_time_"] < time(): del n["_notify_time_"] del g_pool.delayed_notifications[n["subject"]] g_pool.notifications.append(n) # notify each plugin if there are new notifactions: while g_pool.notifications: n = g_pool.notifications.pop(0) for p in g_pool.plugins: p.on_notify(n) # allow each Plugin to do its work. for p in g_pool.plugins: p.update(frame, events) writer.write_video_frame(frame) current_frame.value += 1 writer.close() writer = None duration = time() - start_time effective_fps = float(current_frame.value) / duration result = "Export done: Exported {} frames to {}. This took {} seconds. Exporter ran at {} frames per second." logger.info(result.format(current_frame.value, out_file_path, duration, effective_fps)) return True
def session(rec_dir): system_plugins = [Log_Display, Seek_Bar, Trim_Marks] vis_plugins = sorted([Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross, Vis_Watermark, Vis_Eye_Video_Overlay, Vis_Scan_Path], key=lambda x: x.__name__) analysis_plugins = sorted([Gaze_Position_2D_Fixation_Detector, Pupil_Angle_3D_Fixation_Detector, Manual_Gaze_Correction, Video_Export_Launcher, Offline_Surface_Tracker, Raw_Data_Exporter, Batch_Exporter, Annotation_Player], key=lambda x: x.__name__) other_plugins = sorted([Show_Calibration, Log_History, Marker_Auto_Trim_Marks], key=lambda x: x.__name__) user_plugins = sorted(import_runtime_plugins(os.path.join(user_dir, 'plugins')), key=lambda x: x.__name__) user_launchable_plugins = vis_plugins + analysis_plugins + other_plugins + user_plugins available_plugins = system_plugins + user_launchable_plugins name_by_index = [p.__name__ for p in available_plugins] plugin_by_name = dict(zip(name_by_index, available_plugins)) # Callback functions def on_resize(window, w, h): g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() graph.adjust_size(w, h) gl_utils.adjust_gl_view(w, h) for p in g_pool.plugins: p.on_window_resize(window, w, h) def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_char(window, char): g_pool.gui.update_char(char) def on_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) pos = glfwGetCursorPos(window) pos = normalize(pos, glfwGetWindowSize(window)) pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels for p in g_pool.plugins: p.on_click(pos, button, action) def on_pos(window, x, y): hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0]) g_pool.gui.update_mouse(x*hdpi_factor, y*hdpi_factor) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y*scroll_factor) def on_drop(window, count, paths): for x in range(count): new_rec_dir = paths[x].decode('utf-8') if is_pupil_rec_dir(new_rec_dir): logger.debug("Starting new session with '{}'".format(new_rec_dir)) global rec_dir rec_dir = new_rec_dir glfwSetWindowShouldClose(window, True) else: logger.error("'{}' is not a valid pupil recording".format(new_rec_dir)) tick = delta_t() def get_dt(): return next(tick) update_recording_to_recent(rec_dir) video_path = [f for f in glob(os.path.join(rec_dir, "world.*")) if f[-3:] in ('mp4', 'mkv', 'avi')][0] timestamps_path = os.path.join(rec_dir, "world_timestamps.npy") pupil_data_path = os.path.join(rec_dir, "pupil_data") meta_info = load_meta_info(rec_dir) rec_version = read_rec_version(meta_info) app_version = get_version(version_file) # log info about Pupil Platform and Platform in player.log logger.info('Application Version: {}'.format(app_version)) logger.info('System Info: {}'.format(get_system_info())) timestamps = np.load(timestamps_path) # create container for globally scoped vars g_pool = Global_Container() g_pool.app = 'player' # Initialize capture cap = File_Source(g_pool, video_path, timestamps=list(timestamps)) # load session persistent settings session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings")) if session_settings.get("version", VersionFormat('0.0')) < get_version(version_file): logger.info("Session setting are from older version of this app. I will not use those.") session_settings.clear() width, height = session_settings.get('window_size', cap.frame_size) window_pos = session_settings.get('window_position', window_position_default) main_window = glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - " + rec_dir.split(os.path.sep)[-1], None, None) glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfwMakeContextCurrent(main_window) cygl.utils.init() # load pupil_positions, gaze_positions pupil_data = load_object(pupil_data_path) pupil_list = pupil_data['pupil_positions'] gaze_list = pupil_data['gaze_positions'] g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular' g_pool.version = app_version g_pool.capture = cap g_pool.timestamps = timestamps g_pool.play = False g_pool.new_seek = True g_pool.user_dir = user_dir g_pool.rec_dir = rec_dir g_pool.rec_version = rec_version g_pool.meta_info = meta_info g_pool.min_data_confidence = session_settings.get('min_data_confidence', 0.6) g_pool.pupil_positions_by_frame = correlate_data(pupil_list, g_pool.timestamps) g_pool.gaze_positions_by_frame = correlate_data(gaze_list, g_pool.timestamps) g_pool.fixations_by_frame = [[] for x in g_pool.timestamps] # populated by the fixation detector plugin def next_frame(_): try: cap.seek_to_frame(cap.get_frame_index()) except(FileSeekError): logger.warning("Could not seek to next frame.") else: g_pool.new_seek = True def prev_frame(_): try: cap.seek_to_frame(cap.get_frame_index()-2) except(FileSeekError): logger.warning("Could not seek to previous frame.") else: g_pool.new_seek = True def toggle_play(new_state): if cap.get_frame_index() >= cap.get_frame_count()-5: cap.seek_to_frame(1) # avoid pause set by hitting trimmark pause. logger.warning("End of video - restart at beginning.") g_pool.play = new_state def set_scale(new_scale): g_pool.gui.scale = new_scale g_pool.gui.collect_menus() def set_data_confidence(new_confidence): g_pool.min_data_confidence = new_confidence notification = {'subject': 'min_data_confidence_changed'} notification['_notify_time_'] = time()+.8 g_pool.delayed_notifications[notification['subject']] = notification def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def purge_plugins(): for p in g_pool.plugins: if p.__class__ in user_launchable_plugins: p.alive = False g_pool.plugins.clean() def do_export(_): export_range = slice(g_pool.trim_marks.in_mark, g_pool.trim_marks.out_mark) export_dir = os.path.join(g_pool.rec_dir, 'exports', '{}-{}'.format(export_range.start, export_range.stop)) try: os.makedirs(export_dir) except OSError as e: if e.errno != errno.EEXIST: logger.error("Could not create export dir") raise e else: overwrite_warning = "Previous export for range [{}-{}] already exsits - overwriting." logger.warning(overwrite_warning.format(export_range.start, export_range.stop)) else: logger.info('Created export dir at "{}"'.format(export_dir)) notification = {'subject': 'should_export', 'range': export_range, 'export_dir': export_dir} g_pool.notifications.append(notification) g_pool.gui = ui.UI() g_pool.gui.scale = session_settings.get('gui_scale', 1) g_pool.main_menu = ui.Scrolling_Menu("Settings", pos=(-350, 20), size=(300, 500)) g_pool.main_menu.append(ui.Button("Close Pupil Player", lambda: glfwSetWindowShouldClose(main_window, True))) g_pool.main_menu.append(ui.Slider('scale', g_pool.gui, setter=set_scale, step=.05, min=0.75, max=2.5, label='Interface Size')) g_pool.main_menu.append(ui.Info_Text('Player Version: {}'.format(g_pool.version))) g_pool.main_menu.append(ui.Info_Text('Recording Version: {}'.format(rec_version))) g_pool.main_menu.append(ui.Slider('min_data_confidence', g_pool, setter=set_data_confidence, step=.05, min=0.0, max=1.0, label='Confidence threshold')) selector_label = "Select to load" vis_labels = [" " + p.__name__.replace('_', ' ') for p in vis_plugins] analysis_labels = [" " + p.__name__.replace('_', ' ') for p in analysis_plugins] other_labels = [" " + p.__name__.replace('_', ' ') for p in other_plugins] user_labels = [" " + p.__name__.replace('_', ' ') for p in user_plugins] plugins = ([selector_label, selector_label] + vis_plugins + [selector_label] + analysis_plugins + [selector_label] + other_plugins + [selector_label] + user_plugins) labels = ([selector_label, "Visualization"] + vis_labels + ["Analysis"] + analysis_labels + ["Other"] + other_labels + ["User added"] + user_labels) g_pool.main_menu.append(ui.Selector('Open plugin:', selection=plugins, labels=labels, setter=open_plugin, getter=lambda: selector_label)) g_pool.main_menu.append(ui.Button('Close all plugins', purge_plugins)) g_pool.main_menu.append(ui.Button('Reset window size', lambda: glfwSetWindowSize(main_window, cap.frame_size[0], cap.frame_size[1]))) g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100)) g_pool.play_button = ui.Thumb('play', g_pool, label=chr(0xf04b), setter=toggle_play, hotkey=GLFW_KEY_SPACE, label_font='fontawesome', label_offset_x=5, label_offset_y=0, label_offset_size=-24) g_pool.play_button.on_color[:] = (0, 1., .0, .8) g_pool.forward_button = ui.Thumb('forward', label=chr(0xf04e), getter=lambda: False, setter=next_frame, hotkey=GLFW_KEY_RIGHT, label_font='fontawesome', label_offset_x=5, label_offset_y=0, label_offset_size=-24) g_pool.backward_button = ui.Thumb('backward', label=chr(0xf04a), getter=lambda: False, setter=prev_frame, hotkey=GLFW_KEY_LEFT, label_font='fontawesome', label_offset_x=-5, label_offset_y=0, label_offset_size=-24) g_pool.export_button = ui.Thumb('export', label=chr(0xf063), getter=lambda: False, setter=do_export, hotkey='e', label_font='fontawesome', label_offset_x=0, label_offset_y=2, label_offset_size=-24) g_pool.quickbar.extend([g_pool.play_button, g_pool.forward_button, g_pool.backward_button, g_pool.export_button]) g_pool.gui.append(g_pool.quickbar) g_pool.gui.append(g_pool.main_menu) # we always load these plugins system_plugins = [('Trim_Marks', {}), ('Seek_Bar', {})] default_plugins = [('Log_Display', {}), ('Vis_Scan_Path', {}), ('Vis_Polyline', {}), ('Vis_Circle', {}), ('Video_Export_Launcher', {})] previous_plugins = session_settings.get('loaded_plugins', default_plugins) g_pool.notifications = [] g_pool.delayed_notifications = {} g_pool.plugins = Plugin_List(g_pool, plugin_by_name, system_plugins+previous_plugins) # Register callbacks main_window glfwSetFramebufferSizeCallback(main_window, on_resize) glfwSetKeyCallback(main_window, on_key) glfwSetCharCallback(main_window, on_char) glfwSetMouseButtonCallback(main_window, on_button) glfwSetCursorPosCallback(main_window, on_pos) glfwSetScrollCallback(main_window, on_scroll) glfwSetDropCallback(main_window, on_drop) # trigger on_resize on_resize(main_window, *glfwGetFramebufferSize(main_window)) g_pool.gui.configuration = session_settings.get('ui_config', {}) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() # set up performace graphs: pid = os.getpid() ps = psutil.Process(pid) ts = None cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 110) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 110) fps_graph.update_rate = 5 fps_graph.label = "%0.0f REC FPS" pupil_graph = graph.Bar_Graph(max_val=1.0) pupil_graph.pos = (260, 110) pupil_graph.update_rate = 5 pupil_graph.label = "Confidence: %0.2f" while not glfwWindowShouldClose(main_window): # grab new frame if g_pool.play or g_pool.new_seek: g_pool.new_seek = False try: new_frame = cap.get_frame_nowait() except EndofVideoFileError: # end of video logic: pause at last frame. g_pool.play = False logger.warning("end of video") update_graph = True else: update_graph = False frame = new_frame.copy() events = {} # report time between now and the last loop interation events['dt'] = get_dt() # new positons we make a deepcopy just like the image is a copy. events['gaze_positions'] = deepcopy(g_pool.gaze_positions_by_frame[frame.index]) events['pupil_positions'] = deepcopy(g_pool.pupil_positions_by_frame[frame.index]) if update_graph: # update performace graphs for p in events['pupil_positions']: pupil_graph.add(p['confidence']) t = new_frame.timestamp if ts and ts != t: dt, ts = t-ts, t fps_graph.add(1./dt) else: ts = new_frame.timestamp g_pool.play_button.status_text = str(frame.index) # always update the CPU graph cpu_graph.update() # publish delayed notifiactions when their time has come. for n in list(g_pool.delayed_notifications.values()): if n['_notify_time_'] < time(): del n['_notify_time_'] del g_pool.delayed_notifications[n['subject']] g_pool.notifications.append(n) # notify each plugin if there are new notifactions: while g_pool.notifications: n = g_pool.notifications.pop(0) for p in g_pool.plugins: p.on_notify(n) # allow each Plugin to do its work. for p in g_pool.plugins: p.update(frame, events) # check if a plugin need to be destroyed g_pool.plugins.clean() # render camera image glfwMakeContextCurrent(main_window) gl_utils.make_coord_system_norm_based() g_pool.image_tex.update_from_frame(frame) g_pool.image_tex.draw() gl_utils.make_coord_system_pixel_based(frame.img.shape) # render visual feedback from loaded plugins for p in g_pool.plugins: p.gl_display() graph.push_view() fps_graph.draw() cpu_graph.draw() pupil_graph.draw() graph.pop_view() g_pool.gui.update() # present frames at appropriate speed cap.wait(frame) glfwSwapBuffers(main_window) glfwPollEvents() session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['min_data_confidence'] = g_pool.min_data_confidence session_settings['gui_scale'] = g_pool.gui.scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['window_size'] = glfwGetWindowSize(main_window) session_settings['window_position'] = glfwGetWindowPos(main_window) session_settings['version'] = g_pool.version session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() cap.cleanup() g_pool.gui.terminate() glfwDestroyWindow(main_window)
def session(rec_dir): system_plugins = [Log_Display, Seek_Bar, Trim_Marks] vis_plugins = sorted([ Vis_Circle, Vis_Polyline, Vis_Light_Points, Vis_Cross, Vis_Watermark, Eye_Video_Overlay, Scan_Path ], key=lambda x: x.__name__) analysis_plugins = sorted([ Gaze_Position_2D_Fixation_Detector, Pupil_Angle_3D_Fixation_Detector, Manual_Gaze_Correction, Video_Export_Launcher, Offline_Surface_Tracker, Raw_Data_Exporter, Batch_Exporter, Annotation_Player ], key=lambda x: x.__name__) other_plugins = sorted([Show_Calibration, Log_History], key=lambda x: x.__name__) user_plugins = sorted(import_runtime_plugins( os.path.join(user_dir, 'plugins')), key=lambda x: x.__name__) user_launchable_plugins = vis_plugins + analysis_plugins + other_plugins + user_plugins available_plugins = system_plugins + user_launchable_plugins name_by_index = [p.__name__ for p in available_plugins] index_by_name = dict(zip(name_by_index, range(len(name_by_index)))) plugin_by_name = dict(zip(name_by_index, available_plugins)) # Callback functions def on_resize(window, w, h): g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() graph.adjust_size(w, h) adjust_gl_view(w, h) for p in g_pool.plugins: p.on_window_resize(window, w, h) def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_char(window, char): g_pool.gui.update_char(char) def on_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) pos = glfwGetCursorPos(window) pos = normalize(pos, glfwGetWindowSize(window)) pos = denormalize( pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels for p in g_pool.plugins: p.on_click(pos, button, action) def on_pos(window, x, y): hdpi_factor = float( glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0]) g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * y_scroll_factor) def on_drop(window, count, paths): for x in range(count): new_rec_dir = paths[x] if is_pupil_rec_dir(new_rec_dir): logger.debug("Starting new session with '%s'" % new_rec_dir) global rec_dir rec_dir = new_rec_dir glfwSetWindowShouldClose(window, True) else: logger.error("'%s' is not a valid pupil recording" % new_rec_dir) tick = delta_t() def get_dt(): return next(tick) update_recording_to_recent(rec_dir) video_path = [ f for f in glob(os.path.join(rec_dir, "world.*")) if f[-3:] in ('mp4', 'mkv', 'avi') ][0] timestamps_path = os.path.join(rec_dir, "world_timestamps.npy") pupil_data_path = os.path.join(rec_dir, "pupil_data") meta_info = load_meta_info(rec_dir) rec_version = read_rec_version(meta_info) app_version = get_version(version_file) # log info about Pupil Platform and Platform in player.log logger.info('Application Version: %s' % app_version) logger.info('System Info: %s' % get_system_info()) timestamps = np.load(timestamps_path) # create container for globally scoped vars g_pool = Global_Container() g_pool.app = 'player' # Initialize capture cap = File_Source(g_pool, video_path, timestamps=list(timestamps)) # load session persistent settings session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings")) if session_settings.get("version", VersionFormat('0.0')) < get_version(version_file): logger.info( "Session setting are from older version of this app. I will not use those." ) session_settings.clear() width, height = session_settings.get('window_size', cap.frame_size) window_pos = session_settings.get('window_position', (0, 0)) main_window = glfwCreateWindow( width, height, "Pupil Player: " + meta_info["Recording Name"] + " - " + rec_dir.split(os.path.sep)[-1], None, None) glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfwMakeContextCurrent(main_window) cygl.utils.init() # load pupil_positions, gaze_positions pupil_data = load_object(pupil_data_path) pupil_list = pupil_data['pupil_positions'] gaze_list = pupil_data['gaze_positions'] g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular' g_pool.version = app_version g_pool.capture = cap g_pool.timestamps = timestamps g_pool.play = False g_pool.new_seek = True g_pool.user_dir = user_dir g_pool.rec_dir = rec_dir g_pool.rec_version = rec_version g_pool.meta_info = meta_info g_pool.min_data_confidence = session_settings.get('min_data_confidence', 0.6) g_pool.pupil_positions_by_frame = correlate_data(pupil_list, g_pool.timestamps) g_pool.gaze_positions_by_frame = correlate_data(gaze_list, g_pool.timestamps) g_pool.fixations_by_frame = [[] for x in g_pool.timestamps ] #populated by the fixation detector plugin def next_frame(_): try: cap.seek_to_frame(cap.get_frame_index()) except FileSeekError: logger.warning("Could not seek to next frame.") else: g_pool.new_seek = True def prev_frame(_): try: cap.seek_to_frame(cap.get_frame_index() - 2) except FileSeekError: logger.warning("Could not seek to previous frame.") else: g_pool.new_seek = True def toggle_play(new_state): if cap.get_frame_index() >= cap.get_frame_count() - 5: cap.seek_to_frame(1) #avoid pause set by hitting trimmark pause. logger.warning("End of video - restart at beginning.") g_pool.play = new_state def set_scale(new_scale): g_pool.gui.scale = new_scale g_pool.gui.collect_menus() def set_data_confidence(new_confidence): g_pool.min_data_confidence = new_confidence notification = {'subject': 'min_data_confidence_changed'} notification['_notify_time_'] = time() + .8 g_pool.delayed_notifications[notification['subject']] = notification def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def purge_plugins(): for p in g_pool.plugins: if p.__class__ in user_launchable_plugins: p.alive = False g_pool.plugins.clean() def do_export(_): export_range = slice(g_pool.trim_marks.in_mark, g_pool.trim_marks.out_mark) export_dir = os.path.join( g_pool.rec_dir, 'exports', '%s-%s' % (export_range.start, export_range.stop)) try: os.makedirs(export_dir) except OSError as e: if e.errno != errno.EEXIST: logger.error("Could not create export dir") raise e else: logger.warning( "Previous export for range [%s-%s] already exsits - overwriting." % (export_range.start, export_range.stop)) else: logger.info('Created export dir at "%s"' % export_dir) notification = { 'subject': 'should_export', 'range': export_range, 'export_dir': export_dir } g_pool.notifications.append(notification) g_pool.gui = ui.UI() g_pool.gui.scale = session_settings.get('gui_scale', 1) g_pool.main_menu = ui.Scrolling_Menu("Settings", pos=(-350, 20), size=(300, 500)) g_pool.main_menu.append( ui.Button("Close Pupil Player", lambda: glfwSetWindowShouldClose(main_window, True))) g_pool.main_menu.append( ui.Slider('scale', g_pool.gui, setter=set_scale, step=.05, min=0.75, max=2.5, label='Interface Size')) g_pool.main_menu.append(ui.Info_Text('Player Version: %s' % g_pool.version)) g_pool.main_menu.append(ui.Info_Text('Recording Version: %s' % rec_version)) g_pool.main_menu.append( ui.Slider('min_data_confidence', g_pool, setter=set_data_confidence, step=.05, min=0.0, max=1.0, label='Confidence threshold')) selector_label = "Select to load" vis_labels = [" " + p.__name__.replace('_', ' ') for p in vis_plugins] analysis_labels = [ " " + p.__name__.replace('_', ' ') for p in analysis_plugins ] other_labels = [ " " + p.__name__.replace('_', ' ') for p in other_plugins ] user_labels = [" " + p.__name__.replace('_', ' ') for p in user_plugins] plugins = [selector_label, selector_label] + vis_plugins + [ selector_label ] + analysis_plugins + [selector_label] + other_plugins + [selector_label ] + user_plugins labels = [selector_label, "Visualization"] + vis_labels + [ "Analysis" ] + analysis_labels + ["Other"] + other_labels + ["User added" ] + user_labels g_pool.main_menu.append( ui.Selector('Open plugin:', selection=plugins, labels=labels, setter=open_plugin, getter=lambda: selector_label)) g_pool.main_menu.append(ui.Button('Close all plugins', purge_plugins)) g_pool.main_menu.append( ui.Button( 'Reset window size', lambda: glfwSetWindowSize( main_window, cap.frame_size[0], cap.frame_size[1]))) g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100)) g_pool.play_button = ui.Thumb('play', g_pool, label=unichr(0xf04b).encode('utf-8'), setter=toggle_play, hotkey=GLFW_KEY_SPACE, label_font='fontawesome', label_offset_x=5, label_offset_y=0, label_offset_size=-24) g_pool.play_button.on_color[:] = (0, 1., .0, .8) g_pool.forward_button = ui.Thumb('forward', label=unichr(0xf04e).encode('utf-8'), getter=lambda: False, setter=next_frame, hotkey=GLFW_KEY_RIGHT, label_font='fontawesome', label_offset_x=5, label_offset_y=0, label_offset_size=-24) g_pool.backward_button = ui.Thumb('backward', label=unichr(0xf04a).encode('utf-8'), getter=lambda: False, setter=prev_frame, hotkey=GLFW_KEY_LEFT, label_font='fontawesome', label_offset_x=-5, label_offset_y=0, label_offset_size=-24) g_pool.export_button = ui.Thumb('export', label=unichr(0xf063).encode('utf-8'), getter=lambda: False, setter=do_export, hotkey='e', label_font='fontawesome', label_offset_x=0, label_offset_y=2, label_offset_size=-24) g_pool.quickbar.extend([ g_pool.play_button, g_pool.forward_button, g_pool.backward_button, g_pool.export_button ]) g_pool.gui.append(g_pool.quickbar) g_pool.gui.append(g_pool.main_menu) #we always load these plugins system_plugins = [('Trim_Marks', {}), ('Seek_Bar', {})] default_plugins = [('Log_Display', {}), ('Scan_Path', {}), ('Vis_Polyline', {}), ('Vis_Circle', {}), ('Video_Export_Launcher', {})] previous_plugins = session_settings.get('loaded_plugins', default_plugins) g_pool.notifications = [] g_pool.delayed_notifications = {} g_pool.plugins = Plugin_List(g_pool, plugin_by_name, system_plugins + previous_plugins) # Register callbacks main_window glfwSetFramebufferSizeCallback(main_window, on_resize) glfwSetKeyCallback(main_window, on_key) glfwSetCharCallback(main_window, on_char) glfwSetMouseButtonCallback(main_window, on_button) glfwSetCursorPosCallback(main_window, on_pos) glfwSetScrollCallback(main_window, on_scroll) glfwSetDropCallback(main_window, on_drop) #trigger on_resize on_resize(main_window, *glfwGetFramebufferSize(main_window)) g_pool.gui.configuration = session_settings.get('ui_config', {}) # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() #set up performace graphs: pid = os.getpid() ps = psutil.Process(pid) ts = None cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 110) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 110) fps_graph.update_rate = 5 fps_graph.label = "%0.0f REC FPS" pupil_graph = graph.Bar_Graph(max_val=1.0) pupil_graph.pos = (260, 110) pupil_graph.update_rate = 5 pupil_graph.label = "Confidence: %0.2f" while not glfwWindowShouldClose(main_window): #grab new frame if g_pool.play or g_pool.new_seek: g_pool.new_seek = False try: new_frame = cap.get_frame_nowait() except EndofVideoFileError: #end of video logic: pause at last frame. g_pool.play = False logger.warning("end of video") update_graph = True else: update_graph = False frame = new_frame.copy() events = {} #report time between now and the last loop interation events['dt'] = get_dt() #new positons we make a deepcopy just like the image is a copy. events['gaze_positions'] = deepcopy( g_pool.gaze_positions_by_frame[frame.index]) events['pupil_positions'] = deepcopy( g_pool.pupil_positions_by_frame[frame.index]) if update_graph: #update performace graphs for p in events['pupil_positions']: pupil_graph.add(p['confidence']) t = new_frame.timestamp if ts and ts != t: dt, ts = t - ts, t fps_graph.add(1. / dt) else: ts = new_frame.timestamp g_pool.play_button.status_text = str(frame.index) #always update the CPU graph cpu_graph.update() # publish delayed notifiactions when their time has come. for n in g_pool.delayed_notifications.values(): if n['_notify_time_'] < time(): del n['_notify_time_'] del g_pool.delayed_notifications[n['subject']] g_pool.notifications.append(n) # notify each plugin if there are new notifactions: while g_pool.notifications: n = g_pool.notifications.pop(0) for p in g_pool.plugins: p.on_notify(n) # allow each Plugin to do its work. for p in g_pool.plugins: p.update(frame, events) #check if a plugin need to be destroyed g_pool.plugins.clean() # render camera image glfwMakeContextCurrent(main_window) make_coord_system_norm_based() g_pool.image_tex.update_from_frame(frame) g_pool.image_tex.draw() make_coord_system_pixel_based(frame.img.shape) # render visual feedback from loaded plugins for p in g_pool.plugins: p.gl_display() graph.push_view() fps_graph.draw() cpu_graph.draw() pupil_graph.draw() graph.pop_view() g_pool.gui.update() #present frames at appropriate speed cap.wait(frame) glfwSwapBuffers(main_window) glfwPollEvents() session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['min_data_confidence'] = g_pool.min_data_confidence session_settings['gui_scale'] = g_pool.gui.scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['window_size'] = glfwGetWindowSize(main_window) session_settings['window_position'] = glfwGetWindowPos(main_window) session_settings['version'] = g_pool.version session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() cap.cleanup() g_pool.gui.terminate() glfwDestroyWindow(main_window)