def update_recording_v19_v111(rec_dir): logger.info("Updating recording from v1.9 to v1.11") meta_info = pm.load_meta_info(rec_dir) meta_info["Data Format Version"] = "v1.11" meta_info["Recording UUID"] = meta_info.get("Recording UUID", uuid.uuid4()) update_meta_info(rec_dir, meta_info)
def update_recording_to_recent(rec_dir): meta_info = pm.load_meta_info(rec_dir) update_meta_info(rec_dir, meta_info) if ( meta_info.get("Capture Software", "Pupil Capture") == "Pupil Mobile" and "Data Format Version" not in meta_info ): convert_pupil_mobile_recording_to_v094(rec_dir) meta_info["Data Format Version"] = "v0.9.4" update_meta_info(rec_dir, meta_info) # Reference format: v0.7.4 rec_version = read_rec_version(meta_info) # Convert python2 to python3 if rec_version <= VersionFormat("0.8.7"): update_recording_bytes_to_unicode(rec_dir) if rec_version >= VersionFormat("0.7.4"): pass elif rec_version >= VersionFormat("0.7.3"): update_recording_v073_to_v074(rec_dir) elif rec_version >= VersionFormat("0.5"): update_recording_v05_to_v074(rec_dir) elif rec_version >= VersionFormat("0.4"): update_recording_v04_to_v074(rec_dir) elif rec_version >= VersionFormat("0.3"): update_recording_v03_to_v074(rec_dir) else: logger.Error("This recording is too old. Sorry.") return # Incremental format updates if rec_version < VersionFormat("0.8.2"): update_recording_v074_to_v082(rec_dir) if rec_version < VersionFormat("0.8.3"): update_recording_v082_to_v083(rec_dir) if rec_version < VersionFormat("0.8.6"): update_recording_v083_to_v086(rec_dir) if rec_version < VersionFormat("0.8.7"): update_recording_v086_to_v087(rec_dir) if rec_version < VersionFormat("0.9.1"): update_recording_v087_to_v091(rec_dir) if rec_version < VersionFormat("0.9.3"): update_recording_v091_to_v093(rec_dir) if rec_version < VersionFormat("0.9.4"): update_recording_v093_to_v094(rec_dir) if rec_version < VersionFormat("0.9.13"): update_recording_v094_to_v0913(rec_dir) if rec_version < VersionFormat("1.3"): update_recording_v0913_to_v13(rec_dir) if rec_version < VersionFormat("1.4"): update_recording_v13_v14(rec_dir) # Do this independent of rec_version check_for_worldless_recording(rec_dir) if rec_version < VersionFormat("1.8"): update_recording_v14_v18(rec_dir) if rec_version < VersionFormat("1.9"): update_recording_v18_v19(rec_dir) if rec_version < VersionFormat("1.11"): update_recording_v19_v111(rec_dir) if rec_version < VersionFormat("1.13"): update_recording_v111_v113(rec_dir)
def _export_world_video( rec_dir, user_dir, min_data_confidence, start_frame, end_frame, plugin_initializers, out_file_path, pre_computed_eye_data, ): """ Simulates the generation for the world video and saves a certain time range as a video. It simulates a whole g_pool such that all plugins run as normal. """ from glob import glob from time import time import file_methods as fm import player_methods as pm from av_writer import AV_Writer # we are not importing manual gaze correction. In Player corrections have already been applied. # in batch exporter this plugin makes little sense. from fixation_detector import Offline_Fixation_Detector # Plug-ins from plugin import Plugin_List, import_runtime_plugins from video_capture import EndofVideoError, File_Source from vis_circle import Vis_Circle from vis_cross import Vis_Cross from vis_eye_video_overlay import Vis_Eye_Video_Overlay from vis_light_points import Vis_Light_Points from vis_polyline import Vis_Polyline from vis_scan_path import Vis_Scan_Path from vis_watermark import Vis_Watermark PID = str(os.getpid()) logger = logging.getLogger(__name__ + " with pid: " + PID) start_status = "Starting video export with pid: {}".format(PID) logger.info(start_status) yield start_status, 0 try: vis_plugins = sorted( [ Vis_Circle, Vis_Cross, Vis_Polyline, Vis_Light_Points, Vis_Watermark, Vis_Scan_Path, Vis_Eye_Video_Overlay, ], key=lambda x: x.__name__, ) analysis_plugins = [Offline_Fixation_Detector] user_plugins = sorted( import_runtime_plugins(os.path.join(user_dir, "plugins")), key=lambda x: x.__name__, ) available_plugins = vis_plugins + analysis_plugins + user_plugins name_by_index = [p.__name__ for p in available_plugins] plugin_by_name = dict(zip(name_by_index, available_plugins)) meta_info = pm.load_meta_info(rec_dir) g_pool = GlobalContainer() g_pool.app = "exporter" g_pool.min_data_confidence = min_data_confidence valid_ext = (".mp4", ".mkv", ".avi", ".h264", ".mjpeg", ".fake") try: video_path = next(f for f in glob(os.path.join(rec_dir, "world.*")) if os.path.splitext(f)[1] in valid_ext) except StopIteration: raise FileNotFoundError("No Video world found") cap = File_Source(g_pool, source_path=video_path, fill_gaps=True, timing=None) timestamps = cap.timestamps file_name = os.path.basename(out_file_path) dir_name = os.path.dirname(out_file_path) out_file_path = os.path.expanduser(os.path.join(dir_name, file_name)) if os.path.isfile(out_file_path): logger.warning("Video out file already exsists. I will overwrite!") os.remove(out_file_path) logger.debug("Saving Video to {}".format(out_file_path)) # Trim mark verification # make sure the trim marks (start frame, end frame) make sense: # We define them like python list slices, thus we can test them like such. trimmed_timestamps = timestamps[start_frame:end_frame] if len(trimmed_timestamps) == 0: warn = "Start and end frames are set such that no video will be exported." logger.warning(warn) yield warn, 0.0 return if start_frame is None: start_frame = 0 # these two vars are shared with the launching process and give a job length and progress report. frames_to_export = len(trimmed_timestamps) current_frame = 0 exp_info = ( "Will export from frame {} to frame {}. This means I will export {} frames." ) logger.debug( exp_info.format(start_frame, start_frame + frames_to_export, frames_to_export)) # setup of writer writer = AV_Writer(out_file_path, fps=cap.frame_rate, audio_dir=rec_dir, use_timestamps=True) cap.seek_to_frame(start_frame) start_time = time() g_pool.plugin_by_name = plugin_by_name g_pool.capture = cap g_pool.rec_dir = rec_dir g_pool.user_dir = user_dir g_pool.meta_info = meta_info g_pool.timestamps = timestamps g_pool.delayed_notifications = {} g_pool.notifications = [] for initializers in pre_computed_eye_data.values(): initializers["data"] = [ fm.Serialized_Dict(msgpack_bytes=serialized) for serialized in initializers["data"] ] g_pool.pupil_positions = pm.Bisector(**pre_computed_eye_data["pupil"]) g_pool.gaze_positions = pm.Bisector(**pre_computed_eye_data["gaze"]) g_pool.fixations = pm.Affiliator(**pre_computed_eye_data["fixations"]) # add plugins g_pool.plugins = Plugin_List(g_pool, plugin_initializers) while frames_to_export > current_frame: try: frame = cap.get_frame() except EndofVideoError: break events = {"frame": frame} # new positions and events frame_window = pm.enclosing_window(g_pool.timestamps, frame.index) events["gaze"] = g_pool.gaze_positions.by_ts_window(frame_window) events["pupil"] = g_pool.pupil_positions.by_ts_window(frame_window) # publish delayed notifications when their time has come. for n in list(g_pool.delayed_notifications.values()): if n["_notify_time_"] < time(): del n["_notify_time_"] del g_pool.delayed_notifications[n["subject"]] g_pool.notifications.append(n) # notify each plugin if there are new notifications: while g_pool.notifications: n = g_pool.notifications.pop(0) for p in g_pool.plugins: p.on_notify(n) # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) writer.write_video_frame(frame) current_frame += 1 yield "Exporting with pid {}".format(PID), current_frame writer.close(timestamp_export_format="all") duration = time() - start_time effective_fps = float(current_frame) / duration result = "Export done: Exported {} frames to {}. This took {} seconds. Exporter ran at {} frames per second." logger.info( result.format(current_frame, out_file_path, duration, effective_fps)) yield "Export done. This took {:.0f} seconds.".format( duration), current_frame except GeneratorExit: logger.warning("Video export with pid {} was canceled.".format( os.getpid()))
def _update_info_version_to(new_version, rec_dir): meta_info = pm.load_meta_info(rec_dir) meta_info["Data Format Version"] = new_version update_meta_info(rec_dir, meta_info)
def export(should_terminate,frames_to_export,current_frame, rec_dir,user_dir,start_frame=None,end_frame=None,plugin_initializers=[],out_file_path=None): logger = logging.getLogger(__name__+' with pid: '+str(os.getpid()) ) update_recording_to_recent(rec_dir) video_path = [f for f in glob(os.path.join(rec_dir,"world.*")) if f[-3:] in ('mp4','mkv','avi')][0] timestamps_path = os.path.join(rec_dir, "world_timestamps.npy") pupil_data_path = os.path.join(rec_dir, "pupil_data") meta_info = load_meta_info(rec_dir) rec_version = read_rec_version(meta_info) timestamps = np.load(timestamps_path) cap = File_Capture(video_path,timestamps=timestamps) #Out file path verification, we do this before but if one uses a seperate tool, this will kick in. if out_file_path is None: out_file_path = os.path.join(rec_dir, "world_viz.mp4") else: file_name = os.path.basename(out_file_path) dir_name = os.path.dirname(out_file_path) if not dir_name: dir_name = rec_dir if not file_name: file_name = 'world_viz.mp4' out_file_path = os.path.expanduser(os.path.join(dir_name,file_name)) if os.path.isfile(out_file_path): logger.warning("Video out file already exsists. I will overwrite!") os.remove(out_file_path) logger.debug("Saving Video to %s"%out_file_path) #Trim mark verification #make sure the trim marks (start frame, endframe) make sense: We define them like python list slices,thus we can test them like such. trimmed_timestamps = timestamps[start_frame:end_frame] if len(trimmed_timestamps)==0: logger.warn("Start and end frames are set such that no video will be exported.") return False if start_frame == None: start_frame = 0 #these two vars are shared with the lauching process and give a job length and progress report. frames_to_export.value = len(trimmed_timestamps) current_frame.value = 0 logger.debug("Will export from frame %s to frame %s. This means I will export %s frames."%(start_frame,start_frame+frames_to_export.value,frames_to_export.value)) #setup of writer writer = AV_Writer(out_file_path,fps=cap.frame_rate,use_timestamps=True) cap.seek_to_frame(start_frame) start_time = time() g = Global_Container() g.app = 'exporter' g.capture = cap g.rec_dir = rec_dir g.user_dir = user_dir g.rec_version = rec_version g.timestamps = timestamps # load pupil_positions, gaze_positions pupil_data = load_object(pupil_data_path) pupil_list = pupil_data['pupil_positions'] gaze_list = pupil_data['gaze_positions'] g.pupil_positions_by_frame = correlate_data(pupil_list,g.timestamps) g.gaze_positions_by_frame = correlate_data(gaze_list,g.timestamps) g.fixations_by_frame = [[] for x in g.timestamps] #populated by the fixation detector plugin #add plugins g.plugins = Plugin_List(g,plugin_by_name,plugin_initializers) while frames_to_export.value > current_frame.value: if should_terminate.value: logger.warning("User aborted export. Exported %s frames to %s."%(current_frame.value,out_file_path)) #explicit release of VideoWriter writer.close() writer = None return False try: frame = cap.get_frame_nowait() except EndofVideoFileError: break events = {} #new positons and events events['gaze_positions'] = g.gaze_positions_by_frame[frame.index] events['pupil_positions'] = g.pupil_positions_by_frame[frame.index] # allow each Plugin to do its work. for p in g.plugins: p.update(frame,events) writer.write_video_frame(frame) current_frame.value +=1 writer.close() writer = None duration = time()-start_time effective_fps = float(current_frame.value)/duration logger.info("Export done: Exported %s frames to %s. This took %s seconds. Exporter ran at %s frames per second"%(current_frame.value,out_file_path,duration,effective_fps)) return True
def session(rec_dir): # Callback functions def on_resize(window,w, h): g_pool.gui.update_window(w,h) g_pool.gui.collect_menus() graph.adjust_size(w,h) adjust_gl_view(w,h) for p in g_pool.plugins: p.on_window_resize(window,w,h) def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key,scancode,action,mods) def on_char(window,char): g_pool.gui.update_char(char) def on_button(window,button, action, mods): g_pool.gui.update_button(button,action,mods) pos = glfwGetCursorPos(window) pos = normalize(pos,glfwGetWindowSize(window)) pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels for p in g_pool.plugins: p.on_click(pos,button,action) def on_pos(window,x, y): hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0]) g_pool.gui.update_mouse(x*hdpi_factor,y*hdpi_factor) def on_scroll(window,x,y): g_pool.gui.update_scroll(x,y*y_scroll_factor) def on_drop(window,count,paths): for x in range(count): new_rec_dir = paths[x] if is_pupil_rec_dir(new_rec_dir): logger.debug("Starting new session with '%s'"%new_rec_dir) global rec_dir rec_dir = new_rec_dir glfwSetWindowShouldClose(window,True) else: logger.error("'%s' is not a valid pupil recording"%new_rec_dir) tick = delta_t() def get_dt(): return next(tick) update_recording_to_recent(rec_dir) video_path = [f for f in glob(os.path.join(rec_dir,"world.*")) if f[-3:] in ('mp4','mkv','avi')][0] timestamps_path = os.path.join(rec_dir, "world_timestamps.npy") pupil_data_path = os.path.join(rec_dir, "pupil_data") meta_info = load_meta_info(rec_dir) rec_version = read_rec_version(meta_info) app_version = get_version(version_file) # log info about Pupil Platform and Platform in player.log logger.info('Application Version: %s'%app_version) logger.info('System Info: %s'%get_system_info()) timestamps = np.load(timestamps_path) # Initialize capture cap = File_Capture(video_path,timestamps=list(timestamps)) # load session persistent settings session_settings = Persistent_Dict(os.path.join(user_dir,"user_settings")) if session_settings.get("version",VersionFormat('0.0')) < get_version(version_file): logger.info("Session setting are from older version of this app. I will not use those.") session_settings.clear() width,height = session_settings.get('window_size',cap.frame_size) window_pos = session_settings.get('window_position',(0,0)) main_window = glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - "+ rec_dir.split(os.path.sep)[-1], None, None) glfwSetWindowPos(main_window,window_pos[0],window_pos[1]) glfwMakeContextCurrent(main_window) cygl.utils.init() # load pupil_positions, gaze_positions pupil_data = load_object(pupil_data_path) pupil_list = pupil_data['pupil_positions'] gaze_list = pupil_data['gaze_positions'] # create container for globally scoped vars g_pool = Global_Container() g_pool.app = 'player' g_pool.binocular = meta_info.get('Eye Mode','monocular') == 'binocular' g_pool.version = app_version g_pool.capture = cap g_pool.timestamps = timestamps g_pool.play = False g_pool.new_seek = True g_pool.user_dir = user_dir g_pool.rec_dir = rec_dir g_pool.rec_version = rec_version g_pool.meta_info = meta_info g_pool.pupil_positions_by_frame = correlate_data(pupil_list,g_pool.timestamps) g_pool.gaze_positions_by_frame = correlate_data(gaze_list,g_pool.timestamps) g_pool.fixations_by_frame = [[] for x in g_pool.timestamps] #populated by the fixation detector plugin def next_frame(_): try: cap.seek_to_frame(cap.get_frame_index()) except FileSeekError: logger.warning("Could not seek to next frame.") else: g_pool.new_seek = True def prev_frame(_): try: cap.seek_to_frame(cap.get_frame_index()-2) except FileSeekError: logger.warning("Could not seek to previous frame.") else: g_pool.new_seek = True def set_scale(new_scale): g_pool.gui.scale = new_scale g_pool.gui.collect_menus() def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def purge_plugins(): for p in g_pool.plugins: if p.__class__ in user_launchable_plugins: p.alive = False g_pool.plugins.clean() def do_export(_): export_range = slice(g_pool.trim_marks.in_mark,g_pool.trim_marks.out_mark) export_dir = os.path.join(g_pool.rec_dir,'exports','%s-%s'%(export_range.start,export_range.stop)) try: os.makedirs(export_dir) except OSError as e: if e.errno != errno.EEXIST: logger.error("Could not create export dir") raise e else: logger.warning("Previous export for range [%s-%s] already exsits - overwriting."%(export_range.start,export_range.stop)) else: logger.info('Created export dir at "%s"'%export_dir) notification = {'subject':'should_export','range':export_range,'export_dir':export_dir} g_pool.notifications.append(notification) g_pool.gui = ui.UI() g_pool.gui.scale = session_settings.get('gui_scale',1) g_pool.main_menu = ui.Growing_Menu("Settings",pos=(-350,20),size=(300,400)) g_pool.main_menu.append(ui.Button("Close Pupil Player",lambda:glfwSetWindowShouldClose(main_window,True))) g_pool.main_menu.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=0.75,max=2.5,label='Interface Size')) g_pool.main_menu.append(ui.Info_Text('Player Version: %s'%g_pool.version)) g_pool.main_menu.append(ui.Info_Text('Recording Version: %s'%rec_version)) g_pool.main_menu.append(ui.Selector('Open plugin', selection = user_launchable_plugins, labels = [p.__name__.replace('_',' ') for p in user_launchable_plugins], setter= open_plugin, getter = lambda: "Select to load")) g_pool.main_menu.append(ui.Button('Close all plugins',purge_plugins)) g_pool.main_menu.append(ui.Button('Reset window size',lambda: glfwSetWindowSize(main_window,cap.frame_size[0],cap.frame_size[1])) ) g_pool.quickbar = ui.Stretching_Menu('Quick Bar',(0,100),(120,-100)) g_pool.play_button = ui.Thumb('play',g_pool,label='Play',hotkey=GLFW_KEY_SPACE) g_pool.play_button.on_color[:] = (0,1.,.0,.8) g_pool.forward_button = ui.Thumb('forward',getter = lambda: False,setter= next_frame, hotkey=GLFW_KEY_RIGHT) g_pool.backward_button = ui.Thumb('backward',getter = lambda: False, setter = prev_frame, hotkey=GLFW_KEY_LEFT) g_pool.export_button = ui.Thumb('export',getter = lambda: False, setter = do_export, hotkey='e') g_pool.quickbar.extend([g_pool.play_button,g_pool.forward_button,g_pool.backward_button,g_pool.export_button]) g_pool.gui.append(g_pool.quickbar) g_pool.gui.append(g_pool.main_menu) #we always load these plugins system_plugins = [('Trim_Marks',{}),('Seek_Bar',{})] default_plugins = [('Log_Display',{}),('Scan_Path',{}),('Vis_Polyline',{}),('Vis_Circle',{}),('Video_Export_Launcher',{})] previous_plugins = session_settings.get('loaded_plugins',default_plugins) g_pool.notifications = [] g_pool.delayed_notifications = {} g_pool.plugins = Plugin_List(g_pool,plugin_by_name,system_plugins+previous_plugins) # Register callbacks main_window glfwSetFramebufferSizeCallback(main_window,on_resize) glfwSetKeyCallback(main_window,on_key) glfwSetCharCallback(main_window,on_char) glfwSetMouseButtonCallback(main_window,on_button) glfwSetCursorPosCallback(main_window,on_pos) glfwSetScrollCallback(main_window,on_scroll) glfwSetDropCallback(main_window,on_drop) #trigger on_resize on_resize(main_window, *glfwGetFramebufferSize(main_window)) g_pool.gui.configuration = session_settings.get('ui_config',{}) # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() #set up performace graphs: pid = os.getpid() ps = psutil.Process(pid) ts = cap.get_timestamp()-.03 cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20,110) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140,110) fps_graph.update_rate = 5 fps_graph.label = "%0.0f REC FPS" pupil_graph = graph.Bar_Graph(max_val=1.0) pupil_graph.pos = (260,110) pupil_graph.update_rate = 5 pupil_graph.label = "Confidence: %0.2f" while not glfwWindowShouldClose(main_window): #grab new frame if g_pool.play or g_pool.new_seek: g_pool.new_seek = False try: new_frame = cap.get_frame_nowait() except EndofVideoFileError: #end of video logic: pause at last frame. g_pool.play=False update_graph = True else: update_graph = False frame = new_frame.copy() events = {} #report time between now and the last loop interation events['dt'] = get_dt() #new positons we make a deepcopy just like the image is a copy. events['gaze_positions'] = deepcopy(g_pool.gaze_positions_by_frame[frame.index]) events['pupil_positions'] = deepcopy(g_pool.pupil_positions_by_frame[frame.index]) if update_graph: #update performace graphs for p in events['pupil_positions']: pupil_graph.add(p['confidence']) t = new_frame.timestamp if ts != t: dt,ts = t-ts,t fps_graph.add(1./dt) g_pool.play_button.status_text = str(frame.index) #always update the CPU graph cpu_graph.update() # publish delayed notifiactions when their time has come. for n in g_pool.delayed_notifications.values(): if n['_notify_time_'] < time(): del n['_notify_time_'] del g_pool.delayed_notifications[n['subject']] g_pool.notifications.append(n) # notify each plugin if there are new notifactions: while g_pool.notifications: n = g_pool.notifications.pop(0) for p in g_pool.plugins: p.on_notify(n) # allow each Plugin to do its work. for p in g_pool.plugins: p.update(frame,events) #check if a plugin need to be destroyed g_pool.plugins.clean() # render camera image glfwMakeContextCurrent(main_window) make_coord_system_norm_based() g_pool.image_tex.update_from_frame(frame) g_pool.image_tex.draw() make_coord_system_pixel_based(frame.img.shape) # render visual feedback from loaded plugins for p in g_pool.plugins: p.gl_display() graph.push_view() fps_graph.draw() cpu_graph.draw() pupil_graph.draw() graph.pop_view() g_pool.gui.update() #present frames at appropriate speed cap.wait(frame) glfwSwapBuffers(main_window) glfwPollEvents() session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['gui_scale'] = g_pool.gui.scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['window_size'] = glfwGetWindowSize(main_window) session_settings['window_position'] = glfwGetWindowPos(main_window) session_settings['version'] = g_pool.version session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() cap.close() g_pool.gui.terminate() glfwDestroyWindow(main_window)
def export(should_terminate, frames_to_export, current_frame, rec_dir, user_dir, min_data_confidence, start_frame=None, end_frame=None, plugin_initializers=(), out_file_path=None,pre_computed={}): vis_plugins = sorted([Vis_Circle,Vis_Cross,Vis_Polyline,Vis_Light_Points, Vis_Watermark,Vis_Scan_Path,Vis_Eye_Video_Overlay], key=lambda x: x.__name__) analysis_plugins = sorted([ Pupil_Angle_3D_Fixation_Detector, Gaze_Position_2D_Fixation_Detector], key=lambda x: x.__name__) user_plugins = sorted(import_runtime_plugins(os.path.join(user_dir, 'plugins')), key=lambda x: x.__name__) available_plugins = vis_plugins + analysis_plugins + user_plugins name_by_index = [p.__name__ for p in available_plugins] plugin_by_name = dict(zip(name_by_index, available_plugins)) logger = logging.getLogger(__name__+' with pid: '+str(os.getpid())) update_recording_to_recent(rec_dir) video_path = [f for f in glob(os.path.join(rec_dir, "world.*")) if f[-3:] in ('mp4', 'mkv', 'avi')][0] timestamps_path = os.path.join(rec_dir, "world_timestamps.npy") pupil_data_path = os.path.join(rec_dir, "pupil_data") audio_path = os.path.join(rec_dir, "audio.mp4") meta_info = load_meta_info(rec_dir) g_pool = Global_Container() g_pool.app = 'exporter' g_pool.min_data_confidence = min_data_confidence timestamps = np.load(timestamps_path) cap = File_Source(g_pool, video_path, timestamps=timestamps) # Out file path verification, we do this before but if one uses a seperate tool, this will kick in. if out_file_path is None: out_file_path = os.path.join(rec_dir, "world_viz.mp4") else: file_name = os.path.basename(out_file_path) dir_name = os.path.dirname(out_file_path) if not dir_name: dir_name = rec_dir if not file_name: file_name = 'world_viz.mp4' out_file_path = os.path.expanduser(os.path.join(dir_name, file_name)) if os.path.isfile(out_file_path): logger.warning("Video out file already exsists. I will overwrite!") os.remove(out_file_path) logger.debug("Saving Video to {}".format(out_file_path)) # Trim mark verification # make sure the trim marks (start frame, endframe) make sense: # We define them like python list slices, thus we can test them like such. trimmed_timestamps = timestamps[start_frame:end_frame] if len(trimmed_timestamps) == 0: logger.warn("Start and end frames are set such that no video will be exported.") return False if start_frame is None: start_frame = 0 # these two vars are shared with the lauching process and give a job length and progress report. frames_to_export.value = len(trimmed_timestamps) current_frame.value = 0 exp_info = "Will export from frame {} to frame {}. This means I will export {} frames." logger.debug(exp_info.format(start_frame, start_frame + frames_to_export.value, frames_to_export.value)) # setup of writer writer = AV_Writer(out_file_path, fps=cap.frame_rate, audio_loc=audio_path, use_timestamps=True) cap.seek_to_frame(start_frame) start_time = time() g_pool.capture = cap g_pool.rec_dir = rec_dir g_pool.user_dir = user_dir g_pool.meta_info = meta_info g_pool.timestamps = timestamps g_pool.delayed_notifications = {} g_pool.notifications = [] # load pupil_positions, gaze_positions pupil_data = pre_computed.get("pupil_data") or load_object(pupil_data_path) g_pool.pupil_data = pupil_data g_pool.pupil_positions = pre_computed.get("pupil_positions") or pupil_data['pupil_positions'] g_pool.gaze_positions = pre_computed.get("gaze_positions") or pupil_data['gaze_positions'] g_pool.fixations = [] # populated by the fixation detector plugin g_pool.pupil_positions_by_frame = correlate_data(g_pool.pupil_positions,g_pool.timestamps) g_pool.gaze_positions_by_frame = correlate_data(g_pool.gaze_positions,g_pool.timestamps) g_pool.fixations_by_frame = [[] for x in g_pool.timestamps] # populated by the fixation detector plugin # add plugins g_pool.plugins = Plugin_List(g_pool, plugin_by_name, plugin_initializers) while frames_to_export.value > current_frame.value: if should_terminate.value: logger.warning("User aborted export. Exported {} frames to {}.".format(current_frame.value, out_file_path)) # explicit release of VideoWriter writer.close() writer = None return False try: frame = cap.get_frame() except EndofVideoFileError: break events = {'frame':frame} # new positons and events events['gaze_positions'] = g_pool.gaze_positions_by_frame[frame.index] events['pupil_positions'] = g_pool.pupil_positions_by_frame[frame.index] # publish delayed notifiactions when their time has come. for n in list(g_pool.delayed_notifications.values()): if n['_notify_time_'] < time(): del n['_notify_time_'] del g_pool.delayed_notifications[n['subject']] g_pool.notifications.append(n) # notify each plugin if there are new notifactions: while g_pool.notifications: n = g_pool.notifications.pop(0) for p in g_pool.plugins: p.on_notify(n) # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) writer.write_video_frame(frame) current_frame.value += 1 writer.close() writer = None duration = time()-start_time effective_fps = float(current_frame.value)/duration result = "Export done: Exported {} frames to {}. This took {} seconds. Exporter ran at {} frames per second." logger.info(result.format(current_frame.value, out_file_path, duration, effective_fps)) return True
def export(should_terminate, frames_to_export, current_frame, rec_dir, user_dir, min_data_confidence, start_frame=None, end_frame=None, plugin_initializers=(), out_file_path=None): vis_plugins = sorted([ Vis_Circle, Vis_Cross, Vis_Polyline, Vis_Light_Points, Vis_Watermark, Scan_Path ], key=lambda x: x.__name__) analysis_plugins = sorted([ Manual_Gaze_Correction, Eye_Video_Overlay, Pupil_Angle_3D_Fixation_Detector, Gaze_Position_2D_Fixation_Detector ], key=lambda x: x.__name__) user_plugins = sorted(import_runtime_plugins( os.path.join(user_dir, 'plugins')), key=lambda x: x.__name__) available_plugins = vis_plugins + analysis_plugins + user_plugins name_by_index = [p.__name__ for p in available_plugins] index_by_name = dict(zip(name_by_index, range(len(name_by_index)))) plugin_by_name = dict(zip(name_by_index, available_plugins)) logger = logging.getLogger(__name__ + ' with pid: ' + str(os.getpid())) update_recording_to_recent(rec_dir) video_path = [ f for f in glob(os.path.join(rec_dir, "world.*")) if f[-3:] in ('mp4', 'mkv', 'avi') ][0] timestamps_path = os.path.join(rec_dir, "world_timestamps.npy") pupil_data_path = os.path.join(rec_dir, "pupil_data") meta_info = load_meta_info(rec_dir) rec_version = read_rec_version(meta_info) g_pool = Global_Container() g_pool.app = 'exporter' g_pool.min_data_confidence = min_data_confidence timestamps = np.load(timestamps_path) cap = File_Source(g_pool, video_path, timestamps=timestamps) #Out file path verification, we do this before but if one uses a seperate tool, this will kick in. if out_file_path is None: out_file_path = os.path.join(rec_dir, "world_viz.mp4") else: file_name = os.path.basename(out_file_path) dir_name = os.path.dirname(out_file_path) if not dir_name: dir_name = rec_dir if not file_name: file_name = 'world_viz.mp4' out_file_path = os.path.expanduser(os.path.join(dir_name, file_name)) if os.path.isfile(out_file_path): logger.warning("Video out file already exsists. I will overwrite!") os.remove(out_file_path) logger.debug("Saving Video to %s" % out_file_path) #Trim mark verification #make sure the trim marks (start frame, endframe) make sense: We define them like python list slices,thus we can test them like such. trimmed_timestamps = timestamps[start_frame:end_frame] if len(trimmed_timestamps) == 0: logger.warn( "Start and end frames are set such that no video will be exported." ) return False if start_frame == None: start_frame = 0 #these two vars are shared with the lauching process and give a job length and progress report. frames_to_export.value = len(trimmed_timestamps) current_frame.value = 0 logger.debug( "Will export from frame %s to frame %s. This means I will export %s frames." % (start_frame, start_frame + frames_to_export.value, frames_to_export.value)) #setup of writer writer = AV_Writer(out_file_path, fps=cap.frame_rate, use_timestamps=True) cap.seek_to_frame(start_frame) start_time = time() g_pool.capture = cap g_pool.rec_dir = rec_dir g_pool.user_dir = user_dir g_pool.rec_version = rec_version g_pool.timestamps = timestamps g_pool.delayed_notifications = {} g_pool.notifications = [] # load pupil_positions, gaze_positions pupil_data = load_object(pupil_data_path) pupil_list = pupil_data['pupil_positions'] gaze_list = pupil_data['gaze_positions'] g_pool.pupil_positions_by_frame = correlate_data(pupil_list, g_pool.timestamps) g_pool.gaze_positions_by_frame = correlate_data(gaze_list, g_pool.timestamps) g_pool.fixations_by_frame = [[] for x in g_pool.timestamps ] #populated by the fixation detector plugin #add plugins g_pool.plugins = Plugin_List(g_pool, plugin_by_name, plugin_initializers) while frames_to_export.value > current_frame.value: if should_terminate.value: logger.warning("User aborted export. Exported %s frames to %s." % (current_frame.value, out_file_path)) #explicit release of VideoWriter writer.close() writer = None return False try: frame = cap.get_frame_nowait() except EndofVideoFileError: break events = {} #new positons and events events['gaze_positions'] = g_pool.gaze_positions_by_frame[frame.index] events['pupil_positions'] = g_pool.pupil_positions_by_frame[ frame.index] # publish delayed notifiactions when their time has come. for n in g_pool.delayed_notifications.values(): if n['_notify_time_'] < time(): del n['_notify_time_'] del g_pool.delayed_notifications[n['subject']] g_pool.notifications.append(n) # notify each plugin if there are new notifactions: while g_pool.notifications: n = g_pool.notifications.pop(0) for p in g_pool.plugins: p.on_notify(n) # allow each Plugin to do its work. for p in g_pool.plugins: p.update(frame, events) writer.write_video_frame(frame) current_frame.value += 1 writer.close() writer = None duration = time() - start_time effective_fps = float(current_frame.value) / duration logger.info( "Export done: Exported %s frames to %s. This took %s seconds. Exporter ran at %s frames per second" % (current_frame.value, out_file_path, duration, effective_fps)) return True
def export(should_terminate, frames_to_export, current_frame, rec_dir, user_dir, start_frame=None, end_frame=None, plugin_initializers=[], out_file_path=None): logger = logging.getLogger(__name__ + ' with pid: ' + str(os.getpid())) update_recording_to_recent(rec_dir) video_path = [ f for f in glob(os.path.join(rec_dir, "world.*")) if f[-3:] in ('mp4', 'mkv', 'avi') ][0] timestamps_path = os.path.join(rec_dir, "world_timestamps.npy") pupil_data_path = os.path.join(rec_dir, "pupil_data") meta_info = load_meta_info(rec_dir) rec_version = read_rec_version(meta_info) timestamps = np.load(timestamps_path) cap = File_Capture(video_path, timestamps=timestamps) #Out file path verification, we do this before but if one uses a seperate tool, this will kick in. if out_file_path is None: out_file_path = os.path.join(rec_dir, "world_viz.mp4") else: file_name = os.path.basename(out_file_path) dir_name = os.path.dirname(out_file_path) if not dir_name: dir_name = rec_dir if not file_name: file_name = 'world_viz.mp4' out_file_path = os.path.expanduser(os.path.join(dir_name, file_name)) if os.path.isfile(out_file_path): logger.warning("Video out file already exsists. I will overwrite!") os.remove(out_file_path) logger.debug("Saving Video to %s" % out_file_path) #Trim mark verification #make sure the trim marks (start frame, endframe) make sense: We define them like python list slices,thus we can test them like such. trimmed_timestamps = timestamps[start_frame:end_frame] if len(trimmed_timestamps) == 0: logger.warn( "Start and end frames are set such that no video will be exported." ) return False if start_frame == None: start_frame = 0 #these two vars are shared with the lauching process and give a job length and progress report. frames_to_export.value = len(trimmed_timestamps) current_frame.value = 0 logger.debug( "Will export from frame %s to frame %s. This means I will export %s frames." % (start_frame, start_frame + frames_to_export.value, frames_to_export.value)) #setup of writer writer = AV_Writer(out_file_path, fps=cap.frame_rate, use_timestamps=True) cap.seek_to_frame(start_frame) start_time = time() g = Global_Container() g.app = 'exporter' g.capture = cap g.rec_dir = rec_dir g.user_dir = user_dir g.rec_version = rec_version g.timestamps = timestamps # load pupil_positions, gaze_positions pupil_data = load_object(pupil_data_path) pupil_list = pupil_data['pupil_positions'] gaze_list = pupil_data['gaze_positions'] g.pupil_positions_by_frame = correlate_data(pupil_list, g.timestamps) g.gaze_positions_by_frame = correlate_data(gaze_list, g.timestamps) g.fixations_by_frame = [[] for x in g.timestamps ] #populated by the fixation detector plugin #add plugins g.plugins = Plugin_List(g, plugin_by_name, plugin_initializers) while frames_to_export.value > current_frame.value: if should_terminate.value: logger.warning("User aborted export. Exported %s frames to %s." % (current_frame.value, out_file_path)) #explicit release of VideoWriter writer.close() writer = None return False try: frame = cap.get_frame_nowait() except EndofVideoFileError: break events = {} #new positons and events events['gaze_positions'] = g.gaze_positions_by_frame[frame.index] events['pupil_positions'] = g.pupil_positions_by_frame[frame.index] # allow each Plugin to do its work. for p in g.plugins: p.update(frame, events) writer.write_video_frame(frame) current_frame.value += 1 writer.close() writer = None duration = time() - start_time effective_fps = float(current_frame.value) / duration logger.info( "Export done: Exported %s frames to %s. This took %s seconds. Exporter ran at %s frames per second" % (current_frame.value, out_file_path, duration, effective_fps)) return True
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_version): # general imports import logging import errno from glob import glob from copy import deepcopy from time import time # networking import zmq import zmq_tools # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify',)) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.INFO) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) # imports from file_methods import Persistent_Dict, load_object import numpy as np # display import glfw # check versions for our own depedencies as they are fast-changing from pyglui import __version__ as pyglui_version from pyglui import ui, graph, cygl from pyglui.cygl.utils import Named_Texture import gl_utils # capture from video_capture import File_Source, EndofVideoFileError, FileSeekError # helpers/utils from version_utils import VersionFormat from methods import normalize, denormalize, delta_t, get_system_info from player_methods import correlate_data, is_pupil_rec_dir, load_meta_info # monitoring import psutil # Plug-ins from plugin import Plugin, Plugin_List, import_runtime_plugins, Visualizer_Plugin_Base, Analysis_Plugin_Base, Producer_Plugin_Base from vis_circle import Vis_Circle from vis_cross import Vis_Cross from vis_polyline import Vis_Polyline from vis_light_points import Vis_Light_Points from vis_watermark import Vis_Watermark from vis_fixation import Vis_Fixation from vis_scan_path import Vis_Scan_Path from vis_eye_video_overlay import Vis_Eye_Video_Overlay from seek_bar import Seek_Bar from trim_marks import Trim_Marks from video_export_launcher import Video_Export_Launcher from offline_surface_tracker import Offline_Surface_Tracker from marker_auto_trim_marks import Marker_Auto_Trim_Marks from fixation_detector import Gaze_Position_2D_Fixation_Detector, Pupil_Angle_3D_Fixation_Detector # from manual_gaze_correction import Manual_Gaze_Correction from batch_exporter import Batch_Exporter from log_display import Log_Display from annotations import Annotation_Player from raw_data_exporter import Raw_Data_Exporter from log_history import Log_History from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection from gaze_producers import Gaze_From_Recording, Offline_Calibration assert pyglui_version >= '1.5' runtime_plugins = import_runtime_plugins(os.path.join(user_dir, 'plugins')) system_plugins = [Log_Display, Seek_Bar, Trim_Marks] user_launchable_plugins = [Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross, Vis_Watermark, Vis_Eye_Video_Overlay, Vis_Scan_Path, Gaze_Position_2D_Fixation_Detector, Pupil_Angle_3D_Fixation_Detector, Video_Export_Launcher, Offline_Surface_Tracker, Raw_Data_Exporter, Batch_Exporter, Annotation_Player, Log_History, Marker_Auto_Trim_Marks, Pupil_From_Recording, Offline_Pupil_Detection, Gaze_From_Recording, Offline_Calibration] + runtime_plugins available_plugins = system_plugins + user_launchable_plugins name_by_index = [p.__name__ for p in available_plugins] plugin_by_name = dict(zip(name_by_index, available_plugins)) # Callback functions def on_resize(window, w, h): if gl_utils.is_window_visible(window): hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() for g in g_pool.graphs: g.scale = hdpi_factor g.adjust_window_size(w, h) gl_utils.adjust_gl_view(w, h) for p in g_pool.plugins: p.on_window_resize(window, w, h) def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_char(window, char): g_pool.gui.update_char(char) def on_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0]/glfw.glfwGetWindowSize(window)[0]) g_pool.gui.update_mouse(x*hdpi_factor, y*hdpi_factor) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y*scroll_factor) def on_drop(window, count, paths): for x in range(count): new_rec_dir = paths[x].decode('utf-8') if is_pupil_rec_dir(new_rec_dir): logger.debug("Starting new session with '{}'".format(new_rec_dir)) ipc_pub.notify({"subject": "player_drop_process.should_start", "rec_dir": new_rec_dir}) glfw.glfwSetWindowShouldClose(window, True) else: logger.error("'{}' is not a valid pupil recording".format(new_rec_dir)) tick = delta_t() def get_dt(): return next(tick) video_path = [f for f in glob(os.path.join(rec_dir, "world.*")) if os.path.splitext(f)[1] in ('.mp4', '.mkv', '.avi', '.h264', '.mjpeg')][0] timestamps_path = os.path.join(rec_dir, "world_timestamps.npy") pupil_data_path = os.path.join(rec_dir, "pupil_data") meta_info = load_meta_info(rec_dir) # log info about Pupil Platform and Platform in player.log logger.info('Application Version: {}'.format(app_version)) logger.info('System Info: {}'.format(get_system_info())) timestamps = np.load(timestamps_path) # create container for globally scoped vars g_pool = Global_Container() g_pool.app = 'player' g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url # Initialize capture cap = File_Source(g_pool, video_path, timestamps=list(timestamps)) # load session persistent settings session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings")) if VersionFormat(session_settings.get("version", '0.0')) != app_version: logger.info("Session setting are a different version of this app. I will not use those.") session_settings.clear() width, height = session_settings.get('window_size', cap.frame_size) window_pos = session_settings.get('window_position', window_position_default) glfw.glfwInit() main_window = glfw.glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - " + rec_dir.split(os.path.sep)[-1], None, None) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() def set_scale(new_scale): g_pool.gui_user_scale = new_scale on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) # load pupil_positions, gaze_positions g_pool.pupil_data = load_object(pupil_data_path) g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular' g_pool.version = app_version g_pool.capture = cap g_pool.timestamps = timestamps g_pool.get_timestamp = lambda: 0. g_pool.play = False g_pool.new_seek = True g_pool.user_dir = user_dir g_pool.rec_dir = rec_dir g_pool.meta_info = meta_info g_pool.min_data_confidence = session_settings.get('min_data_confidence', 0.6) g_pool.pupil_positions = [] g_pool.gaze_positions = [] g_pool.fixations = [] g_pool.notifications_by_frame = correlate_data(g_pool.pupil_data['notifications'], g_pool.timestamps) g_pool.pupil_positions_by_frame = [[] for x in g_pool.timestamps] # populated by producer` g_pool.gaze_positions_by_frame = [[] for x in g_pool.timestamps] # populated by producer g_pool.fixations_by_frame = [[] for x in g_pool.timestamps] # populated by the fixation detector plugin def next_frame(_): try: cap.seek_to_frame(cap.get_frame_index() + 1) except(FileSeekError): logger.warning("Could not seek to next frame.") else: g_pool.new_seek = True def prev_frame(_): try: cap.seek_to_frame(cap.get_frame_index() - 1) except(FileSeekError): logger.warning("Could not seek to previous frame.") else: g_pool.new_seek = True def toggle_play(new_state): if cap.get_frame_index() >= cap.get_frame_count()-5: cap.seek_to_frame(1) # avoid pause set by hitting trimmark pause. logger.warning("End of video - restart at beginning.") g_pool.play = new_state def set_data_confidence(new_confidence): g_pool.min_data_confidence = new_confidence notification = {'subject': 'min_data_confidence_changed'} notification['_notify_time_'] = time()+.8 g_pool.ipc_pub.notify(notification) def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def purge_plugins(): for p in g_pool.plugins: if p.__class__ in user_launchable_plugins: p.alive = False g_pool.plugins.clean() def do_export(_): export_range = g_pool.trim_marks.in_mark, g_pool.trim_marks.out_mark export_dir = os.path.join(g_pool.rec_dir, 'exports', '{}-{}'.format(*export_range)) try: os.makedirs(export_dir) except OSError as e: if e.errno != errno.EEXIST: logger.error("Could not create export dir") raise e else: overwrite_warning = "Previous export for range [{}-{}] already exsits - overwriting." logger.warning(overwrite_warning.format(*export_range)) else: logger.info('Created export dir at "{}"'.format(export_dir)) notification = {'subject': 'should_export', 'range': export_range, 'export_dir': export_dir} g_pool.ipc_pub.notify(notification) g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get('gui_scale', 1.) g_pool.main_menu = ui.Scrolling_Menu("Settings", pos=(-350, 20), size=(300, 560)) g_pool.main_menu.append(ui.Button('Reset window size', lambda: glfw.glfwSetWindowSize(main_window, cap.frame_size[0], cap.frame_size[1]))) g_pool.main_menu.append(ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2], label='Interface Size')) g_pool.main_menu.append(ui.Info_Text('Player Version: {}'.format(g_pool.version))) g_pool.main_menu.append(ui.Info_Text('Capture Version: {}'.format(meta_info['Capture Software Version']))) g_pool.main_menu.append(ui.Info_Text('Data Format Version: {}'.format(meta_info['Data Format Version']))) g_pool.main_menu.append(ui.Slider('min_data_confidence', g_pool, setter=set_data_confidence, step=.05, min=0.0, max=1.0, label='Confidence threshold')) g_pool.main_menu.append(ui.Info_Text('Open plugins')) selector_label = "Select to load" def append_selector(label, plugins): plugins.sort(key=lambda p: p.__name__) plugin_labels = [p.__name__.replace('_', ' ') for p in plugins] g_pool.main_menu.append(ui.Selector(label, selection=[selector_label] + plugins, labels=[selector_label] + plugin_labels, setter=open_plugin, getter=lambda: selector_label)) base_plugins = [Visualizer_Plugin_Base, Analysis_Plugin_Base, Producer_Plugin_Base] base_labels = ['Visualizer:', 'Analyser:', 'Data Source:'] launchable = user_launchable_plugins.copy() for base_class, label in zip(base_plugins, base_labels): member_plugins = [] for p in user_launchable_plugins: if issubclass(p, base_class): member_plugins.append(p) launchable.remove(p) append_selector(label, member_plugins) # launchable only contains plugins that could not be assigned to any of the above categories append_selector('Other', launchable) g_pool.main_menu.append(ui.Button('Close all plugins', purge_plugins)) g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100)) g_pool.play_button = ui.Thumb('play', g_pool, label=chr(0xf04b), setter=toggle_play, hotkey=glfw.GLFW_KEY_SPACE, label_font='fontawesome', label_offset_x=5, label_offset_y=0, label_offset_size=-24) g_pool.play_button.on_color[:] = (0, 1., .0, .8) g_pool.forward_button = ui.Thumb('forward', label=chr(0xf04e), getter=lambda: False, setter=next_frame, hotkey=glfw.GLFW_KEY_RIGHT, label_font='fontawesome', label_offset_x=5, label_offset_y=0, label_offset_size=-24) g_pool.backward_button = ui.Thumb('backward', label=chr(0xf04a), getter=lambda: False, setter=prev_frame, hotkey=glfw.GLFW_KEY_LEFT, label_font='fontawesome', label_offset_x=-5, label_offset_y=0, label_offset_size=-24) g_pool.export_button = ui.Thumb('export', label=chr(0xf063), getter=lambda: False, setter=do_export, hotkey='e', label_font='fontawesome', label_offset_x=0, label_offset_y=2, label_offset_size=-24) g_pool.quickbar.extend([g_pool.play_button, g_pool.forward_button, g_pool.backward_button, g_pool.export_button]) g_pool.gui.append(g_pool.quickbar) g_pool.gui.append(g_pool.main_menu) # we always load these plugins system_plugins = [('Trim_Marks', {}), ('Seek_Bar', {})] default_plugins = [('Log_Display', {}), ('Vis_Scan_Path', {}), ('Vis_Polyline', {}), ('Vis_Circle', {}), ('Video_Export_Launcher', {}), ('Pupil_From_Recording', {}), ('Gaze_From_Recording', {})] previous_plugins = session_settings.get('loaded_plugins', default_plugins) g_pool.plugins = Plugin_List(g_pool, plugin_by_name, system_plugins+previous_plugins) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetKeyCallback(main_window, on_key) glfw.glfwSetCharCallback(main_window, on_char) glfw.glfwSetMouseButtonCallback(main_window, on_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) g_pool.gui.configuration = session_settings.get('ui_config', {}) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() # set up performace graphs: pid = os.getpid() ps = psutil.Process(pid) ts = None cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 110) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 110) fps_graph.update_rate = 5 fps_graph.label = "%0.0f REC FPS" pupil_graph = graph.Bar_Graph(max_val=1.0) pupil_graph.pos = (260, 110) pupil_graph.update_rate = 5 pupil_graph.label = "Confidence: %0.2f" g_pool.graphs = [cpu_graph, fps_graph, pupil_graph] # trigger on_resize on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) def handle_notifications(n): subject = n['subject'] if subject == 'start_plugin': g_pool.plugins.add( plugin_by_name[n['name']], args=n.get('args', {})) elif subject.startswith('meta.should_doc'): ipc_pub.notify({'subject': 'meta.doc', 'actor': g_pool.app, 'doc': player.__doc__}) for p in g_pool.plugins: if (p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify): ipc_pub.notify({'subject': 'meta.doc', 'actor': p.class_name, 'doc': p.on_notify.__doc__}) while not glfw.glfwWindowShouldClose(main_window): # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) # grab new frame if g_pool.play or g_pool.new_seek: g_pool.new_seek = False try: new_frame = cap.get_frame() except EndofVideoFileError: # end of video logic: pause at last frame. g_pool.play = False logger.warning("end of video") update_graph = True else: update_graph = False frame = new_frame.copy() events = {} events['frame'] = frame # report time between now and the last loop interation events['dt'] = get_dt() # new positons we make a deepcopy just like the image is a copy. events['gaze_positions'] = deepcopy(g_pool.gaze_positions_by_frame[frame.index]) events['pupil_positions'] = deepcopy(g_pool.pupil_positions_by_frame[frame.index]) if update_graph: # update performace graphs for p in events['pupil_positions']: pupil_graph.add(p['confidence']) t = new_frame.timestamp if ts and ts != t: dt, ts = t-ts, t fps_graph.add(1./dt) else: ts = new_frame.timestamp g_pool.play_button.status_text = str(frame.index) # always update the CPU graph cpu_graph.update() # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() # render camera image glfw.glfwMakeContextCurrent(main_window) gl_utils.make_coord_system_norm_based() g_pool.image_tex.update_from_frame(frame) g_pool.image_tex.draw() gl_utils.make_coord_system_pixel_based(frame.img.shape) # render visual feedback from loaded plugins for p in g_pool.plugins: p.gl_display() fps_graph.draw() cpu_graph.draw() pupil_graph.draw() unused_buttons = g_pool.gui.update() for b in unused_buttons: button,action,mods = b pos = glfw.glfwGetCursorPos(main_window) pos = normalize(pos, glfw.glfwGetWindowSize(main_window)) pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels for p in g_pool.plugins: p.on_click(pos, button, action) # present frames at appropriate speed cap.wait(frame) glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['min_data_confidence'] = g_pool.min_data_confidence session_settings['gui_scale'] = g_pool.gui_user_scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos(main_window) session_settings['version'] = str(g_pool.version) session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() cap.cleanup() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) logger.info("Process shutting down.") ipc_pub.notify({'subject': 'player_process.stopped'})
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_version): # general imports import logging import errno from glob import glob from copy import deepcopy from time import time # networking import zmq import zmq_tools # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify',)) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.INFO) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) # imports from file_methods import Persistent_Dict, load_object # display import glfw # check versions for our own depedencies as they are fast-changing from pyglui import __version__ as pyglui_version from pyglui import ui, graph, cygl from pyglui.cygl.utils import Named_Texture import gl_utils # capture from video_capture import File_Source, EndofVideoFileError, FileSeekError # helpers/utils from version_utils import VersionFormat from methods import normalize, denormalize, delta_t, get_system_info from player_methods import correlate_data, is_pupil_rec_dir, load_meta_info # monitoring import psutil # Plug-ins from plugin import Plugin, Plugin_List, import_runtime_plugins, Visualizer_Plugin_Base, Analysis_Plugin_Base, Producer_Plugin_Base from vis_circle import Vis_Circle from vis_cross import Vis_Cross from vis_polyline import Vis_Polyline from vis_light_points import Vis_Light_Points from vis_watermark import Vis_Watermark from vis_fixation import Vis_Fixation from vis_scan_path import Vis_Scan_Path from vis_eye_video_overlay import Vis_Eye_Video_Overlay from seek_bar import Seek_Bar from trim_marks import Trim_Marks from video_export_launcher import Video_Export_Launcher from offline_surface_tracker import Offline_Surface_Tracker from marker_auto_trim_marks import Marker_Auto_Trim_Marks from fixation_detector import Gaze_Position_2D_Fixation_Detector, Pupil_Angle_3D_Fixation_Detector # from manual_gaze_correction import Manual_Gaze_Correction from batch_exporter import Batch_Exporter from log_display import Log_Display from annotations import Annotation_Player from raw_data_exporter import Raw_Data_Exporter from log_history import Log_History from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection from gaze_producers import Gaze_From_Recording, Offline_Calibration assert pyglui_version >= '1.7' runtime_plugins = import_runtime_plugins(os.path.join(user_dir, 'plugins')) system_plugins = [Log_Display, Seek_Bar, Trim_Marks] user_launchable_plugins = [Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross, Vis_Watermark, Vis_Eye_Video_Overlay, Vis_Scan_Path, Gaze_Position_2D_Fixation_Detector, Pupil_Angle_3D_Fixation_Detector, Video_Export_Launcher, Offline_Surface_Tracker, Raw_Data_Exporter, Batch_Exporter, Annotation_Player, Log_History, Marker_Auto_Trim_Marks, Pupil_From_Recording, Offline_Pupil_Detection, Gaze_From_Recording, Offline_Calibration] + runtime_plugins available_plugins = system_plugins + user_launchable_plugins name_by_index = [p.__name__ for p in available_plugins] plugin_by_name = dict(zip(name_by_index, available_plugins)) # Callback functions def on_resize(window, w, h): if gl_utils.is_window_visible(window): hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() for g in g_pool.graphs: g.scale = hdpi_factor g.adjust_window_size(w, h) gl_utils.adjust_gl_view(w, h) for p in g_pool.plugins: p.on_window_resize(window, w, h) def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0]/glfw.glfwGetWindowSize(window)[0]) g_pool.gui.update_mouse(x*hdpi_factor, y*hdpi_factor) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y*scroll_factor) def on_drop(window, count, paths): for x in range(count): new_rec_dir = paths[x].decode('utf-8') if is_pupil_rec_dir(new_rec_dir): logger.debug("Starting new session with '{}'".format(new_rec_dir)) ipc_pub.notify({"subject": "player_drop_process.should_start", "rec_dir": new_rec_dir}) glfw.glfwSetWindowShouldClose(window, True) else: logger.error("'{}' is not a valid pupil recording".format(new_rec_dir)) tick = delta_t() def get_dt(): return next(tick) video_path = [f for f in glob(os.path.join(rec_dir, "world.*")) if os.path.splitext(f)[1] in ('.mp4', '.mkv', '.avi', '.h264', '.mjpeg')][0] pupil_data_path = os.path.join(rec_dir, "pupil_data") meta_info = load_meta_info(rec_dir) # log info about Pupil Platform and Platform in player.log logger.info('Application Version: {}'.format(app_version)) logger.info('System Info: {}'.format(get_system_info())) # create container for globally scoped vars g_pool = Global_Container() g_pool.app = 'player' g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url # Initialize capture cap = File_Source(g_pool, video_path) # load session persistent settings session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings")) if VersionFormat(session_settings.get("version", '0.0')) != app_version: logger.info("Session setting are a different version of this app. I will not use those.") session_settings.clear() width, height = session_settings.get('window_size', cap.frame_size) window_pos = session_settings.get('window_position', window_position_default) glfw.glfwInit() main_window = glfw.glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - " + rec_dir.split(os.path.sep)[-1], None, None) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() def set_scale(new_scale): g_pool.gui_user_scale = new_scale on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) # load pupil_positions, gaze_positions g_pool.pupil_data = load_object(pupil_data_path) g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular' g_pool.version = app_version g_pool.capture = cap g_pool.timestamps = g_pool.capture.timestamps g_pool.get_timestamp = lambda: 0. g_pool.play = False g_pool.new_seek = True g_pool.user_dir = user_dir g_pool.rec_dir = rec_dir g_pool.meta_info = meta_info g_pool.min_data_confidence = session_settings.get('min_data_confidence', 0.6) g_pool.pupil_positions = [] g_pool.gaze_positions = [] g_pool.fixations = [] g_pool.notifications_by_frame = correlate_data(g_pool.pupil_data['notifications'], g_pool.timestamps) g_pool.pupil_positions_by_frame = [[] for x in g_pool.timestamps] # populated by producer` g_pool.gaze_positions_by_frame = [[] for x in g_pool.timestamps] # populated by producer g_pool.fixations_by_frame = [[] for x in g_pool.timestamps] # populated by the fixation detector plugin def next_frame(_): try: cap.seek_to_frame(cap.get_frame_index() + 1) except(FileSeekError): logger.warning("Could not seek to next frame.") else: g_pool.new_seek = True def prev_frame(_): try: cap.seek_to_frame(cap.get_frame_index() - 1) except(FileSeekError): logger.warning("Could not seek to previous frame.") else: g_pool.new_seek = True def toggle_play(new_state): if cap.get_frame_index() >= cap.get_frame_count()-5: cap.seek_to_frame(1) # avoid pause set by hitting trimmark pause. logger.warning("End of video - restart at beginning.") g_pool.play = new_state def set_data_confidence(new_confidence): g_pool.min_data_confidence = new_confidence notification = {'subject': 'min_data_confidence_changed'} notification['_notify_time_'] = time()+.8 g_pool.ipc_pub.notify(notification) def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def purge_plugins(): for p in g_pool.plugins: if p.__class__ in user_launchable_plugins: p.alive = False g_pool.plugins.clean() def do_export(_): export_range = g_pool.trim_marks.in_mark, g_pool.trim_marks.out_mark export_dir = os.path.join(g_pool.rec_dir, 'exports', '{}-{}'.format(*export_range)) try: os.makedirs(export_dir) except OSError as e: if e.errno != errno.EEXIST: logger.error("Could not create export dir") raise e else: overwrite_warning = "Previous export for range [{}-{}] already exsits - overwriting." logger.warning(overwrite_warning.format(*export_range)) else: logger.info('Created export dir at "{}"'.format(export_dir)) notification = {'subject': 'should_export', 'range': export_range, 'export_dir': export_dir} g_pool.ipc_pub.notify(notification) g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get('gui_scale', 1.) g_pool.main_menu = ui.Scrolling_Menu("Settings", pos=(-350, 20), size=(300, 560)) g_pool.main_menu.append(ui.Button('Reset window size', lambda: glfw.glfwSetWindowSize(main_window, cap.frame_size[0], cap.frame_size[1]))) g_pool.main_menu.append(ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2], label='Interface Size')) g_pool.main_menu.append(ui.Info_Text('Player Version: {}'.format(g_pool.version))) g_pool.main_menu.append(ui.Info_Text('Capture Version: {}'.format(meta_info['Capture Software Version']))) g_pool.main_menu.append(ui.Info_Text('Data Format Version: {}'.format(meta_info['Data Format Version']))) g_pool.main_menu.append(ui.Slider('min_data_confidence', g_pool, setter=set_data_confidence, step=.05, min=0.0, max=1.0, label='Confidence threshold')) g_pool.main_menu.append(ui.Info_Text('Open plugins')) selector_label = "Select to load" def append_selector(label, plugins): plugins.sort(key=lambda p: p.__name__) plugin_labels = [p.__name__.replace('_', ' ') for p in plugins] g_pool.main_menu.append(ui.Selector(label, selection=[selector_label] + plugins, labels=[selector_label] + plugin_labels, setter=open_plugin, getter=lambda: selector_label)) base_plugins = [Visualizer_Plugin_Base, Analysis_Plugin_Base, Producer_Plugin_Base] base_labels = ['Visualizer:', 'Analyser:', 'Data Source:'] launchable = user_launchable_plugins.copy() for base_class, label in zip(base_plugins, base_labels): member_plugins = [] for p in user_launchable_plugins: if issubclass(p, base_class): member_plugins.append(p) launchable.remove(p) append_selector(label, member_plugins) # launchable only contains plugins that could not be assigned to any of the above categories append_selector('Other', launchable) g_pool.main_menu.append(ui.Button('Close all plugins', purge_plugins)) g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100)) g_pool.play_button = ui.Thumb('play', g_pool, label=chr(0xf04b), setter=toggle_play, hotkey=glfw.GLFW_KEY_SPACE, label_font='fontawesome', label_offset_x=5, label_offset_y=0, label_offset_size=-24) g_pool.play_button.on_color[:] = (0, 1., .0, .8) g_pool.forward_button = ui.Thumb('forward', label=chr(0xf04e), getter=lambda: False, setter=next_frame, hotkey=glfw.GLFW_KEY_RIGHT, label_font='fontawesome', label_offset_x=5, label_offset_y=0, label_offset_size=-24) g_pool.backward_button = ui.Thumb('backward', label=chr(0xf04a), getter=lambda: False, setter=prev_frame, hotkey=glfw.GLFW_KEY_LEFT, label_font='fontawesome', label_offset_x=-5, label_offset_y=0, label_offset_size=-24) g_pool.export_button = ui.Thumb('export', label=chr(0xf063), getter=lambda: False, setter=do_export, hotkey='e', label_font='fontawesome', label_offset_x=0, label_offset_y=2, label_offset_size=-24) g_pool.quickbar.extend([g_pool.play_button, g_pool.forward_button, g_pool.backward_button, g_pool.export_button]) g_pool.gui.append(g_pool.quickbar) g_pool.gui.append(g_pool.main_menu) # we always load these plugins system_plugins = [('Trim_Marks', {}), ('Seek_Bar', {})] default_plugins = [('Log_Display', {}), ('Vis_Scan_Path', {}), ('Vis_Polyline', {}), ('Vis_Circle', {}), ('Video_Export_Launcher', {}), ('Pupil_From_Recording', {}), ('Gaze_From_Recording', {})] previous_plugins = session_settings.get('loaded_plugins', default_plugins) g_pool.plugins = Plugin_List(g_pool, plugin_by_name, system_plugins+previous_plugins) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) g_pool.gui.configuration = session_settings.get('ui_config', {}) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() # set up performace graphs: pid = os.getpid() ps = psutil.Process(pid) ts = None cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 110) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 110) fps_graph.update_rate = 5 fps_graph.label = "%0.0f REC FPS" pupil_graph = graph.Bar_Graph(max_val=1.0) pupil_graph.pos = (260, 110) pupil_graph.update_rate = 5 pupil_graph.label = "Confidence: %0.2f" g_pool.graphs = [cpu_graph, fps_graph, pupil_graph] # trigger on_resize on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) def handle_notifications(n): subject = n['subject'] if subject == 'start_plugin': g_pool.plugins.add( plugin_by_name[n['name']], args=n.get('args', {})) elif subject.startswith('meta.should_doc'): ipc_pub.notify({'subject': 'meta.doc', 'actor': g_pool.app, 'doc': player.__doc__}) for p in g_pool.plugins: if (p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify): ipc_pub.notify({'subject': 'meta.doc', 'actor': p.class_name, 'doc': p.on_notify.__doc__}) while not glfw.glfwWindowShouldClose(main_window): # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) # grab new frame if g_pool.play or g_pool.new_seek: g_pool.new_seek = False try: new_frame = cap.get_frame() except EndofVideoFileError: # end of video logic: pause at last frame. g_pool.play = False logger.warning("end of video") update_graph = True else: update_graph = False frame = new_frame.copy() events = {} events['frame'] = frame # report time between now and the last loop interation events['dt'] = get_dt() # new positons we make a deepcopy just like the image is a copy. events['gaze_positions'] = deepcopy(g_pool.gaze_positions_by_frame[frame.index]) events['pupil_positions'] = deepcopy(g_pool.pupil_positions_by_frame[frame.index]) if update_graph: # update performace graphs for p in events['pupil_positions']: pupil_graph.add(p['confidence']) t = new_frame.timestamp if ts and ts != t: dt, ts = t-ts, t fps_graph.add(1./dt) else: ts = new_frame.timestamp g_pool.play_button.status_text = str(frame.index) # always update the CPU graph cpu_graph.update() # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() # render camera image glfw.glfwMakeContextCurrent(main_window) gl_utils.make_coord_system_norm_based() g_pool.image_tex.update_from_ndarray(frame.bgr) g_pool.image_tex.draw() gl_utils.make_coord_system_pixel_based(frame.img.shape) # render visual feedback from loaded plugins for p in g_pool.plugins: p.gl_display() fps_graph.draw() cpu_graph.draw() pupil_graph.draw() unused_elements = g_pool.gui.update() for b in unused_elements.buttons: button, action, mods = b pos = glfw.glfwGetCursorPos(main_window) pos = normalize(pos, glfw.glfwGetWindowSize(main_window)) pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels for p in g_pool.plugins: p.on_click(pos, button, action) for key, scancode, action, mods in unused_elements.keys: for p in g_pool.plugins: p.on_key(key, scancode, action, mods) for char_ in unused_elements.chars: for p in g_pool.plugins: p.on_char(char_) # present frames at appropriate speed cap.wait(frame) glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['min_data_confidence'] = g_pool.min_data_confidence session_settings['gui_scale'] = g_pool.gui_user_scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos(main_window) session_settings['version'] = str(g_pool.version) session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() cap.cleanup() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) logger.info("Process shutting down.") ipc_pub.notify({'subject': 'player_process.stopped'})
def session(rec_dir): plugin_dir = os.path.join(user_dir, 'plugins') if not os.path.isdir(plugin_dir): os.mkdir(plugin_dir) runtime_plugins = import_runtime_plugins(plugin_dir) system_plugins = [Log_Display, Seek_Bar, Trim_Marks] vis_plugins = sorted([Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross, Vis_Watermark, Vis_Eye_Video_Overlay, Vis_Scan_Path], key=lambda x: x.__name__) analysis_plugins = sorted([Gaze_Position_2D_Fixation_Detector, Pupil_Angle_3D_Fixation_Detector, Manual_Gaze_Correction, Video_Export_Launcher, Offline_Surface_Tracker, Raw_Data_Exporter, Batch_Exporter, Annotation_Player], key=lambda x: x.__name__) other_plugins = sorted([Log_History, Marker_Auto_Trim_Marks], key=lambda x: x.__name__) user_plugins = sorted(runtime_plugins, key=lambda x: x.__name__) user_launchable_plugins = vis_plugins + analysis_plugins + other_plugins + user_plugins available_plugins = system_plugins + user_launchable_plugins name_by_index = [p.__name__ for p in available_plugins] plugin_by_name = dict(zip(name_by_index, available_plugins)) # Callback functions def on_resize(window, w, h): if gl_utils.is_window_visible(window): hdpi_factor = float(glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0]) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() for g in g_pool.graphs: g.scale = hdpi_factor g.adjust_window_size(w, h) gl_utils.adjust_gl_view(w, h) for p in g_pool.plugins: p.on_window_resize(window, w, h) def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_char(window, char): g_pool.gui.update_char(char) def on_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) pos = glfwGetCursorPos(window) pos = normalize(pos, glfwGetWindowSize(window)) pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels for p in g_pool.plugins: p.on_click(pos, button, action) def on_pos(window, x, y): hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0]) g_pool.gui.update_mouse(x*hdpi_factor, y*hdpi_factor) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y*scroll_factor) def on_drop(window, count, paths): for x in range(count): new_rec_dir = paths[x].decode('utf-8') if is_pupil_rec_dir(new_rec_dir): logger.debug("Starting new session with '{}'".format(new_rec_dir)) global rec_dir rec_dir = new_rec_dir glfwSetWindowShouldClose(window, True) else: logger.error("'{}' is not a valid pupil recording".format(new_rec_dir)) tick = delta_t() def get_dt(): return next(tick) update_recording_to_recent(rec_dir) video_path = [f for f in glob(os.path.join(rec_dir, "world.*")) if f[-3:] in ('mp4', 'mkv', 'avi')][0] timestamps_path = os.path.join(rec_dir, "world_timestamps.npy") pupil_data_path = os.path.join(rec_dir, "pupil_data") meta_info = load_meta_info(rec_dir) app_version = get_version(version_file) # log info about Pupil Platform and Platform in player.log logger.info('Application Version: {}'.format(app_version)) logger.info('System Info: {}'.format(get_system_info())) timestamps = np.load(timestamps_path) # create container for globally scoped vars g_pool = Global_Container() g_pool.app = 'player' # Initialize capture cap = File_Source(g_pool, video_path, timestamps=list(timestamps)) # load session persistent settings session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings")) if VersionFormat(session_settings.get("version", '0.0')) < get_version(version_file): logger.info("Session setting are from older version of this app. I will not use those.") session_settings.clear() width, height = session_settings.get('window_size', cap.frame_size) window_pos = session_settings.get('window_position', window_position_default) main_window = glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - " + rec_dir.split(os.path.sep)[-1], None, None) glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfwMakeContextCurrent(main_window) cygl.utils.init() def set_scale(new_scale): g_pool.gui_user_scale = new_scale on_resize(main_window, *glfwGetFramebufferSize(main_window)) # load pupil_positions, gaze_positions pupil_data = load_object(pupil_data_path) pupil_list = pupil_data['pupil_positions'] gaze_list = pupil_data['gaze_positions'] g_pool.pupil_data = pupil_data g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular' g_pool.version = app_version g_pool.capture = cap g_pool.timestamps = timestamps g_pool.play = False g_pool.new_seek = True g_pool.user_dir = user_dir g_pool.rec_dir = rec_dir g_pool.meta_info = meta_info g_pool.min_data_confidence = session_settings.get('min_data_confidence', 0.6) g_pool.pupil_positions_by_frame = correlate_data(pupil_list, g_pool.timestamps) g_pool.gaze_positions_by_frame = correlate_data(gaze_list, g_pool.timestamps) g_pool.fixations_by_frame = [[] for x in g_pool.timestamps] # populated by the fixation detector plugin def next_frame(_): try: cap.seek_to_frame(cap.get_frame_index()) except(FileSeekError): logger.warning("Could not seek to next frame.") else: g_pool.new_seek = True def prev_frame(_): try: cap.seek_to_frame(cap.get_frame_index()-2) except(FileSeekError): logger.warning("Could not seek to previous frame.") else: g_pool.new_seek = True def toggle_play(new_state): if cap.get_frame_index() >= cap.get_frame_count()-5: cap.seek_to_frame(1) # avoid pause set by hitting trimmark pause. logger.warning("End of video - restart at beginning.") g_pool.play = new_state def set_data_confidence(new_confidence): g_pool.min_data_confidence = new_confidence notification = {'subject': 'min_data_confidence_changed'} notification['_notify_time_'] = time()+.8 g_pool.delayed_notifications[notification['subject']] = notification def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def purge_plugins(): for p in g_pool.plugins: if p.__class__ in user_launchable_plugins: p.alive = False g_pool.plugins.clean() def do_export(_): export_range = slice(g_pool.trim_marks.in_mark, g_pool.trim_marks.out_mark) export_dir = os.path.join(g_pool.rec_dir, 'exports', '{}-{}'.format(export_range.start, export_range.stop)) try: os.makedirs(export_dir) except OSError as e: if e.errno != errno.EEXIST: logger.error("Could not create export dir") raise e else: overwrite_warning = "Previous export for range [{}-{}] already exsits - overwriting." logger.warning(overwrite_warning.format(export_range.start, export_range.stop)) else: logger.info('Created export dir at "{}"'.format(export_dir)) notification = {'subject': 'should_export', 'range': export_range, 'export_dir': export_dir} g_pool.notifications.append(notification) g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get('gui_scale', 1.) g_pool.main_menu = ui.Scrolling_Menu("Settings", pos=(-350, 20), size=(300, 500)) g_pool.main_menu.append(ui.Button('Reset window size', lambda: glfwSetWindowSize(main_window, cap.frame_size[0], cap.frame_size[1]))) g_pool.main_menu.append(ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2], label='Interface Size')) g_pool.main_menu.append(ui.Info_Text('Player Version: {}'.format(g_pool.version))) g_pool.main_menu.append(ui.Info_Text('Capture Version: {}'.format(meta_info['Capture Software Version']))) g_pool.main_menu.append(ui.Info_Text('Data Format Version: {}'.format(meta_info['Data Format Version']))) g_pool.main_menu.append(ui.Slider('min_data_confidence', g_pool, setter=set_data_confidence, step=.05, min=0.0, max=1.0, label='Confidence threshold')) selector_label = "Select to load" vis_labels = [" " + p.__name__.replace('_', ' ') for p in vis_plugins] analysis_labels = [" " + p.__name__.replace('_', ' ') for p in analysis_plugins] other_labels = [" " + p.__name__.replace('_', ' ') for p in other_plugins] user_labels = [" " + p.__name__.replace('_', ' ') for p in user_plugins] plugins = ([selector_label, selector_label] + vis_plugins + [selector_label] + analysis_plugins + [selector_label] + other_plugins + [selector_label] + user_plugins) labels = ([selector_label, "Visualization"] + vis_labels + ["Analysis"] + analysis_labels + ["Other"] + other_labels + ["User added"] + user_labels) g_pool.main_menu.append(ui.Selector('Open plugin:', selection=plugins, labels=labels, setter=open_plugin, getter=lambda: selector_label)) g_pool.main_menu.append(ui.Button('Close all plugins', purge_plugins)) g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100)) g_pool.play_button = ui.Thumb('play', g_pool, label=chr(0xf04b), setter=toggle_play, hotkey=GLFW_KEY_SPACE, label_font='fontawesome', label_offset_x=5, label_offset_y=0, label_offset_size=-24) g_pool.play_button.on_color[:] = (0, 1., .0, .8) g_pool.forward_button = ui.Thumb('forward', label=chr(0xf04e), getter=lambda: False, setter=next_frame, hotkey=GLFW_KEY_RIGHT, label_font='fontawesome', label_offset_x=5, label_offset_y=0, label_offset_size=-24) g_pool.backward_button = ui.Thumb('backward', label=chr(0xf04a), getter=lambda: False, setter=prev_frame, hotkey=GLFW_KEY_LEFT, label_font='fontawesome', label_offset_x=-5, label_offset_y=0, label_offset_size=-24) g_pool.export_button = ui.Thumb('export', label=chr(0xf063), getter=lambda: False, setter=do_export, hotkey='e', label_font='fontawesome', label_offset_x=0, label_offset_y=2, label_offset_size=-24) g_pool.quickbar.extend([g_pool.play_button, g_pool.forward_button, g_pool.backward_button, g_pool.export_button]) g_pool.gui.append(g_pool.quickbar) g_pool.gui.append(g_pool.main_menu) # we always load these plugins system_plugins = [('Trim_Marks', {}), ('Seek_Bar', {})] default_plugins = [('Log_Display', {}), ('Vis_Scan_Path', {}), ('Vis_Polyline', {}), ('Vis_Circle', {}), ('Video_Export_Launcher', {})] previous_plugins = session_settings.get('loaded_plugins', default_plugins) g_pool.notifications = [] g_pool.delayed_notifications = {} g_pool.plugins = Plugin_List(g_pool, plugin_by_name, system_plugins+previous_plugins) # Register callbacks main_window glfwSetFramebufferSizeCallback(main_window, on_resize) glfwSetKeyCallback(main_window, on_key) glfwSetCharCallback(main_window, on_char) glfwSetMouseButtonCallback(main_window, on_button) glfwSetCursorPosCallback(main_window, on_pos) glfwSetScrollCallback(main_window, on_scroll) glfwSetDropCallback(main_window, on_drop) g_pool.gui.configuration = session_settings.get('ui_config', {}) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() # set up performace graphs: pid = os.getpid() ps = psutil.Process(pid) ts = None cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 110) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 110) fps_graph.update_rate = 5 fps_graph.label = "%0.0f REC FPS" pupil_graph = graph.Bar_Graph(max_val=1.0) pupil_graph.pos = (260, 110) pupil_graph.update_rate = 5 pupil_graph.label = "Confidence: %0.2f" g_pool.graphs = [cpu_graph, fps_graph, pupil_graph] # trigger on_resize on_resize(main_window, *glfwGetFramebufferSize(main_window)) while not glfwWindowShouldClose(main_window): # grab new frame if g_pool.play or g_pool.new_seek: g_pool.new_seek = False try: new_frame = cap.get_frame() except EndofVideoFileError: # end of video logic: pause at last frame. g_pool.play = False logger.warning("end of video") update_graph = True else: update_graph = False frame = new_frame.copy() events = {} events['frame'] = frame # report time between now and the last loop interation events['dt'] = get_dt() # new positons we make a deepcopy just like the image is a copy. events['gaze_positions'] = deepcopy(g_pool.gaze_positions_by_frame[frame.index]) events['pupil_positions'] = deepcopy(g_pool.pupil_positions_by_frame[frame.index]) if update_graph: # update performace graphs for p in events['pupil_positions']: pupil_graph.add(p['confidence']) t = new_frame.timestamp if ts and ts != t: dt, ts = t-ts, t fps_graph.add(1./dt) else: ts = new_frame.timestamp g_pool.play_button.status_text = str(frame.index) # always update the CPU graph cpu_graph.update() # publish delayed notifiactions when their time has come. for n in list(g_pool.delayed_notifications.values()): if n['_notify_time_'] < time(): del n['_notify_time_'] del g_pool.delayed_notifications[n['subject']] g_pool.notifications.append(n) # notify each plugin if there are new notifactions: while g_pool.notifications: n = g_pool.notifications.pop(0) for p in g_pool.plugins: p.on_notify(n) # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() # render camera image glfwMakeContextCurrent(main_window) gl_utils.make_coord_system_norm_based() g_pool.image_tex.update_from_frame(frame) g_pool.image_tex.draw() gl_utils.make_coord_system_pixel_based(frame.img.shape) # render visual feedback from loaded plugins for p in g_pool.plugins: p.gl_display() fps_graph.draw() cpu_graph.draw() pupil_graph.draw() g_pool.gui.update() # present frames at appropriate speed cap.wait(frame) glfwSwapBuffers(main_window) glfwPollEvents() session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['min_data_confidence'] = g_pool.min_data_confidence session_settings['gui_scale'] = g_pool.gui_user_scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['window_size'] = glfwGetWindowSize(main_window) session_settings['window_position'] = glfwGetWindowPos(main_window) session_settings['version'] = str(g_pool.version) session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() cap.cleanup() g_pool.gui.terminate() glfwDestroyWindow(main_window)
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_version): # general imports from time import sleep import logging from glob import glob from time import time, strftime, localtime # networking import zmq import zmq_tools import numpy as np # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify", )) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.NOTSET) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) try: # imports from file_methods import Persistent_Dict, next_export_sub_dir # display import glfw # check versions for our own depedencies as they are fast-changing from pyglui import __version__ as pyglui_version from pyglui import ui, cygl from pyglui.cygl.utils import Named_Texture, RGBA import gl_utils # capture from video_capture import init_playback_source # helpers/utils from version_utils import VersionFormat from methods import normalize, denormalize, delta_t, get_system_info import player_methods as pm from csv_utils import write_key_value_file # Plug-ins from plugin import Plugin, Plugin_List, import_runtime_plugins from plugin_manager import Plugin_Manager from vis_circle import Vis_Circle from vis_cross import Vis_Cross from vis_polyline import Vis_Polyline from vis_light_points import Vis_Light_Points from vis_watermark import Vis_Watermark from vis_fixation import Vis_Fixation # from vis_scan_path import Vis_Scan_Path from vis_eye_video_overlay import Vis_Eye_Video_Overlay from seek_control import Seek_Control from offline_surface_tracker import Offline_Surface_Tracker # from marker_auto_trim_marks import Marker_Auto_Trim_Marks from fixation_detector import Offline_Fixation_Detector from log_display import Log_Display from annotations import Annotation_Player from raw_data_exporter import Raw_Data_Exporter from log_history import Log_History from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection from gaze_producers import Gaze_From_Recording, Offline_Calibration from system_graphs import System_Graphs from system_timelines import System_Timelines from blink_detection import Offline_Blink_Detection from audio_playback import Audio_Playback from video_export.plugins.imotions_exporter import iMotions_Exporter from video_export.plugins.eye_video_exporter import Eye_Video_Exporter from video_export.plugins.world_video_exporter import World_Video_Exporter from background_helper import IPC_Logging_Task_Proxy IPC_Logging_Task_Proxy.push_url = ipc_push_url assert VersionFormat(pyglui_version) >= VersionFormat( "1.23"), "pyglui out of date, please upgrade to newest version" runtime_plugins = import_runtime_plugins( os.path.join(user_dir, "plugins")) system_plugins = [ Log_Display, Seek_Control, Plugin_Manager, System_Graphs, System_Timelines, Audio_Playback, ] user_plugins = [ Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross, Vis_Watermark, Vis_Eye_Video_Overlay, # Vis_Scan_Path, Offline_Fixation_Detector, Offline_Blink_Detection, Offline_Surface_Tracker, Raw_Data_Exporter, Annotation_Player, Log_History, Pupil_From_Recording, Offline_Pupil_Detection, Gaze_From_Recording, World_Video_Exporter, iMotions_Exporter, Eye_Video_Exporter, Offline_Calibration, ] + runtime_plugins plugins = system_plugins + user_plugins # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal hdpi_factor if w == 0 or h == 0: return hdpi_factor = glfw.getHDPIFactor(window) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor window_size = w, h g_pool.camera_render_size = w - int( icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(*window_size) g_pool.gui.collect_menus() for p in g_pool.plugins: p.on_window_resize(window, *g_pool.camera_render_size) def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) pos = x, y pos = normalize(pos, g_pool.camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_pos(pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, count, paths): for x in range(count): new_rec_dir = paths[x].decode("utf-8") if pm.is_pupil_rec_dir(new_rec_dir): logger.debug( "Starting new session with '{}'".format(new_rec_dir)) ipc_pub.notify({ "subject": "player_drop_process.should_start", "rec_dir": new_rec_dir, }) glfw.glfwSetWindowShouldClose(window, True) else: logger.error("'{}' is not a valid pupil recording".format( new_rec_dir)) tick = delta_t() def get_dt(): return next(tick) meta_info = pm.load_meta_info(rec_dir) # log info about Pupil Platform and Platform in player.log logger.info("Application Version: {}".format(app_version)) logger.info("System Info: {}".format(get_system_info())) icon_bar_width = 50 window_size = None hdpi_factor = 1.0 # create container for globally scoped vars g_pool = Global_Container() g_pool.app = "player" g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.plugin_by_name = {p.__name__: p for p in plugins} g_pool.camera_render_size = None valid_ext = (".mp4", ".mkv", ".avi", ".h264", ".mjpeg", ".fake") video_path = [ f for f in glob(os.path.join(rec_dir, "world.*")) if os.path.splitext(f)[1] in valid_ext ][0] init_playback_source(g_pool, timing="external", source_path=video_path, buffered_decoding=True) # load session persistent settings session_settings = Persistent_Dict( os.path.join(user_dir, "user_settings_player")) if VersionFormat(session_settings.get("version", "0.0")) != app_version: logger.info( "Session setting are a different version of this app. I will not use those." ) session_settings.clear() width, height = g_pool.capture.frame_size width += icon_bar_width width, height = session_settings.get("window_size", (width, height)) window_pos = session_settings.get("window_position", window_position_default) window_name = "Pupil Player: {} - {}".format( meta_info["Recording Name"], os.path.split(rec_dir)[-1]) glfw.glfwInit() main_window = glfw.glfwCreateWindow(width, height, window_name, None, None) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window def set_scale(new_scale): g_pool.gui_user_scale = new_scale window_size = ( g_pool.camera_render_size[0] + int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor), glfw.glfwGetFramebufferSize(main_window)[1], ) logger.warning(icon_bar_width * g_pool.gui_user_scale * hdpi_factor) glfw.glfwSetWindowSize(main_window, *window_size) # load pupil_positions, gaze_positions g_pool.binocular = meta_info.get("Eye Mode", "monocular") == "binocular" g_pool.version = app_version g_pool.timestamps = g_pool.capture.timestamps g_pool.get_timestamp = lambda: 0.0 g_pool.user_dir = user_dir g_pool.rec_dir = rec_dir g_pool.meta_info = meta_info g_pool.min_data_confidence = session_settings.get( "min_data_confidence", 0.6) g_pool.min_calibration_confidence = session_settings.get( "min_calibration_confidence", 0.8) # populated by producers g_pool.pupil_positions = pm.Bisector() g_pool.pupil_positions_by_id = (pm.Bisector(), pm.Bisector()) g_pool.gaze_positions = pm.Bisector() g_pool.fixations = pm.Affiliator() def set_data_confidence(new_confidence): g_pool.min_data_confidence = new_confidence notification = {"subject": "min_data_confidence_changed"} notification["_notify_time_"] = time() + 0.8 g_pool.ipc_pub.notify(notification) def do_export(_): left_idx = g_pool.seek_control.trim_left right_idx = g_pool.seek_control.trim_right export_range = left_idx, right_idx + 1 # exclusive range.stop export_dir = os.path.join(g_pool.rec_dir, "exports") export_dir = next_export_sub_dir(export_dir) os.makedirs(export_dir) logger.info('Created export dir at "{}"'.format(export_dir)) export_info = { "Player Software Version": str(g_pool.version), "Data Format Version": meta_info["Data Format Version"], "Export Date": strftime("%d.%m.%Y", localtime()), "Export Time": strftime("%H:%M:%S", localtime()), "Frame Index Range:": g_pool.seek_control.get_frame_index_trim_range_string(), "Relative Time Range": g_pool.seek_control.get_rel_time_trim_range_string(), "Absolute Time Range": g_pool.seek_control.get_abs_time_trim_range_string(), } with open(os.path.join(export_dir, "export_info.csv"), "w") as csv: write_key_value_file(csv, export_info) notification = { "subject": "should_export", "range": export_range, "export_dir": export_dir, } g_pool.ipc_pub.notify(notification) def reset_restart(): logger.warning("Resetting all settings and restarting Player.") glfw.glfwSetWindowShouldClose(main_window, True) ipc_pub.notify({"subject": "clear_settings_process.should_start"}) ipc_pub.notify({ "subject": "player_process.should_start", "rec_dir": rec_dir, "delay": 2.0, }) def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is open the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0) g_pool.menubar = ui.Scrolling_Menu("Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos="left") g_pool.iconbar = ui.Scrolling_Menu("Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden") g_pool.timelines = ui.Container((0, 0), (0, 0), (0, 0)) g_pool.timelines.horizontal_constraint = g_pool.menubar g_pool.user_timelines = ui.Timeline_Menu("User Timelines", pos=(0.0, -150.0), size=(0.0, 0.0), header_pos="headline") g_pool.user_timelines.color = RGBA(a=0.0) g_pool.user_timelines.collapsed = True # add container that constaints itself to the seekbar height vert_constr = ui.Container((0, 0), (0, -50.0), (0, 0)) vert_constr.append(g_pool.user_timelines) g_pool.timelines.append(vert_constr) def set_window_size(): f_width, f_height = g_pool.capture.frame_size f_width += int(icon_bar_width * g_pool.gui.scale) glfw.glfwSetWindowSize(main_window, f_width, f_height) general_settings = ui.Growing_Menu("General", header_pos="headline") general_settings.append(ui.Button("Reset window size", set_window_size)) general_settings.append( ui.Selector( "gui_user_scale", g_pool, setter=set_scale, selection=[0.8, 0.9, 1.0, 1.1, 1.2] + list(np.arange(1.5, 5.1, 0.5)), label="Interface Size", )) general_settings.append( ui.Info_Text("Player Version: {}".format(g_pool.version))) general_settings.append( ui.Info_Text("Capture Version: {}".format( meta_info["Capture Software Version"]))) general_settings.append( ui.Info_Text("Data Format Version: {}".format( meta_info["Data Format Version"]))) general_settings.append( ui.Info_Text( "High level data, e.g. fixations, or visualizations only consider gaze data that has an equal or higher confidence than the minimum data confidence." )) general_settings.append( ui.Slider( "min_data_confidence", g_pool, setter=set_data_confidence, step=0.05, min=0.0, max=1.0, label="Minimum data confidence", )) general_settings.append( ui.Button("Restart with default settings", reset_restart)) g_pool.menubar.append(general_settings) icon = ui.Icon( "collapsed", general_settings, label=chr(0xE8B8), on_val=False, off_val=True, setter=toggle_general_settings, label_font="pupil_icons", ) icon.tooltip = "General Settings" g_pool.iconbar.append(icon) user_plugin_separator = ui.Separator() user_plugin_separator.order = 0.35 g_pool.iconbar.append(user_plugin_separator) g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100), (100, -100)) g_pool.export_button = ui.Thumb( "export", label=chr(0xE2C5), getter=lambda: False, setter=do_export, hotkey="e", label_font="pupil_icons", ) g_pool.quickbar.extend([g_pool.export_button]) g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.timelines) g_pool.gui.append(g_pool.iconbar) g_pool.gui.append(g_pool.quickbar) # we always load these plugins default_plugins = [ ("Plugin_Manager", {}), ("Seek_Control", {}), ("Log_Display", {}), ("Raw_Data_Exporter", {}), ("Vis_Polyline", {}), ("Vis_Circle", {}), ("System_Graphs", {}), ("System_Timelines", {}), ("World_Video_Exporter", {}), ("Pupil_From_Recording", {}), ("Gaze_From_Recording", {}), ("Audio_Playback", {}), ] g_pool.plugins = Plugin_List( g_pool, session_settings.get("loaded_plugins", default_plugins)) # Manually add g_pool.capture to the plugin list g_pool.plugins._plugins.append(g_pool.capture) g_pool.plugins._plugins.sort(key=lambda p: p.order) g_pool.capture.init_ui() general_settings.insert( -1, ui.Text_Input( "rel_time_trim_section", getter=g_pool.seek_control.get_rel_time_trim_range_string, setter=g_pool.seek_control.set_rel_time_trim_range_string, label="Relative time range to export", ), ) general_settings.insert( -1, ui.Text_Input( "frame_idx_trim_section", getter=g_pool.seek_control.get_frame_index_trim_range_string, setter=g_pool.seek_control.set_frame_index_trim_range_string, label="Frame index range to export", ), ) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) toggle_general_settings(True) g_pool.gui.configuration = session_settings.get("ui_config", {}) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() # trigger on_resize on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) def handle_notifications(n): subject = n["subject"] if subject == "start_plugin": g_pool.plugins.add(g_pool.plugin_by_name[n["name"]], args=n.get("args", {})) elif subject.startswith("meta.should_doc"): ipc_pub.notify({ "subject": "meta.doc", "actor": g_pool.app, "doc": player.__doc__ }) for p in g_pool.plugins: if (p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify): ipc_pub.notify({ "subject": "meta.doc", "actor": p.class_name, "doc": p.on_notify.__doc__, }) while not glfw.glfwWindowShouldClose(main_window): # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) events = {} # report time between now and the last loop interation events["dt"] = get_dt() # pupil and gaze positions are added by their respective producer plugins events["pupil"] = [] events["gaze"] = [] # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() glfw.glfwMakeContextCurrent(main_window) # render visual feedback from loaded plugins if gl_utils.is_window_visible(main_window): gl_utils.glViewport(0, 0, *g_pool.camera_render_size) g_pool.capture.gl_display() for p in g_pool.plugins: p.gl_display() gl_utils.glViewport(0, 0, *window_size) try: clipboard = glfw.glfwGetClipboardString( main_window).decode() except AttributeError: # clipbaord is None, might happen on startup clipboard = "" g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard and user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.glfwSetClipboardString(main_window, user_input.clipboard.encode()) for b in user_input.buttons: button, action, mods = b x, y = glfw.glfwGetCursorPos(main_window) pos = x * hdpi_factor, y * hdpi_factor pos = normalize(pos, g_pool.camera_render_size) pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_click(pos, button, action) for key, scancode, action, mods in user_input.keys: for p in g_pool.plugins: p.on_key(key, scancode, action, mods) for char_ in user_input.chars: for p in g_pool.plugins: p.on_char(char_) # present frames at appropriate speed g_pool.seek_control.wait(events["frame"].timestamp) glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() session_settings["loaded_plugins"] = g_pool.plugins.get_initializers() session_settings["min_data_confidence"] = g_pool.min_data_confidence session_settings[ "min_calibration_confidence"] = g_pool.min_calibration_confidence session_settings["gui_scale"] = g_pool.gui_user_scale session_settings["ui_config"] = g_pool.gui.configuration session_settings["window_position"] = glfw.glfwGetWindowPos( main_window) session_settings["version"] = str(g_pool.version) session_window_size = glfw.glfwGetWindowSize(main_window) if 0 not in session_window_size: session_settings["window_size"] = session_window_size session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) except: import traceback trace = traceback.format_exc() logger.error("Process Player crashed with trace:\n{}".format(trace)) finally: logger.info("Process shutting down.") ipc_pub.notify({"subject": "player_process.stopped"}) sleep(1.0)
def export(rec_dir, user_dir, min_data_confidence, start_frame=None, end_frame=None, plugin_initializers=(), out_file_path=None, pre_computed={}): logger = logging.getLogger(__name__ + ' with pid: ' + str(os.getpid())) start_status = 'Starting video export with pid: {}'.format(os.getpid()) print(start_status) yield start_status, 0 try: update_recording_to_recent(rec_dir) vis_plugins = sorted([ Vis_Circle, Vis_Cross, Vis_Polyline, Vis_Light_Points, Vis_Watermark, Vis_Scan_Path, Vis_Eye_Video_Overlay ], key=lambda x: x.__name__) analysis_plugins = [Offline_Fixation_Detector] user_plugins = sorted(import_runtime_plugins( os.path.join(user_dir, 'plugins')), key=lambda x: x.__name__) available_plugins = vis_plugins + analysis_plugins + user_plugins name_by_index = [p.__name__ for p in available_plugins] plugin_by_name = dict(zip(name_by_index, available_plugins)) update_recording_to_recent(rec_dir) video_path = [ f for f in glob(os.path.join(rec_dir, "world.*")) if os.path.splitext(f)[-1] in ('.mp4', '.mkv', '.avi', '.mjpeg') ][0] pupil_data_path = os.path.join(rec_dir, "pupil_data") audio_path = os.path.join(rec_dir, "audio.mp4") meta_info = load_meta_info(rec_dir) g_pool = Global_Container() g_pool.app = 'exporter' g_pool.min_data_confidence = min_data_confidence cap = File_Source(g_pool, video_path) timestamps = cap.timestamps # Out file path verification, we do this before but if one uses a separate tool, this will kick in. if out_file_path is None: out_file_path = os.path.join(rec_dir, "world_viz.mp4") else: file_name = os.path.basename(out_file_path) dir_name = os.path.dirname(out_file_path) if not dir_name: dir_name = rec_dir if not file_name: file_name = 'world_viz.mp4' out_file_path = os.path.expanduser( os.path.join(dir_name, file_name)) if os.path.isfile(out_file_path): logger.warning("Video out file already exsists. I will overwrite!") os.remove(out_file_path) logger.debug("Saving Video to {}".format(out_file_path)) # Trim mark verification # make sure the trim marks (start frame, endframe) make sense: # We define them like python list slices, thus we can test them like such. trimmed_timestamps = timestamps[start_frame:end_frame] if len(trimmed_timestamps) == 0: warn = "Start and end frames are set such that no video will be exported." logger.warning(warn) yield warn, 0. return if start_frame is None: start_frame = 0 # these two vars are shared with the lauching process and give a job length and progress report. frames_to_export = len(trimmed_timestamps) current_frame = 0 exp_info = "Will export from frame {} to frame {}. This means I will export {} frames." logger.debug( exp_info.format(start_frame, start_frame + frames_to_export, frames_to_export)) # setup of writer writer = AV_Writer(out_file_path, fps=cap.frame_rate, audio_loc=audio_path, use_timestamps=True) cap.seek_to_frame(start_frame) start_time = time() g_pool.plugin_by_name = plugin_by_name g_pool.capture = cap g_pool.rec_dir = rec_dir g_pool.user_dir = user_dir g_pool.meta_info = meta_info g_pool.timestamps = timestamps g_pool.delayed_notifications = {} g_pool.notifications = [] # load pupil_positions, gaze_positions pupil_data = pre_computed.get("pupil_data") or load_object( pupil_data_path) g_pool.pupil_data = pupil_data g_pool.pupil_positions = pre_computed.get( "pupil_positions") or pupil_data['pupil_positions'] g_pool.gaze_positions = pre_computed.get( "gaze_positions") or pupil_data['gaze_positions'] g_pool.fixations = [] # populated by the fixation detector plugin g_pool.pupil_positions_by_frame = correlate_data( g_pool.pupil_positions, g_pool.timestamps) g_pool.gaze_positions_by_frame = correlate_data( g_pool.gaze_positions, g_pool.timestamps) g_pool.fixations_by_frame = [ [] for x in g_pool.timestamps ] # populated by the fixation detector plugin # add plugins g_pool.plugins = Plugin_List(g_pool, plugin_initializers) while frames_to_export > current_frame: try: frame = cap.get_frame() except EndofVideoFileError: break events = {'frame': frame} # new positons and events events['gaze_positions'] = g_pool.gaze_positions_by_frame[ frame.index] events['pupil_positions'] = g_pool.pupil_positions_by_frame[ frame.index] # publish delayed notifiactions when their time has come. for n in list(g_pool.delayed_notifications.values()): if n['_notify_time_'] < time(): del n['_notify_time_'] del g_pool.delayed_notifications[n['subject']] g_pool.notifications.append(n) # notify each plugin if there are new notifactions: while g_pool.notifications: n = g_pool.notifications.pop(0) for p in g_pool.plugins: p.on_notify(n) # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) writer.write_video_frame(frame) current_frame += 1 yield 'Exporting', current_frame writer.close() writer = None duration = time() - start_time effective_fps = float(current_frame) / duration result = "Export done: Exported {} frames to {}. This took {} seconds. Exporter ran at {} frames per second." print( result.format(current_frame, out_file_path, duration, effective_fps)) yield 'Export done. This took {:.0f} seconds.'.format( duration), current_frame except GeneratorExit: print('Video export with pid {} was canceled.'.format(os.getpid())) except: from time import sleep import traceback trace = traceback.format_exc() print('Process Export (pid: {}) crashed with trace:\n{}'.format( os.getpid(), trace)) sleep(1.0)
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_version): # general imports from time import sleep import logging import errno from glob import glob from time import time # networking import zmq import zmq_tools import numpy as np # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify', )) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.INFO) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) try: # imports from file_methods import Persistent_Dict, load_object # display import glfw # check versions for our own depedencies as they are fast-changing from pyglui import __version__ as pyglui_version from pyglui import ui, cygl from pyglui.cygl.utils import Named_Texture, RGBA import gl_utils # capture from video_capture import File_Source, EndofVideoFileError # helpers/utils from version_utils import VersionFormat from methods import normalize, denormalize, delta_t, get_system_info from player_methods import correlate_data, is_pupil_rec_dir, load_meta_info # Plug-ins from plugin import Plugin, Plugin_List, import_runtime_plugins from plugin_manager import Plugin_Manager from vis_circle import Vis_Circle from vis_cross import Vis_Cross from vis_polyline import Vis_Polyline from vis_light_points import Vis_Light_Points from vis_watermark import Vis_Watermark from vis_fixation import Vis_Fixation from vis_scan_path import Vis_Scan_Path from vis_eye_video_overlay import Vis_Eye_Video_Overlay from seek_control import Seek_Control from video_export_launcher import Video_Export_Launcher from offline_surface_tracker import Offline_Surface_Tracker # from marker_auto_trim_marks import Marker_Auto_Trim_Marks from fixation_detector import Offline_Fixation_Detector from batch_exporter import Batch_Exporter, Batch_Export from log_display import Log_Display from annotations import Annotation_Player from raw_data_exporter import Raw_Data_Exporter from log_history import Log_History from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection from gaze_producers import Gaze_From_Recording, Offline_Calibration from system_graphs import System_Graphs from system_timelines import System_Timelines from blink_detection import Offline_Blink_Detection assert VersionFormat(pyglui_version) >= VersionFormat( '1.17'), 'pyglui out of date, please upgrade to newest version' runtime_plugins = import_runtime_plugins( os.path.join(user_dir, 'plugins')) system_plugins = [ Log_Display, Seek_Control, Plugin_Manager, System_Graphs, Batch_Export, System_Timelines ] user_plugins = [ Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross, Vis_Watermark, Vis_Eye_Video_Overlay, Vis_Scan_Path, Offline_Fixation_Detector, Offline_Blink_Detection, Batch_Exporter, Video_Export_Launcher, Offline_Surface_Tracker, Raw_Data_Exporter, Annotation_Player, Log_History, Pupil_From_Recording, Offline_Pupil_Detection, Gaze_From_Recording, Offline_Calibration ] + runtime_plugins plugins = system_plugins + user_plugins # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal hdpi_factor hdpi_factor = float( glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor window_size = w, h g_pool.camera_render_size = w - int( icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(*window_size) g_pool.gui.collect_menus() for p in g_pool.plugins: p.on_window_resize(window, *g_pool.camera_render_size) def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) pos = x, y pos = normalize(pos, g_pool.camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_pos(pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, count, paths): for x in range(count): new_rec_dir = paths[x].decode('utf-8') if is_pupil_rec_dir(new_rec_dir): logger.debug( "Starting new session with '{}'".format(new_rec_dir)) ipc_pub.notify({ "subject": "player_drop_process.should_start", "rec_dir": new_rec_dir }) glfw.glfwSetWindowShouldClose(window, True) else: logger.error("'{}' is not a valid pupil recording".format( new_rec_dir)) tick = delta_t() def get_dt(): return next(tick) video_path = [ f for f in glob(os.path.join(rec_dir, "world.*")) if os.path.splitext(f)[1] in ('.mp4', '.mkv', '.avi', '.h264', '.mjpeg') ][0] pupil_data_path = os.path.join(rec_dir, "pupil_data") meta_info = load_meta_info(rec_dir) # log info about Pupil Platform and Platform in player.log logger.info('Application Version: {}'.format(app_version)) logger.info('System Info: {}'.format(get_system_info())) icon_bar_width = 50 window_size = None hdpi_factor = 1.0 # create container for globally scoped vars g_pool = Global_Container() g_pool.app = 'player' g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.plugin_by_name = {p.__name__: p for p in plugins} g_pool.camera_render_size = None # sets itself to g_pool.capture File_Source(g_pool, video_path) # load session persistent settings session_settings = Persistent_Dict( os.path.join(user_dir, "user_settings_player")) if VersionFormat(session_settings.get("version", '0.0')) != app_version: logger.info( "Session setting are a different version of this app. I will not use those." ) session_settings.clear() g_pool.capture.playback_speed = session_settings.get( 'playback_speed', 1.) width, height = session_settings.get('window_size', g_pool.capture.frame_size) window_pos = session_settings.get('window_position', window_position_default) glfw.glfwInit() main_window = glfw.glfwCreateWindow( width, height, "Pupil Player: " + meta_info["Recording Name"] + " - " + rec_dir.split(os.path.sep)[-1], None, None) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window def set_scale(new_scale): g_pool.gui_user_scale = new_scale window_size = ( g_pool.camera_render_size[0] + int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor), glfw.glfwGetFramebufferSize(main_window)[1]) logger.warning(icon_bar_width * g_pool.gui_user_scale * hdpi_factor) glfw.glfwSetWindowSize(main_window, *window_size) # load pupil_positions, gaze_positions g_pool.pupil_data = load_object(pupil_data_path) g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular' g_pool.version = app_version g_pool.timestamps = g_pool.capture.timestamps g_pool.get_timestamp = lambda: 0. g_pool.new_seek = True g_pool.user_dir = user_dir g_pool.rec_dir = rec_dir g_pool.meta_info = meta_info g_pool.min_data_confidence = session_settings.get( 'min_data_confidence', 0.6) g_pool.pupil_positions = [] g_pool.gaze_positions = [] g_pool.fixations = [] g_pool.notifications_by_frame = correlate_data( g_pool.pupil_data['notifications'], g_pool.timestamps) g_pool.pupil_positions_by_frame = [[] for x in g_pool.timestamps ] # populated by producer` g_pool.gaze_positions_by_frame = [[] for x in g_pool.timestamps ] # populated by producer g_pool.fixations_by_frame = [ [] for x in g_pool.timestamps ] # populated by the fixation detector plugin def set_data_confidence(new_confidence): g_pool.min_data_confidence = new_confidence notification = {'subject': 'min_data_confidence_changed'} notification['_notify_time_'] = time() + .8 g_pool.ipc_pub.notify(notification) def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def purge_plugins(): for p in g_pool.plugins: if p.__class__ in user_plugins: p.alive = False g_pool.plugins.clean() def do_export(_): export_range = g_pool.seek_control.trim_left, g_pool.seek_control.trim_right export_dir = os.path.join(g_pool.rec_dir, 'exports', '{}-{}'.format(*export_range)) try: os.makedirs(export_dir) except OSError as e: if e.errno != errno.EEXIST: logger.error("Could not create export dir") raise e else: overwrite_warning = "Previous export for range [{}-{}] already exists - overwriting." logger.warning(overwrite_warning.format(*export_range)) else: logger.info('Created export dir at "{}"'.format(export_dir)) notification = { 'subject': 'should_export', 'range': export_range, 'export_dir': export_dir } g_pool.ipc_pub.notify(notification) def reset_restart(): logger.warning("Resetting all settings and restarting Player.") glfw.glfwSetWindowShouldClose(main_window, True) ipc_pub.notify({'subject': 'clear_settings_process.should_start'}) ipc_pub.notify({ 'subject': 'player_process.should_start', 'rec_dir': rec_dir, 'delay': 2. }) def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is open the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get('gui_scale', 1.) g_pool.menubar = ui.Scrolling_Menu("Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos='left') g_pool.iconbar = ui.Scrolling_Menu("Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos='hidden') g_pool.timelines = ui.Container((0, 0), (0, 0), (0, 0)) g_pool.timelines.horizontal_constraint = g_pool.menubar g_pool.user_timelines = ui.Timeline_Menu('User Timelines', pos=(0., -150.), size=(0., 0.), header_pos='headline') g_pool.user_timelines.color = RGBA(a=0.) g_pool.user_timelines.collapsed = True # add container that constaints itself to the seekbar height vert_constr = ui.Container((0, 0), (0, -50.), (0, 0)) vert_constr.append(g_pool.user_timelines) g_pool.timelines.append(vert_constr) general_settings = ui.Growing_Menu('General', header_pos='headline') general_settings.append( ui.Button( 'Reset window size', lambda: glfw.glfwSetWindowSize( main_window, g_pool.capture.frame_size[0], g_pool.capture. frame_size[1]))) general_settings.append( ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2] + list(np.arange(1.5, 5.1, .5)), label='Interface Size')) general_settings.append( ui.Info_Text('Player Version: {}'.format(g_pool.version))) general_settings.append( ui.Info_Text('Capture Version: {}'.format( meta_info['Capture Software Version']))) general_settings.append( ui.Info_Text('Data Format Version: {}'.format( meta_info['Data Format Version']))) general_settings.append( ui.Slider('min_data_confidence', g_pool, setter=set_data_confidence, step=.05, min=0.0, max=1.0, label='Confidence threshold')) general_settings.append( ui.Button('Restart with default settings', reset_restart)) g_pool.menubar.append(general_settings) icon = ui.Icon('collapsed', general_settings, label=chr(0xe8b8), on_val=False, off_val=True, setter=toggle_general_settings, label_font='pupil_icons') icon.tooltip = 'General Settings' g_pool.iconbar.append(icon) user_plugin_separator = ui.Separator() user_plugin_separator.order = 0.35 g_pool.iconbar.append(user_plugin_separator) g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (100, -100)) g_pool.export_button = ui.Thumb('export', label=chr(0xe2c5), getter=lambda: False, setter=do_export, hotkey='e', label_font='pupil_icons') g_pool.quickbar.extend([g_pool.export_button]) g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.timelines) g_pool.gui.append(g_pool.iconbar) g_pool.gui.append(g_pool.quickbar) # we always load these plugins default_plugins = [('Plugin_Manager', {}), ('Seek_Control', {}), ('Log_Display', {}), ('Raw_Data_Exporter', {}), ('Vis_Polyline', {}), ('Vis_Circle', {}), ('System_Graphs', {}), ('System_Timelines', {}), ('Video_Export_Launcher', {}), ('Pupil_From_Recording', {}), ('Gaze_From_Recording', {})] g_pool.plugins = Plugin_List( g_pool, session_settings.get('loaded_plugins', default_plugins)) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) toggle_general_settings(True) g_pool.gui.configuration = session_settings.get('ui_config', {}) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() # trigger on_resize on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) def handle_notifications(n): subject = n['subject'] if subject == 'start_plugin': g_pool.plugins.add(g_pool.plugin_by_name[n['name']], args=n.get('args', {})) elif subject.startswith('meta.should_doc'): ipc_pub.notify({ 'subject': 'meta.doc', 'actor': g_pool.app, 'doc': player.__doc__ }) for p in g_pool.plugins: if (p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify): ipc_pub.notify({ 'subject': 'meta.doc', 'actor': p.class_name, 'doc': p.on_notify.__doc__ }) while not glfw.glfwWindowShouldClose(main_window): # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) # grab new frame if g_pool.capture.play or g_pool.new_seek: g_pool.new_seek = False try: new_frame = g_pool.capture.get_frame() except EndofVideoFileError: # end of video logic: pause at last frame. g_pool.capture.play = False logger.warning("end of video") frame = new_frame.copy() events = {} events['frame'] = frame # report time between now and the last loop interation events['dt'] = get_dt() # pupil and gaze positions are added by their respective producer plugins events['pupil_positions'] = [] events['gaze_positions'] = [] # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() glfw.glfwMakeContextCurrent(main_window) # render visual feedback from loaded plugins if gl_utils.is_window_visible(main_window): gl_utils.glViewport(0, 0, *g_pool.camera_render_size) g_pool.capture._recent_frame = frame g_pool.capture.gl_display() for p in g_pool.plugins: p.gl_display() gl_utils.glViewport(0, 0, *window_size) try: clipboard = glfw.glfwGetClipboardString( main_window).decode() except AttributeError: # clipbaord is None, might happen on startup clipboard = '' g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard and user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.glfwSetClipboardString(main_window, user_input.clipboard.encode()) for b in user_input.buttons: button, action, mods = b x, y = glfw.glfwGetCursorPos(main_window) pos = x * hdpi_factor, y * hdpi_factor pos = normalize(pos, g_pool.camera_render_size) pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_click(pos, button, action) for key, scancode, action, mods in user_input.keys: for p in g_pool.plugins: p.on_key(key, scancode, action, mods) for char_ in user_input.chars: for p in g_pool.plugins: p.on_char(char_) glfw.glfwSwapBuffers(main_window) # present frames at appropriate speed g_pool.capture.wait(frame) glfw.glfwPollEvents() session_settings['playback_speed'] = g_pool.capture.playback_speed session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['min_data_confidence'] = g_pool.min_data_confidence session_settings['gui_scale'] = g_pool.gui_user_scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos( main_window) session_settings['version'] = str(g_pool.version) session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.capture.cleanup() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) except: import traceback trace = traceback.format_exc() logger.error('Process Player crashed with trace:\n{}'.format(trace)) finally: logger.info("Process shutting down.") ipc_pub.notify({'subject': 'player_process.stopped'}) sleep(1.0)
def export( rec_dir, user_dir, min_data_confidence, start_frame=None, end_frame=None, plugin_initializers=(), out_file_path=None, pre_computed={}, ): PID = str(os.getpid()) logger = logging.getLogger(__name__ + " with pid: " + PID) start_status = "Starting video export with pid: {}".format(PID) print(start_status) yield start_status, 0 try: pm.update_recording_to_recent(rec_dir) vis_plugins = sorted( [ Vis_Circle, Vis_Cross, Vis_Polyline, Vis_Light_Points, Vis_Watermark, Vis_Scan_Path, Vis_Eye_Video_Overlay, ], key=lambda x: x.__name__, ) analysis_plugins = [Offline_Fixation_Detector] user_plugins = sorted( import_runtime_plugins(os.path.join(user_dir, "plugins")), key=lambda x: x.__name__, ) available_plugins = vis_plugins + analysis_plugins + user_plugins name_by_index = [p.__name__ for p in available_plugins] plugin_by_name = dict(zip(name_by_index, available_plugins)) pm.update_recording_to_recent(rec_dir) audio_path = os.path.join(rec_dir, "audio.mp4") meta_info = pm.load_meta_info(rec_dir) g_pool = Global_Container() g_pool.app = "exporter" g_pool.min_data_confidence = min_data_confidence valid_ext = (".mp4", ".mkv", ".avi", ".h264", ".mjpeg", ".fake") video_path = [ f for f in glob(os.path.join(rec_dir, "world.*")) if os.path.splitext(f)[1] in valid_ext ][0] cap = init_playback_source(g_pool, source_path=video_path, timing=None) timestamps = cap.timestamps # Out file path verification, we do this before but if one uses a separate tool, this will kick in. if out_file_path is None: out_file_path = os.path.join(rec_dir, "world_viz.mp4") else: file_name = os.path.basename(out_file_path) dir_name = os.path.dirname(out_file_path) if not dir_name: dir_name = rec_dir if not file_name: file_name = "world_viz.mp4" out_file_path = os.path.expanduser( os.path.join(dir_name, file_name)) if os.path.isfile(out_file_path): logger.warning("Video out file already exsists. I will overwrite!") os.remove(out_file_path) logger.debug("Saving Video to {}".format(out_file_path)) # Trim mark verification # make sure the trim marks (start frame, endframe) make sense: # We define them like python list slices, thus we can test them like such. trimmed_timestamps = timestamps[start_frame:end_frame] if len(trimmed_timestamps) == 0: warn = "Start and end frames are set such that no video will be exported." logger.warning(warn) yield warn, 0.0 return if start_frame is None: start_frame = 0 # these two vars are shared with the lauching process and give a job length and progress report. frames_to_export = len(trimmed_timestamps) current_frame = 0 exp_info = ( "Will export from frame {} to frame {}. This means I will export {} frames." ) logger.debug( exp_info.format(start_frame, start_frame + frames_to_export, frames_to_export)) # setup of writer writer = AV_Writer(out_file_path, fps=cap.frame_rate, audio_loc=audio_path, use_timestamps=True) cap.seek_to_frame(start_frame) start_time = time() g_pool.plugin_by_name = plugin_by_name g_pool.capture = cap g_pool.rec_dir = rec_dir g_pool.user_dir = user_dir g_pool.meta_info = meta_info g_pool.timestamps = timestamps g_pool.delayed_notifications = {} g_pool.notifications = [] for initializers in pre_computed.values(): initializers["data"] = [ fm.Serialized_Dict(msgpack_bytes=serialized) for serialized in initializers["data"] ] g_pool.pupil_positions = pm.Bisector(**pre_computed["pupil"]) g_pool.gaze_positions = pm.Bisector(**pre_computed["gaze"]) g_pool.fixations = pm.Affiliator(**pre_computed["fixations"]) # add plugins g_pool.plugins = Plugin_List(g_pool, plugin_initializers) while frames_to_export > current_frame: try: frame = cap.get_frame() except EndofVideoError: break events = {"frame": frame} # new positons and events frame_window = pm.enclosing_window(g_pool.timestamps, frame.index) events["gaze"] = g_pool.gaze_positions.by_ts_window(frame_window) events["pupil"] = g_pool.pupil_positions.by_ts_window(frame_window) # publish delayed notifiactions when their time has come. for n in list(g_pool.delayed_notifications.values()): if n["_notify_time_"] < time(): del n["_notify_time_"] del g_pool.delayed_notifications[n["subject"]] g_pool.notifications.append(n) # notify each plugin if there are new notifactions: while g_pool.notifications: n = g_pool.notifications.pop(0) for p in g_pool.plugins: p.on_notify(n) # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) writer.write_video_frame(frame) current_frame += 1 yield "Exporting with pid {}".format(PID), current_frame writer.close() writer = None duration = time() - start_time effective_fps = float(current_frame) / duration result = "Export done: Exported {} frames to {}. This took {} seconds. Exporter ran at {} frames per second." print( result.format(current_frame, out_file_path, duration, effective_fps)) yield "Export done. This took {:.0f} seconds.".format( duration), current_frame except GeneratorExit: print("Video export with pid {} was canceled.".format(os.getpid())) except Exception as e: from time import sleep import traceback trace = traceback.format_exc() print("Process Export (pid: {}) crashed with trace:\n{}".format( os.getpid(), trace)) yield e sleep(1.0)
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_version): # general imports from time import sleep import logging from glob import glob from time import time, strftime, localtime # networking import zmq import zmq_tools import numpy as np # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",)) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.NOTSET) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) try: from background_helper import IPC_Logging_Task_Proxy IPC_Logging_Task_Proxy.push_url = ipc_push_url from tasklib.background.patches import IPCLoggingPatch IPCLoggingPatch.ipc_push_url = ipc_push_url # imports from file_methods import Persistent_Dict, next_export_sub_dir # display import glfw # check versions for our own depedencies as they are fast-changing from pyglui import __version__ as pyglui_version from pyglui import ui, cygl from pyglui.cygl.utils import Named_Texture, RGBA import gl_utils # capture from video_capture import File_Source # helpers/utils from version_utils import VersionFormat from methods import normalize, denormalize, delta_t, get_system_info import player_methods as pm from csv_utils import write_key_value_file # Plug-ins from plugin import Plugin, Plugin_List, import_runtime_plugins from plugin_manager import Plugin_Manager from vis_circle import Vis_Circle from vis_cross import Vis_Cross from vis_polyline import Vis_Polyline from vis_light_points import Vis_Light_Points from vis_watermark import Vis_Watermark from vis_fixation import Vis_Fixation # from vis_scan_path import Vis_Scan_Path from seek_control import Seek_Control from surface_tracker import Surface_Tracker_Offline # from marker_auto_trim_marks import Marker_Auto_Trim_Marks from fixation_detector import Offline_Fixation_Detector from eye_movement import Offline_Eye_Movement_Detector from log_display import Log_Display from annotations import Annotation_Player from raw_data_exporter import Raw_Data_Exporter from log_history import Log_History from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection from gaze_producer.gaze_from_recording import GazeFromRecording from gaze_producer.gaze_from_offline_calibration import ( GazeFromOfflineCalibration, ) from system_graphs import System_Graphs from system_timelines import System_Timelines from blink_detection import Offline_Blink_Detection from audio_playback import Audio_Playback from video_export.plugins.imotions_exporter import iMotions_Exporter from video_export.plugins.eye_video_exporter import Eye_Video_Exporter from video_export.plugins.world_video_exporter import World_Video_Exporter from video_capture import File_Source from video_overlay.plugins import Video_Overlay, Eye_Overlay assert VersionFormat(pyglui_version) >= VersionFormat( "1.23" ), "pyglui out of date, please upgrade to newest version" runtime_plugins = import_runtime_plugins(os.path.join(user_dir, "plugins")) system_plugins = [ Log_Display, Seek_Control, Plugin_Manager, System_Graphs, System_Timelines, Audio_Playback, ] user_plugins = [ Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross, Vis_Watermark, Eye_Overlay, Video_Overlay, # Vis_Scan_Path, Offline_Fixation_Detector, Offline_Eye_Movement_Detector, Offline_Blink_Detection, Surface_Tracker_Offline, Raw_Data_Exporter, Annotation_Player, Log_History, Pupil_From_Recording, Offline_Pupil_Detection, GazeFromRecording, GazeFromOfflineCalibration, World_Video_Exporter, iMotions_Exporter, Eye_Video_Exporter, ] + runtime_plugins if platform.system() != "Windows": # Head pose tracking is currently not available on Windows from head_pose_tracker.offline_head_pose_tracker import ( Offline_Head_Pose_Tracker, ) user_plugins.append(Offline_Head_Pose_Tracker) plugins = system_plugins + user_plugins # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal hdpi_factor if w == 0 or h == 0: return hdpi_factor = glfw.getHDPIFactor(window) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor window_size = w, h g_pool.camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(*window_size) g_pool.gui.collect_menus() for p in g_pool.plugins: p.on_window_resize(window, *g_pool.camera_render_size) def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) pos = x, y pos = normalize(pos, g_pool.camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_pos(pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, count, paths): paths = [paths[x].decode("utf-8") for x in range(count)] for path in paths: if pm.is_pupil_rec_dir(path): _restart_with_recording(path) return # call `on_drop` callbacks until a plugin indicates # that it has consumed the event (by returning True) any(p.on_drop(paths) for p in g_pool.plugins) def _restart_with_recording(rec_dir): logger.debug("Starting new session with '{}'".format(rec_dir)) ipc_pub.notify( {"subject": "player_drop_process.should_start", "rec_dir": rec_dir} ) glfw.glfwSetWindowShouldClose(g_pool.main_window, True) tick = delta_t() def get_dt(): return next(tick) meta_info = pm.load_meta_info(rec_dir) # log info about Pupil Platform and Platform in player.log logger.info("Application Version: {}".format(app_version)) logger.info("System Info: {}".format(get_system_info())) icon_bar_width = 50 window_size = None hdpi_factor = 1.0 # create container for globally scoped vars g_pool = SimpleNamespace() g_pool.app = "player" g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.plugin_by_name = {p.__name__: p for p in plugins} g_pool.camera_render_size = None valid_ext = (".mp4", ".mkv", ".avi", ".h264", ".mjpeg", ".fake") video_path = [ f for f in glob(os.path.join(rec_dir, "world.*")) if os.path.splitext(f)[1] in valid_ext ][0] File_Source( g_pool, timing="external", source_path=video_path, buffered_decoding=True, fill_gaps=True, ) # load session persistent settings session_settings = Persistent_Dict( os.path.join(user_dir, "user_settings_player") ) if VersionFormat(session_settings.get("version", "0.0")) != app_version: logger.info( "Session setting are a different version of this app. I will not use those." ) session_settings.clear() width, height = g_pool.capture.frame_size width += icon_bar_width width, height = session_settings.get("window_size", (width, height)) window_pos = session_settings.get("window_position", window_position_default) window_name = "Pupil Player: {} - {}".format( meta_info["Recording Name"], os.path.split(rec_dir)[-1] ) glfw.glfwInit() main_window = glfw.glfwCreateWindow(width, height, window_name, None, None) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window def set_scale(new_scale): g_pool.gui_user_scale = new_scale window_size = ( g_pool.camera_render_size[0] + int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor), glfw.glfwGetFramebufferSize(main_window)[1], ) logger.warning(icon_bar_width * g_pool.gui_user_scale * hdpi_factor) glfw.glfwSetWindowSize(main_window, *window_size) # load pupil_positions, gaze_positions g_pool.binocular = meta_info.get("Eye Mode", "monocular") == "binocular" g_pool.version = app_version g_pool.timestamps = g_pool.capture.timestamps g_pool.get_timestamp = lambda: 0.0 g_pool.user_dir = user_dir g_pool.rec_dir = rec_dir g_pool.meta_info = meta_info g_pool.min_data_confidence = session_settings.get("min_data_confidence", MIN_DATA_CONFIDENCE_DEFAULT) g_pool.min_calibration_confidence = session_settings.get( "min_calibration_confidence", MIN_CALIBRATION_CONFIDENCE_DEFAULT ) # populated by producers g_pool.pupil_positions = pm.Bisector() g_pool.pupil_positions_by_id = (pm.Bisector(), pm.Bisector()) g_pool.gaze_positions = pm.Bisector() g_pool.fixations = pm.Affiliator() g_pool.eye_movements = pm.Affiliator() def set_data_confidence(new_confidence): g_pool.min_data_confidence = new_confidence notification = {"subject": "min_data_confidence_changed"} notification["_notify_time_"] = time() + 0.8 g_pool.ipc_pub.notify(notification) def do_export(_): left_idx = g_pool.seek_control.trim_left right_idx = g_pool.seek_control.trim_right export_range = left_idx, right_idx + 1 # exclusive range.stop export_ts_window = pm.exact_window(g_pool.timestamps, (left_idx, right_idx)) export_dir = os.path.join(g_pool.rec_dir, "exports") export_dir = next_export_sub_dir(export_dir) os.makedirs(export_dir) logger.info('Created export dir at "{}"'.format(export_dir)) export_info = { "Player Software Version": str(g_pool.version), "Data Format Version": meta_info["Data Format Version"], "Export Date": strftime("%d.%m.%Y", localtime()), "Export Time": strftime("%H:%M:%S", localtime()), "Frame Index Range:": g_pool.seek_control.get_frame_index_trim_range_string(), "Relative Time Range": g_pool.seek_control.get_rel_time_trim_range_string(), "Absolute Time Range": g_pool.seek_control.get_abs_time_trim_range_string(), } with open(os.path.join(export_dir, "export_info.csv"), "w") as csv: write_key_value_file(csv, export_info) notification = { "subject": "should_export", "range": export_range, "ts_window": export_ts_window, "export_dir": export_dir, } g_pool.ipc_pub.notify(notification) def reset_restart(): logger.warning("Resetting all settings and restarting Player.") glfw.glfwSetWindowShouldClose(main_window, True) ipc_pub.notify({"subject": "clear_settings_process.should_start"}) ipc_pub.notify( { "subject": "player_process.should_start", "rec_dir": rec_dir, "delay": 2.0, } ) def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is open the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0) g_pool.menubar = ui.Scrolling_Menu( "Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos="left" ) g_pool.iconbar = ui.Scrolling_Menu( "Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden" ) g_pool.timelines = ui.Container((0, 0), (0, 0), (0, 0)) g_pool.timelines.horizontal_constraint = g_pool.menubar g_pool.user_timelines = ui.Timeline_Menu( "User Timelines", pos=(0.0, -150.0), size=(0.0, 0.0), header_pos="headline" ) g_pool.user_timelines.color = RGBA(a=0.0) g_pool.user_timelines.collapsed = True # add container that constaints itself to the seekbar height vert_constr = ui.Container((0, 0), (0, -50.0), (0, 0)) vert_constr.append(g_pool.user_timelines) g_pool.timelines.append(vert_constr) def set_window_size(): f_width, f_height = g_pool.capture.frame_size f_width += int(icon_bar_width * g_pool.gui.scale) glfw.glfwSetWindowSize(main_window, f_width, f_height) general_settings = ui.Growing_Menu("General", header_pos="headline") general_settings.append(ui.Button("Reset window size", set_window_size)) general_settings.append( ui.Selector( "gui_user_scale", g_pool, setter=set_scale, selection=[0.8, 0.9, 1.0, 1.1, 1.2] + list(np.arange(1.5, 5.1, 0.5)), label="Interface Size", ) ) general_settings.append( ui.Info_Text("Player Version: {}".format(g_pool.version)) ) general_settings.append( ui.Info_Text( "Capture Version: {}".format(meta_info["Capture Software Version"]) ) ) general_settings.append( ui.Info_Text( "Data Format Version: {}".format(meta_info["Data Format Version"]) ) ) general_settings.append( ui.Info_Text( "High level data, e.g. fixations, or visualizations only consider gaze data that has an equal or higher confidence than the minimum data confidence." ) ) general_settings.append( ui.Slider( "min_data_confidence", g_pool, setter=set_data_confidence, step=0.05, min=0.0, max=1.0, label="Minimum data confidence", ) ) general_settings.append( ui.Button("Restart with default settings", reset_restart) ) g_pool.menubar.append(general_settings) icon = ui.Icon( "collapsed", general_settings, label=chr(0xE8B8), on_val=False, off_val=True, setter=toggle_general_settings, label_font="pupil_icons", ) icon.tooltip = "General Settings" g_pool.iconbar.append(icon) user_plugin_separator = ui.Separator() user_plugin_separator.order = 0.35 g_pool.iconbar.append(user_plugin_separator) g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100), (100, -100)) g_pool.export_button = ui.Thumb( "export", label=chr(0xE2C5), getter=lambda: False, setter=do_export, hotkey="e", label_font="pupil_icons", ) g_pool.quickbar.extend([g_pool.export_button]) g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.timelines) g_pool.gui.append(g_pool.iconbar) g_pool.gui.append(g_pool.quickbar) # we always load these plugins default_plugins = [ ("Plugin_Manager", {}), ("Seek_Control", {}), ("Log_Display", {}), ("Raw_Data_Exporter", {}), ("Vis_Polyline", {}), ("Vis_Circle", {}), ("System_Graphs", {}), ("System_Timelines", {}), ("World_Video_Exporter", {}), ("Pupil_From_Recording", {}), ("GazeFromRecording", {}), ("Audio_Playback", {}), ] g_pool.plugins = Plugin_List( g_pool, session_settings.get("loaded_plugins", default_plugins) ) # Manually add g_pool.capture to the plugin list g_pool.plugins._plugins.append(g_pool.capture) g_pool.plugins._plugins.sort(key=lambda p: p.order) g_pool.capture.init_ui() general_settings.insert( -1, ui.Text_Input( "rel_time_trim_section", getter=g_pool.seek_control.get_rel_time_trim_range_string, setter=g_pool.seek_control.set_rel_time_trim_range_string, label="Relative time range to export", ), ) general_settings.insert( -1, ui.Text_Input( "frame_idx_trim_section", getter=g_pool.seek_control.get_frame_index_trim_range_string, setter=g_pool.seek_control.set_frame_index_trim_range_string, label="Frame index range to export", ), ) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) toggle_general_settings(True) g_pool.gui.configuration = session_settings.get("ui_config", {}) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() # trigger on_resize on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) def handle_notifications(n): subject = n["subject"] if subject == "start_plugin": g_pool.plugins.add( g_pool.plugin_by_name[n["name"]], args=n.get("args", {}) ) elif subject.startswith("meta.should_doc"): ipc_pub.notify( {"subject": "meta.doc", "actor": g_pool.app, "doc": player.__doc__} ) for p in g_pool.plugins: if ( p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify ): ipc_pub.notify( { "subject": "meta.doc", "actor": p.class_name, "doc": p.on_notify.__doc__, } ) while not glfw.glfwWindowShouldClose(main_window): # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) events = {} # report time between now and the last loop interation events["dt"] = get_dt() # pupil and gaze positions are added by their respective producer plugins events["pupil"] = [] events["gaze"] = [] # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() glfw.glfwMakeContextCurrent(main_window) # render visual feedback from loaded plugins if gl_utils.is_window_visible(main_window): gl_utils.glViewport(0, 0, *g_pool.camera_render_size) g_pool.capture.gl_display() for p in g_pool.plugins: p.gl_display() gl_utils.glViewport(0, 0, *window_size) try: clipboard = glfw.glfwGetClipboardString(main_window).decode() except AttributeError: # clipbaord is None, might happen on startup clipboard = "" g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard and user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.glfwSetClipboardString( main_window, user_input.clipboard.encode() ) for b in user_input.buttons: button, action, mods = b x, y = glfw.glfwGetCursorPos(main_window) pos = x * hdpi_factor, y * hdpi_factor pos = normalize(pos, g_pool.camera_render_size) pos = denormalize(pos, g_pool.capture.frame_size) # call `on_click` callbacks until a plugin indicates # that it has consumed the event (by returning True) any(p.on_click(pos, button, action) for p in g_pool.plugins) for key, scancode, action, mods in user_input.keys: # call `on_key` callbacks until a plugin indicates # that it has consumed the event (by returning True) any(p.on_key(key, scancode, action, mods) for p in g_pool.plugins) for char_ in user_input.chars: # call `char_` callbacks until a plugin indicates # that it has consumed the event (by returning True) any(p.on_char(char_) for p in g_pool.plugins) # present frames at appropriate speed g_pool.seek_control.wait(events["frame"].timestamp) glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() session_settings["loaded_plugins"] = g_pool.plugins.get_initializers() session_settings["min_data_confidence"] = g_pool.min_data_confidence session_settings[ "min_calibration_confidence" ] = g_pool.min_calibration_confidence session_settings["gui_scale"] = g_pool.gui_user_scale session_settings["ui_config"] = g_pool.gui.configuration session_settings["window_position"] = glfw.glfwGetWindowPos(main_window) session_settings["version"] = str(g_pool.version) session_window_size = glfw.glfwGetWindowSize(main_window) if 0 not in session_window_size: session_settings["window_size"] = session_window_size session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) except: import traceback trace = traceback.format_exc() logger.error("Process Player crashed with trace:\n{}".format(trace)) finally: logger.info("Process shutting down.") ipc_pub.notify({"subject": "player_process.stopped"}) sleep(1.0)
def _export_world_video( rec_dir, user_dir, min_data_confidence, start_frame, end_frame, plugin_initializers, out_file_path, pre_computed_eye_data, ): """ Simulates the generation for the world video and saves a certain time range as a video. It simulates a whole g_pool such that all plugins run as normal. """ from glob import glob from time import time import file_methods as fm import player_methods as pm from av_writer import AV_Writer # we are not importing manual gaze correction. In Player corrections have already been applied. # in batch exporter this plugin makes little sense. from fixation_detector import Offline_Fixation_Detector from eye_movement import Offline_Eye_Movement_Detector # Plug-ins from plugin import Plugin_List, import_runtime_plugins from video_capture import EndofVideoError, File_Source from video_overlay.plugins import Video_Overlay, Eye_Overlay from vis_circle import Vis_Circle from vis_cross import Vis_Cross from vis_light_points import Vis_Light_Points from vis_polyline import Vis_Polyline from vis_scan_path import Vis_Scan_Path from vis_watermark import Vis_Watermark PID = str(os.getpid()) logger = logging.getLogger(__name__ + " with pid: " + PID) start_status = "Starting video export with pid: {}".format(PID) logger.info(start_status) yield start_status, 0 try: vis_plugins = sorted( [ Vis_Circle, Vis_Cross, Vis_Polyline, Vis_Light_Points, Vis_Watermark, Vis_Scan_Path, Eye_Overlay, Video_Overlay, ], key=lambda x: x.__name__, ) analysis_plugins = [Offline_Fixation_Detector, Offline_Eye_Movement_Detector] user_plugins = sorted( import_runtime_plugins(os.path.join(user_dir, "plugins")), key=lambda x: x.__name__, ) available_plugins = vis_plugins + analysis_plugins + user_plugins name_by_index = [p.__name__ for p in available_plugins] plugin_by_name = dict(zip(name_by_index, available_plugins)) meta_info = pm.load_meta_info(rec_dir) g_pool = GlobalContainer() g_pool.app = "exporter" g_pool.min_data_confidence = min_data_confidence valid_ext = (".mp4", ".mkv", ".avi", ".h264", ".mjpeg", ".fake") try: video_path = next( f for f in glob(os.path.join(rec_dir, "world.*")) if os.path.splitext(f)[1] in valid_ext ) except StopIteration: raise FileNotFoundError("No Video world found") cap = File_Source(g_pool, source_path=video_path, fill_gaps=True, timing=None) timestamps = cap.timestamps file_name = os.path.basename(out_file_path) dir_name = os.path.dirname(out_file_path) out_file_path = os.path.expanduser(os.path.join(dir_name, file_name)) if os.path.isfile(out_file_path): logger.warning("Video out file already exsists. I will overwrite!") os.remove(out_file_path) logger.debug("Saving Video to {}".format(out_file_path)) # Trim mark verification # make sure the trim marks (start frame, end frame) make sense: # We define them like python list slices, thus we can test them like such. trimmed_timestamps = timestamps[start_frame:end_frame] if len(trimmed_timestamps) == 0: warn = "Start and end frames are set such that no video will be exported." logger.warning(warn) yield warn, 0.0 return if start_frame is None: start_frame = 0 # these two vars are shared with the launching process and give a job length and progress report. frames_to_export = len(trimmed_timestamps) current_frame = 0 exp_info = ( "Will export from frame {} to frame {}. This means I will export {} frames." ) logger.debug( exp_info.format( start_frame, start_frame + frames_to_export, frames_to_export ) ) # setup of writer writer = AV_Writer( out_file_path, fps=cap.frame_rate, audio_dir=rec_dir, use_timestamps=True ) cap.seek_to_frame(start_frame) start_time = time() g_pool.plugin_by_name = plugin_by_name g_pool.capture = cap g_pool.rec_dir = rec_dir g_pool.user_dir = user_dir g_pool.meta_info = meta_info g_pool.timestamps = timestamps g_pool.delayed_notifications = {} g_pool.notifications = [] for initializers in pre_computed_eye_data.values(): initializers["data"] = [ fm.Serialized_Dict(msgpack_bytes=serialized) for serialized in initializers["data"] ] g_pool.pupil_positions = pm.Bisector(**pre_computed_eye_data["pupil"]) g_pool.pupil_positions_by_id = ( pm.Bisector(**pre_computed_eye_data["pupil_by_id_0"]), pm.Bisector(**pre_computed_eye_data["pupil_by_id_1"]), ) g_pool.gaze_positions = pm.Bisector(**pre_computed_eye_data["gaze"]) g_pool.fixations = pm.Affiliator(**pre_computed_eye_data["fixations"]) # add plugins g_pool.plugins = Plugin_List(g_pool, plugin_initializers) while frames_to_export > current_frame: try: frame = cap.get_frame() except EndofVideoError: break events = {"frame": frame} # new positions and events frame_window = pm.enclosing_window(g_pool.timestamps, frame.index) events["gaze"] = g_pool.gaze_positions.by_ts_window(frame_window) events["pupil"] = g_pool.pupil_positions.by_ts_window(frame_window) # publish delayed notifications when their time has come. for n in list(g_pool.delayed_notifications.values()): if n["_notify_time_"] < time(): del n["_notify_time_"] del g_pool.delayed_notifications[n["subject"]] g_pool.notifications.append(n) # notify each plugin if there are new notifications: while g_pool.notifications: n = g_pool.notifications.pop(0) for p in g_pool.plugins: p.on_notify(n) # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) writer.write_video_frame(frame) current_frame += 1 yield "Exporting with pid {}".format(PID), current_frame writer.close(timestamp_export_format="all") duration = time() - start_time effective_fps = float(current_frame) / duration result = "Export done: Exported {} frames to {}. This took {} seconds. Exporter ran at {} frames per second." logger.info( result.format(current_frame, out_file_path, duration, effective_fps) ) yield "Export done. This took {:.0f} seconds.".format(duration), current_frame except GeneratorExit: logger.warning("Video export with pid {} was canceled.".format(os.getpid()))