def recent_events(self, events): frame = events.get("frame") if not frame: return self.last_frame_ts = frame.timestamp if frame.index != self.last_frame_index: self.last_frame_index = frame.index frame_window = pm.enclosing_window(self.g_pool.timestamps, frame.index) events = self.annotations.by_ts_window(frame_window) for event in events: logger.info( "{} annotation @ {}".format(event["label"], event["timestamp"]) )
def map_section(self, section, all_world_timestamps, all_gaze_events, camera_model): try: location_cache = self.location_cache[section] except TypeError: return [] section_gaze_on_surf = [] for frame_idx, location in enumerate(location_cache): frame_idx += section.start if location and location.detected: frame_window = player_methods.enclosing_window( all_world_timestamps, frame_idx ) gaze_events = all_gaze_events.by_ts_window(frame_window) gaze_on_surf = self.map_gaze_and_fixation_events( gaze_events, camera_model, trans_matrix=location.img_to_surf_trans ) else: gaze_on_surf = [] section_gaze_on_surf.append(gaze_on_surf) return section_gaze_on_surf
def recent_events(self, events): # TODO: comments or method extraction if "frame" in events: frame_idx = events["frame"].index window = pm.enclosing_window(self.g_pool.timestamps, frame_idx) events["gaze"] = self.g_pool.gaze_positions.by_ts_window(window)
def recent_events(self, events): if "frame" in events: frm_idx = events["frame"].index window = pm.enclosing_window(self.g_pool.timestamps, frm_idx) events["pupil"] = self.g_pool.pupil_positions.by_ts_window(window)
def _export_world_video( rec_dir, user_dir, min_data_confidence, start_frame, end_frame, plugin_initializers, out_file_path, pre_computed_eye_data, ): """ Simulates the generation for the world video and saves a certain time range as a video. It simulates a whole g_pool such that all plugins run as normal. """ from glob import glob from time import time import file_methods as fm import player_methods as pm from av_writer import AV_Writer # we are not importing manual gaze correction. In Player corrections have already been applied. # in batch exporter this plugin makes little sense. from fixation_detector import Offline_Fixation_Detector # Plug-ins from plugin import Plugin_List, import_runtime_plugins from video_capture import EndofVideoError, init_playback_source from vis_circle import Vis_Circle from vis_cross import Vis_Cross from vis_eye_video_overlay import Vis_Eye_Video_Overlay from vis_light_points import Vis_Light_Points from vis_polyline import Vis_Polyline from vis_scan_path import Vis_Scan_Path from vis_watermark import Vis_Watermark PID = str(os.getpid()) logger = logging.getLogger(__name__ + " with pid: " + PID) start_status = "Starting video export with pid: {}".format(PID) logger.info(start_status) yield start_status, 0 try: vis_plugins = sorted( [ Vis_Circle, Vis_Cross, Vis_Polyline, Vis_Light_Points, Vis_Watermark, Vis_Scan_Path, Vis_Eye_Video_Overlay, ], key=lambda x: x.__name__, ) analysis_plugins = [Offline_Fixation_Detector] user_plugins = sorted( import_runtime_plugins(os.path.join(user_dir, "plugins")), key=lambda x: x.__name__, ) available_plugins = vis_plugins + analysis_plugins + user_plugins name_by_index = [p.__name__ for p in available_plugins] plugin_by_name = dict(zip(name_by_index, available_plugins)) audio_path = os.path.join(rec_dir, "audio.mp4") meta_info = pm.load_meta_info(rec_dir) g_pool = GlobalContainer() g_pool.app = "exporter" g_pool.min_data_confidence = min_data_confidence valid_ext = (".mp4", ".mkv", ".avi", ".h264", ".mjpeg", ".fake") try: video_path = next(f for f in glob(os.path.join(rec_dir, "world.*")) if os.path.splitext(f)[1] in valid_ext) except StopIteration: raise FileNotFoundError("No Video world found") cap = init_playback_source(g_pool, source_path=video_path, timing=None) timestamps = cap.timestamps file_name = os.path.basename(out_file_path) dir_name = os.path.dirname(out_file_path) out_file_path = os.path.expanduser(os.path.join(dir_name, file_name)) if os.path.isfile(out_file_path): logger.warning("Video out file already exsists. I will overwrite!") os.remove(out_file_path) logger.debug("Saving Video to {}".format(out_file_path)) # Trim mark verification # make sure the trim marks (start frame, end frame) make sense: # We define them like python list slices, thus we can test them like such. trimmed_timestamps = timestamps[start_frame:end_frame] if len(trimmed_timestamps) == 0: warn = "Start and end frames are set such that no video will be exported." logger.warning(warn) yield warn, 0.0 return if start_frame is None: start_frame = 0 # these two vars are shared with the launching process and give a job length and progress report. frames_to_export = len(trimmed_timestamps) current_frame = 0 exp_info = ( "Will export from frame {} to frame {}. This means I will export {} frames." ) logger.debug( exp_info.format(start_frame, start_frame + frames_to_export, frames_to_export)) # setup of writer writer = AV_Writer(out_file_path, fps=cap.frame_rate, audio_loc=audio_path, use_timestamps=True) cap.seek_to_frame(start_frame) start_time = time() g_pool.plugin_by_name = plugin_by_name g_pool.capture = cap g_pool.rec_dir = rec_dir g_pool.user_dir = user_dir g_pool.meta_info = meta_info g_pool.timestamps = timestamps g_pool.delayed_notifications = {} g_pool.notifications = [] for initializers in pre_computed_eye_data.values(): initializers["data"] = [ fm.Serialized_Dict(msgpack_bytes=serialized) for serialized in initializers["data"] ] g_pool.pupil_positions = pm.Bisector(**pre_computed_eye_data["pupil"]) g_pool.gaze_positions = pm.Bisector(**pre_computed_eye_data["gaze"]) g_pool.fixations = pm.Affiliator(**pre_computed_eye_data["fixations"]) # add plugins g_pool.plugins = Plugin_List(g_pool, plugin_initializers) while frames_to_export > current_frame: try: frame = cap.get_frame() except EndofVideoError: break events = {"frame": frame} # new positions and events frame_window = pm.enclosing_window(g_pool.timestamps, frame.index) events["gaze"] = g_pool.gaze_positions.by_ts_window(frame_window) events["pupil"] = g_pool.pupil_positions.by_ts_window(frame_window) # publish delayed notifications when their time has come. for n in list(g_pool.delayed_notifications.values()): if n["_notify_time_"] < time(): del n["_notify_time_"] del g_pool.delayed_notifications[n["subject"]] g_pool.notifications.append(n) # notify each plugin if there are new notifications: while g_pool.notifications: n = g_pool.notifications.pop(0) for p in g_pool.plugins: p.on_notify(n) # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) writer.write_video_frame(frame) current_frame += 1 yield "Exporting with pid {}".format(PID), current_frame writer.close(timestamp_export_format="all") duration = time() - start_time effective_fps = float(current_frame) / duration result = "Export done: Exported {} frames to {}. This took {} seconds. Exporter ran at {} frames per second." logger.info( result.format(current_frame, out_file_path, duration, effective_fps)) yield "Export done. This took {:.0f} seconds.".format( duration), current_frame except GeneratorExit: logger.warning("Video export with pid {} was canceled.".format( os.getpid()))
def recent_events(self, events): if self.bg_task: for progress, fixation_result in self.bg_task.fetch(): self.status = progress if fixation_result: serialized, start_ts, stop_ts = fixation_result self.fixation_data.append( fm.Serialized_Dict(msgpack_bytes=serialized)) self.fixation_start_ts.append(start_ts) self.fixation_stop_ts.append(stop_ts) if self.fixation_data: current_ts = self.fixation_stop_ts[-1] progress = ((current_ts - self.g_pool.timestamps[0]) / (self.g_pool.timestamps[-1] - self.g_pool.timestamps[0])) self.menu_icon.indicator_stop = progress if self.bg_task.completed: self.status = "{} fixations detected".format( len(self.fixation_data)) self.correlate_and_publish() self.bg_task = None self.menu_icon.indicator_stop = 0. frame = events.get('frame') if not frame: return self.last_frame_idx = frame.index frame_window = pm.enclosing_window(self.g_pool.timestamps, frame.index) fixations = self.g_pool.fixations.by_ts_window(frame_window) events['fixations'] = fixations if self.show_fixations: for f in fixations: x = int(f['norm_pos'][0] * frame.width) y = int((1. - f['norm_pos'][1]) * frame.height) pm.transparent_circle(frame.img, (x, y), radius=25., color=(0., 1., 1., 1.), thickness=3) cv2.putText(frame.img, '{}'.format(f['id']), (x + 30, y), cv2.FONT_HERSHEY_DUPLEX, 0.8, (255, 150, 100)) if self.current_fixation_details and self.prev_index != frame.index: info = '' for f in fixations: info += 'Current fixation, {} of {}\n'.format( f['id'], len(self.g_pool.fixations)) info += ' Confidence: {:.2f}\n'.format(f['confidence']) info += ' Duration: {:.2f} milliseconds\n'.format( f['duration']) info += ' Dispersion: {:.3f} degrees\n'.format( f['dispersion']) info += ' Frame range: {}-{}\n'.format( f['start_frame_index'] + 1, f['end_frame_index'] + 1) info += ' 2d gaze pos: x={:.3f}, y={:.3f}\n'.format( *f['norm_pos']) if 'gaze_point_3d' in f: info += ' 3d gaze pos: x={:.3f}, y={:.3f}, z={:.3f}\n'.format( *f['gaze_point_3d']) else: info += ' 3d gaze pos: N/A\n' if f['id'] > 1: prev_f = self.g_pool.fixations[f['id'] - 2] time_lapsed = f['timestamp'] - prev_f[ 'timestamp'] + prev_f['duration'] / 1000 info += ' Time since prev. fixation: {:.2f} seconds\n'.format( time_lapsed) else: info += ' Time since prev. fixation: N/A\n' if f['id'] < len(self.g_pool.fixations): next_f = self.g_pool.fixations[f['id']] time_lapsed = next_f['timestamp'] - f[ 'timestamp'] + f['duration'] / 1000 info += ' Time to next fixation: {:.2f} seconds\n'.format( time_lapsed) else: info += ' Time to next fixation: N/A\n' self.current_fixation_details.text = info self.prev_index = frame.index
def get_current_frame_window(self): frame_index = self.get_current_frame_index() frame_window = pm.enclosing_window(self.g_pool.timestamps, frame_index) return frame_window
def export(rec_dir, user_dir, min_data_confidence, start_frame=None, end_frame=None, plugin_initializers=(), out_file_path=None, pre_computed={}): PID = str(os.getpid()) logger = logging.getLogger(__name__+' with pid: '+PID) start_status = 'Starting video export with pid: {}'.format(PID) print(start_status) yield start_status, 0 try: pm.update_recording_to_recent(rec_dir) vis_plugins = sorted([Vis_Circle, Vis_Cross, Vis_Polyline, Vis_Light_Points, Vis_Watermark, Vis_Scan_Path, Vis_Eye_Video_Overlay], key=lambda x: x.__name__) analysis_plugins = [Offline_Fixation_Detector] user_plugins = sorted(import_runtime_plugins(os.path.join(user_dir, 'plugins')), key=lambda x: x.__name__) available_plugins = vis_plugins + analysis_plugins + user_plugins name_by_index = [p.__name__ for p in available_plugins] plugin_by_name = dict(zip(name_by_index, available_plugins)) pm.update_recording_to_recent(rec_dir) audio_path = os.path.join(rec_dir, "audio.mp4") meta_info = pm.load_meta_info(rec_dir) g_pool = Global_Container() g_pool.app = 'exporter' g_pool.min_data_confidence = min_data_confidence valid_ext = ('.mp4', '.mkv', '.avi', '.h264', '.mjpeg', '.fake') video_path = [f for f in glob(os.path.join(rec_dir, "world.*")) if os.path.splitext(f)[1] in valid_ext][0] cap = init_playback_source(g_pool, source_path=video_path, timing=None) timestamps = cap.timestamps # Out file path verification, we do this before but if one uses a separate tool, this will kick in. if out_file_path is None: out_file_path = os.path.join(rec_dir, "world_viz.mp4") else: file_name = os.path.basename(out_file_path) dir_name = os.path.dirname(out_file_path) if not dir_name: dir_name = rec_dir if not file_name: file_name = 'world_viz.mp4' out_file_path = os.path.expanduser(os.path.join(dir_name, file_name)) if os.path.isfile(out_file_path): logger.warning("Video out file already exsists. I will overwrite!") os.remove(out_file_path) logger.debug("Saving Video to {}".format(out_file_path)) # Trim mark verification # make sure the trim marks (start frame, endframe) make sense: # We define them like python list slices, thus we can test them like such. trimmed_timestamps = timestamps[start_frame:end_frame] if len(trimmed_timestamps) == 0: warn = "Start and end frames are set such that no video will be exported." logger.warning(warn) yield warn, 0. return if start_frame is None: start_frame = 0 # these two vars are shared with the lauching process and give a job length and progress report. frames_to_export = len(trimmed_timestamps) current_frame = 0 exp_info = "Will export from frame {} to frame {}. This means I will export {} frames." logger.debug(exp_info.format(start_frame, start_frame + frames_to_export, frames_to_export)) # setup of writer writer = AV_Writer(out_file_path, fps=cap.frame_rate, audio_loc=audio_path, use_timestamps=True) cap.seek_to_frame(start_frame) start_time = time() g_pool.plugin_by_name = plugin_by_name g_pool.capture = cap g_pool.rec_dir = rec_dir g_pool.user_dir = user_dir g_pool.meta_info = meta_info g_pool.timestamps = timestamps g_pool.delayed_notifications = {} g_pool.notifications = [] for initializers in pre_computed.values(): initializers['data'] = [fm.Serialized_Dict(msgpack_bytes=serialized) for serialized in initializers['data']] g_pool.pupil_positions = pm.Bisector(**pre_computed["pupil"]) g_pool.gaze_positions = pm.Bisector(**pre_computed["gaze"]) g_pool.fixations = pm.Affiliator(**pre_computed["fixations"]) # add plugins g_pool.plugins = Plugin_List(g_pool, plugin_initializers) while frames_to_export > current_frame: try: frame = cap.get_frame() except EndofVideoError: break events = {'frame': frame} # new positons and events frame_window = pm.enclosing_window(g_pool.timestamps, frame.index) events['gaze'] = g_pool.gaze_positions.by_ts_window(frame_window) events['pupil'] = g_pool.pupil_positions.by_ts_window(frame_window) # publish delayed notifiactions when their time has come. for n in list(g_pool.delayed_notifications.values()): if n['_notify_time_'] < time(): del n['_notify_time_'] del g_pool.delayed_notifications[n['subject']] g_pool.notifications.append(n) # notify each plugin if there are new notifactions: while g_pool.notifications: n = g_pool.notifications.pop(0) for p in g_pool.plugins: p.on_notify(n) # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) writer.write_video_frame(frame) current_frame += 1 yield 'Exporting with pid {}'.format(PID), current_frame writer.close() writer = None duration = time()-start_time effective_fps = float(current_frame)/duration result = "Export done: Exported {} frames to {}. This took {} seconds. Exporter ran at {} frames per second." print(result.format(current_frame, out_file_path, duration, effective_fps)) yield 'Export done. This took {:.0f} seconds.'.format(duration), current_frame except GeneratorExit: print('Video export with pid {} was canceled.'.format(os.getpid())) except Exception as e: from time import sleep import traceback trace = traceback.format_exc() print('Process Export (pid: {}) crashed with trace:\n{}'.format(os.getpid(), trace)) yield e sleep(1.0)
def find_markers_in_frame(index): window = pm.enclosing_window(timestamps, index) return markers_bisector.by_ts_window(window)
def recent_events(self, events): if 'frame' in events: frm_idx = events['frame'].index window = pm.enclosing_window(self.g_pool.timestamps, frm_idx) events['gaze'] = self.g_pool.gaze_positions.by_ts_window(window)
def recent_events(self, events): if self.bg_task: for progress, fixation_result in self.bg_task.fetch(): self.status = progress if fixation_result: serialized, start_ts, stop_ts = fixation_result self.fixation_data.append( fm.Serialized_Dict(msgpack_bytes=serialized) ) self.fixation_start_ts.append(start_ts) self.fixation_stop_ts.append(stop_ts) if self.fixation_data: current_ts = self.fixation_stop_ts[-1] progress = (current_ts - self.g_pool.timestamps[0]) / ( self.g_pool.timestamps[-1] - self.g_pool.timestamps[0] ) self.menu_icon.indicator_stop = progress if self.bg_task.completed: self.status = "{} fixations detected".format(len(self.fixation_data)) self.correlate_and_publish() self.bg_task = None self.menu_icon.indicator_stop = 0.0 frame = events.get("frame") if not frame: return self.last_frame_idx = frame.index frame_window = pm.enclosing_window(self.g_pool.timestamps, frame.index) fixations = self.g_pool.fixations.by_ts_window(frame_window) events["fixations"] = fixations if self.show_fixations: for f in fixations: x = int(f["norm_pos"][0] * frame.width) y = int((1.0 - f["norm_pos"][1]) * frame.height) pm.transparent_circle( frame.img, (x, y), radius=25.0, color=(0.0, 1.0, 1.0, 1.0), thickness=3, ) cv2.putText( frame.img, "{}".format(f["id"]), (x + 30, y), cv2.FONT_HERSHEY_DUPLEX, 0.8, (255, 150, 100), ) if self.current_fixation_details and self.prev_index != frame.index: info = "" for f in fixations: info += "Current fixation, {} of {}\n".format( f["id"], len(self.g_pool.fixations) ) info += " Confidence: {:.2f}\n".format(f["confidence"]) info += " Duration: {:.2f} milliseconds\n".format(f["duration"]) info += " Dispersion: {:.3f} degrees\n".format(f["dispersion"]) info += " Frame range: {}-{}\n".format( f["start_frame_index"] + 1, f["end_frame_index"] + 1 ) info += " 2d gaze pos: x={:.3f}, y={:.3f}\n".format(*f["norm_pos"]) if "gaze_point_3d" in f: info += " 3d gaze pos: x={:.3f}, y={:.3f}, z={:.3f}\n".format( *f["gaze_point_3d"] ) else: info += " 3d gaze pos: N/A\n" if f["id"] > 1: prev_f = self.g_pool.fixations[f["id"] - 2] time_lapsed = ( f["timestamp"] - prev_f["timestamp"] + prev_f["duration"] / 1000 ) info += " Time since prev. fixation: {:.2f} seconds\n".format( time_lapsed ) else: info += " Time since prev. fixation: N/A\n" if f["id"] < len(self.g_pool.fixations): next_f = self.g_pool.fixations[f["id"]] time_lapsed = ( next_f["timestamp"] - f["timestamp"] + f["duration"] / 1000 ) info += " Time to next fixation: {:.2f} seconds\n".format( time_lapsed ) else: info += " Time to next fixation: N/A\n" self.current_fixation_details.text = info self.prev_index = frame.index
def segments_in_frame(self, frame) -> t.Iterable[Classified_Segment]: frame_window = pm.enclosing_window(self._timestamps, frame.index) return self.segments_in_timestamp_window(frame_window)
def export_data(self, export_range, export_dir): user_warned_3d_only = False rec_start = self._get_recording_start_date() im_dir = os.path.join(export_dir, "iMotions_{}".format(rec_start)) try: self.add_export_job( export_range, im_dir, plugin_name="iMotions", input_name="world", output_name="scene", process_frame=_process_frame, export_timestamps=False, ) except FileNotFoundError: logger.info( "'world' video not found. Export continues with gaze data.") info_src = os.path.join(self.g_pool.rec_dir, "info.csv") info_dest = os.path.join(im_dir, "iMotions_info.csv") copy2(info_src, info_dest) # copy info.csv file with open(os.path.join(im_dir, "gaze.tlv"), "w", encoding="utf-8", newline="") as csvfile: csv_writer = csv.writer(csvfile, delimiter="\t") csv_writer.writerow(( "GazeTimeStamp", "MediaTimeStamp", "MediaFrameIndex", "Gaze3dX", "Gaze3dY", "Gaze3dZ", "Gaze2dX", "Gaze2dY", "PupilDiaLeft", "PupilDiaRight", "Confidence", )) for media_idx in range(*export_range): media_timestamp = self.g_pool.timestamps[media_idx] media_window = pm.enclosing_window(self.g_pool.timestamps, media_idx) for g in self.g_pool.gaze_positions.by_ts_window(media_window): try: pupil_dia = {} for p in g["base_data"]: pupil_dia[p["id"]] = p["diameter_3d"] pixel_pos = denormalize(g["norm_pos"], self.g_pool.capture.frame_size, flip_y=True) undistorted3d = self.g_pool.capture.intrinsics.unprojectPoints( pixel_pos) undistorted2d = self.g_pool.capture.intrinsics.projectPoints( undistorted3d, use_distortion=False) data = ( g["timestamp"], media_timestamp, media_idx - export_range[0], *g["gaze_point_3d"], # Gaze3dX/Y/Z *undistorted2d.flat, # Gaze2dX/Y pupil_dia.get(1, 0.0), # PupilDiaLeft pupil_dia.get(0, 0.0), # PupilDiaRight g["confidence"], ) # Confidence except KeyError: if not user_warned_3d_only: logger.error( "Currently, the iMotions export only supports 3d gaze data" ) user_warned_3d_only = True continue csv_writer.writerow(data)
def fixations_on_srf_by_frame_idx(self, frame_index, m_from_screen): frame_window = pm.enclosing_window(self.g_pool.timestamps, frame_index) return self.map_data_to_surface( self.g_pool.fixations.by_ts_window(frame_window), m_from_screen)
def recent_events(self, events): if self.bg_task: for progress, fixation_result in self.bg_task.fetch(): self.status = progress if fixation_result: serialized, start_ts, stop_ts = fixation_result self.fixation_data.append( fm.Serialized_Dict(msgpack_bytes=serialized)) self.fixation_start_ts.append(start_ts) self.fixation_stop_ts.append(stop_ts) if self.fixation_data: current_ts = self.fixation_stop_ts[-1] progress = (current_ts - self.g_pool.timestamps[0]) / ( self.g_pool.timestamps[-1] - self.g_pool.timestamps[0]) self.menu_icon.indicator_stop = progress if self.bg_task.completed: self.status = "{} fixations detected".format( len(self.fixation_data)) self.correlate_and_publish() self.bg_task = None self.menu_icon.indicator_stop = 0.0 frame = events.get("frame") if not frame: return self.last_frame_idx = frame.index frame_window = pm.enclosing_window(self.g_pool.timestamps, frame.index) fixations = self.g_pool.fixations.by_ts_window(frame_window) events["fixations"] = fixations if self.show_fixations: for f in fixations: x = int(f["norm_pos"][0] * frame.width) y = int((1.0 - f["norm_pos"][1]) * frame.height) pm.transparent_circle( frame.img, (x, y), radius=25.0, color=(0.0, 1.0, 1.0, 1.0), thickness=3, ) cv2.putText( frame.img, "{}".format(f["id"]), (x + 30, y), cv2.FONT_HERSHEY_DUPLEX, 0.8, (255, 150, 100), ) if self.current_fixation_details and self.prev_index != frame.index: info = "" for f in fixations: info += "Current fixation, {} of {}\n".format( f["id"], len(self.g_pool.fixations)) info += " Confidence: {:.2f}\n".format(f["confidence"]) info += " Duration: {:.2f} milliseconds\n".format( f["duration"]) info += " Dispersion: {:.3f} degrees\n".format( f["dispersion"]) info += " Frame range: {}-{}\n".format( f["start_frame_index"] + 1, f["end_frame_index"] + 1) info += " 2d gaze pos: x={:.3f}, y={:.3f}\n".format( *f["norm_pos"]) if "gaze_point_3d" in f: info += " 3d gaze pos: x={:.3f}, y={:.3f}, z={:.3f}\n".format( *f["gaze_point_3d"]) else: info += " 3d gaze pos: N/A\n" if f["id"] > 1: prev_f = self.g_pool.fixations[f["id"] - 2] time_lapsed = (f["timestamp"] - prev_f["timestamp"] + prev_f["duration"] / 1000) info += " Time since prev. fixation: {:.2f} seconds\n".format( time_lapsed) else: info += " Time since prev. fixation: N/A\n" if f["id"] < len(self.g_pool.fixations): next_f = self.g_pool.fixations[f["id"]] time_lapsed = (next_f["timestamp"] - f["timestamp"] + f["duration"] / 1000) info += " Time to next fixation: {:.2f} seconds\n".format( time_lapsed) else: info += " Time to next fixation: N/A\n" self.current_fixation_details.text = info self.prev_index = frame.index
def _export_world_video( rec_dir, user_dir, min_data_confidence, start_frame, end_frame, plugin_initializers, out_file_path, pre_computed_eye_data, ): """ Simulates the generation for the world video and saves a certain time range as a video. It simulates a whole g_pool such that all plugins run as normal. """ from glob import glob from time import time import file_methods as fm import player_methods as pm from av_writer import AV_Writer # we are not importing manual gaze correction. In Player corrections have already been applied. # in batch exporter this plugin makes little sense. from fixation_detector import Offline_Fixation_Detector from eye_movement import Offline_Eye_Movement_Detector # Plug-ins from plugin import Plugin_List, import_runtime_plugins from video_capture import EndofVideoError, File_Source from video_overlay.plugins import Video_Overlay, Eye_Overlay from vis_circle import Vis_Circle from vis_cross import Vis_Cross from vis_light_points import Vis_Light_Points from vis_polyline import Vis_Polyline from vis_scan_path import Vis_Scan_Path from vis_watermark import Vis_Watermark PID = str(os.getpid()) logger = logging.getLogger(__name__ + " with pid: " + PID) start_status = "Starting video export with pid: {}".format(PID) logger.info(start_status) yield start_status, 0 try: vis_plugins = sorted( [ Vis_Circle, Vis_Cross, Vis_Polyline, Vis_Light_Points, Vis_Watermark, Vis_Scan_Path, Eye_Overlay, Video_Overlay, ], key=lambda x: x.__name__, ) analysis_plugins = [Offline_Fixation_Detector, Offline_Eye_Movement_Detector] user_plugins = sorted( import_runtime_plugins(os.path.join(user_dir, "plugins")), key=lambda x: x.__name__, ) available_plugins = vis_plugins + analysis_plugins + user_plugins name_by_index = [p.__name__ for p in available_plugins] plugin_by_name = dict(zip(name_by_index, available_plugins)) meta_info = pm.load_meta_info(rec_dir) g_pool = GlobalContainer() g_pool.app = "exporter" g_pool.min_data_confidence = min_data_confidence valid_ext = (".mp4", ".mkv", ".avi", ".h264", ".mjpeg", ".fake") try: video_path = next( f for f in glob(os.path.join(rec_dir, "world.*")) if os.path.splitext(f)[1] in valid_ext ) except StopIteration: raise FileNotFoundError("No Video world found") cap = File_Source(g_pool, source_path=video_path, fill_gaps=True, timing=None) timestamps = cap.timestamps file_name = os.path.basename(out_file_path) dir_name = os.path.dirname(out_file_path) out_file_path = os.path.expanduser(os.path.join(dir_name, file_name)) if os.path.isfile(out_file_path): logger.warning("Video out file already exsists. I will overwrite!") os.remove(out_file_path) logger.debug("Saving Video to {}".format(out_file_path)) # Trim mark verification # make sure the trim marks (start frame, end frame) make sense: # We define them like python list slices, thus we can test them like such. trimmed_timestamps = timestamps[start_frame:end_frame] if len(trimmed_timestamps) == 0: warn = "Start and end frames are set such that no video will be exported." logger.warning(warn) yield warn, 0.0 return if start_frame is None: start_frame = 0 # these two vars are shared with the launching process and give a job length and progress report. frames_to_export = len(trimmed_timestamps) current_frame = 0 exp_info = ( "Will export from frame {} to frame {}. This means I will export {} frames." ) logger.debug( exp_info.format( start_frame, start_frame + frames_to_export, frames_to_export ) ) # setup of writer writer = AV_Writer( out_file_path, fps=cap.frame_rate, audio_dir=rec_dir, use_timestamps=True ) cap.seek_to_frame(start_frame) start_time = time() g_pool.plugin_by_name = plugin_by_name g_pool.capture = cap g_pool.rec_dir = rec_dir g_pool.user_dir = user_dir g_pool.meta_info = meta_info g_pool.timestamps = timestamps g_pool.delayed_notifications = {} g_pool.notifications = [] for initializers in pre_computed_eye_data.values(): initializers["data"] = [ fm.Serialized_Dict(msgpack_bytes=serialized) for serialized in initializers["data"] ] g_pool.pupil_positions = pm.Bisector(**pre_computed_eye_data["pupil"]) g_pool.pupil_positions_by_id = ( pm.Bisector(**pre_computed_eye_data["pupil_by_id_0"]), pm.Bisector(**pre_computed_eye_data["pupil_by_id_1"]), ) g_pool.gaze_positions = pm.Bisector(**pre_computed_eye_data["gaze"]) g_pool.fixations = pm.Affiliator(**pre_computed_eye_data["fixations"]) # add plugins g_pool.plugins = Plugin_List(g_pool, plugin_initializers) while frames_to_export > current_frame: try: frame = cap.get_frame() except EndofVideoError: break events = {"frame": frame} # new positions and events frame_window = pm.enclosing_window(g_pool.timestamps, frame.index) events["gaze"] = g_pool.gaze_positions.by_ts_window(frame_window) events["pupil"] = g_pool.pupil_positions.by_ts_window(frame_window) # publish delayed notifications when their time has come. for n in list(g_pool.delayed_notifications.values()): if n["_notify_time_"] < time(): del n["_notify_time_"] del g_pool.delayed_notifications[n["subject"]] g_pool.notifications.append(n) # notify each plugin if there are new notifications: while g_pool.notifications: n = g_pool.notifications.pop(0) for p in g_pool.plugins: p.on_notify(n) # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) writer.write_video_frame(frame) current_frame += 1 yield "Exporting with pid {}".format(PID), current_frame writer.close(timestamp_export_format="all") duration = time() - start_time effective_fps = float(current_frame) / duration result = "Export done: Exported {} frames to {}. This took {} seconds. Exporter ran at {} frames per second." logger.info( result.format(current_frame, out_file_path, duration, effective_fps) ) yield "Export done. This took {:.0f} seconds.".format(duration), current_frame except GeneratorExit: logger.warning("Video export with pid {} was canceled.".format(os.getpid()))