Esempio n. 1
0
def update_recording_v19_v111(rec_dir):
    logger.info("Updating recording from v1.9 to v1.11")

    meta_info = pm.load_meta_info(rec_dir)
    meta_info["Data Format Version"] = "v1.11"
    meta_info["Recording UUID"] = meta_info.get("Recording UUID", uuid.uuid4())
    update_meta_info(rec_dir, meta_info)
Esempio n. 2
0
def update_recording_to_recent(rec_dir):

    meta_info = pm.load_meta_info(rec_dir)
    update_meta_info(rec_dir, meta_info)

    if (
        meta_info.get("Capture Software", "Pupil Capture") == "Pupil Mobile"
        and "Data Format Version" not in meta_info
    ):
        convert_pupil_mobile_recording_to_v094(rec_dir)
        meta_info["Data Format Version"] = "v0.9.4"
        update_meta_info(rec_dir, meta_info)

    # Reference format: v0.7.4
    rec_version = read_rec_version(meta_info)

    # Convert python2 to python3
    if rec_version <= VersionFormat("0.8.7"):
        update_recording_bytes_to_unicode(rec_dir)

    if rec_version >= VersionFormat("0.7.4"):
        pass
    elif rec_version >= VersionFormat("0.7.3"):
        update_recording_v073_to_v074(rec_dir)
    elif rec_version >= VersionFormat("0.5"):
        update_recording_v05_to_v074(rec_dir)
    elif rec_version >= VersionFormat("0.4"):
        update_recording_v04_to_v074(rec_dir)
    elif rec_version >= VersionFormat("0.3"):
        update_recording_v03_to_v074(rec_dir)
    else:
        logger.Error("This recording is too old. Sorry.")
        return

    # Incremental format updates
    if rec_version < VersionFormat("0.8.2"):
        update_recording_v074_to_v082(rec_dir)
    if rec_version < VersionFormat("0.8.3"):
        update_recording_v082_to_v083(rec_dir)
    if rec_version < VersionFormat("0.8.6"):
        update_recording_v083_to_v086(rec_dir)
    if rec_version < VersionFormat("0.8.7"):
        update_recording_v086_to_v087(rec_dir)
    if rec_version < VersionFormat("0.9.1"):
        update_recording_v087_to_v091(rec_dir)
    if rec_version < VersionFormat("0.9.3"):
        update_recording_v091_to_v093(rec_dir)
    if rec_version < VersionFormat("0.9.4"):
        update_recording_v093_to_v094(rec_dir)
    if rec_version < VersionFormat("0.9.13"):
        update_recording_v094_to_v0913(rec_dir)
    if rec_version < VersionFormat("1.3"):
        update_recording_v0913_to_v13(rec_dir)
    if rec_version < VersionFormat("1.4"):
        update_recording_v13_v14(rec_dir)

    # Do this independent of rec_version
    check_for_worldless_recording(rec_dir)

    if rec_version < VersionFormat("1.8"):
        update_recording_v14_v18(rec_dir)
    if rec_version < VersionFormat("1.9"):
        update_recording_v18_v19(rec_dir)
    if rec_version < VersionFormat("1.11"):
        update_recording_v19_v111(rec_dir)
    if rec_version < VersionFormat("1.13"):
        update_recording_v111_v113(rec_dir)
Esempio n. 3
0
def _export_world_video(
    rec_dir,
    user_dir,
    min_data_confidence,
    start_frame,
    end_frame,
    plugin_initializers,
    out_file_path,
    pre_computed_eye_data,
):
    """
    Simulates the generation for the world video and saves a certain time range as a video.
    It simulates a whole g_pool such that all plugins run as normal.
    """
    from glob import glob
    from time import time

    import file_methods as fm
    import player_methods as pm
    from av_writer import AV_Writer

    # we are not importing manual gaze correction. In Player corrections have already been applied.
    # in batch exporter this plugin makes little sense.
    from fixation_detector import Offline_Fixation_Detector

    # Plug-ins
    from plugin import Plugin_List, import_runtime_plugins
    from video_capture import EndofVideoError, File_Source
    from vis_circle import Vis_Circle
    from vis_cross import Vis_Cross
    from vis_eye_video_overlay import Vis_Eye_Video_Overlay
    from vis_light_points import Vis_Light_Points
    from vis_polyline import Vis_Polyline
    from vis_scan_path import Vis_Scan_Path
    from vis_watermark import Vis_Watermark

    PID = str(os.getpid())
    logger = logging.getLogger(__name__ + " with pid: " + PID)
    start_status = "Starting video export with pid: {}".format(PID)
    logger.info(start_status)
    yield start_status, 0

    try:
        vis_plugins = sorted(
            [
                Vis_Circle,
                Vis_Cross,
                Vis_Polyline,
                Vis_Light_Points,
                Vis_Watermark,
                Vis_Scan_Path,
                Vis_Eye_Video_Overlay,
            ],
            key=lambda x: x.__name__,
        )
        analysis_plugins = [Offline_Fixation_Detector]
        user_plugins = sorted(
            import_runtime_plugins(os.path.join(user_dir, "plugins")),
            key=lambda x: x.__name__,
        )

        available_plugins = vis_plugins + analysis_plugins + user_plugins
        name_by_index = [p.__name__ for p in available_plugins]
        plugin_by_name = dict(zip(name_by_index, available_plugins))

        meta_info = pm.load_meta_info(rec_dir)

        g_pool = GlobalContainer()
        g_pool.app = "exporter"
        g_pool.min_data_confidence = min_data_confidence

        valid_ext = (".mp4", ".mkv", ".avi", ".h264", ".mjpeg", ".fake")
        try:
            video_path = next(f for f in glob(os.path.join(rec_dir, "world.*"))
                              if os.path.splitext(f)[1] in valid_ext)
        except StopIteration:
            raise FileNotFoundError("No Video world found")
        cap = File_Source(g_pool,
                          source_path=video_path,
                          fill_gaps=True,
                          timing=None)

        timestamps = cap.timestamps

        file_name = os.path.basename(out_file_path)
        dir_name = os.path.dirname(out_file_path)
        out_file_path = os.path.expanduser(os.path.join(dir_name, file_name))

        if os.path.isfile(out_file_path):
            logger.warning("Video out file already exsists. I will overwrite!")
            os.remove(out_file_path)
        logger.debug("Saving Video to {}".format(out_file_path))

        # Trim mark verification
        # make sure the trim marks (start frame, end frame) make sense:
        # We define them like python list slices, thus we can test them like such.
        trimmed_timestamps = timestamps[start_frame:end_frame]
        if len(trimmed_timestamps) == 0:
            warn = "Start and end frames are set such that no video will be exported."
            logger.warning(warn)
            yield warn, 0.0
            return

        if start_frame is None:
            start_frame = 0

        # these two vars are shared with the launching process and give a job length and progress report.
        frames_to_export = len(trimmed_timestamps)
        current_frame = 0
        exp_info = (
            "Will export from frame {} to frame {}. This means I will export {} frames."
        )
        logger.debug(
            exp_info.format(start_frame, start_frame + frames_to_export,
                            frames_to_export))

        # setup of writer
        writer = AV_Writer(out_file_path,
                           fps=cap.frame_rate,
                           audio_dir=rec_dir,
                           use_timestamps=True)

        cap.seek_to_frame(start_frame)

        start_time = time()

        g_pool.plugin_by_name = plugin_by_name
        g_pool.capture = cap
        g_pool.rec_dir = rec_dir
        g_pool.user_dir = user_dir
        g_pool.meta_info = meta_info
        g_pool.timestamps = timestamps
        g_pool.delayed_notifications = {}
        g_pool.notifications = []

        for initializers in pre_computed_eye_data.values():
            initializers["data"] = [
                fm.Serialized_Dict(msgpack_bytes=serialized)
                for serialized in initializers["data"]
            ]

        g_pool.pupil_positions = pm.Bisector(**pre_computed_eye_data["pupil"])
        g_pool.gaze_positions = pm.Bisector(**pre_computed_eye_data["gaze"])
        g_pool.fixations = pm.Affiliator(**pre_computed_eye_data["fixations"])

        # add plugins
        g_pool.plugins = Plugin_List(g_pool, plugin_initializers)

        while frames_to_export > current_frame:
            try:
                frame = cap.get_frame()
            except EndofVideoError:
                break

            events = {"frame": frame}
            # new positions and events
            frame_window = pm.enclosing_window(g_pool.timestamps, frame.index)
            events["gaze"] = g_pool.gaze_positions.by_ts_window(frame_window)
            events["pupil"] = g_pool.pupil_positions.by_ts_window(frame_window)

            # publish delayed notifications when their time has come.
            for n in list(g_pool.delayed_notifications.values()):
                if n["_notify_time_"] < time():
                    del n["_notify_time_"]
                    del g_pool.delayed_notifications[n["subject"]]
                    g_pool.notifications.append(n)

            # notify each plugin if there are new notifications:
            while g_pool.notifications:
                n = g_pool.notifications.pop(0)
                for p in g_pool.plugins:
                    p.on_notify(n)

            # allow each Plugin to do its work.
            for p in g_pool.plugins:
                p.recent_events(events)

            writer.write_video_frame(frame)
            current_frame += 1
            yield "Exporting with pid {}".format(PID), current_frame

        writer.close(timestamp_export_format="all")

        duration = time() - start_time
        effective_fps = float(current_frame) / duration

        result = "Export done: Exported {} frames to {}. This took {} seconds. Exporter ran at {} frames per second."
        logger.info(
            result.format(current_frame, out_file_path, duration,
                          effective_fps))
        yield "Export done. This took {:.0f} seconds.".format(
            duration), current_frame

    except GeneratorExit:
        logger.warning("Video export with pid {} was canceled.".format(
            os.getpid()))
Esempio n. 4
0
def _update_info_version_to(new_version, rec_dir):
    meta_info = pm.load_meta_info(rec_dir)
    meta_info["Data Format Version"] = new_version
    update_meta_info(rec_dir, meta_info)
Esempio n. 5
0
def export(should_terminate,frames_to_export,current_frame, rec_dir,user_dir,start_frame=None,end_frame=None,plugin_initializers=[],out_file_path=None):

    logger = logging.getLogger(__name__+' with pid: '+str(os.getpid()) )

    update_recording_to_recent(rec_dir)

    video_path = [f for f in glob(os.path.join(rec_dir,"world.*")) if f[-3:] in ('mp4','mkv','avi')][0]
    timestamps_path = os.path.join(rec_dir, "world_timestamps.npy")
    pupil_data_path = os.path.join(rec_dir, "pupil_data")

    meta_info = load_meta_info(rec_dir)
    rec_version = read_rec_version(meta_info)

    timestamps = np.load(timestamps_path)
    cap = File_Capture(video_path,timestamps=timestamps)


    #Out file path verification, we do this before but if one uses a seperate tool, this will kick in.
    if out_file_path is None:
        out_file_path = os.path.join(rec_dir, "world_viz.mp4")
    else:
        file_name =  os.path.basename(out_file_path)
        dir_name = os.path.dirname(out_file_path)
        if not dir_name:
            dir_name = rec_dir
        if not file_name:
            file_name = 'world_viz.mp4'
        out_file_path = os.path.expanduser(os.path.join(dir_name,file_name))

    if os.path.isfile(out_file_path):
        logger.warning("Video out file already exsists. I will overwrite!")
        os.remove(out_file_path)
    logger.debug("Saving Video to %s"%out_file_path)


    #Trim mark verification
    #make sure the trim marks (start frame, endframe) make sense: We define them like python list slices,thus we can test them like such.
    trimmed_timestamps = timestamps[start_frame:end_frame]
    if len(trimmed_timestamps)==0:
        logger.warn("Start and end frames are set such that no video will be exported.")
        return False

    if start_frame == None:
        start_frame = 0

    #these two vars are shared with the lauching process and give a job length and progress report.
    frames_to_export.value = len(trimmed_timestamps)
    current_frame.value = 0
    logger.debug("Will export from frame %s to frame %s. This means I will export %s frames."%(start_frame,start_frame+frames_to_export.value,frames_to_export.value))

    #setup of writer
    writer = AV_Writer(out_file_path,fps=cap.frame_rate,use_timestamps=True)

    cap.seek_to_frame(start_frame)

    start_time = time()

    g = Global_Container()
    g.app = 'exporter'
    g.capture = cap
    g.rec_dir = rec_dir
    g.user_dir = user_dir
    g.rec_version = rec_version
    g.timestamps = timestamps


    # load pupil_positions, gaze_positions
    pupil_data = load_object(pupil_data_path)
    pupil_list = pupil_data['pupil_positions']
    gaze_list = pupil_data['gaze_positions']

    g.pupil_positions_by_frame = correlate_data(pupil_list,g.timestamps)
    g.gaze_positions_by_frame = correlate_data(gaze_list,g.timestamps)
    g.fixations_by_frame = [[] for x in g.timestamps] #populated by the fixation detector plugin

    #add plugins
    g.plugins = Plugin_List(g,plugin_by_name,plugin_initializers)

    while frames_to_export.value > current_frame.value:

        if should_terminate.value:
            logger.warning("User aborted export. Exported %s frames to %s."%(current_frame.value,out_file_path))

            #explicit release of VideoWriter
            writer.close()
            writer = None
            return False

        try:
            frame = cap.get_frame_nowait()
        except EndofVideoFileError:
            break

        events = {}
        #new positons and events
        events['gaze_positions'] = g.gaze_positions_by_frame[frame.index]
        events['pupil_positions'] = g.pupil_positions_by_frame[frame.index]

        # allow each Plugin to do its work.
        for p in g.plugins:
            p.update(frame,events)

        writer.write_video_frame(frame)
        current_frame.value +=1

    writer.close()
    writer = None

    duration = time()-start_time
    effective_fps = float(current_frame.value)/duration

    logger.info("Export done: Exported %s frames to %s. This took %s seconds. Exporter ran at %s frames per second"%(current_frame.value,out_file_path,duration,effective_fps))
    return True
Esempio n. 6
0
def session(rec_dir):


    # Callback functions
    def on_resize(window,w, h):
        g_pool.gui.update_window(w,h)
        g_pool.gui.collect_menus()
        graph.adjust_size(w,h)
        adjust_gl_view(w,h)
        for p in g_pool.plugins:
            p.on_window_resize(window,w,h)

    def on_key(window, key, scancode, action, mods):
        g_pool.gui.update_key(key,scancode,action,mods)

    def on_char(window,char):
        g_pool.gui.update_char(char)

    def on_button(window,button, action, mods):
        g_pool.gui.update_button(button,action,mods)
        pos = glfwGetCursorPos(window)
        pos = normalize(pos,glfwGetWindowSize(window))
        pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
        for p in g_pool.plugins:
            p.on_click(pos,button,action)

    def on_pos(window,x, y):
        hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0])
        g_pool.gui.update_mouse(x*hdpi_factor,y*hdpi_factor)

    def on_scroll(window,x,y):
        g_pool.gui.update_scroll(x,y*y_scroll_factor)


    def on_drop(window,count,paths):
        for x in range(count):
            new_rec_dir =  paths[x]
            if is_pupil_rec_dir(new_rec_dir):
                logger.debug("Starting new session with '%s'"%new_rec_dir)
                global rec_dir
                rec_dir = new_rec_dir
                glfwSetWindowShouldClose(window,True)
            else:
                logger.error("'%s' is not a valid pupil recording"%new_rec_dir)




    tick = delta_t()
    def get_dt():
        return next(tick)

    update_recording_to_recent(rec_dir)

    video_path = [f for f in glob(os.path.join(rec_dir,"world.*")) if f[-3:] in ('mp4','mkv','avi')][0]
    timestamps_path = os.path.join(rec_dir, "world_timestamps.npy")
    pupil_data_path = os.path.join(rec_dir, "pupil_data")

    meta_info = load_meta_info(rec_dir)
    rec_version = read_rec_version(meta_info)
    app_version = get_version(version_file)

    # log info about Pupil Platform and Platform in player.log
    logger.info('Application Version: %s'%app_version)
    logger.info('System Info: %s'%get_system_info())

    timestamps = np.load(timestamps_path)
    # Initialize capture
    cap = File_Capture(video_path,timestamps=list(timestamps))

    # load session persistent settings
    session_settings = Persistent_Dict(os.path.join(user_dir,"user_settings"))
    if session_settings.get("version",VersionFormat('0.0')) < get_version(version_file):
        logger.info("Session setting are from older version of this app. I will not use those.")
        session_settings.clear()

    width,height = session_settings.get('window_size',cap.frame_size)
    window_pos = session_settings.get('window_position',(0,0))
    main_window = glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - "+ rec_dir.split(os.path.sep)[-1], None, None)
    glfwSetWindowPos(main_window,window_pos[0],window_pos[1])
    glfwMakeContextCurrent(main_window)
    cygl.utils.init()

    # load pupil_positions, gaze_positions
    pupil_data = load_object(pupil_data_path)
    pupil_list = pupil_data['pupil_positions']
    gaze_list = pupil_data['gaze_positions']

    # create container for globally scoped vars
    g_pool = Global_Container()
    g_pool.app = 'player'
    g_pool.binocular = meta_info.get('Eye Mode','monocular') == 'binocular'
    g_pool.version = app_version
    g_pool.capture = cap
    g_pool.timestamps = timestamps
    g_pool.play = False
    g_pool.new_seek = True
    g_pool.user_dir = user_dir
    g_pool.rec_dir = rec_dir
    g_pool.rec_version = rec_version
    g_pool.meta_info = meta_info
    g_pool.pupil_positions_by_frame = correlate_data(pupil_list,g_pool.timestamps)
    g_pool.gaze_positions_by_frame = correlate_data(gaze_list,g_pool.timestamps)
    g_pool.fixations_by_frame = [[] for x in g_pool.timestamps] #populated by the fixation detector plugin

    def next_frame(_):
        try:
            cap.seek_to_frame(cap.get_frame_index())
        except FileSeekError:
            logger.warning("Could not seek to next frame.")
        else:
            g_pool.new_seek = True

    def prev_frame(_):
        try:
            cap.seek_to_frame(cap.get_frame_index()-2)
        except FileSeekError:
            logger.warning("Could not seek to previous frame.")
        else:
            g_pool.new_seek = True

    def set_scale(new_scale):
        g_pool.gui.scale = new_scale
        g_pool.gui.collect_menus()

    def open_plugin(plugin):
        if plugin ==  "Select to load":
            return
        g_pool.plugins.add(plugin)

    def purge_plugins():
        for p in g_pool.plugins:
            if p.__class__ in user_launchable_plugins:
                p.alive = False
        g_pool.plugins.clean()

    def do_export(_):
        export_range = slice(g_pool.trim_marks.in_mark,g_pool.trim_marks.out_mark)
        export_dir = os.path.join(g_pool.rec_dir,'exports','%s-%s'%(export_range.start,export_range.stop))
        try:
            os.makedirs(export_dir)
        except OSError as e:
            if e.errno != errno.EEXIST:
                logger.error("Could not create export dir")
                raise e
            else:
                logger.warning("Previous export for range [%s-%s] already exsits - overwriting."%(export_range.start,export_range.stop))
        else:
            logger.info('Created export dir at "%s"'%export_dir)

        notification = {'subject':'should_export','range':export_range,'export_dir':export_dir}
        g_pool.notifications.append(notification)

    g_pool.gui = ui.UI()
    g_pool.gui.scale = session_settings.get('gui_scale',1)
    g_pool.main_menu = ui.Growing_Menu("Settings",pos=(-350,20),size=(300,400))
    g_pool.main_menu.append(ui.Button("Close Pupil Player",lambda:glfwSetWindowShouldClose(main_window,True)))
    g_pool.main_menu.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=0.75,max=2.5,label='Interface Size'))
    g_pool.main_menu.append(ui.Info_Text('Player Version: %s'%g_pool.version))
    g_pool.main_menu.append(ui.Info_Text('Recording Version: %s'%rec_version))
    g_pool.main_menu.append(ui.Selector('Open plugin', selection = user_launchable_plugins,
                                        labels = [p.__name__.replace('_',' ') for p in user_launchable_plugins],
                                        setter= open_plugin, getter = lambda: "Select to load"))
    g_pool.main_menu.append(ui.Button('Close all plugins',purge_plugins))
    g_pool.main_menu.append(ui.Button('Reset window size',lambda: glfwSetWindowSize(main_window,cap.frame_size[0],cap.frame_size[1])) )
    g_pool.quickbar = ui.Stretching_Menu('Quick Bar',(0,100),(120,-100))
    g_pool.play_button = ui.Thumb('play',g_pool,label='Play',hotkey=GLFW_KEY_SPACE)
    g_pool.play_button.on_color[:] = (0,1.,.0,.8)
    g_pool.forward_button = ui.Thumb('forward',getter = lambda: False,setter= next_frame, hotkey=GLFW_KEY_RIGHT)
    g_pool.backward_button = ui.Thumb('backward',getter = lambda: False, setter = prev_frame, hotkey=GLFW_KEY_LEFT)
    g_pool.export_button = ui.Thumb('export',getter = lambda: False, setter = do_export, hotkey='e')
    g_pool.quickbar.extend([g_pool.play_button,g_pool.forward_button,g_pool.backward_button,g_pool.export_button])
    g_pool.gui.append(g_pool.quickbar)
    g_pool.gui.append(g_pool.main_menu)


    #we always load these plugins
    system_plugins = [('Trim_Marks',{}),('Seek_Bar',{})]
    default_plugins = [('Log_Display',{}),('Scan_Path',{}),('Vis_Polyline',{}),('Vis_Circle',{}),('Video_Export_Launcher',{})]
    previous_plugins = session_settings.get('loaded_plugins',default_plugins)
    g_pool.notifications = []
    g_pool.delayed_notifications = {}
    g_pool.plugins = Plugin_List(g_pool,plugin_by_name,system_plugins+previous_plugins)


    # Register callbacks main_window
    glfwSetFramebufferSizeCallback(main_window,on_resize)
    glfwSetKeyCallback(main_window,on_key)
    glfwSetCharCallback(main_window,on_char)
    glfwSetMouseButtonCallback(main_window,on_button)
    glfwSetCursorPosCallback(main_window,on_pos)
    glfwSetScrollCallback(main_window,on_scroll)
    glfwSetDropCallback(main_window,on_drop)
    #trigger on_resize
    on_resize(main_window, *glfwGetFramebufferSize(main_window))

    g_pool.gui.configuration = session_settings.get('ui_config',{})

    # gl_state settings
    basic_gl_setup()
    g_pool.image_tex = Named_Texture()

    #set up performace graphs:
    pid = os.getpid()
    ps = psutil.Process(pid)
    ts = cap.get_timestamp()-.03

    cpu_graph = graph.Bar_Graph()
    cpu_graph.pos = (20,110)
    cpu_graph.update_fn = ps.cpu_percent
    cpu_graph.update_rate = 5
    cpu_graph.label = 'CPU %0.1f'

    fps_graph = graph.Bar_Graph()
    fps_graph.pos = (140,110)
    fps_graph.update_rate = 5
    fps_graph.label = "%0.0f REC FPS"

    pupil_graph = graph.Bar_Graph(max_val=1.0)
    pupil_graph.pos = (260,110)
    pupil_graph.update_rate = 5
    pupil_graph.label = "Confidence: %0.2f"

    while not glfwWindowShouldClose(main_window):


        #grab new frame
        if g_pool.play or g_pool.new_seek:
            g_pool.new_seek = False
            try:
                new_frame = cap.get_frame_nowait()
            except EndofVideoFileError:
                #end of video logic: pause at last frame.
                g_pool.play=False
            update_graph = True
        else:
            update_graph = False


        frame = new_frame.copy()
        events = {}
        #report time between now and the last loop interation
        events['dt'] = get_dt()
        #new positons we make a deepcopy just like the image is a copy.
        events['gaze_positions'] = deepcopy(g_pool.gaze_positions_by_frame[frame.index])
        events['pupil_positions'] = deepcopy(g_pool.pupil_positions_by_frame[frame.index])

        if update_graph:
            #update performace graphs
            for p in  events['pupil_positions']:
                pupil_graph.add(p['confidence'])

            t = new_frame.timestamp
            if ts != t:
                dt,ts = t-ts,t
            fps_graph.add(1./dt)

            g_pool.play_button.status_text = str(frame.index)
        #always update the CPU graph
        cpu_graph.update()


        # publish delayed notifiactions when their time has come.
        for n in g_pool.delayed_notifications.values():
            if n['_notify_time_'] < time():
                del n['_notify_time_']
                del g_pool.delayed_notifications[n['subject']]
                g_pool.notifications.append(n)

        # notify each plugin if there are new notifactions:
        while g_pool.notifications:
            n = g_pool.notifications.pop(0)
            for p in g_pool.plugins:
                p.on_notify(n)

        # allow each Plugin to do its work.
        for p in g_pool.plugins:
            p.update(frame,events)

        #check if a plugin need to be destroyed
        g_pool.plugins.clean()

        # render camera image
        glfwMakeContextCurrent(main_window)
        make_coord_system_norm_based()
        g_pool.image_tex.update_from_frame(frame)
        g_pool.image_tex.draw()
        make_coord_system_pixel_based(frame.img.shape)
        # render visual feedback from loaded plugins
        for p in g_pool.plugins:
            p.gl_display()

        graph.push_view()
        fps_graph.draw()
        cpu_graph.draw()
        pupil_graph.draw()
        graph.pop_view()
        g_pool.gui.update()

        #present frames at appropriate speed
        cap.wait(frame)

        glfwSwapBuffers(main_window)
        glfwPollEvents()

    session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
    session_settings['gui_scale'] = g_pool.gui.scale
    session_settings['ui_config'] = g_pool.gui.configuration
    session_settings['window_size'] = glfwGetWindowSize(main_window)
    session_settings['window_position'] = glfwGetWindowPos(main_window)
    session_settings['version'] = g_pool.version
    session_settings.close()

    # de-init all running plugins
    for p in g_pool.plugins:
        p.alive = False
    g_pool.plugins.clean()

    cap.close()
    g_pool.gui.terminate()
    glfwDestroyWindow(main_window)
Esempio n. 7
0
def export(should_terminate, frames_to_export, current_frame, rec_dir, user_dir, min_data_confidence,
           start_frame=None, end_frame=None, plugin_initializers=(), out_file_path=None,pre_computed={}):

    vis_plugins = sorted([Vis_Circle,Vis_Cross,Vis_Polyline,Vis_Light_Points,
        Vis_Watermark,Vis_Scan_Path,Vis_Eye_Video_Overlay], key=lambda x: x.__name__)
    analysis_plugins = sorted([ Pupil_Angle_3D_Fixation_Detector,
                               Gaze_Position_2D_Fixation_Detector], key=lambda x: x.__name__)
    user_plugins = sorted(import_runtime_plugins(os.path.join(user_dir, 'plugins')), key=lambda x: x.__name__)

    available_plugins = vis_plugins + analysis_plugins + user_plugins
    name_by_index = [p.__name__ for p in available_plugins]
    plugin_by_name = dict(zip(name_by_index, available_plugins))

    logger = logging.getLogger(__name__+' with pid: '+str(os.getpid()))

    update_recording_to_recent(rec_dir)

    video_path = [f for f in glob(os.path.join(rec_dir, "world.*")) if f[-3:] in ('mp4', 'mkv', 'avi')][0]
    timestamps_path = os.path.join(rec_dir, "world_timestamps.npy")
    pupil_data_path = os.path.join(rec_dir, "pupil_data")
    audio_path = os.path.join(rec_dir, "audio.mp4")

    meta_info = load_meta_info(rec_dir)

    g_pool = Global_Container()
    g_pool.app = 'exporter'
    g_pool.min_data_confidence = min_data_confidence
    timestamps = np.load(timestamps_path)
    cap = File_Source(g_pool, video_path, timestamps=timestamps)

    # Out file path verification, we do this before but if one uses a seperate tool, this will kick in.
    if out_file_path is None:
        out_file_path = os.path.join(rec_dir, "world_viz.mp4")
    else:
        file_name = os.path.basename(out_file_path)
        dir_name = os.path.dirname(out_file_path)
        if not dir_name:
            dir_name = rec_dir
        if not file_name:
            file_name = 'world_viz.mp4'
        out_file_path = os.path.expanduser(os.path.join(dir_name, file_name))

    if os.path.isfile(out_file_path):
        logger.warning("Video out file already exsists. I will overwrite!")
        os.remove(out_file_path)
    logger.debug("Saving Video to {}".format(out_file_path))

    # Trim mark verification
    # make sure the trim marks (start frame, endframe) make sense:
    # We define them like python list slices, thus we can test them like such.
    trimmed_timestamps = timestamps[start_frame:end_frame]
    if len(trimmed_timestamps) == 0:
        logger.warn("Start and end frames are set such that no video will be exported.")
        return False

    if start_frame is None:
        start_frame = 0

    # these two vars are shared with the lauching process and give a job length and progress report.
    frames_to_export.value = len(trimmed_timestamps)
    current_frame.value = 0
    exp_info = "Will export from frame {} to frame {}. This means I will export {} frames."
    logger.debug(exp_info.format(start_frame, start_frame + frames_to_export.value, frames_to_export.value))

    # setup of writer
    writer = AV_Writer(out_file_path, fps=cap.frame_rate, audio_loc=audio_path, use_timestamps=True)

    cap.seek_to_frame(start_frame)

    start_time = time()

    g_pool.capture = cap
    g_pool.rec_dir = rec_dir
    g_pool.user_dir = user_dir
    g_pool.meta_info = meta_info
    g_pool.timestamps = timestamps
    g_pool.delayed_notifications = {}
    g_pool.notifications = []
    # load pupil_positions, gaze_positions
    pupil_data = pre_computed.get("pupil_data") or load_object(pupil_data_path)
    g_pool.pupil_data = pupil_data
    g_pool.pupil_positions = pre_computed.get("pupil_positions") or pupil_data['pupil_positions']
    g_pool.gaze_positions = pre_computed.get("gaze_positions") or pupil_data['gaze_positions']
    g_pool.fixations = [] # populated by the fixation detector plugin

    g_pool.pupil_positions_by_frame = correlate_data(g_pool.pupil_positions,g_pool.timestamps)
    g_pool.gaze_positions_by_frame = correlate_data(g_pool.gaze_positions,g_pool.timestamps)
    g_pool.fixations_by_frame = [[] for x in g_pool.timestamps]  # populated by the fixation detector plugin

    # add plugins
    g_pool.plugins = Plugin_List(g_pool, plugin_by_name, plugin_initializers)

    while frames_to_export.value > current_frame.value:

        if should_terminate.value:
            logger.warning("User aborted export. Exported {} frames to {}.".format(current_frame.value, out_file_path))

            # explicit release of VideoWriter
            writer.close()
            writer = None
            return False

        try:
            frame = cap.get_frame()
        except EndofVideoFileError:
            break

        events = {'frame':frame}
        # new positons and events
        events['gaze_positions'] = g_pool.gaze_positions_by_frame[frame.index]
        events['pupil_positions'] = g_pool.pupil_positions_by_frame[frame.index]

        # publish delayed notifiactions when their time has come.
        for n in list(g_pool.delayed_notifications.values()):
            if n['_notify_time_'] < time():
                del n['_notify_time_']
                del g_pool.delayed_notifications[n['subject']]
                g_pool.notifications.append(n)

        # notify each plugin if there are new notifactions:
        while g_pool.notifications:
            n = g_pool.notifications.pop(0)
            for p in g_pool.plugins:
                p.on_notify(n)

        # allow each Plugin to do its work.
        for p in g_pool.plugins:
            p.recent_events(events)

        writer.write_video_frame(frame)
        current_frame.value += 1

    writer.close()
    writer = None

    duration = time()-start_time
    effective_fps = float(current_frame.value)/duration

    result = "Export done: Exported {} frames to {}. This took {} seconds. Exporter ran at {} frames per second."
    logger.info(result.format(current_frame.value, out_file_path, duration, effective_fps))
    return True
Esempio n. 8
0
def export(should_terminate,
           frames_to_export,
           current_frame,
           rec_dir,
           user_dir,
           min_data_confidence,
           start_frame=None,
           end_frame=None,
           plugin_initializers=(),
           out_file_path=None):

    vis_plugins = sorted([
        Vis_Circle, Vis_Cross, Vis_Polyline, Vis_Light_Points, Vis_Watermark,
        Scan_Path
    ],
                         key=lambda x: x.__name__)
    analysis_plugins = sorted([
        Manual_Gaze_Correction, Eye_Video_Overlay,
        Pupil_Angle_3D_Fixation_Detector, Gaze_Position_2D_Fixation_Detector
    ],
                              key=lambda x: x.__name__)
    user_plugins = sorted(import_runtime_plugins(
        os.path.join(user_dir, 'plugins')),
                          key=lambda x: x.__name__)
    available_plugins = vis_plugins + analysis_plugins + user_plugins
    name_by_index = [p.__name__ for p in available_plugins]
    index_by_name = dict(zip(name_by_index, range(len(name_by_index))))
    plugin_by_name = dict(zip(name_by_index, available_plugins))

    logger = logging.getLogger(__name__ + ' with pid: ' + str(os.getpid()))

    update_recording_to_recent(rec_dir)

    video_path = [
        f for f in glob(os.path.join(rec_dir, "world.*"))
        if f[-3:] in ('mp4', 'mkv', 'avi')
    ][0]
    timestamps_path = os.path.join(rec_dir, "world_timestamps.npy")
    pupil_data_path = os.path.join(rec_dir, "pupil_data")

    meta_info = load_meta_info(rec_dir)
    rec_version = read_rec_version(meta_info)

    g_pool = Global_Container()
    g_pool.app = 'exporter'
    g_pool.min_data_confidence = min_data_confidence
    timestamps = np.load(timestamps_path)
    cap = File_Source(g_pool, video_path, timestamps=timestamps)

    #Out file path verification, we do this before but if one uses a seperate tool, this will kick in.
    if out_file_path is None:
        out_file_path = os.path.join(rec_dir, "world_viz.mp4")
    else:
        file_name = os.path.basename(out_file_path)
        dir_name = os.path.dirname(out_file_path)
        if not dir_name:
            dir_name = rec_dir
        if not file_name:
            file_name = 'world_viz.mp4'
        out_file_path = os.path.expanduser(os.path.join(dir_name, file_name))

    if os.path.isfile(out_file_path):
        logger.warning("Video out file already exsists. I will overwrite!")
        os.remove(out_file_path)
    logger.debug("Saving Video to %s" % out_file_path)

    #Trim mark verification
    #make sure the trim marks (start frame, endframe) make sense: We define them like python list slices,thus we can test them like such.
    trimmed_timestamps = timestamps[start_frame:end_frame]
    if len(trimmed_timestamps) == 0:
        logger.warn(
            "Start and end frames are set such that no video will be exported."
        )
        return False

    if start_frame == None:
        start_frame = 0

    #these two vars are shared with the lauching process and give a job length and progress report.
    frames_to_export.value = len(trimmed_timestamps)
    current_frame.value = 0
    logger.debug(
        "Will export from frame %s to frame %s. This means I will export %s frames."
        % (start_frame, start_frame + frames_to_export.value,
           frames_to_export.value))

    #setup of writer
    writer = AV_Writer(out_file_path, fps=cap.frame_rate, use_timestamps=True)

    cap.seek_to_frame(start_frame)

    start_time = time()

    g_pool.capture = cap
    g_pool.rec_dir = rec_dir
    g_pool.user_dir = user_dir
    g_pool.rec_version = rec_version
    g_pool.timestamps = timestamps
    g_pool.delayed_notifications = {}
    g_pool.notifications = []

    # load pupil_positions, gaze_positions
    pupil_data = load_object(pupil_data_path)
    pupil_list = pupil_data['pupil_positions']
    gaze_list = pupil_data['gaze_positions']
    g_pool.pupil_positions_by_frame = correlate_data(pupil_list,
                                                     g_pool.timestamps)
    g_pool.gaze_positions_by_frame = correlate_data(gaze_list,
                                                    g_pool.timestamps)
    g_pool.fixations_by_frame = [[] for x in g_pool.timestamps
                                 ]  #populated by the fixation detector plugin

    #add plugins
    g_pool.plugins = Plugin_List(g_pool, plugin_by_name, plugin_initializers)

    while frames_to_export.value > current_frame.value:

        if should_terminate.value:
            logger.warning("User aborted export. Exported %s frames to %s." %
                           (current_frame.value, out_file_path))

            #explicit release of VideoWriter
            writer.close()
            writer = None
            return False

        try:
            frame = cap.get_frame_nowait()
        except EndofVideoFileError:
            break

        events = {}
        #new positons and events
        events['gaze_positions'] = g_pool.gaze_positions_by_frame[frame.index]
        events['pupil_positions'] = g_pool.pupil_positions_by_frame[
            frame.index]

        # publish delayed notifiactions when their time has come.
        for n in g_pool.delayed_notifications.values():
            if n['_notify_time_'] < time():
                del n['_notify_time_']
                del g_pool.delayed_notifications[n['subject']]
                g_pool.notifications.append(n)

        # notify each plugin if there are new notifactions:
        while g_pool.notifications:
            n = g_pool.notifications.pop(0)
            for p in g_pool.plugins:
                p.on_notify(n)

        # allow each Plugin to do its work.
        for p in g_pool.plugins:
            p.update(frame, events)

        writer.write_video_frame(frame)
        current_frame.value += 1

    writer.close()
    writer = None

    duration = time() - start_time
    effective_fps = float(current_frame.value) / duration

    logger.info(
        "Export done: Exported %s frames to %s. This took %s seconds. Exporter ran at %s frames per second"
        % (current_frame.value, out_file_path, duration, effective_fps))
    return True
Esempio n. 9
0
def export(should_terminate,
           frames_to_export,
           current_frame,
           rec_dir,
           user_dir,
           start_frame=None,
           end_frame=None,
           plugin_initializers=[],
           out_file_path=None):

    logger = logging.getLogger(__name__ + ' with pid: ' + str(os.getpid()))

    update_recording_to_recent(rec_dir)

    video_path = [
        f for f in glob(os.path.join(rec_dir, "world.*"))
        if f[-3:] in ('mp4', 'mkv', 'avi')
    ][0]
    timestamps_path = os.path.join(rec_dir, "world_timestamps.npy")
    pupil_data_path = os.path.join(rec_dir, "pupil_data")

    meta_info = load_meta_info(rec_dir)
    rec_version = read_rec_version(meta_info)

    timestamps = np.load(timestamps_path)
    cap = File_Capture(video_path, timestamps=timestamps)

    #Out file path verification, we do this before but if one uses a seperate tool, this will kick in.
    if out_file_path is None:
        out_file_path = os.path.join(rec_dir, "world_viz.mp4")
    else:
        file_name = os.path.basename(out_file_path)
        dir_name = os.path.dirname(out_file_path)
        if not dir_name:
            dir_name = rec_dir
        if not file_name:
            file_name = 'world_viz.mp4'
        out_file_path = os.path.expanduser(os.path.join(dir_name, file_name))

    if os.path.isfile(out_file_path):
        logger.warning("Video out file already exsists. I will overwrite!")
        os.remove(out_file_path)
    logger.debug("Saving Video to %s" % out_file_path)

    #Trim mark verification
    #make sure the trim marks (start frame, endframe) make sense: We define them like python list slices,thus we can test them like such.
    trimmed_timestamps = timestamps[start_frame:end_frame]
    if len(trimmed_timestamps) == 0:
        logger.warn(
            "Start and end frames are set such that no video will be exported."
        )
        return False

    if start_frame == None:
        start_frame = 0

    #these two vars are shared with the lauching process and give a job length and progress report.
    frames_to_export.value = len(trimmed_timestamps)
    current_frame.value = 0
    logger.debug(
        "Will export from frame %s to frame %s. This means I will export %s frames."
        % (start_frame, start_frame + frames_to_export.value,
           frames_to_export.value))

    #setup of writer
    writer = AV_Writer(out_file_path, fps=cap.frame_rate, use_timestamps=True)

    cap.seek_to_frame(start_frame)

    start_time = time()

    g = Global_Container()
    g.app = 'exporter'
    g.capture = cap
    g.rec_dir = rec_dir
    g.user_dir = user_dir
    g.rec_version = rec_version
    g.timestamps = timestamps

    # load pupil_positions, gaze_positions
    pupil_data = load_object(pupil_data_path)
    pupil_list = pupil_data['pupil_positions']
    gaze_list = pupil_data['gaze_positions']

    g.pupil_positions_by_frame = correlate_data(pupil_list, g.timestamps)
    g.gaze_positions_by_frame = correlate_data(gaze_list, g.timestamps)
    g.fixations_by_frame = [[] for x in g.timestamps
                            ]  #populated by the fixation detector plugin

    #add plugins
    g.plugins = Plugin_List(g, plugin_by_name, plugin_initializers)

    while frames_to_export.value > current_frame.value:

        if should_terminate.value:
            logger.warning("User aborted export. Exported %s frames to %s." %
                           (current_frame.value, out_file_path))

            #explicit release of VideoWriter
            writer.close()
            writer = None
            return False

        try:
            frame = cap.get_frame_nowait()
        except EndofVideoFileError:
            break

        events = {}
        #new positons and events
        events['gaze_positions'] = g.gaze_positions_by_frame[frame.index]
        events['pupil_positions'] = g.pupil_positions_by_frame[frame.index]

        # allow each Plugin to do its work.
        for p in g.plugins:
            p.update(frame, events)

        writer.write_video_frame(frame)
        current_frame.value += 1

    writer.close()
    writer = None

    duration = time() - start_time
    effective_fps = float(current_frame.value) / duration

    logger.info(
        "Export done: Exported %s frames to %s. This took %s seconds. Exporter ran at %s frames per second"
        % (current_frame.value, out_file_path, duration, effective_fps))
    return True
Esempio n. 10
0
def player(rec_dir, ipc_pub_url, ipc_sub_url,
           ipc_push_url, user_dir, app_version):
    # general imports
    import logging
    import errno
    from glob import glob
    from copy import deepcopy
    from time import time
    # networking
    import zmq
    import zmq_tools

    # zmq ipc setup
    zmq_ctx = zmq.Context()
    ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify',))

    # log setup
    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    logger.setLevel(logging.INFO)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)



    # imports
    from file_methods import Persistent_Dict, load_object
    import numpy as np

    # display
    import glfw
    # check versions for our own depedencies as they are fast-changing
    from pyglui import __version__ as pyglui_version

    from pyglui import ui, graph, cygl
    from pyglui.cygl.utils import Named_Texture
    import gl_utils
    # capture
    from video_capture import File_Source, EndofVideoFileError, FileSeekError

    # helpers/utils
    from version_utils import VersionFormat
    from methods import normalize, denormalize, delta_t, get_system_info
    from player_methods import correlate_data, is_pupil_rec_dir, load_meta_info

    # monitoring
    import psutil

    # Plug-ins
    from plugin import Plugin, Plugin_List, import_runtime_plugins, Visualizer_Plugin_Base, Analysis_Plugin_Base, Producer_Plugin_Base
    from vis_circle import Vis_Circle
    from vis_cross import Vis_Cross
    from vis_polyline import Vis_Polyline
    from vis_light_points import Vis_Light_Points
    from vis_watermark import Vis_Watermark
    from vis_fixation import Vis_Fixation
    from vis_scan_path import Vis_Scan_Path
    from vis_eye_video_overlay import Vis_Eye_Video_Overlay
    from seek_bar import Seek_Bar
    from trim_marks import Trim_Marks
    from video_export_launcher import Video_Export_Launcher
    from offline_surface_tracker import Offline_Surface_Tracker
    from marker_auto_trim_marks import Marker_Auto_Trim_Marks
    from fixation_detector import Gaze_Position_2D_Fixation_Detector, Pupil_Angle_3D_Fixation_Detector
    # from manual_gaze_correction import Manual_Gaze_Correction
    from batch_exporter import Batch_Exporter
    from log_display import Log_Display
    from annotations import Annotation_Player
    from raw_data_exporter import Raw_Data_Exporter
    from log_history import Log_History
    from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection
    from gaze_producers import Gaze_From_Recording, Offline_Calibration

    assert pyglui_version >= '1.5'

    runtime_plugins = import_runtime_plugins(os.path.join(user_dir, 'plugins'))
    system_plugins = [Log_Display, Seek_Bar, Trim_Marks]
    user_launchable_plugins = [Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross, Vis_Watermark,
                               Vis_Eye_Video_Overlay, Vis_Scan_Path, Gaze_Position_2D_Fixation_Detector,
                               Pupil_Angle_3D_Fixation_Detector, Video_Export_Launcher,
                               Offline_Surface_Tracker, Raw_Data_Exporter, Batch_Exporter, Annotation_Player,
                               Log_History, Marker_Auto_Trim_Marks, Pupil_From_Recording, Offline_Pupil_Detection,
                               Gaze_From_Recording, Offline_Calibration] + runtime_plugins

    available_plugins = system_plugins + user_launchable_plugins
    name_by_index = [p.__name__ for p in available_plugins]
    plugin_by_name = dict(zip(name_by_index, available_plugins))

    # Callback functions
    def on_resize(window, w, h):
        if gl_utils.is_window_visible(window):
            hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0])
            g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
            g_pool.gui.update_window(w, h)
            g_pool.gui.collect_menus()
            for g in g_pool.graphs:
                g.scale = hdpi_factor
                g.adjust_window_size(w, h)
            gl_utils.adjust_gl_view(w, h)
            for p in g_pool.plugins:
                p.on_window_resize(window, w, h)

    def on_key(window, key, scancode, action, mods):
        g_pool.gui.update_key(key, scancode, action, mods)

    def on_char(window, char):
        g_pool.gui.update_char(char)

    def on_button(window, button, action, mods):
        g_pool.gui.update_button(button, action, mods)

    def on_pos(window, x, y):
        hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0]/glfw.glfwGetWindowSize(window)[0])
        g_pool.gui.update_mouse(x*hdpi_factor, y*hdpi_factor)

    def on_scroll(window, x, y):
        g_pool.gui.update_scroll(x, y*scroll_factor)

    def on_drop(window, count, paths):
        for x in range(count):
            new_rec_dir = paths[x].decode('utf-8')
            if is_pupil_rec_dir(new_rec_dir):
                logger.debug("Starting new session with '{}'".format(new_rec_dir))
                ipc_pub.notify({"subject": "player_drop_process.should_start", "rec_dir": new_rec_dir})
                glfw.glfwSetWindowShouldClose(window, True)
            else:
                logger.error("'{}' is not a valid pupil recording".format(new_rec_dir))

    tick = delta_t()

    def get_dt():
        return next(tick)

    video_path = [f for f in glob(os.path.join(rec_dir, "world.*"))
                  if os.path.splitext(f)[1] in ('.mp4', '.mkv', '.avi', '.h264', '.mjpeg')][0]
    timestamps_path = os.path.join(rec_dir, "world_timestamps.npy")
    pupil_data_path = os.path.join(rec_dir, "pupil_data")

    meta_info = load_meta_info(rec_dir)

    # log info about Pupil Platform and Platform in player.log
    logger.info('Application Version: {}'.format(app_version))
    logger.info('System Info: {}'.format(get_system_info()))

    timestamps = np.load(timestamps_path)

    # create container for globally scoped vars
    g_pool = Global_Container()
    g_pool.app = 'player'
    g_pool.zmq_ctx = zmq_ctx
    g_pool.ipc_pub = ipc_pub
    g_pool.ipc_pub_url = ipc_pub_url
    g_pool.ipc_sub_url = ipc_sub_url
    g_pool.ipc_push_url = ipc_push_url

    # Initialize capture
    cap = File_Source(g_pool, video_path, timestamps=list(timestamps))

    # load session persistent settings
    session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings"))
    if VersionFormat(session_settings.get("version", '0.0')) != app_version:
        logger.info("Session setting are a different version of this app. I will not use those.")
        session_settings.clear()

    width, height = session_settings.get('window_size', cap.frame_size)
    window_pos = session_settings.get('window_position', window_position_default)
    glfw.glfwInit()
    main_window = glfw.glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - "
                                   + rec_dir.split(os.path.sep)[-1], None, None)
    glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
    glfw.glfwMakeContextCurrent(main_window)
    cygl.utils.init()

    def set_scale(new_scale):
        g_pool.gui_user_scale = new_scale
        on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

    # load pupil_positions, gaze_positions
    g_pool.pupil_data = load_object(pupil_data_path)
    g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular'
    g_pool.version = app_version
    g_pool.capture = cap
    g_pool.timestamps = timestamps
    g_pool.get_timestamp = lambda: 0.
    g_pool.play = False
    g_pool.new_seek = True
    g_pool.user_dir = user_dir
    g_pool.rec_dir = rec_dir
    g_pool.meta_info = meta_info
    g_pool.min_data_confidence = session_settings.get('min_data_confidence', 0.6)

    g_pool.pupil_positions = []
    g_pool.gaze_positions = []
    g_pool.fixations = []

    g_pool.notifications_by_frame = correlate_data(g_pool.pupil_data['notifications'], g_pool.timestamps)
    g_pool.pupil_positions_by_frame = [[] for x in g_pool.timestamps] # populated by producer`
    g_pool.gaze_positions_by_frame = [[] for x in g_pool.timestamps] # populated by producer
    g_pool.fixations_by_frame = [[] for x in g_pool.timestamps]  # populated by the fixation detector plugin

    def next_frame(_):
        try:
            cap.seek_to_frame(cap.get_frame_index() + 1)
        except(FileSeekError):
            logger.warning("Could not seek to next frame.")
        else:
            g_pool.new_seek = True

    def prev_frame(_):
        try:
            cap.seek_to_frame(cap.get_frame_index() - 1)
        except(FileSeekError):
            logger.warning("Could not seek to previous frame.")
        else:
            g_pool.new_seek = True

    def toggle_play(new_state):
        if cap.get_frame_index() >= cap.get_frame_count()-5:
            cap.seek_to_frame(1)  # avoid pause set by hitting trimmark pause.
            logger.warning("End of video - restart at beginning.")
        g_pool.play = new_state

    def set_data_confidence(new_confidence):
        g_pool.min_data_confidence = new_confidence
        notification = {'subject': 'min_data_confidence_changed'}
        notification['_notify_time_'] = time()+.8
        g_pool.ipc_pub.notify(notification)

    def open_plugin(plugin):
        if plugin == "Select to load":
            return
        g_pool.plugins.add(plugin)

    def purge_plugins():
        for p in g_pool.plugins:
            if p.__class__ in user_launchable_plugins:
                p.alive = False
        g_pool.plugins.clean()

    def do_export(_):
        export_range = g_pool.trim_marks.in_mark, g_pool.trim_marks.out_mark
        export_dir = os.path.join(g_pool.rec_dir, 'exports', '{}-{}'.format(*export_range))
        try:
            os.makedirs(export_dir)
        except OSError as e:
            if e.errno != errno.EEXIST:
                logger.error("Could not create export dir")
                raise e
            else:
                overwrite_warning = "Previous export for range [{}-{}] already exsits - overwriting."
                logger.warning(overwrite_warning.format(*export_range))
        else:
            logger.info('Created export dir at "{}"'.format(export_dir))

        notification = {'subject': 'should_export', 'range': export_range, 'export_dir': export_dir}
        g_pool.ipc_pub.notify(notification)

    g_pool.gui = ui.UI()
    g_pool.gui_user_scale = session_settings.get('gui_scale', 1.)
    g_pool.main_menu = ui.Scrolling_Menu("Settings", pos=(-350, 20), size=(300, 560))
    g_pool.main_menu.append(ui.Button('Reset window size',
                                      lambda: glfw.glfwSetWindowSize(main_window, cap.frame_size[0], cap.frame_size[1])))
    g_pool.main_menu.append(ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2], label='Interface Size'))
    g_pool.main_menu.append(ui.Info_Text('Player Version: {}'.format(g_pool.version)))
    g_pool.main_menu.append(ui.Info_Text('Capture Version: {}'.format(meta_info['Capture Software Version'])))
    g_pool.main_menu.append(ui.Info_Text('Data Format Version: {}'.format(meta_info['Data Format Version'])))
    g_pool.main_menu.append(ui.Slider('min_data_confidence', g_pool, setter=set_data_confidence,
                                      step=.05, min=0.0, max=1.0, label='Confidence threshold'))

    g_pool.main_menu.append(ui.Info_Text('Open plugins'))

    selector_label = "Select to load"

    def append_selector(label, plugins):
        plugins.sort(key=lambda p: p.__name__)
        plugin_labels = [p.__name__.replace('_', ' ') for p in plugins]
        g_pool.main_menu.append(ui.Selector(label,
                                            selection=[selector_label] + plugins,
                                            labels=[selector_label] + plugin_labels,
                                            setter=open_plugin,
                                            getter=lambda: selector_label))

    base_plugins = [Visualizer_Plugin_Base, Analysis_Plugin_Base, Producer_Plugin_Base]
    base_labels = ['Visualizer:', 'Analyser:', 'Data Source:']
    launchable = user_launchable_plugins.copy()
    for base_class, label in zip(base_plugins, base_labels):
        member_plugins = []
        for p in user_launchable_plugins:
            if issubclass(p, base_class):
                member_plugins.append(p)
                launchable.remove(p)
        append_selector(label, member_plugins)

    # launchable only contains plugins that could not be assigned to any of the above categories
    append_selector('Other', launchable)

    g_pool.main_menu.append(ui.Button('Close all plugins', purge_plugins))
    g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))
    g_pool.play_button = ui.Thumb('play',
                                  g_pool,
                                  label=chr(0xf04b),
                                  setter=toggle_play,
                                  hotkey=glfw.GLFW_KEY_SPACE,
                                  label_font='fontawesome',
                                  label_offset_x=5,
                                  label_offset_y=0,
                                  label_offset_size=-24)
    g_pool.play_button.on_color[:] = (0, 1., .0, .8)
    g_pool.forward_button = ui.Thumb('forward',
                                     label=chr(0xf04e),
                                     getter=lambda: False,
                                     setter=next_frame,
                                     hotkey=glfw.GLFW_KEY_RIGHT,
                                     label_font='fontawesome',
                                     label_offset_x=5,
                                     label_offset_y=0,
                                     label_offset_size=-24)
    g_pool.backward_button = ui.Thumb('backward',
                                      label=chr(0xf04a),
                                      getter=lambda: False,
                                      setter=prev_frame,
                                      hotkey=glfw.GLFW_KEY_LEFT,
                                      label_font='fontawesome',
                                      label_offset_x=-5,
                                      label_offset_y=0,
                                      label_offset_size=-24)
    g_pool.export_button = ui.Thumb('export',
                                    label=chr(0xf063),
                                    getter=lambda: False,
                                    setter=do_export,
                                    hotkey='e',
                                    label_font='fontawesome',
                                    label_offset_x=0,
                                    label_offset_y=2,
                                    label_offset_size=-24)
    g_pool.quickbar.extend([g_pool.play_button, g_pool.forward_button, g_pool.backward_button, g_pool.export_button])
    g_pool.gui.append(g_pool.quickbar)
    g_pool.gui.append(g_pool.main_menu)

    # we always load these plugins
    system_plugins = [('Trim_Marks', {}), ('Seek_Bar', {})]
    default_plugins = [('Log_Display', {}), ('Vis_Scan_Path', {}), ('Vis_Polyline', {}),
                       ('Vis_Circle', {}), ('Video_Export_Launcher', {}),
                       ('Pupil_From_Recording', {}), ('Gaze_From_Recording', {})]
    previous_plugins = session_settings.get('loaded_plugins', default_plugins)
    g_pool.plugins = Plugin_List(g_pool, plugin_by_name, system_plugins+previous_plugins)


    # Register callbacks main_window
    glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
    glfw.glfwSetKeyCallback(main_window, on_key)
    glfw.glfwSetCharCallback(main_window, on_char)
    glfw.glfwSetMouseButtonCallback(main_window, on_button)
    glfw.glfwSetCursorPosCallback(main_window, on_pos)
    glfw.glfwSetScrollCallback(main_window, on_scroll)
    glfw.glfwSetDropCallback(main_window, on_drop)

    g_pool.gui.configuration = session_settings.get('ui_config', {})

    # gl_state settings
    gl_utils.basic_gl_setup()
    g_pool.image_tex = Named_Texture()

    # set up performace graphs:
    pid = os.getpid()
    ps = psutil.Process(pid)
    ts = None

    cpu_graph = graph.Bar_Graph()
    cpu_graph.pos = (20, 110)
    cpu_graph.update_fn = ps.cpu_percent
    cpu_graph.update_rate = 5
    cpu_graph.label = 'CPU %0.1f'

    fps_graph = graph.Bar_Graph()
    fps_graph.pos = (140, 110)
    fps_graph.update_rate = 5
    fps_graph.label = "%0.0f REC FPS"

    pupil_graph = graph.Bar_Graph(max_val=1.0)
    pupil_graph.pos = (260, 110)
    pupil_graph.update_rate = 5
    pupil_graph.label = "Confidence: %0.2f"
    g_pool.graphs = [cpu_graph, fps_graph, pupil_graph]

    # trigger on_resize
    on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

    def handle_notifications(n):
        subject = n['subject']
        if subject == 'start_plugin':
            g_pool.plugins.add(
                plugin_by_name[n['name']], args=n.get('args', {}))
        elif subject.startswith('meta.should_doc'):
            ipc_pub.notify({'subject': 'meta.doc',
                            'actor': g_pool.app,
                            'doc': player.__doc__})
            for p in g_pool.plugins:
                if (p.on_notify.__doc__
                        and p.__class__.on_notify != Plugin.on_notify):
                    ipc_pub.notify({'subject': 'meta.doc',
                                    'actor': p.class_name,
                                    'doc': p.on_notify.__doc__})

    while not glfw.glfwWindowShouldClose(main_window):

        # fetch newest notifications
        new_notifications = []
        while notify_sub.new_data:
            t, n = notify_sub.recv()
            new_notifications.append(n)

        # notify each plugin if there are new notifications:
        for n in new_notifications:
            handle_notifications(n)
            for p in g_pool.plugins:
                p.on_notify(n)

        # grab new frame
        if g_pool.play or g_pool.new_seek:
            g_pool.new_seek = False
            try:
                new_frame = cap.get_frame()
            except EndofVideoFileError:
                # end of video logic: pause at last frame.
                g_pool.play = False
                logger.warning("end of video")
            update_graph = True
        else:
            update_graph = False

        frame = new_frame.copy()
        events = {}
        events['frame'] = frame
        # report time between now and the last loop interation
        events['dt'] = get_dt()
        # new positons we make a deepcopy just like the image is a copy.
        events['gaze_positions'] = deepcopy(g_pool.gaze_positions_by_frame[frame.index])
        events['pupil_positions'] = deepcopy(g_pool.pupil_positions_by_frame[frame.index])

        if update_graph:
            # update performace graphs
            for p in events['pupil_positions']:
                pupil_graph.add(p['confidence'])

            t = new_frame.timestamp
            if ts and ts != t:
                dt, ts = t-ts, t
                fps_graph.add(1./dt)
            else:
                ts = new_frame.timestamp

            g_pool.play_button.status_text = str(frame.index)
        # always update the CPU graph
        cpu_graph.update()

        # allow each Plugin to do its work.
        for p in g_pool.plugins:
            p.recent_events(events)

        # check if a plugin need to be destroyed
        g_pool.plugins.clean()

        # render camera image
        glfw.glfwMakeContextCurrent(main_window)
        gl_utils.make_coord_system_norm_based()
        g_pool.image_tex.update_from_frame(frame)
        g_pool.image_tex.draw()
        gl_utils.make_coord_system_pixel_based(frame.img.shape)
        # render visual feedback from loaded plugins
        for p in g_pool.plugins:
            p.gl_display()

        fps_graph.draw()
        cpu_graph.draw()
        pupil_graph.draw()
        unused_buttons = g_pool.gui.update()
        for b in unused_buttons:
            button,action,mods = b
            pos = glfw.glfwGetCursorPos(main_window)
            pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
            pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0]))  # Position in img pixels
            for p in g_pool.plugins:
                p.on_click(pos, button, action)

        # present frames at appropriate speed
        cap.wait(frame)

        glfw.glfwSwapBuffers(main_window)
        glfw.glfwPollEvents()

    session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
    session_settings['min_data_confidence'] = g_pool.min_data_confidence
    session_settings['gui_scale'] = g_pool.gui_user_scale
    session_settings['ui_config'] = g_pool.gui.configuration
    session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
    session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
    session_settings['version'] = str(g_pool.version)
    session_settings.close()

    # de-init all running plugins
    for p in g_pool.plugins:
        p.alive = False
    g_pool.plugins.clean()

    cap.cleanup()
    g_pool.gui.terminate()
    glfw.glfwDestroyWindow(main_window)

    logger.info("Process shutting down.")
    ipc_pub.notify({'subject': 'player_process.stopped'})
Esempio n. 11
0
def player(rec_dir, ipc_pub_url, ipc_sub_url,
           ipc_push_url, user_dir, app_version):
    # general imports
    import logging
    import errno
    from glob import glob
    from copy import deepcopy
    from time import time
    # networking
    import zmq
    import zmq_tools

    # zmq ipc setup
    zmq_ctx = zmq.Context()
    ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify',))

    # log setup
    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    logger.setLevel(logging.INFO)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)



    # imports
    from file_methods import Persistent_Dict, load_object

    # display
    import glfw
    # check versions for our own depedencies as they are fast-changing
    from pyglui import __version__ as pyglui_version

    from pyglui import ui, graph, cygl
    from pyglui.cygl.utils import Named_Texture
    import gl_utils
    # capture
    from video_capture import File_Source, EndofVideoFileError, FileSeekError

    # helpers/utils
    from version_utils import VersionFormat
    from methods import normalize, denormalize, delta_t, get_system_info
    from player_methods import correlate_data, is_pupil_rec_dir, load_meta_info

    # monitoring
    import psutil

    # Plug-ins
    from plugin import Plugin, Plugin_List, import_runtime_plugins, Visualizer_Plugin_Base, Analysis_Plugin_Base, Producer_Plugin_Base
    from vis_circle import Vis_Circle
    from vis_cross import Vis_Cross
    from vis_polyline import Vis_Polyline
    from vis_light_points import Vis_Light_Points
    from vis_watermark import Vis_Watermark
    from vis_fixation import Vis_Fixation
    from vis_scan_path import Vis_Scan_Path
    from vis_eye_video_overlay import Vis_Eye_Video_Overlay
    from seek_bar import Seek_Bar
    from trim_marks import Trim_Marks
    from video_export_launcher import Video_Export_Launcher
    from offline_surface_tracker import Offline_Surface_Tracker
    from marker_auto_trim_marks import Marker_Auto_Trim_Marks
    from fixation_detector import Gaze_Position_2D_Fixation_Detector, Pupil_Angle_3D_Fixation_Detector
    # from manual_gaze_correction import Manual_Gaze_Correction
    from batch_exporter import Batch_Exporter
    from log_display import Log_Display
    from annotations import Annotation_Player
    from raw_data_exporter import Raw_Data_Exporter
    from log_history import Log_History
    from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection
    from gaze_producers import Gaze_From_Recording, Offline_Calibration

    assert pyglui_version >= '1.7'

    runtime_plugins = import_runtime_plugins(os.path.join(user_dir, 'plugins'))
    system_plugins = [Log_Display, Seek_Bar, Trim_Marks]
    user_launchable_plugins = [Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross, Vis_Watermark,
                               Vis_Eye_Video_Overlay, Vis_Scan_Path, Gaze_Position_2D_Fixation_Detector,
                               Pupil_Angle_3D_Fixation_Detector, Video_Export_Launcher,
                               Offline_Surface_Tracker, Raw_Data_Exporter, Batch_Exporter, Annotation_Player,
                               Log_History, Marker_Auto_Trim_Marks, Pupil_From_Recording, Offline_Pupil_Detection,
                               Gaze_From_Recording, Offline_Calibration] + runtime_plugins

    available_plugins = system_plugins + user_launchable_plugins
    name_by_index = [p.__name__ for p in available_plugins]
    plugin_by_name = dict(zip(name_by_index, available_plugins))

    # Callback functions
    def on_resize(window, w, h):
        if gl_utils.is_window_visible(window):
            hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0])
            g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
            g_pool.gui.update_window(w, h)
            g_pool.gui.collect_menus()
            for g in g_pool.graphs:
                g.scale = hdpi_factor
                g.adjust_window_size(w, h)
            gl_utils.adjust_gl_view(w, h)
            for p in g_pool.plugins:
                p.on_window_resize(window, w, h)

    def on_window_key(window, key, scancode, action, mods):
        g_pool.gui.update_key(key, scancode, action, mods)

    def on_window_char(window, char):
        g_pool.gui.update_char(char)

    def on_window_mouse_button(window, button, action, mods):
        g_pool.gui.update_button(button, action, mods)

    def on_pos(window, x, y):
        hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0]/glfw.glfwGetWindowSize(window)[0])
        g_pool.gui.update_mouse(x*hdpi_factor, y*hdpi_factor)

    def on_scroll(window, x, y):
        g_pool.gui.update_scroll(x, y*scroll_factor)

    def on_drop(window, count, paths):
        for x in range(count):
            new_rec_dir = paths[x].decode('utf-8')
            if is_pupil_rec_dir(new_rec_dir):
                logger.debug("Starting new session with '{}'".format(new_rec_dir))
                ipc_pub.notify({"subject": "player_drop_process.should_start", "rec_dir": new_rec_dir})
                glfw.glfwSetWindowShouldClose(window, True)
            else:
                logger.error("'{}' is not a valid pupil recording".format(new_rec_dir))

    tick = delta_t()

    def get_dt():
        return next(tick)

    video_path = [f for f in glob(os.path.join(rec_dir, "world.*"))
                  if os.path.splitext(f)[1] in ('.mp4', '.mkv', '.avi', '.h264', '.mjpeg')][0]
    pupil_data_path = os.path.join(rec_dir, "pupil_data")

    meta_info = load_meta_info(rec_dir)

    # log info about Pupil Platform and Platform in player.log
    logger.info('Application Version: {}'.format(app_version))
    logger.info('System Info: {}'.format(get_system_info()))

    # create container for globally scoped vars
    g_pool = Global_Container()
    g_pool.app = 'player'
    g_pool.zmq_ctx = zmq_ctx
    g_pool.ipc_pub = ipc_pub
    g_pool.ipc_pub_url = ipc_pub_url
    g_pool.ipc_sub_url = ipc_sub_url
    g_pool.ipc_push_url = ipc_push_url

    # Initialize capture
    cap = File_Source(g_pool, video_path)

    # load session persistent settings
    session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings"))
    if VersionFormat(session_settings.get("version", '0.0')) != app_version:
        logger.info("Session setting are a different version of this app. I will not use those.")
        session_settings.clear()

    width, height = session_settings.get('window_size', cap.frame_size)
    window_pos = session_settings.get('window_position', window_position_default)
    glfw.glfwInit()
    main_window = glfw.glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - "
                                   + rec_dir.split(os.path.sep)[-1], None, None)
    glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
    glfw.glfwMakeContextCurrent(main_window)
    cygl.utils.init()

    def set_scale(new_scale):
        g_pool.gui_user_scale = new_scale
        on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

    # load pupil_positions, gaze_positions
    g_pool.pupil_data = load_object(pupil_data_path)
    g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular'
    g_pool.version = app_version
    g_pool.capture = cap
    g_pool.timestamps = g_pool.capture.timestamps
    g_pool.get_timestamp = lambda: 0.
    g_pool.play = False
    g_pool.new_seek = True
    g_pool.user_dir = user_dir
    g_pool.rec_dir = rec_dir
    g_pool.meta_info = meta_info
    g_pool.min_data_confidence = session_settings.get('min_data_confidence', 0.6)

    g_pool.pupil_positions = []
    g_pool.gaze_positions = []
    g_pool.fixations = []

    g_pool.notifications_by_frame = correlate_data(g_pool.pupil_data['notifications'], g_pool.timestamps)
    g_pool.pupil_positions_by_frame = [[] for x in g_pool.timestamps] # populated by producer`
    g_pool.gaze_positions_by_frame = [[] for x in g_pool.timestamps] # populated by producer
    g_pool.fixations_by_frame = [[] for x in g_pool.timestamps]  # populated by the fixation detector plugin

    def next_frame(_):
        try:
            cap.seek_to_frame(cap.get_frame_index() + 1)
        except(FileSeekError):
            logger.warning("Could not seek to next frame.")
        else:
            g_pool.new_seek = True

    def prev_frame(_):
        try:
            cap.seek_to_frame(cap.get_frame_index() - 1)
        except(FileSeekError):
            logger.warning("Could not seek to previous frame.")
        else:
            g_pool.new_seek = True

    def toggle_play(new_state):
        if cap.get_frame_index() >= cap.get_frame_count()-5:
            cap.seek_to_frame(1)  # avoid pause set by hitting trimmark pause.
            logger.warning("End of video - restart at beginning.")
        g_pool.play = new_state

    def set_data_confidence(new_confidence):
        g_pool.min_data_confidence = new_confidence
        notification = {'subject': 'min_data_confidence_changed'}
        notification['_notify_time_'] = time()+.8
        g_pool.ipc_pub.notify(notification)

    def open_plugin(plugin):
        if plugin == "Select to load":
            return
        g_pool.plugins.add(plugin)

    def purge_plugins():
        for p in g_pool.plugins:
            if p.__class__ in user_launchable_plugins:
                p.alive = False
        g_pool.plugins.clean()

    def do_export(_):
        export_range = g_pool.trim_marks.in_mark, g_pool.trim_marks.out_mark
        export_dir = os.path.join(g_pool.rec_dir, 'exports', '{}-{}'.format(*export_range))
        try:
            os.makedirs(export_dir)
        except OSError as e:
            if e.errno != errno.EEXIST:
                logger.error("Could not create export dir")
                raise e
            else:
                overwrite_warning = "Previous export for range [{}-{}] already exsits - overwriting."
                logger.warning(overwrite_warning.format(*export_range))
        else:
            logger.info('Created export dir at "{}"'.format(export_dir))

        notification = {'subject': 'should_export', 'range': export_range, 'export_dir': export_dir}
        g_pool.ipc_pub.notify(notification)

    g_pool.gui = ui.UI()
    g_pool.gui_user_scale = session_settings.get('gui_scale', 1.)
    g_pool.main_menu = ui.Scrolling_Menu("Settings", pos=(-350, 20), size=(300, 560))
    g_pool.main_menu.append(ui.Button('Reset window size',
                                      lambda: glfw.glfwSetWindowSize(main_window, cap.frame_size[0], cap.frame_size[1])))
    g_pool.main_menu.append(ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2], label='Interface Size'))
    g_pool.main_menu.append(ui.Info_Text('Player Version: {}'.format(g_pool.version)))
    g_pool.main_menu.append(ui.Info_Text('Capture Version: {}'.format(meta_info['Capture Software Version'])))
    g_pool.main_menu.append(ui.Info_Text('Data Format Version: {}'.format(meta_info['Data Format Version'])))
    g_pool.main_menu.append(ui.Slider('min_data_confidence', g_pool, setter=set_data_confidence,
                                      step=.05, min=0.0, max=1.0, label='Confidence threshold'))

    g_pool.main_menu.append(ui.Info_Text('Open plugins'))

    selector_label = "Select to load"

    def append_selector(label, plugins):
        plugins.sort(key=lambda p: p.__name__)
        plugin_labels = [p.__name__.replace('_', ' ') for p in plugins]
        g_pool.main_menu.append(ui.Selector(label,
                                            selection=[selector_label] + plugins,
                                            labels=[selector_label] + plugin_labels,
                                            setter=open_plugin,
                                            getter=lambda: selector_label))

    base_plugins = [Visualizer_Plugin_Base, Analysis_Plugin_Base, Producer_Plugin_Base]
    base_labels = ['Visualizer:', 'Analyser:', 'Data Source:']
    launchable = user_launchable_plugins.copy()
    for base_class, label in zip(base_plugins, base_labels):
        member_plugins = []
        for p in user_launchable_plugins:
            if issubclass(p, base_class):
                member_plugins.append(p)
                launchable.remove(p)
        append_selector(label, member_plugins)

    # launchable only contains plugins that could not be assigned to any of the above categories
    append_selector('Other', launchable)

    g_pool.main_menu.append(ui.Button('Close all plugins', purge_plugins))
    g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))
    g_pool.play_button = ui.Thumb('play',
                                  g_pool,
                                  label=chr(0xf04b),
                                  setter=toggle_play,
                                  hotkey=glfw.GLFW_KEY_SPACE,
                                  label_font='fontawesome',
                                  label_offset_x=5,
                                  label_offset_y=0,
                                  label_offset_size=-24)
    g_pool.play_button.on_color[:] = (0, 1., .0, .8)
    g_pool.forward_button = ui.Thumb('forward',
                                     label=chr(0xf04e),
                                     getter=lambda: False,
                                     setter=next_frame,
                                     hotkey=glfw.GLFW_KEY_RIGHT,
                                     label_font='fontawesome',
                                     label_offset_x=5,
                                     label_offset_y=0,
                                     label_offset_size=-24)
    g_pool.backward_button = ui.Thumb('backward',
                                      label=chr(0xf04a),
                                      getter=lambda: False,
                                      setter=prev_frame,
                                      hotkey=glfw.GLFW_KEY_LEFT,
                                      label_font='fontawesome',
                                      label_offset_x=-5,
                                      label_offset_y=0,
                                      label_offset_size=-24)
    g_pool.export_button = ui.Thumb('export',
                                    label=chr(0xf063),
                                    getter=lambda: False,
                                    setter=do_export,
                                    hotkey='e',
                                    label_font='fontawesome',
                                    label_offset_x=0,
                                    label_offset_y=2,
                                    label_offset_size=-24)
    g_pool.quickbar.extend([g_pool.play_button, g_pool.forward_button, g_pool.backward_button, g_pool.export_button])
    g_pool.gui.append(g_pool.quickbar)
    g_pool.gui.append(g_pool.main_menu)

    # we always load these plugins
    system_plugins = [('Trim_Marks', {}), ('Seek_Bar', {})]
    default_plugins = [('Log_Display', {}), ('Vis_Scan_Path', {}), ('Vis_Polyline', {}),
                       ('Vis_Circle', {}), ('Video_Export_Launcher', {}),
                       ('Pupil_From_Recording', {}), ('Gaze_From_Recording', {})]
    previous_plugins = session_settings.get('loaded_plugins', default_plugins)
    g_pool.plugins = Plugin_List(g_pool, plugin_by_name, system_plugins+previous_plugins)


    # Register callbacks main_window
    glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
    glfw.glfwSetKeyCallback(main_window, on_window_key)
    glfw.glfwSetCharCallback(main_window, on_window_char)
    glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
    glfw.glfwSetCursorPosCallback(main_window, on_pos)
    glfw.glfwSetScrollCallback(main_window, on_scroll)
    glfw.glfwSetDropCallback(main_window, on_drop)

    g_pool.gui.configuration = session_settings.get('ui_config', {})

    # gl_state settings
    gl_utils.basic_gl_setup()
    g_pool.image_tex = Named_Texture()

    # set up performace graphs:
    pid = os.getpid()
    ps = psutil.Process(pid)
    ts = None

    cpu_graph = graph.Bar_Graph()
    cpu_graph.pos = (20, 110)
    cpu_graph.update_fn = ps.cpu_percent
    cpu_graph.update_rate = 5
    cpu_graph.label = 'CPU %0.1f'

    fps_graph = graph.Bar_Graph()
    fps_graph.pos = (140, 110)
    fps_graph.update_rate = 5
    fps_graph.label = "%0.0f REC FPS"

    pupil_graph = graph.Bar_Graph(max_val=1.0)
    pupil_graph.pos = (260, 110)
    pupil_graph.update_rate = 5
    pupil_graph.label = "Confidence: %0.2f"
    g_pool.graphs = [cpu_graph, fps_graph, pupil_graph]

    # trigger on_resize
    on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

    def handle_notifications(n):
        subject = n['subject']
        if subject == 'start_plugin':
            g_pool.plugins.add(
                plugin_by_name[n['name']], args=n.get('args', {}))
        elif subject.startswith('meta.should_doc'):
            ipc_pub.notify({'subject': 'meta.doc',
                            'actor': g_pool.app,
                            'doc': player.__doc__})
            for p in g_pool.plugins:
                if (p.on_notify.__doc__
                        and p.__class__.on_notify != Plugin.on_notify):
                    ipc_pub.notify({'subject': 'meta.doc',
                                    'actor': p.class_name,
                                    'doc': p.on_notify.__doc__})

    while not glfw.glfwWindowShouldClose(main_window):

        # fetch newest notifications
        new_notifications = []
        while notify_sub.new_data:
            t, n = notify_sub.recv()
            new_notifications.append(n)

        # notify each plugin if there are new notifications:
        for n in new_notifications:
            handle_notifications(n)
            for p in g_pool.plugins:
                p.on_notify(n)

        # grab new frame
        if g_pool.play or g_pool.new_seek:
            g_pool.new_seek = False
            try:
                new_frame = cap.get_frame()
            except EndofVideoFileError:
                # end of video logic: pause at last frame.
                g_pool.play = False
                logger.warning("end of video")
            update_graph = True
        else:
            update_graph = False

        frame = new_frame.copy()
        events = {}
        events['frame'] = frame
        # report time between now and the last loop interation
        events['dt'] = get_dt()
        # new positons we make a deepcopy just like the image is a copy.
        events['gaze_positions'] = deepcopy(g_pool.gaze_positions_by_frame[frame.index])
        events['pupil_positions'] = deepcopy(g_pool.pupil_positions_by_frame[frame.index])

        if update_graph:
            # update performace graphs
            for p in events['pupil_positions']:
                pupil_graph.add(p['confidence'])

            t = new_frame.timestamp
            if ts and ts != t:
                dt, ts = t-ts, t
                fps_graph.add(1./dt)
            else:
                ts = new_frame.timestamp

            g_pool.play_button.status_text = str(frame.index)
        # always update the CPU graph
        cpu_graph.update()

        # allow each Plugin to do its work.
        for p in g_pool.plugins:
            p.recent_events(events)

        # check if a plugin need to be destroyed
        g_pool.plugins.clean()

        # render camera image
        glfw.glfwMakeContextCurrent(main_window)
        gl_utils.make_coord_system_norm_based()
        g_pool.image_tex.update_from_ndarray(frame.bgr)
        g_pool.image_tex.draw()
        gl_utils.make_coord_system_pixel_based(frame.img.shape)
        # render visual feedback from loaded plugins
        for p in g_pool.plugins:
            p.gl_display()

        fps_graph.draw()
        cpu_graph.draw()
        pupil_graph.draw()
        unused_elements = g_pool.gui.update()
        for b in unused_elements.buttons:
            button, action, mods = b
            pos = glfw.glfwGetCursorPos(main_window)
            pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
            pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0]))  # Position in img pixels
            for p in g_pool.plugins:
                p.on_click(pos, button, action)

        for key, scancode, action, mods in unused_elements.keys:
            for p in g_pool.plugins:
                p.on_key(key, scancode, action, mods)

        for char_ in unused_elements.chars:
            for p in g_pool.plugins:
                p.on_char(char_)

        # present frames at appropriate speed
        cap.wait(frame)

        glfw.glfwSwapBuffers(main_window)
        glfw.glfwPollEvents()

    session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
    session_settings['min_data_confidence'] = g_pool.min_data_confidence
    session_settings['gui_scale'] = g_pool.gui_user_scale
    session_settings['ui_config'] = g_pool.gui.configuration
    session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
    session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
    session_settings['version'] = str(g_pool.version)
    session_settings.close()

    # de-init all running plugins
    for p in g_pool.plugins:
        p.alive = False
    g_pool.plugins.clean()

    cap.cleanup()
    g_pool.gui.terminate()
    glfw.glfwDestroyWindow(main_window)

    logger.info("Process shutting down.")
    ipc_pub.notify({'subject': 'player_process.stopped'})
Esempio n. 12
0
def session(rec_dir):
    plugin_dir = os.path.join(user_dir, 'plugins')
    if not os.path.isdir(plugin_dir):
        os.mkdir(plugin_dir)
    runtime_plugins = import_runtime_plugins(plugin_dir)

    system_plugins = [Log_Display, Seek_Bar, Trim_Marks]
    vis_plugins = sorted([Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross,
                          Vis_Watermark, Vis_Eye_Video_Overlay, Vis_Scan_Path], key=lambda x: x.__name__)

    analysis_plugins = sorted([Gaze_Position_2D_Fixation_Detector, Pupil_Angle_3D_Fixation_Detector,
                               Manual_Gaze_Correction, Video_Export_Launcher, Offline_Surface_Tracker,
                               Raw_Data_Exporter, Batch_Exporter, Annotation_Player], key=lambda x: x.__name__)

    other_plugins = sorted([Log_History, Marker_Auto_Trim_Marks], key=lambda x: x.__name__)
    user_plugins = sorted(runtime_plugins, key=lambda x: x.__name__)

    user_launchable_plugins = vis_plugins + analysis_plugins + other_plugins + user_plugins
    available_plugins = system_plugins + user_launchable_plugins
    name_by_index = [p.__name__ for p in available_plugins]
    plugin_by_name = dict(zip(name_by_index, available_plugins))

    # Callback functions
    def on_resize(window, w, h):
        if gl_utils.is_window_visible(window):
            hdpi_factor = float(glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0])
            g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
            g_pool.gui.update_window(w, h)
            g_pool.gui.collect_menus()
            for g in g_pool.graphs:
                g.scale = hdpi_factor
                g.adjust_window_size(w, h)
            gl_utils.adjust_gl_view(w, h)
            for p in g_pool.plugins:
                p.on_window_resize(window, w, h)

    def on_key(window, key, scancode, action, mods):
        g_pool.gui.update_key(key, scancode, action, mods)

    def on_char(window, char):
        g_pool.gui.update_char(char)

    def on_button(window, button, action, mods):
        g_pool.gui.update_button(button, action, mods)
        pos = glfwGetCursorPos(window)
        pos = normalize(pos, glfwGetWindowSize(window))
        pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0]))  # Position in img pixels
        for p in g_pool.plugins:
            p.on_click(pos, button, action)

    def on_pos(window, x, y):
        hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0])
        g_pool.gui.update_mouse(x*hdpi_factor, y*hdpi_factor)

    def on_scroll(window, x, y):
        g_pool.gui.update_scroll(x, y*scroll_factor)

    def on_drop(window, count, paths):
        for x in range(count):
            new_rec_dir = paths[x].decode('utf-8')
            if is_pupil_rec_dir(new_rec_dir):
                logger.debug("Starting new session with '{}'".format(new_rec_dir))
                global rec_dir
                rec_dir = new_rec_dir
                glfwSetWindowShouldClose(window, True)
            else:
                logger.error("'{}' is not a valid pupil recording".format(new_rec_dir))

    tick = delta_t()

    def get_dt():
        return next(tick)

    update_recording_to_recent(rec_dir)

    video_path = [f for f in glob(os.path.join(rec_dir, "world.*")) if f[-3:] in ('mp4', 'mkv', 'avi')][0]
    timestamps_path = os.path.join(rec_dir, "world_timestamps.npy")
    pupil_data_path = os.path.join(rec_dir, "pupil_data")

    meta_info = load_meta_info(rec_dir)
    app_version = get_version(version_file)

    # log info about Pupil Platform and Platform in player.log
    logger.info('Application Version: {}'.format(app_version))
    logger.info('System Info: {}'.format(get_system_info()))

    timestamps = np.load(timestamps_path)

    # create container for globally scoped vars
    g_pool = Global_Container()
    g_pool.app = 'player'

    # Initialize capture
    cap = File_Source(g_pool, video_path, timestamps=list(timestamps))

    # load session persistent settings
    session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings"))
    if VersionFormat(session_settings.get("version", '0.0')) < get_version(version_file):
        logger.info("Session setting are from older version of this app. I will not use those.")
        session_settings.clear()

    width, height = session_settings.get('window_size', cap.frame_size)
    window_pos = session_settings.get('window_position', window_position_default)
    main_window = glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - "
                                   + rec_dir.split(os.path.sep)[-1], None, None)
    glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
    glfwMakeContextCurrent(main_window)
    cygl.utils.init()

    def set_scale(new_scale):
        g_pool.gui_user_scale = new_scale
        on_resize(main_window, *glfwGetFramebufferSize(main_window))

    # load pupil_positions, gaze_positions
    pupil_data = load_object(pupil_data_path)
    pupil_list = pupil_data['pupil_positions']
    gaze_list = pupil_data['gaze_positions']
    g_pool.pupil_data = pupil_data
    g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular'
    g_pool.version = app_version
    g_pool.capture = cap
    g_pool.timestamps = timestamps
    g_pool.play = False
    g_pool.new_seek = True
    g_pool.user_dir = user_dir
    g_pool.rec_dir = rec_dir
    g_pool.meta_info = meta_info
    g_pool.min_data_confidence = session_settings.get('min_data_confidence', 0.6)
    g_pool.pupil_positions_by_frame = correlate_data(pupil_list, g_pool.timestamps)
    g_pool.gaze_positions_by_frame = correlate_data(gaze_list, g_pool.timestamps)
    g_pool.fixations_by_frame = [[] for x in g_pool.timestamps]  # populated by the fixation detector plugin

    def next_frame(_):
        try:
            cap.seek_to_frame(cap.get_frame_index())
        except(FileSeekError):
            logger.warning("Could not seek to next frame.")
        else:
            g_pool.new_seek = True

    def prev_frame(_):
        try:
            cap.seek_to_frame(cap.get_frame_index()-2)
        except(FileSeekError):
            logger.warning("Could not seek to previous frame.")
        else:
            g_pool.new_seek = True

    def toggle_play(new_state):
        if cap.get_frame_index() >= cap.get_frame_count()-5:
            cap.seek_to_frame(1)  # avoid pause set by hitting trimmark pause.
            logger.warning("End of video - restart at beginning.")
        g_pool.play = new_state

    def set_data_confidence(new_confidence):
        g_pool.min_data_confidence = new_confidence
        notification = {'subject': 'min_data_confidence_changed'}
        notification['_notify_time_'] = time()+.8
        g_pool.delayed_notifications[notification['subject']] = notification

    def open_plugin(plugin):
        if plugin == "Select to load":
            return
        g_pool.plugins.add(plugin)

    def purge_plugins():
        for p in g_pool.plugins:
            if p.__class__ in user_launchable_plugins:
                p.alive = False
        g_pool.plugins.clean()

    def do_export(_):
        export_range = slice(g_pool.trim_marks.in_mark, g_pool.trim_marks.out_mark)
        export_dir = os.path.join(g_pool.rec_dir, 'exports', '{}-{}'.format(export_range.start, export_range.stop))
        try:
            os.makedirs(export_dir)
        except OSError as e:
            if e.errno != errno.EEXIST:
                logger.error("Could not create export dir")
                raise e
            else:
                overwrite_warning = "Previous export for range [{}-{}] already exsits - overwriting."
                logger.warning(overwrite_warning.format(export_range.start, export_range.stop))
        else:
            logger.info('Created export dir at "{}"'.format(export_dir))

        notification = {'subject': 'should_export', 'range': export_range, 'export_dir': export_dir}
        g_pool.notifications.append(notification)

    g_pool.gui = ui.UI()
    g_pool.gui_user_scale = session_settings.get('gui_scale', 1.)
    g_pool.main_menu = ui.Scrolling_Menu("Settings", pos=(-350, 20), size=(300, 500))
    g_pool.main_menu.append(ui.Button('Reset window size',
                                      lambda: glfwSetWindowSize(main_window, cap.frame_size[0], cap.frame_size[1])))
    g_pool.main_menu.append(ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2], label='Interface Size'))
    g_pool.main_menu.append(ui.Info_Text('Player Version: {}'.format(g_pool.version)))
    g_pool.main_menu.append(ui.Info_Text('Capture Version: {}'.format(meta_info['Capture Software Version'])))
    g_pool.main_menu.append(ui.Info_Text('Data Format Version: {}'.format(meta_info['Data Format Version'])))
    g_pool.main_menu.append(ui.Slider('min_data_confidence', g_pool, setter=set_data_confidence,
                                      step=.05, min=0.0, max=1.0, label='Confidence threshold'))

    selector_label = "Select to load"

    vis_labels = ["   " + p.__name__.replace('_', ' ') for p in vis_plugins]
    analysis_labels = ["   " + p.__name__.replace('_', ' ') for p in analysis_plugins]
    other_labels = ["   " + p.__name__.replace('_', ' ') for p in other_plugins]
    user_labels = ["   " + p.__name__.replace('_', ' ') for p in user_plugins]

    plugins = ([selector_label, selector_label] + vis_plugins + [selector_label] + analysis_plugins + [selector_label]
               + other_plugins + [selector_label] + user_plugins)
    labels = ([selector_label, "Visualization"] + vis_labels + ["Analysis"] + analysis_labels + ["Other"]
              + other_labels + ["User added"] + user_labels)

    g_pool.main_menu.append(ui.Selector('Open plugin:',
                                        selection=plugins,
                                        labels=labels,
                                        setter=open_plugin,
                                        getter=lambda: selector_label))

    g_pool.main_menu.append(ui.Button('Close all plugins', purge_plugins))
    g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))
    g_pool.play_button = ui.Thumb('play',
                                  g_pool,
                                  label=chr(0xf04b),
                                  setter=toggle_play,
                                  hotkey=GLFW_KEY_SPACE,
                                  label_font='fontawesome',
                                  label_offset_x=5,
                                  label_offset_y=0,
                                  label_offset_size=-24)
    g_pool.play_button.on_color[:] = (0, 1., .0, .8)
    g_pool.forward_button = ui.Thumb('forward',
                                     label=chr(0xf04e),
                                     getter=lambda: False,
                                     setter=next_frame,
                                     hotkey=GLFW_KEY_RIGHT,
                                     label_font='fontawesome',
                                     label_offset_x=5,
                                     label_offset_y=0,
                                     label_offset_size=-24)
    g_pool.backward_button = ui.Thumb('backward',
                                      label=chr(0xf04a),
                                      getter=lambda: False,
                                      setter=prev_frame,
                                      hotkey=GLFW_KEY_LEFT,
                                      label_font='fontawesome',
                                      label_offset_x=-5,
                                      label_offset_y=0,
                                      label_offset_size=-24)
    g_pool.export_button = ui.Thumb('export',
                                    label=chr(0xf063),
                                    getter=lambda: False,
                                    setter=do_export,
                                    hotkey='e',
                                    label_font='fontawesome',
                                    label_offset_x=0,
                                    label_offset_y=2,
                                    label_offset_size=-24)
    g_pool.quickbar.extend([g_pool.play_button, g_pool.forward_button, g_pool.backward_button, g_pool.export_button])
    g_pool.gui.append(g_pool.quickbar)
    g_pool.gui.append(g_pool.main_menu)

    # we always load these plugins
    system_plugins = [('Trim_Marks', {}), ('Seek_Bar', {})]
    default_plugins = [('Log_Display', {}), ('Vis_Scan_Path', {}), ('Vis_Polyline', {}), ('Vis_Circle', {}), ('Video_Export_Launcher', {})]
    previous_plugins = session_settings.get('loaded_plugins', default_plugins)
    g_pool.notifications = []
    g_pool.delayed_notifications = {}
    g_pool.plugins = Plugin_List(g_pool, plugin_by_name, system_plugins+previous_plugins)


    # Register callbacks main_window
    glfwSetFramebufferSizeCallback(main_window, on_resize)
    glfwSetKeyCallback(main_window, on_key)
    glfwSetCharCallback(main_window, on_char)
    glfwSetMouseButtonCallback(main_window, on_button)
    glfwSetCursorPosCallback(main_window, on_pos)
    glfwSetScrollCallback(main_window, on_scroll)
    glfwSetDropCallback(main_window, on_drop)

    g_pool.gui.configuration = session_settings.get('ui_config', {})

    # gl_state settings
    gl_utils.basic_gl_setup()
    g_pool.image_tex = Named_Texture()

    # set up performace graphs:
    pid = os.getpid()
    ps = psutil.Process(pid)
    ts = None

    cpu_graph = graph.Bar_Graph()
    cpu_graph.pos = (20, 110)
    cpu_graph.update_fn = ps.cpu_percent
    cpu_graph.update_rate = 5
    cpu_graph.label = 'CPU %0.1f'

    fps_graph = graph.Bar_Graph()
    fps_graph.pos = (140, 110)
    fps_graph.update_rate = 5
    fps_graph.label = "%0.0f REC FPS"

    pupil_graph = graph.Bar_Graph(max_val=1.0)
    pupil_graph.pos = (260, 110)
    pupil_graph.update_rate = 5
    pupil_graph.label = "Confidence: %0.2f"
    g_pool.graphs = [cpu_graph, fps_graph, pupil_graph]

    # trigger on_resize
    on_resize(main_window, *glfwGetFramebufferSize(main_window))

    while not glfwWindowShouldClose(main_window):
        # grab new frame
        if g_pool.play or g_pool.new_seek:
            g_pool.new_seek = False
            try:
                new_frame = cap.get_frame()
            except EndofVideoFileError:
                # end of video logic: pause at last frame.
                g_pool.play = False
                logger.warning("end of video")
            update_graph = True
        else:
            update_graph = False

        frame = new_frame.copy()
        events = {}
        events['frame'] = frame
        # report time between now and the last loop interation
        events['dt'] = get_dt()
        # new positons we make a deepcopy just like the image is a copy.
        events['gaze_positions'] = deepcopy(g_pool.gaze_positions_by_frame[frame.index])
        events['pupil_positions'] = deepcopy(g_pool.pupil_positions_by_frame[frame.index])

        if update_graph:
            # update performace graphs
            for p in events['pupil_positions']:
                pupil_graph.add(p['confidence'])

            t = new_frame.timestamp
            if ts and ts != t:
                dt, ts = t-ts, t
                fps_graph.add(1./dt)
            else:
               ts = new_frame.timestamp

            g_pool.play_button.status_text = str(frame.index)
        # always update the CPU graph
        cpu_graph.update()

        # publish delayed notifiactions when their time has come.
        for n in list(g_pool.delayed_notifications.values()):
            if n['_notify_time_'] < time():
                del n['_notify_time_']
                del g_pool.delayed_notifications[n['subject']]
                g_pool.notifications.append(n)

        # notify each plugin if there are new notifactions:
        while g_pool.notifications:
            n = g_pool.notifications.pop(0)
            for p in g_pool.plugins:
                p.on_notify(n)

        # allow each Plugin to do its work.
        for p in g_pool.plugins:
            p.recent_events(events)

        # check if a plugin need to be destroyed
        g_pool.plugins.clean()

        # render camera image
        glfwMakeContextCurrent(main_window)
        gl_utils.make_coord_system_norm_based()
        g_pool.image_tex.update_from_frame(frame)
        g_pool.image_tex.draw()
        gl_utils.make_coord_system_pixel_based(frame.img.shape)
        # render visual feedback from loaded plugins
        for p in g_pool.plugins:
            p.gl_display()

        fps_graph.draw()
        cpu_graph.draw()
        pupil_graph.draw()
        g_pool.gui.update()

        # present frames at appropriate speed
        cap.wait(frame)

        glfwSwapBuffers(main_window)
        glfwPollEvents()

    session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
    session_settings['min_data_confidence'] = g_pool.min_data_confidence
    session_settings['gui_scale'] = g_pool.gui_user_scale
    session_settings['ui_config'] = g_pool.gui.configuration
    session_settings['window_size'] = glfwGetWindowSize(main_window)
    session_settings['window_position'] = glfwGetWindowPos(main_window)
    session_settings['version'] = str(g_pool.version)
    session_settings.close()

    # de-init all running plugins
    for p in g_pool.plugins:
        p.alive = False
    g_pool.plugins.clean()

    cap.cleanup()
    g_pool.gui.terminate()
    glfwDestroyWindow(main_window)
Esempio n. 13
0
def session(rec_dir):
    plugin_dir = os.path.join(user_dir, 'plugins')
    if not os.path.isdir(plugin_dir):
        os.mkdir(plugin_dir)
    runtime_plugins = import_runtime_plugins(plugin_dir)

    system_plugins = [Log_Display, Seek_Bar, Trim_Marks]
    vis_plugins = sorted([Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross,
                          Vis_Watermark, Vis_Eye_Video_Overlay, Vis_Scan_Path], key=lambda x: x.__name__)

    analysis_plugins = sorted([Gaze_Position_2D_Fixation_Detector, Pupil_Angle_3D_Fixation_Detector,
                               Manual_Gaze_Correction, Video_Export_Launcher, Offline_Surface_Tracker,
                               Raw_Data_Exporter, Batch_Exporter, Annotation_Player], key=lambda x: x.__name__)

    other_plugins = sorted([Log_History, Marker_Auto_Trim_Marks], key=lambda x: x.__name__)
    user_plugins = sorted(runtime_plugins, key=lambda x: x.__name__)

    user_launchable_plugins = vis_plugins + analysis_plugins + other_plugins + user_plugins
    available_plugins = system_plugins + user_launchable_plugins
    name_by_index = [p.__name__ for p in available_plugins]
    plugin_by_name = dict(zip(name_by_index, available_plugins))

    # Callback functions
    def on_resize(window, w, h):
        if gl_utils.is_window_visible(window):
            hdpi_factor = float(glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0])
            g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
            g_pool.gui.update_window(w, h)
            g_pool.gui.collect_menus()
            for g in g_pool.graphs:
                g.scale = hdpi_factor
                g.adjust_window_size(w, h)
            gl_utils.adjust_gl_view(w, h)
            for p in g_pool.plugins:
                p.on_window_resize(window, w, h)

    def on_key(window, key, scancode, action, mods):
        g_pool.gui.update_key(key, scancode, action, mods)

    def on_char(window, char):
        g_pool.gui.update_char(char)

    def on_button(window, button, action, mods):
        g_pool.gui.update_button(button, action, mods)
        pos = glfwGetCursorPos(window)
        pos = normalize(pos, glfwGetWindowSize(window))
        pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0]))  # Position in img pixels
        for p in g_pool.plugins:
            p.on_click(pos, button, action)

    def on_pos(window, x, y):
        hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0])
        g_pool.gui.update_mouse(x*hdpi_factor, y*hdpi_factor)

    def on_scroll(window, x, y):
        g_pool.gui.update_scroll(x, y*scroll_factor)

    def on_drop(window, count, paths):
        for x in range(count):
            new_rec_dir = paths[x].decode('utf-8')
            if is_pupil_rec_dir(new_rec_dir):
                logger.debug("Starting new session with '{}'".format(new_rec_dir))
                global rec_dir
                rec_dir = new_rec_dir
                glfwSetWindowShouldClose(window, True)
            else:
                logger.error("'{}' is not a valid pupil recording".format(new_rec_dir))

    tick = delta_t()

    def get_dt():
        return next(tick)

    update_recording_to_recent(rec_dir)

    video_path = [f for f in glob(os.path.join(rec_dir, "world.*")) if f[-3:] in ('mp4', 'mkv', 'avi')][0]
    timestamps_path = os.path.join(rec_dir, "world_timestamps.npy")
    pupil_data_path = os.path.join(rec_dir, "pupil_data")

    meta_info = load_meta_info(rec_dir)
    app_version = get_version(version_file)

    # log info about Pupil Platform and Platform in player.log
    logger.info('Application Version: {}'.format(app_version))
    logger.info('System Info: {}'.format(get_system_info()))

    timestamps = np.load(timestamps_path)

    # create container for globally scoped vars
    g_pool = Global_Container()
    g_pool.app = 'player'

    # Initialize capture
    cap = File_Source(g_pool, video_path, timestamps=list(timestamps))

    # load session persistent settings
    session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings"))
    if VersionFormat(session_settings.get("version", '0.0')) < get_version(version_file):
        logger.info("Session setting are from older version of this app. I will not use those.")
        session_settings.clear()

    width, height = session_settings.get('window_size', cap.frame_size)
    window_pos = session_settings.get('window_position', window_position_default)
    main_window = glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - "
                                   + rec_dir.split(os.path.sep)[-1], None, None)
    glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
    glfwMakeContextCurrent(main_window)
    cygl.utils.init()

    def set_scale(new_scale):
        g_pool.gui_user_scale = new_scale
        on_resize(main_window, *glfwGetFramebufferSize(main_window))

    # load pupil_positions, gaze_positions
    pupil_data = load_object(pupil_data_path)
    pupil_list = pupil_data['pupil_positions']
    gaze_list = pupil_data['gaze_positions']
    g_pool.pupil_data = pupil_data
    g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular'
    g_pool.version = app_version
    g_pool.capture = cap
    g_pool.timestamps = timestamps
    g_pool.play = False
    g_pool.new_seek = True
    g_pool.user_dir = user_dir
    g_pool.rec_dir = rec_dir
    g_pool.meta_info = meta_info
    g_pool.min_data_confidence = session_settings.get('min_data_confidence', 0.6)
    g_pool.pupil_positions_by_frame = correlate_data(pupil_list, g_pool.timestamps)
    g_pool.gaze_positions_by_frame = correlate_data(gaze_list, g_pool.timestamps)
    g_pool.fixations_by_frame = [[] for x in g_pool.timestamps]  # populated by the fixation detector plugin

    def next_frame(_):
        try:
            cap.seek_to_frame(cap.get_frame_index())
        except(FileSeekError):
            logger.warning("Could not seek to next frame.")
        else:
            g_pool.new_seek = True

    def prev_frame(_):
        try:
            cap.seek_to_frame(cap.get_frame_index()-2)
        except(FileSeekError):
            logger.warning("Could not seek to previous frame.")
        else:
            g_pool.new_seek = True

    def toggle_play(new_state):
        if cap.get_frame_index() >= cap.get_frame_count()-5:
            cap.seek_to_frame(1)  # avoid pause set by hitting trimmark pause.
            logger.warning("End of video - restart at beginning.")
        g_pool.play = new_state

    def set_data_confidence(new_confidence):
        g_pool.min_data_confidence = new_confidence
        notification = {'subject': 'min_data_confidence_changed'}
        notification['_notify_time_'] = time()+.8
        g_pool.delayed_notifications[notification['subject']] = notification

    def open_plugin(plugin):
        if plugin == "Select to load":
            return
        g_pool.plugins.add(plugin)

    def purge_plugins():
        for p in g_pool.plugins:
            if p.__class__ in user_launchable_plugins:
                p.alive = False
        g_pool.plugins.clean()

    def do_export(_):
        export_range = slice(g_pool.trim_marks.in_mark, g_pool.trim_marks.out_mark)
        export_dir = os.path.join(g_pool.rec_dir, 'exports', '{}-{}'.format(export_range.start, export_range.stop))
        try:
            os.makedirs(export_dir)
        except OSError as e:
            if e.errno != errno.EEXIST:
                logger.error("Could not create export dir")
                raise e
            else:
                overwrite_warning = "Previous export for range [{}-{}] already exsits - overwriting."
                logger.warning(overwrite_warning.format(export_range.start, export_range.stop))
        else:
            logger.info('Created export dir at "{}"'.format(export_dir))

        notification = {'subject': 'should_export', 'range': export_range, 'export_dir': export_dir}
        g_pool.notifications.append(notification)

    g_pool.gui = ui.UI()
    g_pool.gui_user_scale = session_settings.get('gui_scale', 1.)
    g_pool.main_menu = ui.Scrolling_Menu("Settings", pos=(-350, 20), size=(300, 500))
    g_pool.main_menu.append(ui.Button('Reset window size',
                                      lambda: glfwSetWindowSize(main_window, cap.frame_size[0], cap.frame_size[1])))
    g_pool.main_menu.append(ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2], label='Interface Size'))
    g_pool.main_menu.append(ui.Info_Text('Player Version: {}'.format(g_pool.version)))
    g_pool.main_menu.append(ui.Info_Text('Capture Version: {}'.format(meta_info['Capture Software Version'])))
    g_pool.main_menu.append(ui.Info_Text('Data Format Version: {}'.format(meta_info['Data Format Version'])))
    g_pool.main_menu.append(ui.Slider('min_data_confidence', g_pool, setter=set_data_confidence,
                                      step=.05, min=0.0, max=1.0, label='Confidence threshold'))

    selector_label = "Select to load"

    vis_labels = ["   " + p.__name__.replace('_', ' ') for p in vis_plugins]
    analysis_labels = ["   " + p.__name__.replace('_', ' ') for p in analysis_plugins]
    other_labels = ["   " + p.__name__.replace('_', ' ') for p in other_plugins]
    user_labels = ["   " + p.__name__.replace('_', ' ') for p in user_plugins]

    plugins = ([selector_label, selector_label] + vis_plugins + [selector_label] + analysis_plugins + [selector_label]
               + other_plugins + [selector_label] + user_plugins)
    labels = ([selector_label, "Visualization"] + vis_labels + ["Analysis"] + analysis_labels + ["Other"]
              + other_labels + ["User added"] + user_labels)

    g_pool.main_menu.append(ui.Selector('Open plugin:',
                                        selection=plugins,
                                        labels=labels,
                                        setter=open_plugin,
                                        getter=lambda: selector_label))

    g_pool.main_menu.append(ui.Button('Close all plugins', purge_plugins))
    g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))
    g_pool.play_button = ui.Thumb('play',
                                  g_pool,
                                  label=chr(0xf04b),
                                  setter=toggle_play,
                                  hotkey=GLFW_KEY_SPACE,
                                  label_font='fontawesome',
                                  label_offset_x=5,
                                  label_offset_y=0,
                                  label_offset_size=-24)
    g_pool.play_button.on_color[:] = (0, 1., .0, .8)
    g_pool.forward_button = ui.Thumb('forward',
                                     label=chr(0xf04e),
                                     getter=lambda: False,
                                     setter=next_frame,
                                     hotkey=GLFW_KEY_RIGHT,
                                     label_font='fontawesome',
                                     label_offset_x=5,
                                     label_offset_y=0,
                                     label_offset_size=-24)
    g_pool.backward_button = ui.Thumb('backward',
                                      label=chr(0xf04a),
                                      getter=lambda: False,
                                      setter=prev_frame,
                                      hotkey=GLFW_KEY_LEFT,
                                      label_font='fontawesome',
                                      label_offset_x=-5,
                                      label_offset_y=0,
                                      label_offset_size=-24)
    g_pool.export_button = ui.Thumb('export',
                                    label=chr(0xf063),
                                    getter=lambda: False,
                                    setter=do_export,
                                    hotkey='e',
                                    label_font='fontawesome',
                                    label_offset_x=0,
                                    label_offset_y=2,
                                    label_offset_size=-24)
    g_pool.quickbar.extend([g_pool.play_button, g_pool.forward_button, g_pool.backward_button, g_pool.export_button])
    g_pool.gui.append(g_pool.quickbar)
    g_pool.gui.append(g_pool.main_menu)

    # we always load these plugins
    system_plugins = [('Trim_Marks', {}), ('Seek_Bar', {})]
    default_plugins = [('Log_Display', {}), ('Vis_Scan_Path', {}), ('Vis_Polyline', {}), ('Vis_Circle', {}), ('Video_Export_Launcher', {})]
    previous_plugins = session_settings.get('loaded_plugins', default_plugins)
    g_pool.notifications = []
    g_pool.delayed_notifications = {}
    g_pool.plugins = Plugin_List(g_pool, plugin_by_name, system_plugins+previous_plugins)


    # Register callbacks main_window
    glfwSetFramebufferSizeCallback(main_window, on_resize)
    glfwSetKeyCallback(main_window, on_key)
    glfwSetCharCallback(main_window, on_char)
    glfwSetMouseButtonCallback(main_window, on_button)
    glfwSetCursorPosCallback(main_window, on_pos)
    glfwSetScrollCallback(main_window, on_scroll)
    glfwSetDropCallback(main_window, on_drop)

    g_pool.gui.configuration = session_settings.get('ui_config', {})

    # gl_state settings
    gl_utils.basic_gl_setup()
    g_pool.image_tex = Named_Texture()

    # set up performace graphs:
    pid = os.getpid()
    ps = psutil.Process(pid)
    ts = None

    cpu_graph = graph.Bar_Graph()
    cpu_graph.pos = (20, 110)
    cpu_graph.update_fn = ps.cpu_percent
    cpu_graph.update_rate = 5
    cpu_graph.label = 'CPU %0.1f'

    fps_graph = graph.Bar_Graph()
    fps_graph.pos = (140, 110)
    fps_graph.update_rate = 5
    fps_graph.label = "%0.0f REC FPS"

    pupil_graph = graph.Bar_Graph(max_val=1.0)
    pupil_graph.pos = (260, 110)
    pupil_graph.update_rate = 5
    pupil_graph.label = "Confidence: %0.2f"
    g_pool.graphs = [cpu_graph, fps_graph, pupil_graph]

    # trigger on_resize
    on_resize(main_window, *glfwGetFramebufferSize(main_window))

    while not glfwWindowShouldClose(main_window):
        # grab new frame
        if g_pool.play or g_pool.new_seek:
            g_pool.new_seek = False
            try:
                new_frame = cap.get_frame()
            except EndofVideoFileError:
                # end of video logic: pause at last frame.
                g_pool.play = False
                logger.warning("end of video")
            update_graph = True
        else:
            update_graph = False

        frame = new_frame.copy()
        events = {}
        events['frame'] = frame
        # report time between now and the last loop interation
        events['dt'] = get_dt()
        # new positons we make a deepcopy just like the image is a copy.
        events['gaze_positions'] = deepcopy(g_pool.gaze_positions_by_frame[frame.index])
        events['pupil_positions'] = deepcopy(g_pool.pupil_positions_by_frame[frame.index])

        if update_graph:
            # update performace graphs
            for p in events['pupil_positions']:
                pupil_graph.add(p['confidence'])

            t = new_frame.timestamp
            if ts and ts != t:
                dt, ts = t-ts, t
                fps_graph.add(1./dt)
            else:
               ts = new_frame.timestamp

            g_pool.play_button.status_text = str(frame.index)
        # always update the CPU graph
        cpu_graph.update()

        # publish delayed notifiactions when their time has come.
        for n in list(g_pool.delayed_notifications.values()):
            if n['_notify_time_'] < time():
                del n['_notify_time_']
                del g_pool.delayed_notifications[n['subject']]
                g_pool.notifications.append(n)

        # notify each plugin if there are new notifactions:
        while g_pool.notifications:
            n = g_pool.notifications.pop(0)
            for p in g_pool.plugins:
                p.on_notify(n)

        # allow each Plugin to do its work.
        for p in g_pool.plugins:
            p.recent_events(events)

        # check if a plugin need to be destroyed
        g_pool.plugins.clean()

        # render camera image
        glfwMakeContextCurrent(main_window)
        gl_utils.make_coord_system_norm_based()
        g_pool.image_tex.update_from_frame(frame)
        g_pool.image_tex.draw()
        gl_utils.make_coord_system_pixel_based(frame.img.shape)
        # render visual feedback from loaded plugins
        for p in g_pool.plugins:
            p.gl_display()

        fps_graph.draw()
        cpu_graph.draw()
        pupil_graph.draw()
        g_pool.gui.update()

        # present frames at appropriate speed
        cap.wait(frame)

        glfwSwapBuffers(main_window)
        glfwPollEvents()

    session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
    session_settings['min_data_confidence'] = g_pool.min_data_confidence
    session_settings['gui_scale'] = g_pool.gui_user_scale
    session_settings['ui_config'] = g_pool.gui.configuration
    session_settings['window_size'] = glfwGetWindowSize(main_window)
    session_settings['window_position'] = glfwGetWindowPos(main_window)
    session_settings['version'] = str(g_pool.version)
    session_settings.close()

    # de-init all running plugins
    for p in g_pool.plugins:
        p.alive = False
    g_pool.plugins.clean()

    cap.cleanup()
    g_pool.gui.terminate()
    glfwDestroyWindow(main_window)
Esempio n. 14
0
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir,
           app_version):
    # general imports
    from time import sleep
    import logging
    from glob import glob
    from time import time, strftime, localtime

    # networking
    import zmq
    import zmq_tools

    import numpy as np

    # zmq ipc setup
    zmq_ctx = zmq.Context()
    ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,
                                        ipc_sub_url,
                                        topics=("notify", ))

    # log setup
    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    logger.setLevel(logging.NOTSET)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    try:

        # imports
        from file_methods import Persistent_Dict, next_export_sub_dir

        # display
        import glfw

        # check versions for our own depedencies as they are fast-changing
        from pyglui import __version__ as pyglui_version

        from pyglui import ui, cygl
        from pyglui.cygl.utils import Named_Texture, RGBA
        import gl_utils

        # capture
        from video_capture import init_playback_source

        # helpers/utils
        from version_utils import VersionFormat
        from methods import normalize, denormalize, delta_t, get_system_info
        import player_methods as pm
        from csv_utils import write_key_value_file

        # Plug-ins
        from plugin import Plugin, Plugin_List, import_runtime_plugins
        from plugin_manager import Plugin_Manager
        from vis_circle import Vis_Circle
        from vis_cross import Vis_Cross
        from vis_polyline import Vis_Polyline
        from vis_light_points import Vis_Light_Points
        from vis_watermark import Vis_Watermark
        from vis_fixation import Vis_Fixation

        # from vis_scan_path import Vis_Scan_Path
        from vis_eye_video_overlay import Vis_Eye_Video_Overlay
        from seek_control import Seek_Control
        from offline_surface_tracker import Offline_Surface_Tracker

        # from marker_auto_trim_marks import Marker_Auto_Trim_Marks
        from fixation_detector import Offline_Fixation_Detector
        from log_display import Log_Display
        from annotations import Annotation_Player
        from raw_data_exporter import Raw_Data_Exporter
        from log_history import Log_History
        from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection
        from gaze_producers import Gaze_From_Recording, Offline_Calibration
        from system_graphs import System_Graphs
        from system_timelines import System_Timelines
        from blink_detection import Offline_Blink_Detection
        from audio_playback import Audio_Playback
        from video_export.plugins.imotions_exporter import iMotions_Exporter
        from video_export.plugins.eye_video_exporter import Eye_Video_Exporter
        from video_export.plugins.world_video_exporter import World_Video_Exporter

        from background_helper import IPC_Logging_Task_Proxy

        IPC_Logging_Task_Proxy.push_url = ipc_push_url

        assert VersionFormat(pyglui_version) >= VersionFormat(
            "1.23"), "pyglui out of date, please upgrade to newest version"

        runtime_plugins = import_runtime_plugins(
            os.path.join(user_dir, "plugins"))
        system_plugins = [
            Log_Display,
            Seek_Control,
            Plugin_Manager,
            System_Graphs,
            System_Timelines,
            Audio_Playback,
        ]
        user_plugins = [
            Vis_Circle,
            Vis_Fixation,
            Vis_Polyline,
            Vis_Light_Points,
            Vis_Cross,
            Vis_Watermark,
            Vis_Eye_Video_Overlay,
            # Vis_Scan_Path,
            Offline_Fixation_Detector,
            Offline_Blink_Detection,
            Offline_Surface_Tracker,
            Raw_Data_Exporter,
            Annotation_Player,
            Log_History,
            Pupil_From_Recording,
            Offline_Pupil_Detection,
            Gaze_From_Recording,
            World_Video_Exporter,
            iMotions_Exporter,
            Eye_Video_Exporter,
            Offline_Calibration,
        ] + runtime_plugins

        plugins = system_plugins + user_plugins

        # Callback functions
        def on_resize(window, w, h):
            nonlocal window_size
            nonlocal hdpi_factor
            if w == 0 or h == 0:
                return
            hdpi_factor = glfw.getHDPIFactor(window)
            g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
            window_size = w, h
            g_pool.camera_render_size = w - int(
                icon_bar_width * g_pool.gui.scale), h
            g_pool.gui.update_window(*window_size)
            g_pool.gui.collect_menus()
            for p in g_pool.plugins:
                p.on_window_resize(window, *g_pool.camera_render_size)

        def on_window_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_window_char(window, char):
            g_pool.gui.update_char(char)

        def on_window_mouse_button(window, button, action, mods):
            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            x, y = x * hdpi_factor, y * hdpi_factor
            g_pool.gui.update_mouse(x, y)
            pos = x, y
            pos = normalize(pos, g_pool.camera_render_size)
            # Position in img pixels
            pos = denormalize(pos, g_pool.capture.frame_size)
            for p in g_pool.plugins:
                p.on_pos(pos)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        def on_drop(window, count, paths):
            for x in range(count):
                new_rec_dir = paths[x].decode("utf-8")
                if pm.is_pupil_rec_dir(new_rec_dir):
                    logger.debug(
                        "Starting new session with '{}'".format(new_rec_dir))
                    ipc_pub.notify({
                        "subject": "player_drop_process.should_start",
                        "rec_dir": new_rec_dir,
                    })
                    glfw.glfwSetWindowShouldClose(window, True)
                else:
                    logger.error("'{}' is not a valid pupil recording".format(
                        new_rec_dir))

        tick = delta_t()

        def get_dt():
            return next(tick)

        meta_info = pm.load_meta_info(rec_dir)

        # log info about Pupil Platform and Platform in player.log
        logger.info("Application Version: {}".format(app_version))
        logger.info("System Info: {}".format(get_system_info()))

        icon_bar_width = 50
        window_size = None
        hdpi_factor = 1.0

        # create container for globally scoped vars
        g_pool = Global_Container()
        g_pool.app = "player"
        g_pool.zmq_ctx = zmq_ctx
        g_pool.ipc_pub = ipc_pub
        g_pool.ipc_pub_url = ipc_pub_url
        g_pool.ipc_sub_url = ipc_sub_url
        g_pool.ipc_push_url = ipc_push_url
        g_pool.plugin_by_name = {p.__name__: p for p in plugins}
        g_pool.camera_render_size = None

        valid_ext = (".mp4", ".mkv", ".avi", ".h264", ".mjpeg", ".fake")
        video_path = [
            f for f in glob(os.path.join(rec_dir, "world.*"))
            if os.path.splitext(f)[1] in valid_ext
        ][0]
        init_playback_source(g_pool,
                             timing="external",
                             source_path=video_path,
                             buffered_decoding=True)

        # load session persistent settings
        session_settings = Persistent_Dict(
            os.path.join(user_dir, "user_settings_player"))
        if VersionFormat(session_settings.get("version",
                                              "0.0")) != app_version:
            logger.info(
                "Session setting are a different version of this app. I will not use those."
            )
            session_settings.clear()

        width, height = g_pool.capture.frame_size
        width += icon_bar_width
        width, height = session_settings.get("window_size", (width, height))

        window_pos = session_settings.get("window_position",
                                          window_position_default)
        window_name = "Pupil Player: {} - {}".format(
            meta_info["Recording Name"],
            os.path.split(rec_dir)[-1])

        glfw.glfwInit()
        main_window = glfw.glfwCreateWindow(width, height, window_name, None,
                                            None)
        glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
        glfw.glfwMakeContextCurrent(main_window)
        cygl.utils.init()
        g_pool.main_window = main_window

        def set_scale(new_scale):
            g_pool.gui_user_scale = new_scale
            window_size = (
                g_pool.camera_render_size[0] +
                int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor),
                glfw.glfwGetFramebufferSize(main_window)[1],
            )
            logger.warning(icon_bar_width * g_pool.gui_user_scale *
                           hdpi_factor)
            glfw.glfwSetWindowSize(main_window, *window_size)

        # load pupil_positions, gaze_positions
        g_pool.binocular = meta_info.get("Eye Mode",
                                         "monocular") == "binocular"
        g_pool.version = app_version
        g_pool.timestamps = g_pool.capture.timestamps
        g_pool.get_timestamp = lambda: 0.0
        g_pool.user_dir = user_dir
        g_pool.rec_dir = rec_dir
        g_pool.meta_info = meta_info
        g_pool.min_data_confidence = session_settings.get(
            "min_data_confidence", 0.6)
        g_pool.min_calibration_confidence = session_settings.get(
            "min_calibration_confidence", 0.8)

        # populated by producers
        g_pool.pupil_positions = pm.Bisector()
        g_pool.pupil_positions_by_id = (pm.Bisector(), pm.Bisector())
        g_pool.gaze_positions = pm.Bisector()
        g_pool.fixations = pm.Affiliator()

        def set_data_confidence(new_confidence):
            g_pool.min_data_confidence = new_confidence
            notification = {"subject": "min_data_confidence_changed"}
            notification["_notify_time_"] = time() + 0.8
            g_pool.ipc_pub.notify(notification)

        def do_export(_):
            left_idx = g_pool.seek_control.trim_left
            right_idx = g_pool.seek_control.trim_right
            export_range = left_idx, right_idx + 1  # exclusive range.stop

            export_dir = os.path.join(g_pool.rec_dir, "exports")
            export_dir = next_export_sub_dir(export_dir)

            os.makedirs(export_dir)
            logger.info('Created export dir at "{}"'.format(export_dir))

            export_info = {
                "Player Software Version":
                str(g_pool.version),
                "Data Format Version":
                meta_info["Data Format Version"],
                "Export Date":
                strftime("%d.%m.%Y", localtime()),
                "Export Time":
                strftime("%H:%M:%S", localtime()),
                "Frame Index Range:":
                g_pool.seek_control.get_frame_index_trim_range_string(),
                "Relative Time Range":
                g_pool.seek_control.get_rel_time_trim_range_string(),
                "Absolute Time Range":
                g_pool.seek_control.get_abs_time_trim_range_string(),
            }
            with open(os.path.join(export_dir, "export_info.csv"), "w") as csv:
                write_key_value_file(csv, export_info)

            notification = {
                "subject": "should_export",
                "range": export_range,
                "export_dir": export_dir,
            }
            g_pool.ipc_pub.notify(notification)

        def reset_restart():
            logger.warning("Resetting all settings and restarting Player.")
            glfw.glfwSetWindowShouldClose(main_window, True)
            ipc_pub.notify({"subject": "clear_settings_process.should_start"})
            ipc_pub.notify({
                "subject": "player_process.should_start",
                "rec_dir": rec_dir,
                "delay": 2.0,
            })

        def toggle_general_settings(collapsed):
            # this is the menu toggle logic.
            # Only one menu can be open.
            # If no menu is open the menubar should collapse.
            g_pool.menubar.collapsed = collapsed
            for m in g_pool.menubar.elements:
                m.collapsed = True
            general_settings.collapsed = collapsed

        g_pool.gui = ui.UI()
        g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0)
        g_pool.menubar = ui.Scrolling_Menu("Settings",
                                           pos=(-500, 0),
                                           size=(-icon_bar_width, 0),
                                           header_pos="left")
        g_pool.iconbar = ui.Scrolling_Menu("Icons",
                                           pos=(-icon_bar_width, 0),
                                           size=(0, 0),
                                           header_pos="hidden")
        g_pool.timelines = ui.Container((0, 0), (0, 0), (0, 0))
        g_pool.timelines.horizontal_constraint = g_pool.menubar
        g_pool.user_timelines = ui.Timeline_Menu("User Timelines",
                                                 pos=(0.0, -150.0),
                                                 size=(0.0, 0.0),
                                                 header_pos="headline")
        g_pool.user_timelines.color = RGBA(a=0.0)
        g_pool.user_timelines.collapsed = True
        # add container that constaints itself to the seekbar height
        vert_constr = ui.Container((0, 0), (0, -50.0), (0, 0))
        vert_constr.append(g_pool.user_timelines)
        g_pool.timelines.append(vert_constr)

        def set_window_size():
            f_width, f_height = g_pool.capture.frame_size
            f_width += int(icon_bar_width * g_pool.gui.scale)
            glfw.glfwSetWindowSize(main_window, f_width, f_height)

        general_settings = ui.Growing_Menu("General", header_pos="headline")
        general_settings.append(ui.Button("Reset window size",
                                          set_window_size))
        general_settings.append(
            ui.Selector(
                "gui_user_scale",
                g_pool,
                setter=set_scale,
                selection=[0.8, 0.9, 1.0, 1.1, 1.2] +
                list(np.arange(1.5, 5.1, 0.5)),
                label="Interface Size",
            ))
        general_settings.append(
            ui.Info_Text("Player Version: {}".format(g_pool.version)))
        general_settings.append(
            ui.Info_Text("Capture Version: {}".format(
                meta_info["Capture Software Version"])))
        general_settings.append(
            ui.Info_Text("Data Format Version: {}".format(
                meta_info["Data Format Version"])))

        general_settings.append(
            ui.Info_Text(
                "High level data, e.g. fixations, or visualizations only consider gaze data that has an equal or higher confidence than the minimum data confidence."
            ))
        general_settings.append(
            ui.Slider(
                "min_data_confidence",
                g_pool,
                setter=set_data_confidence,
                step=0.05,
                min=0.0,
                max=1.0,
                label="Minimum data confidence",
            ))

        general_settings.append(
            ui.Button("Restart with default settings", reset_restart))

        g_pool.menubar.append(general_settings)
        icon = ui.Icon(
            "collapsed",
            general_settings,
            label=chr(0xE8B8),
            on_val=False,
            off_val=True,
            setter=toggle_general_settings,
            label_font="pupil_icons",
        )
        icon.tooltip = "General Settings"
        g_pool.iconbar.append(icon)

        user_plugin_separator = ui.Separator()
        user_plugin_separator.order = 0.35
        g_pool.iconbar.append(user_plugin_separator)

        g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100),
                                             (100, -100))
        g_pool.export_button = ui.Thumb(
            "export",
            label=chr(0xE2C5),
            getter=lambda: False,
            setter=do_export,
            hotkey="e",
            label_font="pupil_icons",
        )
        g_pool.quickbar.extend([g_pool.export_button])
        g_pool.gui.append(g_pool.menubar)
        g_pool.gui.append(g_pool.timelines)
        g_pool.gui.append(g_pool.iconbar)
        g_pool.gui.append(g_pool.quickbar)

        # we always load these plugins
        default_plugins = [
            ("Plugin_Manager", {}),
            ("Seek_Control", {}),
            ("Log_Display", {}),
            ("Raw_Data_Exporter", {}),
            ("Vis_Polyline", {}),
            ("Vis_Circle", {}),
            ("System_Graphs", {}),
            ("System_Timelines", {}),
            ("World_Video_Exporter", {}),
            ("Pupil_From_Recording", {}),
            ("Gaze_From_Recording", {}),
            ("Audio_Playback", {}),
        ]

        g_pool.plugins = Plugin_List(
            g_pool, session_settings.get("loaded_plugins", default_plugins))

        # Manually add g_pool.capture to the plugin list
        g_pool.plugins._plugins.append(g_pool.capture)
        g_pool.plugins._plugins.sort(key=lambda p: p.order)
        g_pool.capture.init_ui()

        general_settings.insert(
            -1,
            ui.Text_Input(
                "rel_time_trim_section",
                getter=g_pool.seek_control.get_rel_time_trim_range_string,
                setter=g_pool.seek_control.set_rel_time_trim_range_string,
                label="Relative time range to export",
            ),
        )
        general_settings.insert(
            -1,
            ui.Text_Input(
                "frame_idx_trim_section",
                getter=g_pool.seek_control.get_frame_index_trim_range_string,
                setter=g_pool.seek_control.set_frame_index_trim_range_string,
                label="Frame index range to export",
            ),
        )

        # Register callbacks main_window
        glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
        glfw.glfwSetKeyCallback(main_window, on_window_key)
        glfw.glfwSetCharCallback(main_window, on_window_char)
        glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
        glfw.glfwSetCursorPosCallback(main_window, on_pos)
        glfw.glfwSetScrollCallback(main_window, on_scroll)
        glfw.glfwSetDropCallback(main_window, on_drop)

        toggle_general_settings(True)

        g_pool.gui.configuration = session_settings.get("ui_config", {})

        # gl_state settings
        gl_utils.basic_gl_setup()
        g_pool.image_tex = Named_Texture()

        # trigger on_resize
        on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

        def handle_notifications(n):
            subject = n["subject"]
            if subject == "start_plugin":
                g_pool.plugins.add(g_pool.plugin_by_name[n["name"]],
                                   args=n.get("args", {}))
            elif subject.startswith("meta.should_doc"):
                ipc_pub.notify({
                    "subject": "meta.doc",
                    "actor": g_pool.app,
                    "doc": player.__doc__
                })
                for p in g_pool.plugins:
                    if (p.on_notify.__doc__
                            and p.__class__.on_notify != Plugin.on_notify):
                        ipc_pub.notify({
                            "subject": "meta.doc",
                            "actor": p.class_name,
                            "doc": p.on_notify.__doc__,
                        })

        while not glfw.glfwWindowShouldClose(main_window):

            # fetch newest notifications
            new_notifications = []
            while notify_sub.new_data:
                t, n = notify_sub.recv()
                new_notifications.append(n)

            # notify each plugin if there are new notifications:
            for n in new_notifications:
                handle_notifications(n)
                for p in g_pool.plugins:
                    p.on_notify(n)

            events = {}
            # report time between now and the last loop interation
            events["dt"] = get_dt()

            # pupil and gaze positions are added by their respective producer plugins
            events["pupil"] = []
            events["gaze"] = []

            # allow each Plugin to do its work.
            for p in g_pool.plugins:
                p.recent_events(events)

            # check if a plugin need to be destroyed
            g_pool.plugins.clean()

            glfw.glfwMakeContextCurrent(main_window)
            # render visual feedback from loaded plugins
            if gl_utils.is_window_visible(main_window):

                gl_utils.glViewport(0, 0, *g_pool.camera_render_size)
                g_pool.capture.gl_display()
                for p in g_pool.plugins:
                    p.gl_display()

                gl_utils.glViewport(0, 0, *window_size)

                try:
                    clipboard = glfw.glfwGetClipboardString(
                        main_window).decode()
                except AttributeError:  # clipbaord is None, might happen on startup
                    clipboard = ""
                g_pool.gui.update_clipboard(clipboard)
                user_input = g_pool.gui.update()
                if user_input.clipboard and user_input.clipboard != clipboard:
                    # only write to clipboard if content changed
                    glfw.glfwSetClipboardString(main_window,
                                                user_input.clipboard.encode())

                for b in user_input.buttons:
                    button, action, mods = b
                    x, y = glfw.glfwGetCursorPos(main_window)
                    pos = x * hdpi_factor, y * hdpi_factor
                    pos = normalize(pos, g_pool.camera_render_size)
                    pos = denormalize(pos, g_pool.capture.frame_size)
                    for p in g_pool.plugins:
                        p.on_click(pos, button, action)

                for key, scancode, action, mods in user_input.keys:
                    for p in g_pool.plugins:
                        p.on_key(key, scancode, action, mods)

                for char_ in user_input.chars:
                    for p in g_pool.plugins:
                        p.on_char(char_)

                # present frames at appropriate speed
                g_pool.seek_control.wait(events["frame"].timestamp)
                glfw.glfwSwapBuffers(main_window)

            glfw.glfwPollEvents()

        session_settings["loaded_plugins"] = g_pool.plugins.get_initializers()
        session_settings["min_data_confidence"] = g_pool.min_data_confidence
        session_settings[
            "min_calibration_confidence"] = g_pool.min_calibration_confidence
        session_settings["gui_scale"] = g_pool.gui_user_scale
        session_settings["ui_config"] = g_pool.gui.configuration
        session_settings["window_position"] = glfw.glfwGetWindowPos(
            main_window)
        session_settings["version"] = str(g_pool.version)

        session_window_size = glfw.glfwGetWindowSize(main_window)
        if 0 not in session_window_size:
            session_settings["window_size"] = session_window_size

        session_settings.close()

        # de-init all running plugins
        for p in g_pool.plugins:
            p.alive = False
        g_pool.plugins.clean()

        g_pool.gui.terminate()
        glfw.glfwDestroyWindow(main_window)

    except:
        import traceback

        trace = traceback.format_exc()
        logger.error("Process Player crashed with trace:\n{}".format(trace))
    finally:
        logger.info("Process shutting down.")
        ipc_pub.notify({"subject": "player_process.stopped"})
        sleep(1.0)
Esempio n. 15
0
def export(rec_dir,
           user_dir,
           min_data_confidence,
           start_frame=None,
           end_frame=None,
           plugin_initializers=(),
           out_file_path=None,
           pre_computed={}):

    logger = logging.getLogger(__name__ + ' with pid: ' + str(os.getpid()))
    start_status = 'Starting video export with pid: {}'.format(os.getpid())
    print(start_status)
    yield start_status, 0

    try:
        update_recording_to_recent(rec_dir)

        vis_plugins = sorted([
            Vis_Circle, Vis_Cross, Vis_Polyline, Vis_Light_Points,
            Vis_Watermark, Vis_Scan_Path, Vis_Eye_Video_Overlay
        ],
                             key=lambda x: x.__name__)
        analysis_plugins = [Offline_Fixation_Detector]
        user_plugins = sorted(import_runtime_plugins(
            os.path.join(user_dir, 'plugins')),
                              key=lambda x: x.__name__)

        available_plugins = vis_plugins + analysis_plugins + user_plugins
        name_by_index = [p.__name__ for p in available_plugins]
        plugin_by_name = dict(zip(name_by_index, available_plugins))

        update_recording_to_recent(rec_dir)

        video_path = [
            f for f in glob(os.path.join(rec_dir, "world.*"))
            if os.path.splitext(f)[-1] in ('.mp4', '.mkv', '.avi', '.mjpeg')
        ][0]
        pupil_data_path = os.path.join(rec_dir, "pupil_data")
        audio_path = os.path.join(rec_dir, "audio.mp4")

        meta_info = load_meta_info(rec_dir)

        g_pool = Global_Container()
        g_pool.app = 'exporter'
        g_pool.min_data_confidence = min_data_confidence
        cap = File_Source(g_pool, video_path)
        timestamps = cap.timestamps

        # Out file path verification, we do this before but if one uses a separate tool, this will kick in.
        if out_file_path is None:
            out_file_path = os.path.join(rec_dir, "world_viz.mp4")
        else:
            file_name = os.path.basename(out_file_path)
            dir_name = os.path.dirname(out_file_path)
            if not dir_name:
                dir_name = rec_dir
            if not file_name:
                file_name = 'world_viz.mp4'
            out_file_path = os.path.expanduser(
                os.path.join(dir_name, file_name))

        if os.path.isfile(out_file_path):
            logger.warning("Video out file already exsists. I will overwrite!")
            os.remove(out_file_path)
        logger.debug("Saving Video to {}".format(out_file_path))

        # Trim mark verification
        # make sure the trim marks (start frame, endframe) make sense:
        # We define them like python list slices, thus we can test them like such.
        trimmed_timestamps = timestamps[start_frame:end_frame]
        if len(trimmed_timestamps) == 0:
            warn = "Start and end frames are set such that no video will be exported."
            logger.warning(warn)
            yield warn, 0.
            return

        if start_frame is None:
            start_frame = 0

        # these two vars are shared with the lauching process and give a job length and progress report.
        frames_to_export = len(trimmed_timestamps)
        current_frame = 0
        exp_info = "Will export from frame {} to frame {}. This means I will export {} frames."
        logger.debug(
            exp_info.format(start_frame, start_frame + frames_to_export,
                            frames_to_export))

        # setup of writer
        writer = AV_Writer(out_file_path,
                           fps=cap.frame_rate,
                           audio_loc=audio_path,
                           use_timestamps=True)

        cap.seek_to_frame(start_frame)

        start_time = time()

        g_pool.plugin_by_name = plugin_by_name
        g_pool.capture = cap
        g_pool.rec_dir = rec_dir
        g_pool.user_dir = user_dir
        g_pool.meta_info = meta_info
        g_pool.timestamps = timestamps
        g_pool.delayed_notifications = {}
        g_pool.notifications = []
        # load pupil_positions, gaze_positions
        pupil_data = pre_computed.get("pupil_data") or load_object(
            pupil_data_path)
        g_pool.pupil_data = pupil_data
        g_pool.pupil_positions = pre_computed.get(
            "pupil_positions") or pupil_data['pupil_positions']
        g_pool.gaze_positions = pre_computed.get(
            "gaze_positions") or pupil_data['gaze_positions']
        g_pool.fixations = []  # populated by the fixation detector plugin

        g_pool.pupil_positions_by_frame = correlate_data(
            g_pool.pupil_positions, g_pool.timestamps)
        g_pool.gaze_positions_by_frame = correlate_data(
            g_pool.gaze_positions, g_pool.timestamps)
        g_pool.fixations_by_frame = [
            [] for x in g_pool.timestamps
        ]  # populated by the fixation detector plugin

        # add plugins
        g_pool.plugins = Plugin_List(g_pool, plugin_initializers)

        while frames_to_export > current_frame:
            try:
                frame = cap.get_frame()
            except EndofVideoFileError:
                break

            events = {'frame': frame}
            # new positons and events
            events['gaze_positions'] = g_pool.gaze_positions_by_frame[
                frame.index]
            events['pupil_positions'] = g_pool.pupil_positions_by_frame[
                frame.index]

            # publish delayed notifiactions when their time has come.
            for n in list(g_pool.delayed_notifications.values()):
                if n['_notify_time_'] < time():
                    del n['_notify_time_']
                    del g_pool.delayed_notifications[n['subject']]
                    g_pool.notifications.append(n)

            # notify each plugin if there are new notifactions:
            while g_pool.notifications:
                n = g_pool.notifications.pop(0)
                for p in g_pool.plugins:
                    p.on_notify(n)

            # allow each Plugin to do its work.
            for p in g_pool.plugins:
                p.recent_events(events)

            writer.write_video_frame(frame)
            current_frame += 1
            yield 'Exporting', current_frame

        writer.close()
        writer = None

        duration = time() - start_time
        effective_fps = float(current_frame) / duration

        result = "Export done: Exported {} frames to {}. This took {} seconds. Exporter ran at {} frames per second."
        print(
            result.format(current_frame, out_file_path, duration,
                          effective_fps))
        yield 'Export done. This took {:.0f} seconds.'.format(
            duration), current_frame

    except GeneratorExit:
        print('Video export with pid {} was canceled.'.format(os.getpid()))
    except:
        from time import sleep
        import traceback
        trace = traceback.format_exc()
        print('Process Export (pid: {}) crashed with trace:\n{}'.format(
            os.getpid(), trace))
        sleep(1.0)
Esempio n. 16
0
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir,
           app_version):
    # general imports
    from time import sleep
    import logging
    import errno
    from glob import glob
    from time import time
    # networking
    import zmq
    import zmq_tools

    import numpy as np

    # zmq ipc setup
    zmq_ctx = zmq.Context()
    ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,
                                        ipc_sub_url,
                                        topics=('notify', ))

    # log setup
    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    logger.setLevel(logging.INFO)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    try:

        # imports
        from file_methods import Persistent_Dict, load_object

        # display
        import glfw
        # check versions for our own depedencies as they are fast-changing
        from pyglui import __version__ as pyglui_version

        from pyglui import ui, cygl
        from pyglui.cygl.utils import Named_Texture, RGBA
        import gl_utils
        # capture
        from video_capture import File_Source, EndofVideoFileError

        # helpers/utils
        from version_utils import VersionFormat
        from methods import normalize, denormalize, delta_t, get_system_info
        from player_methods import correlate_data, is_pupil_rec_dir, load_meta_info

        # Plug-ins
        from plugin import Plugin, Plugin_List, import_runtime_plugins
        from plugin_manager import Plugin_Manager
        from vis_circle import Vis_Circle
        from vis_cross import Vis_Cross
        from vis_polyline import Vis_Polyline
        from vis_light_points import Vis_Light_Points
        from vis_watermark import Vis_Watermark
        from vis_fixation import Vis_Fixation
        from vis_scan_path import Vis_Scan_Path
        from vis_eye_video_overlay import Vis_Eye_Video_Overlay
        from seek_control import Seek_Control
        from video_export_launcher import Video_Export_Launcher
        from offline_surface_tracker import Offline_Surface_Tracker
        # from marker_auto_trim_marks import Marker_Auto_Trim_Marks
        from fixation_detector import Offline_Fixation_Detector
        from batch_exporter import Batch_Exporter, Batch_Export
        from log_display import Log_Display
        from annotations import Annotation_Player
        from raw_data_exporter import Raw_Data_Exporter
        from log_history import Log_History
        from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection
        from gaze_producers import Gaze_From_Recording, Offline_Calibration
        from system_graphs import System_Graphs
        from system_timelines import System_Timelines
        from blink_detection import Offline_Blink_Detection

        assert VersionFormat(pyglui_version) >= VersionFormat(
            '1.17'), 'pyglui out of date, please upgrade to newest version'

        runtime_plugins = import_runtime_plugins(
            os.path.join(user_dir, 'plugins'))
        system_plugins = [
            Log_Display, Seek_Control, Plugin_Manager, System_Graphs,
            Batch_Export, System_Timelines
        ]
        user_plugins = [
            Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points,
            Vis_Cross, Vis_Watermark, Vis_Eye_Video_Overlay, Vis_Scan_Path,
            Offline_Fixation_Detector, Offline_Blink_Detection, Batch_Exporter,
            Video_Export_Launcher, Offline_Surface_Tracker, Raw_Data_Exporter,
            Annotation_Player, Log_History, Pupil_From_Recording,
            Offline_Pupil_Detection, Gaze_From_Recording, Offline_Calibration
        ] + runtime_plugins

        plugins = system_plugins + user_plugins

        # Callback functions
        def on_resize(window, w, h):
            nonlocal window_size
            nonlocal hdpi_factor

            hdpi_factor = float(
                glfw.glfwGetFramebufferSize(window)[0] /
                glfw.glfwGetWindowSize(window)[0])
            g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
            window_size = w, h
            g_pool.camera_render_size = w - int(
                icon_bar_width * g_pool.gui.scale), h
            g_pool.gui.update_window(*window_size)
            g_pool.gui.collect_menus()
            for p in g_pool.plugins:
                p.on_window_resize(window, *g_pool.camera_render_size)

        def on_window_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_window_char(window, char):
            g_pool.gui.update_char(char)

        def on_window_mouse_button(window, button, action, mods):
            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            x, y = x * hdpi_factor, y * hdpi_factor
            g_pool.gui.update_mouse(x, y)
            pos = x, y
            pos = normalize(pos, g_pool.camera_render_size)
            # Position in img pixels
            pos = denormalize(pos, g_pool.capture.frame_size)
            for p in g_pool.plugins:
                p.on_pos(pos)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        def on_drop(window, count, paths):
            for x in range(count):
                new_rec_dir = paths[x].decode('utf-8')
                if is_pupil_rec_dir(new_rec_dir):
                    logger.debug(
                        "Starting new session with '{}'".format(new_rec_dir))
                    ipc_pub.notify({
                        "subject": "player_drop_process.should_start",
                        "rec_dir": new_rec_dir
                    })
                    glfw.glfwSetWindowShouldClose(window, True)
                else:
                    logger.error("'{}' is not a valid pupil recording".format(
                        new_rec_dir))

        tick = delta_t()

        def get_dt():
            return next(tick)

        video_path = [
            f for f in glob(os.path.join(rec_dir, "world.*"))
            if os.path.splitext(f)[1] in ('.mp4', '.mkv', '.avi', '.h264',
                                          '.mjpeg')
        ][0]
        pupil_data_path = os.path.join(rec_dir, "pupil_data")

        meta_info = load_meta_info(rec_dir)

        # log info about Pupil Platform and Platform in player.log
        logger.info('Application Version: {}'.format(app_version))
        logger.info('System Info: {}'.format(get_system_info()))

        icon_bar_width = 50
        window_size = None
        hdpi_factor = 1.0

        # create container for globally scoped vars
        g_pool = Global_Container()
        g_pool.app = 'player'
        g_pool.zmq_ctx = zmq_ctx
        g_pool.ipc_pub = ipc_pub
        g_pool.ipc_pub_url = ipc_pub_url
        g_pool.ipc_sub_url = ipc_sub_url
        g_pool.ipc_push_url = ipc_push_url
        g_pool.plugin_by_name = {p.__name__: p for p in plugins}
        g_pool.camera_render_size = None

        # sets itself to g_pool.capture
        File_Source(g_pool, video_path)

        # load session persistent settings
        session_settings = Persistent_Dict(
            os.path.join(user_dir, "user_settings_player"))
        if VersionFormat(session_settings.get("version",
                                              '0.0')) != app_version:
            logger.info(
                "Session setting are a different version of this app. I will not use those."
            )
            session_settings.clear()

        g_pool.capture.playback_speed = session_settings.get(
            'playback_speed', 1.)

        width, height = session_settings.get('window_size',
                                             g_pool.capture.frame_size)
        window_pos = session_settings.get('window_position',
                                          window_position_default)
        glfw.glfwInit()
        main_window = glfw.glfwCreateWindow(
            width, height, "Pupil Player: " + meta_info["Recording Name"] +
            " - " + rec_dir.split(os.path.sep)[-1], None, None)
        glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
        glfw.glfwMakeContextCurrent(main_window)
        cygl.utils.init()
        g_pool.main_window = main_window

        def set_scale(new_scale):
            g_pool.gui_user_scale = new_scale
            window_size = (
                g_pool.camera_render_size[0] +
                int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor),
                glfw.glfwGetFramebufferSize(main_window)[1])
            logger.warning(icon_bar_width * g_pool.gui_user_scale *
                           hdpi_factor)
            glfw.glfwSetWindowSize(main_window, *window_size)

        # load pupil_positions, gaze_positions
        g_pool.pupil_data = load_object(pupil_data_path)
        g_pool.binocular = meta_info.get('Eye Mode',
                                         'monocular') == 'binocular'
        g_pool.version = app_version
        g_pool.timestamps = g_pool.capture.timestamps
        g_pool.get_timestamp = lambda: 0.
        g_pool.new_seek = True
        g_pool.user_dir = user_dir
        g_pool.rec_dir = rec_dir
        g_pool.meta_info = meta_info
        g_pool.min_data_confidence = session_settings.get(
            'min_data_confidence', 0.6)

        g_pool.pupil_positions = []
        g_pool.gaze_positions = []
        g_pool.fixations = []

        g_pool.notifications_by_frame = correlate_data(
            g_pool.pupil_data['notifications'], g_pool.timestamps)
        g_pool.pupil_positions_by_frame = [[] for x in g_pool.timestamps
                                           ]  # populated by producer`
        g_pool.gaze_positions_by_frame = [[] for x in g_pool.timestamps
                                          ]  # populated by producer
        g_pool.fixations_by_frame = [
            [] for x in g_pool.timestamps
        ]  # populated by the fixation detector plugin

        def set_data_confidence(new_confidence):
            g_pool.min_data_confidence = new_confidence
            notification = {'subject': 'min_data_confidence_changed'}
            notification['_notify_time_'] = time() + .8
            g_pool.ipc_pub.notify(notification)

        def open_plugin(plugin):
            if plugin == "Select to load":
                return
            g_pool.plugins.add(plugin)

        def purge_plugins():
            for p in g_pool.plugins:
                if p.__class__ in user_plugins:
                    p.alive = False
            g_pool.plugins.clean()

        def do_export(_):
            export_range = g_pool.seek_control.trim_left, g_pool.seek_control.trim_right
            export_dir = os.path.join(g_pool.rec_dir, 'exports',
                                      '{}-{}'.format(*export_range))
            try:
                os.makedirs(export_dir)
            except OSError as e:
                if e.errno != errno.EEXIST:
                    logger.error("Could not create export dir")
                    raise e
                else:
                    overwrite_warning = "Previous export for range [{}-{}] already exists - overwriting."
                    logger.warning(overwrite_warning.format(*export_range))
            else:
                logger.info('Created export dir at "{}"'.format(export_dir))

            notification = {
                'subject': 'should_export',
                'range': export_range,
                'export_dir': export_dir
            }
            g_pool.ipc_pub.notify(notification)

        def reset_restart():
            logger.warning("Resetting all settings and restarting Player.")
            glfw.glfwSetWindowShouldClose(main_window, True)
            ipc_pub.notify({'subject': 'clear_settings_process.should_start'})
            ipc_pub.notify({
                'subject': 'player_process.should_start',
                'rec_dir': rec_dir,
                'delay': 2.
            })

        def toggle_general_settings(collapsed):
            # this is the menu toggle logic.
            # Only one menu can be open.
            # If no menu is open the menubar should collapse.
            g_pool.menubar.collapsed = collapsed
            for m in g_pool.menubar.elements:
                m.collapsed = True
            general_settings.collapsed = collapsed

        g_pool.gui = ui.UI()
        g_pool.gui_user_scale = session_settings.get('gui_scale', 1.)
        g_pool.menubar = ui.Scrolling_Menu("Settings",
                                           pos=(-500, 0),
                                           size=(-icon_bar_width, 0),
                                           header_pos='left')
        g_pool.iconbar = ui.Scrolling_Menu("Icons",
                                           pos=(-icon_bar_width, 0),
                                           size=(0, 0),
                                           header_pos='hidden')
        g_pool.timelines = ui.Container((0, 0), (0, 0), (0, 0))
        g_pool.timelines.horizontal_constraint = g_pool.menubar
        g_pool.user_timelines = ui.Timeline_Menu('User Timelines',
                                                 pos=(0., -150.),
                                                 size=(0., 0.),
                                                 header_pos='headline')
        g_pool.user_timelines.color = RGBA(a=0.)
        g_pool.user_timelines.collapsed = True
        # add container that constaints itself to the seekbar height
        vert_constr = ui.Container((0, 0), (0, -50.), (0, 0))
        vert_constr.append(g_pool.user_timelines)
        g_pool.timelines.append(vert_constr)

        general_settings = ui.Growing_Menu('General', header_pos='headline')
        general_settings.append(
            ui.Button(
                'Reset window size', lambda: glfw.glfwSetWindowSize(
                    main_window, g_pool.capture.frame_size[0], g_pool.capture.
                    frame_size[1])))
        general_settings.append(
            ui.Selector('gui_user_scale',
                        g_pool,
                        setter=set_scale,
                        selection=[.8, .9, 1., 1.1, 1.2] +
                        list(np.arange(1.5, 5.1, .5)),
                        label='Interface Size'))
        general_settings.append(
            ui.Info_Text('Player Version: {}'.format(g_pool.version)))
        general_settings.append(
            ui.Info_Text('Capture Version: {}'.format(
                meta_info['Capture Software Version'])))
        general_settings.append(
            ui.Info_Text('Data Format Version: {}'.format(
                meta_info['Data Format Version'])))
        general_settings.append(
            ui.Slider('min_data_confidence',
                      g_pool,
                      setter=set_data_confidence,
                      step=.05,
                      min=0.0,
                      max=1.0,
                      label='Confidence threshold'))
        general_settings.append(
            ui.Button('Restart with default settings', reset_restart))

        g_pool.menubar.append(general_settings)
        icon = ui.Icon('collapsed',
                       general_settings,
                       label=chr(0xe8b8),
                       on_val=False,
                       off_val=True,
                       setter=toggle_general_settings,
                       label_font='pupil_icons')
        icon.tooltip = 'General Settings'
        g_pool.iconbar.append(icon)

        user_plugin_separator = ui.Separator()
        user_plugin_separator.order = 0.35
        g_pool.iconbar.append(user_plugin_separator)

        g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100),
                                             (100, -100))
        g_pool.export_button = ui.Thumb('export',
                                        label=chr(0xe2c5),
                                        getter=lambda: False,
                                        setter=do_export,
                                        hotkey='e',
                                        label_font='pupil_icons')
        g_pool.quickbar.extend([g_pool.export_button])
        g_pool.gui.append(g_pool.menubar)
        g_pool.gui.append(g_pool.timelines)
        g_pool.gui.append(g_pool.iconbar)
        g_pool.gui.append(g_pool.quickbar)

        # we always load these plugins
        default_plugins = [('Plugin_Manager', {}), ('Seek_Control', {}),
                           ('Log_Display', {}), ('Raw_Data_Exporter', {}),
                           ('Vis_Polyline', {}), ('Vis_Circle', {}),
                           ('System_Graphs', {}), ('System_Timelines', {}),
                           ('Video_Export_Launcher', {}),
                           ('Pupil_From_Recording', {}),
                           ('Gaze_From_Recording', {})]
        g_pool.plugins = Plugin_List(
            g_pool, session_settings.get('loaded_plugins', default_plugins))

        # Register callbacks main_window
        glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
        glfw.glfwSetKeyCallback(main_window, on_window_key)
        glfw.glfwSetCharCallback(main_window, on_window_char)
        glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
        glfw.glfwSetCursorPosCallback(main_window, on_pos)
        glfw.glfwSetScrollCallback(main_window, on_scroll)
        glfw.glfwSetDropCallback(main_window, on_drop)

        toggle_general_settings(True)

        g_pool.gui.configuration = session_settings.get('ui_config', {})

        # gl_state settings
        gl_utils.basic_gl_setup()
        g_pool.image_tex = Named_Texture()

        # trigger on_resize
        on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

        def handle_notifications(n):
            subject = n['subject']
            if subject == 'start_plugin':
                g_pool.plugins.add(g_pool.plugin_by_name[n['name']],
                                   args=n.get('args', {}))
            elif subject.startswith('meta.should_doc'):
                ipc_pub.notify({
                    'subject': 'meta.doc',
                    'actor': g_pool.app,
                    'doc': player.__doc__
                })
                for p in g_pool.plugins:
                    if (p.on_notify.__doc__
                            and p.__class__.on_notify != Plugin.on_notify):
                        ipc_pub.notify({
                            'subject': 'meta.doc',
                            'actor': p.class_name,
                            'doc': p.on_notify.__doc__
                        })

        while not glfw.glfwWindowShouldClose(main_window):

            # fetch newest notifications
            new_notifications = []
            while notify_sub.new_data:
                t, n = notify_sub.recv()
                new_notifications.append(n)

            # notify each plugin if there are new notifications:
            for n in new_notifications:
                handle_notifications(n)
                for p in g_pool.plugins:
                    p.on_notify(n)

            # grab new frame
            if g_pool.capture.play or g_pool.new_seek:
                g_pool.new_seek = False
                try:
                    new_frame = g_pool.capture.get_frame()
                except EndofVideoFileError:
                    # end of video logic: pause at last frame.
                    g_pool.capture.play = False
                    logger.warning("end of video")

            frame = new_frame.copy()
            events = {}
            events['frame'] = frame
            # report time between now and the last loop interation
            events['dt'] = get_dt()

            # pupil and gaze positions are added by their respective producer plugins
            events['pupil_positions'] = []
            events['gaze_positions'] = []

            # allow each Plugin to do its work.
            for p in g_pool.plugins:
                p.recent_events(events)

            # check if a plugin need to be destroyed
            g_pool.plugins.clean()

            glfw.glfwMakeContextCurrent(main_window)
            # render visual feedback from loaded plugins
            if gl_utils.is_window_visible(main_window):

                gl_utils.glViewport(0, 0, *g_pool.camera_render_size)
                g_pool.capture._recent_frame = frame
                g_pool.capture.gl_display()
                for p in g_pool.plugins:
                    p.gl_display()

                gl_utils.glViewport(0, 0, *window_size)

                try:
                    clipboard = glfw.glfwGetClipboardString(
                        main_window).decode()
                except AttributeError:  # clipbaord is None, might happen on startup
                    clipboard = ''
                g_pool.gui.update_clipboard(clipboard)
                user_input = g_pool.gui.update()
                if user_input.clipboard and user_input.clipboard != clipboard:
                    # only write to clipboard if content changed
                    glfw.glfwSetClipboardString(main_window,
                                                user_input.clipboard.encode())

                for b in user_input.buttons:
                    button, action, mods = b
                    x, y = glfw.glfwGetCursorPos(main_window)
                    pos = x * hdpi_factor, y * hdpi_factor
                    pos = normalize(pos, g_pool.camera_render_size)
                    pos = denormalize(pos, g_pool.capture.frame_size)
                    for p in g_pool.plugins:
                        p.on_click(pos, button, action)

                for key, scancode, action, mods in user_input.keys:
                    for p in g_pool.plugins:
                        p.on_key(key, scancode, action, mods)

                for char_ in user_input.chars:
                    for p in g_pool.plugins:
                        p.on_char(char_)

                glfw.glfwSwapBuffers(main_window)

            # present frames at appropriate speed
            g_pool.capture.wait(frame)
            glfw.glfwPollEvents()

        session_settings['playback_speed'] = g_pool.capture.playback_speed
        session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
        session_settings['min_data_confidence'] = g_pool.min_data_confidence
        session_settings['gui_scale'] = g_pool.gui_user_scale
        session_settings['ui_config'] = g_pool.gui.configuration
        session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
        session_settings['window_position'] = glfw.glfwGetWindowPos(
            main_window)
        session_settings['version'] = str(g_pool.version)
        session_settings.close()

        # de-init all running plugins
        for p in g_pool.plugins:
            p.alive = False
        g_pool.plugins.clean()

        g_pool.capture.cleanup()
        g_pool.gui.terminate()
        glfw.glfwDestroyWindow(main_window)

    except:
        import traceback
        trace = traceback.format_exc()
        logger.error('Process Player crashed with trace:\n{}'.format(trace))
    finally:
        logger.info("Process shutting down.")
        ipc_pub.notify({'subject': 'player_process.stopped'})
        sleep(1.0)
Esempio n. 17
0
def export(
        rec_dir,
        user_dir,
        min_data_confidence,
        start_frame=None,
        end_frame=None,
        plugin_initializers=(),
        out_file_path=None,
        pre_computed={},
):

    PID = str(os.getpid())
    logger = logging.getLogger(__name__ + " with pid: " + PID)
    start_status = "Starting video export with pid: {}".format(PID)
    print(start_status)
    yield start_status, 0

    try:
        pm.update_recording_to_recent(rec_dir)

        vis_plugins = sorted(
            [
                Vis_Circle,
                Vis_Cross,
                Vis_Polyline,
                Vis_Light_Points,
                Vis_Watermark,
                Vis_Scan_Path,
                Vis_Eye_Video_Overlay,
            ],
            key=lambda x: x.__name__,
        )
        analysis_plugins = [Offline_Fixation_Detector]
        user_plugins = sorted(
            import_runtime_plugins(os.path.join(user_dir, "plugins")),
            key=lambda x: x.__name__,
        )

        available_plugins = vis_plugins + analysis_plugins + user_plugins
        name_by_index = [p.__name__ for p in available_plugins]
        plugin_by_name = dict(zip(name_by_index, available_plugins))

        pm.update_recording_to_recent(rec_dir)

        audio_path = os.path.join(rec_dir, "audio.mp4")

        meta_info = pm.load_meta_info(rec_dir)

        g_pool = Global_Container()
        g_pool.app = "exporter"
        g_pool.min_data_confidence = min_data_confidence

        valid_ext = (".mp4", ".mkv", ".avi", ".h264", ".mjpeg", ".fake")
        video_path = [
            f for f in glob(os.path.join(rec_dir, "world.*"))
            if os.path.splitext(f)[1] in valid_ext
        ][0]
        cap = init_playback_source(g_pool, source_path=video_path, timing=None)

        timestamps = cap.timestamps

        # Out file path verification, we do this before but if one uses a separate tool, this will kick in.
        if out_file_path is None:
            out_file_path = os.path.join(rec_dir, "world_viz.mp4")
        else:
            file_name = os.path.basename(out_file_path)
            dir_name = os.path.dirname(out_file_path)
            if not dir_name:
                dir_name = rec_dir
            if not file_name:
                file_name = "world_viz.mp4"
            out_file_path = os.path.expanduser(
                os.path.join(dir_name, file_name))

        if os.path.isfile(out_file_path):
            logger.warning("Video out file already exsists. I will overwrite!")
            os.remove(out_file_path)
        logger.debug("Saving Video to {}".format(out_file_path))

        # Trim mark verification
        # make sure the trim marks (start frame, endframe) make sense:
        # We define them like python list slices, thus we can test them like such.
        trimmed_timestamps = timestamps[start_frame:end_frame]
        if len(trimmed_timestamps) == 0:
            warn = "Start and end frames are set such that no video will be exported."
            logger.warning(warn)
            yield warn, 0.0
            return

        if start_frame is None:
            start_frame = 0

        # these two vars are shared with the lauching process and give a job length and progress report.
        frames_to_export = len(trimmed_timestamps)
        current_frame = 0
        exp_info = (
            "Will export from frame {} to frame {}. This means I will export {} frames."
        )
        logger.debug(
            exp_info.format(start_frame, start_frame + frames_to_export,
                            frames_to_export))

        # setup of writer
        writer = AV_Writer(out_file_path,
                           fps=cap.frame_rate,
                           audio_loc=audio_path,
                           use_timestamps=True)

        cap.seek_to_frame(start_frame)

        start_time = time()

        g_pool.plugin_by_name = plugin_by_name
        g_pool.capture = cap
        g_pool.rec_dir = rec_dir
        g_pool.user_dir = user_dir
        g_pool.meta_info = meta_info
        g_pool.timestamps = timestamps
        g_pool.delayed_notifications = {}
        g_pool.notifications = []

        for initializers in pre_computed.values():
            initializers["data"] = [
                fm.Serialized_Dict(msgpack_bytes=serialized)
                for serialized in initializers["data"]
            ]

        g_pool.pupil_positions = pm.Bisector(**pre_computed["pupil"])
        g_pool.gaze_positions = pm.Bisector(**pre_computed["gaze"])
        g_pool.fixations = pm.Affiliator(**pre_computed["fixations"])

        # add plugins
        g_pool.plugins = Plugin_List(g_pool, plugin_initializers)

        while frames_to_export > current_frame:
            try:
                frame = cap.get_frame()
            except EndofVideoError:
                break

            events = {"frame": frame}
            # new positons and events
            frame_window = pm.enclosing_window(g_pool.timestamps, frame.index)
            events["gaze"] = g_pool.gaze_positions.by_ts_window(frame_window)
            events["pupil"] = g_pool.pupil_positions.by_ts_window(frame_window)

            # publish delayed notifiactions when their time has come.
            for n in list(g_pool.delayed_notifications.values()):
                if n["_notify_time_"] < time():
                    del n["_notify_time_"]
                    del g_pool.delayed_notifications[n["subject"]]
                    g_pool.notifications.append(n)

            # notify each plugin if there are new notifactions:
            while g_pool.notifications:
                n = g_pool.notifications.pop(0)
                for p in g_pool.plugins:
                    p.on_notify(n)

            # allow each Plugin to do its work.
            for p in g_pool.plugins:
                p.recent_events(events)

            writer.write_video_frame(frame)
            current_frame += 1
            yield "Exporting with pid {}".format(PID), current_frame

        writer.close()
        writer = None

        duration = time() - start_time
        effective_fps = float(current_frame) / duration

        result = "Export done: Exported {} frames to {}. This took {} seconds. Exporter ran at {} frames per second."
        print(
            result.format(current_frame, out_file_path, duration,
                          effective_fps))
        yield "Export done. This took {:.0f} seconds.".format(
            duration), current_frame

    except GeneratorExit:
        print("Video export with pid {} was canceled.".format(os.getpid()))
    except Exception as e:
        from time import sleep
        import traceback

        trace = traceback.format_exc()
        print("Process Export (pid: {}) crashed with trace:\n{}".format(
            os.getpid(), trace))
        yield e
        sleep(1.0)
Esempio n. 18
0
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_version):
    # general imports
    from time import sleep
    import logging
    from glob import glob
    from time import time, strftime, localtime

    # networking
    import zmq
    import zmq_tools

    import numpy as np

    # zmq ipc setup
    zmq_ctx = zmq.Context()
    ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",))

    # log setup
    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    logger.setLevel(logging.NOTSET)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    try:
        from background_helper import IPC_Logging_Task_Proxy

        IPC_Logging_Task_Proxy.push_url = ipc_push_url

        from tasklib.background.patches import IPCLoggingPatch

        IPCLoggingPatch.ipc_push_url = ipc_push_url

        # imports
        from file_methods import Persistent_Dict, next_export_sub_dir

        # display
        import glfw

        # check versions for our own depedencies as they are fast-changing
        from pyglui import __version__ as pyglui_version

        from pyglui import ui, cygl
        from pyglui.cygl.utils import Named_Texture, RGBA
        import gl_utils

        # capture
        from video_capture import File_Source

        # helpers/utils
        from version_utils import VersionFormat
        from methods import normalize, denormalize, delta_t, get_system_info
        import player_methods as pm
        from csv_utils import write_key_value_file

        # Plug-ins
        from plugin import Plugin, Plugin_List, import_runtime_plugins
        from plugin_manager import Plugin_Manager
        from vis_circle import Vis_Circle
        from vis_cross import Vis_Cross
        from vis_polyline import Vis_Polyline
        from vis_light_points import Vis_Light_Points
        from vis_watermark import Vis_Watermark
        from vis_fixation import Vis_Fixation

        # from vis_scan_path import Vis_Scan_Path
        from seek_control import Seek_Control
        from surface_tracker import Surface_Tracker_Offline

        # from marker_auto_trim_marks import Marker_Auto_Trim_Marks
        from fixation_detector import Offline_Fixation_Detector
        from eye_movement import Offline_Eye_Movement_Detector
        from log_display import Log_Display
        from annotations import Annotation_Player
        from raw_data_exporter import Raw_Data_Exporter
        from log_history import Log_History
        from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection
        from gaze_producer.gaze_from_recording import GazeFromRecording
        from gaze_producer.gaze_from_offline_calibration import (
            GazeFromOfflineCalibration,
        )
        from system_graphs import System_Graphs
        from system_timelines import System_Timelines
        from blink_detection import Offline_Blink_Detection
        from audio_playback import Audio_Playback
        from video_export.plugins.imotions_exporter import iMotions_Exporter
        from video_export.plugins.eye_video_exporter import Eye_Video_Exporter
        from video_export.plugins.world_video_exporter import World_Video_Exporter
        from video_capture import File_Source
        from video_overlay.plugins import Video_Overlay, Eye_Overlay

        assert VersionFormat(pyglui_version) >= VersionFormat(
            "1.23"
        ), "pyglui out of date, please upgrade to newest version"

        runtime_plugins = import_runtime_plugins(os.path.join(user_dir, "plugins"))
        system_plugins = [
            Log_Display,
            Seek_Control,
            Plugin_Manager,
            System_Graphs,
            System_Timelines,
            Audio_Playback,
        ]
        user_plugins = [
            Vis_Circle,
            Vis_Fixation,
            Vis_Polyline,
            Vis_Light_Points,
            Vis_Cross,
            Vis_Watermark,
            Eye_Overlay,
            Video_Overlay,
            # Vis_Scan_Path,
            Offline_Fixation_Detector,
            Offline_Eye_Movement_Detector,
            Offline_Blink_Detection,
            Surface_Tracker_Offline,
            Raw_Data_Exporter,
            Annotation_Player,
            Log_History,
            Pupil_From_Recording,
            Offline_Pupil_Detection,
            GazeFromRecording,
            GazeFromOfflineCalibration,
            World_Video_Exporter,
            iMotions_Exporter,
            Eye_Video_Exporter,
        ] + runtime_plugins

        if platform.system() != "Windows":
            # Head pose tracking is currently not available on Windows
            from head_pose_tracker.offline_head_pose_tracker import (
                Offline_Head_Pose_Tracker,
            )

            user_plugins.append(Offline_Head_Pose_Tracker)

        plugins = system_plugins + user_plugins

        # Callback functions
        def on_resize(window, w, h):
            nonlocal window_size
            nonlocal hdpi_factor
            if w == 0 or h == 0:
                return
            hdpi_factor = glfw.getHDPIFactor(window)
            g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
            window_size = w, h
            g_pool.camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h
            g_pool.gui.update_window(*window_size)
            g_pool.gui.collect_menus()
            for p in g_pool.plugins:
                p.on_window_resize(window, *g_pool.camera_render_size)

        def on_window_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_window_char(window, char):
            g_pool.gui.update_char(char)

        def on_window_mouse_button(window, button, action, mods):
            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            x, y = x * hdpi_factor, y * hdpi_factor
            g_pool.gui.update_mouse(x, y)
            pos = x, y
            pos = normalize(pos, g_pool.camera_render_size)
            # Position in img pixels
            pos = denormalize(pos, g_pool.capture.frame_size)
            for p in g_pool.plugins:
                p.on_pos(pos)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        def on_drop(window, count, paths):
            paths = [paths[x].decode("utf-8") for x in range(count)]
            for path in paths:
                if pm.is_pupil_rec_dir(path):
                    _restart_with_recording(path)
                    return
            # call `on_drop` callbacks until a plugin indicates
            # that it has consumed the event (by returning True)
            any(p.on_drop(paths) for p in g_pool.plugins)

        def _restart_with_recording(rec_dir):
            logger.debug("Starting new session with '{}'".format(rec_dir))
            ipc_pub.notify(
                {"subject": "player_drop_process.should_start", "rec_dir": rec_dir}
            )
            glfw.glfwSetWindowShouldClose(g_pool.main_window, True)

        tick = delta_t()

        def get_dt():
            return next(tick)

        meta_info = pm.load_meta_info(rec_dir)

        # log info about Pupil Platform and Platform in player.log
        logger.info("Application Version: {}".format(app_version))
        logger.info("System Info: {}".format(get_system_info()))

        icon_bar_width = 50
        window_size = None
        hdpi_factor = 1.0

        # create container for globally scoped vars
        g_pool = SimpleNamespace()
        g_pool.app = "player"
        g_pool.zmq_ctx = zmq_ctx
        g_pool.ipc_pub = ipc_pub
        g_pool.ipc_pub_url = ipc_pub_url
        g_pool.ipc_sub_url = ipc_sub_url
        g_pool.ipc_push_url = ipc_push_url
        g_pool.plugin_by_name = {p.__name__: p for p in plugins}
        g_pool.camera_render_size = None

        valid_ext = (".mp4", ".mkv", ".avi", ".h264", ".mjpeg", ".fake")
        video_path = [
            f
            for f in glob(os.path.join(rec_dir, "world.*"))
            if os.path.splitext(f)[1] in valid_ext
        ][0]
        File_Source(
            g_pool,
            timing="external",
            source_path=video_path,
            buffered_decoding=True,
            fill_gaps=True,
        )

        # load session persistent settings
        session_settings = Persistent_Dict(
            os.path.join(user_dir, "user_settings_player")
        )
        if VersionFormat(session_settings.get("version", "0.0")) != app_version:
            logger.info(
                "Session setting are a different version of this app. I will not use those."
            )
            session_settings.clear()

        width, height = g_pool.capture.frame_size
        width += icon_bar_width
        width, height = session_settings.get("window_size", (width, height))

        window_pos = session_settings.get("window_position", window_position_default)
        window_name = "Pupil Player: {} - {}".format(
            meta_info["Recording Name"], os.path.split(rec_dir)[-1]
        )

        glfw.glfwInit()
        main_window = glfw.glfwCreateWindow(width, height, window_name, None, None)
        glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
        glfw.glfwMakeContextCurrent(main_window)
        cygl.utils.init()
        g_pool.main_window = main_window

        def set_scale(new_scale):
            g_pool.gui_user_scale = new_scale
            window_size = (
                g_pool.camera_render_size[0]
                + int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor),
                glfw.glfwGetFramebufferSize(main_window)[1],
            )
            logger.warning(icon_bar_width * g_pool.gui_user_scale * hdpi_factor)
            glfw.glfwSetWindowSize(main_window, *window_size)

        # load pupil_positions, gaze_positions
        g_pool.binocular = meta_info.get("Eye Mode", "monocular") == "binocular"
        g_pool.version = app_version
        g_pool.timestamps = g_pool.capture.timestamps
        g_pool.get_timestamp = lambda: 0.0
        g_pool.user_dir = user_dir
        g_pool.rec_dir = rec_dir
        g_pool.meta_info = meta_info
        g_pool.min_data_confidence = session_settings.get("min_data_confidence", MIN_DATA_CONFIDENCE_DEFAULT)
        g_pool.min_calibration_confidence = session_settings.get(
            "min_calibration_confidence", MIN_CALIBRATION_CONFIDENCE_DEFAULT
        )

        # populated by producers
        g_pool.pupil_positions = pm.Bisector()
        g_pool.pupil_positions_by_id = (pm.Bisector(), pm.Bisector())
        g_pool.gaze_positions = pm.Bisector()
        g_pool.fixations = pm.Affiliator()
        g_pool.eye_movements = pm.Affiliator()

        def set_data_confidence(new_confidence):
            g_pool.min_data_confidence = new_confidence
            notification = {"subject": "min_data_confidence_changed"}
            notification["_notify_time_"] = time() + 0.8
            g_pool.ipc_pub.notify(notification)

        def do_export(_):
            left_idx = g_pool.seek_control.trim_left
            right_idx = g_pool.seek_control.trim_right
            export_range = left_idx, right_idx + 1  # exclusive range.stop
            export_ts_window = pm.exact_window(g_pool.timestamps, (left_idx, right_idx))

            export_dir = os.path.join(g_pool.rec_dir, "exports")
            export_dir = next_export_sub_dir(export_dir)

            os.makedirs(export_dir)
            logger.info('Created export dir at "{}"'.format(export_dir))

            export_info = {
                "Player Software Version": str(g_pool.version),
                "Data Format Version": meta_info["Data Format Version"],
                "Export Date": strftime("%d.%m.%Y", localtime()),
                "Export Time": strftime("%H:%M:%S", localtime()),
                "Frame Index Range:": g_pool.seek_control.get_frame_index_trim_range_string(),
                "Relative Time Range": g_pool.seek_control.get_rel_time_trim_range_string(),
                "Absolute Time Range": g_pool.seek_control.get_abs_time_trim_range_string(),
            }
            with open(os.path.join(export_dir, "export_info.csv"), "w") as csv:
                write_key_value_file(csv, export_info)

            notification = {
                "subject": "should_export",
                "range": export_range,
                "ts_window": export_ts_window,
                "export_dir": export_dir,
            }
            g_pool.ipc_pub.notify(notification)

        def reset_restart():
            logger.warning("Resetting all settings and restarting Player.")
            glfw.glfwSetWindowShouldClose(main_window, True)
            ipc_pub.notify({"subject": "clear_settings_process.should_start"})
            ipc_pub.notify(
                {
                    "subject": "player_process.should_start",
                    "rec_dir": rec_dir,
                    "delay": 2.0,
                }
            )

        def toggle_general_settings(collapsed):
            # this is the menu toggle logic.
            # Only one menu can be open.
            # If no menu is open the menubar should collapse.
            g_pool.menubar.collapsed = collapsed
            for m in g_pool.menubar.elements:
                m.collapsed = True
            general_settings.collapsed = collapsed

        g_pool.gui = ui.UI()
        g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0)
        g_pool.menubar = ui.Scrolling_Menu(
            "Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos="left"
        )
        g_pool.iconbar = ui.Scrolling_Menu(
            "Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden"
        )
        g_pool.timelines = ui.Container((0, 0), (0, 0), (0, 0))
        g_pool.timelines.horizontal_constraint = g_pool.menubar
        g_pool.user_timelines = ui.Timeline_Menu(
            "User Timelines", pos=(0.0, -150.0), size=(0.0, 0.0), header_pos="headline"
        )
        g_pool.user_timelines.color = RGBA(a=0.0)
        g_pool.user_timelines.collapsed = True
        # add container that constaints itself to the seekbar height
        vert_constr = ui.Container((0, 0), (0, -50.0), (0, 0))
        vert_constr.append(g_pool.user_timelines)
        g_pool.timelines.append(vert_constr)

        def set_window_size():
            f_width, f_height = g_pool.capture.frame_size
            f_width += int(icon_bar_width * g_pool.gui.scale)
            glfw.glfwSetWindowSize(main_window, f_width, f_height)

        general_settings = ui.Growing_Menu("General", header_pos="headline")
        general_settings.append(ui.Button("Reset window size", set_window_size))
        general_settings.append(
            ui.Selector(
                "gui_user_scale",
                g_pool,
                setter=set_scale,
                selection=[0.8, 0.9, 1.0, 1.1, 1.2] + list(np.arange(1.5, 5.1, 0.5)),
                label="Interface Size",
            )
        )
        general_settings.append(
            ui.Info_Text("Player Version: {}".format(g_pool.version))
        )
        general_settings.append(
            ui.Info_Text(
                "Capture Version: {}".format(meta_info["Capture Software Version"])
            )
        )
        general_settings.append(
            ui.Info_Text(
                "Data Format Version: {}".format(meta_info["Data Format Version"])
            )
        )

        general_settings.append(
            ui.Info_Text(
                "High level data, e.g. fixations, or visualizations only consider gaze data that has an equal or higher confidence than the minimum data confidence."
            )
        )
        general_settings.append(
            ui.Slider(
                "min_data_confidence",
                g_pool,
                setter=set_data_confidence,
                step=0.05,
                min=0.0,
                max=1.0,
                label="Minimum data confidence",
            )
        )

        general_settings.append(
            ui.Button("Restart with default settings", reset_restart)
        )

        g_pool.menubar.append(general_settings)
        icon = ui.Icon(
            "collapsed",
            general_settings,
            label=chr(0xE8B8),
            on_val=False,
            off_val=True,
            setter=toggle_general_settings,
            label_font="pupil_icons",
        )
        icon.tooltip = "General Settings"
        g_pool.iconbar.append(icon)

        user_plugin_separator = ui.Separator()
        user_plugin_separator.order = 0.35
        g_pool.iconbar.append(user_plugin_separator)

        g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100), (100, -100))
        g_pool.export_button = ui.Thumb(
            "export",
            label=chr(0xE2C5),
            getter=lambda: False,
            setter=do_export,
            hotkey="e",
            label_font="pupil_icons",
        )
        g_pool.quickbar.extend([g_pool.export_button])
        g_pool.gui.append(g_pool.menubar)
        g_pool.gui.append(g_pool.timelines)
        g_pool.gui.append(g_pool.iconbar)
        g_pool.gui.append(g_pool.quickbar)

        # we always load these plugins
        default_plugins = [
            ("Plugin_Manager", {}),
            ("Seek_Control", {}),
            ("Log_Display", {}),
            ("Raw_Data_Exporter", {}),
            ("Vis_Polyline", {}),
            ("Vis_Circle", {}),
            ("System_Graphs", {}),
            ("System_Timelines", {}),
            ("World_Video_Exporter", {}),
            ("Pupil_From_Recording", {}),
            ("GazeFromRecording", {}),
            ("Audio_Playback", {}),
        ]

        g_pool.plugins = Plugin_List(
            g_pool, session_settings.get("loaded_plugins", default_plugins)
        )

        # Manually add g_pool.capture to the plugin list
        g_pool.plugins._plugins.append(g_pool.capture)
        g_pool.plugins._plugins.sort(key=lambda p: p.order)
        g_pool.capture.init_ui()

        general_settings.insert(
            -1,
            ui.Text_Input(
                "rel_time_trim_section",
                getter=g_pool.seek_control.get_rel_time_trim_range_string,
                setter=g_pool.seek_control.set_rel_time_trim_range_string,
                label="Relative time range to export",
            ),
        )
        general_settings.insert(
            -1,
            ui.Text_Input(
                "frame_idx_trim_section",
                getter=g_pool.seek_control.get_frame_index_trim_range_string,
                setter=g_pool.seek_control.set_frame_index_trim_range_string,
                label="Frame index range to export",
            ),
        )

        # Register callbacks main_window
        glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
        glfw.glfwSetKeyCallback(main_window, on_window_key)
        glfw.glfwSetCharCallback(main_window, on_window_char)
        glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
        glfw.glfwSetCursorPosCallback(main_window, on_pos)
        glfw.glfwSetScrollCallback(main_window, on_scroll)
        glfw.glfwSetDropCallback(main_window, on_drop)

        toggle_general_settings(True)

        g_pool.gui.configuration = session_settings.get("ui_config", {})

        # gl_state settings
        gl_utils.basic_gl_setup()
        g_pool.image_tex = Named_Texture()

        # trigger on_resize
        on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

        def handle_notifications(n):
            subject = n["subject"]
            if subject == "start_plugin":
                g_pool.plugins.add(
                    g_pool.plugin_by_name[n["name"]], args=n.get("args", {})
                )
            elif subject.startswith("meta.should_doc"):
                ipc_pub.notify(
                    {"subject": "meta.doc", "actor": g_pool.app, "doc": player.__doc__}
                )
                for p in g_pool.plugins:
                    if (
                        p.on_notify.__doc__
                        and p.__class__.on_notify != Plugin.on_notify
                    ):
                        ipc_pub.notify(
                            {
                                "subject": "meta.doc",
                                "actor": p.class_name,
                                "doc": p.on_notify.__doc__,
                            }
                        )

        while not glfw.glfwWindowShouldClose(main_window):

            # fetch newest notifications
            new_notifications = []
            while notify_sub.new_data:
                t, n = notify_sub.recv()
                new_notifications.append(n)

            # notify each plugin if there are new notifications:
            for n in new_notifications:
                handle_notifications(n)
                for p in g_pool.plugins:
                    p.on_notify(n)

            events = {}
            # report time between now and the last loop interation
            events["dt"] = get_dt()

            # pupil and gaze positions are added by their respective producer plugins
            events["pupil"] = []
            events["gaze"] = []

            # allow each Plugin to do its work.
            for p in g_pool.plugins:
                p.recent_events(events)

            # check if a plugin need to be destroyed
            g_pool.plugins.clean()

            glfw.glfwMakeContextCurrent(main_window)
            # render visual feedback from loaded plugins
            if gl_utils.is_window_visible(main_window):

                gl_utils.glViewport(0, 0, *g_pool.camera_render_size)
                g_pool.capture.gl_display()
                for p in g_pool.plugins:
                    p.gl_display()

                gl_utils.glViewport(0, 0, *window_size)

                try:
                    clipboard = glfw.glfwGetClipboardString(main_window).decode()
                except AttributeError:  # clipbaord is None, might happen on startup
                    clipboard = ""
                g_pool.gui.update_clipboard(clipboard)
                user_input = g_pool.gui.update()
                if user_input.clipboard and user_input.clipboard != clipboard:
                    # only write to clipboard if content changed
                    glfw.glfwSetClipboardString(
                        main_window, user_input.clipboard.encode()
                    )

                for b in user_input.buttons:
                    button, action, mods = b
                    x, y = glfw.glfwGetCursorPos(main_window)
                    pos = x * hdpi_factor, y * hdpi_factor
                    pos = normalize(pos, g_pool.camera_render_size)
                    pos = denormalize(pos, g_pool.capture.frame_size)

                    # call `on_click` callbacks until a plugin indicates
                    # that it has consumed the event (by returning True)
                    any(p.on_click(pos, button, action) for p in g_pool.plugins)

                for key, scancode, action, mods in user_input.keys:
                    # call `on_key` callbacks until a plugin indicates
                    # that it has consumed the event (by returning True)
                    any(p.on_key(key, scancode, action, mods) for p in g_pool.plugins)

                for char_ in user_input.chars:
                    # call `char_` callbacks until a plugin indicates
                    # that it has consumed the event (by returning True)
                    any(p.on_char(char_) for p in g_pool.plugins)

                # present frames at appropriate speed
                g_pool.seek_control.wait(events["frame"].timestamp)
                glfw.glfwSwapBuffers(main_window)

            glfw.glfwPollEvents()

        session_settings["loaded_plugins"] = g_pool.plugins.get_initializers()
        session_settings["min_data_confidence"] = g_pool.min_data_confidence
        session_settings[
            "min_calibration_confidence"
        ] = g_pool.min_calibration_confidence
        session_settings["gui_scale"] = g_pool.gui_user_scale
        session_settings["ui_config"] = g_pool.gui.configuration
        session_settings["window_position"] = glfw.glfwGetWindowPos(main_window)
        session_settings["version"] = str(g_pool.version)

        session_window_size = glfw.glfwGetWindowSize(main_window)
        if 0 not in session_window_size:
            session_settings["window_size"] = session_window_size

        session_settings.close()

        # de-init all running plugins
        for p in g_pool.plugins:
            p.alive = False
        g_pool.plugins.clean()

        g_pool.gui.terminate()
        glfw.glfwDestroyWindow(main_window)

    except:
        import traceback

        trace = traceback.format_exc()
        logger.error("Process Player crashed with trace:\n{}".format(trace))
    finally:
        logger.info("Process shutting down.")
        ipc_pub.notify({"subject": "player_process.stopped"})
        sleep(1.0)
Esempio n. 19
0
def _export_world_video(
    rec_dir,
    user_dir,
    min_data_confidence,
    start_frame,
    end_frame,
    plugin_initializers,
    out_file_path,
    pre_computed_eye_data,
):
    """
    Simulates the generation for the world video and saves a certain time range as a video.
    It simulates a whole g_pool such that all plugins run as normal.
    """
    from glob import glob
    from time import time

    import file_methods as fm
    import player_methods as pm
    from av_writer import AV_Writer

    # we are not importing manual gaze correction. In Player corrections have already been applied.
    # in batch exporter this plugin makes little sense.
    from fixation_detector import Offline_Fixation_Detector
    from eye_movement import Offline_Eye_Movement_Detector

    # Plug-ins
    from plugin import Plugin_List, import_runtime_plugins
    from video_capture import EndofVideoError, File_Source
    from video_overlay.plugins import Video_Overlay, Eye_Overlay
    from vis_circle import Vis_Circle
    from vis_cross import Vis_Cross
    from vis_light_points import Vis_Light_Points
    from vis_polyline import Vis_Polyline
    from vis_scan_path import Vis_Scan_Path
    from vis_watermark import Vis_Watermark

    PID = str(os.getpid())
    logger = logging.getLogger(__name__ + " with pid: " + PID)
    start_status = "Starting video export with pid: {}".format(PID)
    logger.info(start_status)
    yield start_status, 0

    try:
        vis_plugins = sorted(
            [
                Vis_Circle,
                Vis_Cross,
                Vis_Polyline,
                Vis_Light_Points,
                Vis_Watermark,
                Vis_Scan_Path,
                Eye_Overlay,
                Video_Overlay,
            ],
            key=lambda x: x.__name__,
        )
        analysis_plugins = [Offline_Fixation_Detector, Offline_Eye_Movement_Detector]
        user_plugins = sorted(
            import_runtime_plugins(os.path.join(user_dir, "plugins")),
            key=lambda x: x.__name__,
        )

        available_plugins = vis_plugins + analysis_plugins + user_plugins
        name_by_index = [p.__name__ for p in available_plugins]
        plugin_by_name = dict(zip(name_by_index, available_plugins))

        meta_info = pm.load_meta_info(rec_dir)

        g_pool = GlobalContainer()
        g_pool.app = "exporter"
        g_pool.min_data_confidence = min_data_confidence

        valid_ext = (".mp4", ".mkv", ".avi", ".h264", ".mjpeg", ".fake")
        try:
            video_path = next(
                f
                for f in glob(os.path.join(rec_dir, "world.*"))
                if os.path.splitext(f)[1] in valid_ext
            )
        except StopIteration:
            raise FileNotFoundError("No Video world found")
        cap = File_Source(g_pool, source_path=video_path, fill_gaps=True, timing=None)

        timestamps = cap.timestamps

        file_name = os.path.basename(out_file_path)
        dir_name = os.path.dirname(out_file_path)
        out_file_path = os.path.expanduser(os.path.join(dir_name, file_name))

        if os.path.isfile(out_file_path):
            logger.warning("Video out file already exsists. I will overwrite!")
            os.remove(out_file_path)
        logger.debug("Saving Video to {}".format(out_file_path))

        # Trim mark verification
        # make sure the trim marks (start frame, end frame) make sense:
        # We define them like python list slices, thus we can test them like such.
        trimmed_timestamps = timestamps[start_frame:end_frame]
        if len(trimmed_timestamps) == 0:
            warn = "Start and end frames are set such that no video will be exported."
            logger.warning(warn)
            yield warn, 0.0
            return

        if start_frame is None:
            start_frame = 0

        # these two vars are shared with the launching process and give a job length and progress report.
        frames_to_export = len(trimmed_timestamps)
        current_frame = 0
        exp_info = (
            "Will export from frame {} to frame {}. This means I will export {} frames."
        )
        logger.debug(
            exp_info.format(
                start_frame, start_frame + frames_to_export, frames_to_export
            )
        )

        # setup of writer
        writer = AV_Writer(
            out_file_path, fps=cap.frame_rate, audio_dir=rec_dir, use_timestamps=True
        )

        cap.seek_to_frame(start_frame)

        start_time = time()

        g_pool.plugin_by_name = plugin_by_name
        g_pool.capture = cap
        g_pool.rec_dir = rec_dir
        g_pool.user_dir = user_dir
        g_pool.meta_info = meta_info
        g_pool.timestamps = timestamps
        g_pool.delayed_notifications = {}
        g_pool.notifications = []

        for initializers in pre_computed_eye_data.values():
            initializers["data"] = [
                fm.Serialized_Dict(msgpack_bytes=serialized)
                for serialized in initializers["data"]
            ]

        g_pool.pupil_positions = pm.Bisector(**pre_computed_eye_data["pupil"])
        g_pool.pupil_positions_by_id = (
            pm.Bisector(**pre_computed_eye_data["pupil_by_id_0"]),
            pm.Bisector(**pre_computed_eye_data["pupil_by_id_1"]),
        )
        g_pool.gaze_positions = pm.Bisector(**pre_computed_eye_data["gaze"])
        g_pool.fixations = pm.Affiliator(**pre_computed_eye_data["fixations"])

        # add plugins
        g_pool.plugins = Plugin_List(g_pool, plugin_initializers)

        while frames_to_export > current_frame:
            try:
                frame = cap.get_frame()
            except EndofVideoError:
                break

            events = {"frame": frame}
            # new positions and events
            frame_window = pm.enclosing_window(g_pool.timestamps, frame.index)
            events["gaze"] = g_pool.gaze_positions.by_ts_window(frame_window)
            events["pupil"] = g_pool.pupil_positions.by_ts_window(frame_window)

            # publish delayed notifications when their time has come.
            for n in list(g_pool.delayed_notifications.values()):
                if n["_notify_time_"] < time():
                    del n["_notify_time_"]
                    del g_pool.delayed_notifications[n["subject"]]
                    g_pool.notifications.append(n)

            # notify each plugin if there are new notifications:
            while g_pool.notifications:
                n = g_pool.notifications.pop(0)
                for p in g_pool.plugins:
                    p.on_notify(n)

            # allow each Plugin to do its work.
            for p in g_pool.plugins:
                p.recent_events(events)

            writer.write_video_frame(frame)
            current_frame += 1
            yield "Exporting with pid {}".format(PID), current_frame

        writer.close(timestamp_export_format="all")

        duration = time() - start_time
        effective_fps = float(current_frame) / duration

        result = "Export done: Exported {} frames to {}. This took {} seconds. Exporter ran at {} frames per second."
        logger.info(
            result.format(current_frame, out_file_path, duration, effective_fps)
        )
        yield "Export done. This took {:.0f} seconds.".format(duration), current_frame

    except GeneratorExit:
        logger.warning("Video export with pid {} was canceled.".format(os.getpid()))