Exemple #1
0
def export(should_terminate,
           frames_to_export,
           current_frame,
           rec_dir,
           user_dir,
           min_data_confidence,
           start_frame=None,
           end_frame=None,
           plugin_initializers=(),
           out_file_path=None):

    vis_plugins = sorted([
        Vis_Circle, Vis_Cross, Vis_Polyline, Vis_Light_Points, Vis_Watermark,
        Scan_Path
    ],
                         key=lambda x: x.__name__)
    analysis_plugins = sorted([
        Manual_Gaze_Correction, Eye_Video_Overlay,
        Pupil_Angle_3D_Fixation_Detector, Gaze_Position_2D_Fixation_Detector
    ],
                              key=lambda x: x.__name__)
    user_plugins = sorted(import_runtime_plugins(
        os.path.join(user_dir, 'plugins')),
                          key=lambda x: x.__name__)
    available_plugins = vis_plugins + analysis_plugins + user_plugins
    name_by_index = [p.__name__ for p in available_plugins]
    index_by_name = dict(zip(name_by_index, range(len(name_by_index))))
    plugin_by_name = dict(zip(name_by_index, available_plugins))

    logger = logging.getLogger(__name__ + ' with pid: ' + str(os.getpid()))

    update_recording_to_recent(rec_dir)

    video_path = [
        f for f in glob(os.path.join(rec_dir, "world.*"))
        if f[-3:] in ('mp4', 'mkv', 'avi')
    ][0]
    timestamps_path = os.path.join(rec_dir, "world_timestamps.npy")
    pupil_data_path = os.path.join(rec_dir, "pupil_data")

    meta_info = load_meta_info(rec_dir)
    rec_version = read_rec_version(meta_info)

    g_pool = Global_Container()
    g_pool.app = 'exporter'
    g_pool.min_data_confidence = min_data_confidence
    timestamps = np.load(timestamps_path)
    cap = File_Source(g_pool, video_path, timestamps=timestamps)

    #Out file path verification, we do this before but if one uses a seperate tool, this will kick in.
    if out_file_path is None:
        out_file_path = os.path.join(rec_dir, "world_viz.mp4")
    else:
        file_name = os.path.basename(out_file_path)
        dir_name = os.path.dirname(out_file_path)
        if not dir_name:
            dir_name = rec_dir
        if not file_name:
            file_name = 'world_viz.mp4'
        out_file_path = os.path.expanduser(os.path.join(dir_name, file_name))

    if os.path.isfile(out_file_path):
        logger.warning("Video out file already exsists. I will overwrite!")
        os.remove(out_file_path)
    logger.debug("Saving Video to %s" % out_file_path)

    #Trim mark verification
    #make sure the trim marks (start frame, endframe) make sense: We define them like python list slices,thus we can test them like such.
    trimmed_timestamps = timestamps[start_frame:end_frame]
    if len(trimmed_timestamps) == 0:
        logger.warn(
            "Start and end frames are set such that no video will be exported."
        )
        return False

    if start_frame == None:
        start_frame = 0

    #these two vars are shared with the lauching process and give a job length and progress report.
    frames_to_export.value = len(trimmed_timestamps)
    current_frame.value = 0
    logger.debug(
        "Will export from frame %s to frame %s. This means I will export %s frames."
        % (start_frame, start_frame + frames_to_export.value,
           frames_to_export.value))

    #setup of writer
    writer = AV_Writer(out_file_path, fps=cap.frame_rate, use_timestamps=True)

    cap.seek_to_frame(start_frame)

    start_time = time()

    g_pool.capture = cap
    g_pool.rec_dir = rec_dir
    g_pool.user_dir = user_dir
    g_pool.rec_version = rec_version
    g_pool.timestamps = timestamps
    g_pool.delayed_notifications = {}
    g_pool.notifications = []

    # load pupil_positions, gaze_positions
    pupil_data = load_object(pupil_data_path)
    pupil_list = pupil_data['pupil_positions']
    gaze_list = pupil_data['gaze_positions']
    g_pool.pupil_positions_by_frame = correlate_data(pupil_list,
                                                     g_pool.timestamps)
    g_pool.gaze_positions_by_frame = correlate_data(gaze_list,
                                                    g_pool.timestamps)
    g_pool.fixations_by_frame = [[] for x in g_pool.timestamps
                                 ]  #populated by the fixation detector plugin

    #add plugins
    g_pool.plugins = Plugin_List(g_pool, plugin_by_name, plugin_initializers)

    while frames_to_export.value > current_frame.value:

        if should_terminate.value:
            logger.warning("User aborted export. Exported %s frames to %s." %
                           (current_frame.value, out_file_path))

            #explicit release of VideoWriter
            writer.close()
            writer = None
            return False

        try:
            frame = cap.get_frame_nowait()
        except EndofVideoFileError:
            break

        events = {}
        #new positons and events
        events['gaze_positions'] = g_pool.gaze_positions_by_frame[frame.index]
        events['pupil_positions'] = g_pool.pupil_positions_by_frame[
            frame.index]

        # publish delayed notifiactions when their time has come.
        for n in g_pool.delayed_notifications.values():
            if n['_notify_time_'] < time():
                del n['_notify_time_']
                del g_pool.delayed_notifications[n['subject']]
                g_pool.notifications.append(n)

        # notify each plugin if there are new notifactions:
        while g_pool.notifications:
            n = g_pool.notifications.pop(0)
            for p in g_pool.plugins:
                p.on_notify(n)

        # allow each Plugin to do its work.
        for p in g_pool.plugins:
            p.update(frame, events)

        writer.write_video_frame(frame)
        current_frame.value += 1

    writer.close()
    writer = None

    duration = time() - start_time
    effective_fps = float(current_frame.value) / duration

    logger.info(
        "Export done: Exported %s frames to %s. This took %s seconds. Exporter ran at %s frames per second"
        % (current_frame.value, out_file_path, duration, effective_fps))
    return True
Exemple #2
0
def export(
    should_terminate,
    frames_to_export,
    current_frame,
    rec_dir,
    user_dir,
    min_data_confidence,
    start_frame=None,
    end_frame=None,
    plugin_initializers=(),
    out_file_path=None,
):

    vis_plugins = sorted(
        [Vis_Circle, Vis_Cross, Vis_Polyline, Vis_Light_Points, Vis_Watermark, Vis_Scan_Path, Vis_Eye_Video_Overlay],
        key=lambda x: x.__name__,
    )
    analysis_plugins = sorted(
        [Manual_Gaze_Correction, Pupil_Angle_3D_Fixation_Detector, Gaze_Position_2D_Fixation_Detector],
        key=lambda x: x.__name__,
    )
    user_plugins = sorted(import_runtime_plugins(os.path.join(user_dir, "plugins")), key=lambda x: x.__name__)

    available_plugins = vis_plugins + analysis_plugins + user_plugins
    name_by_index = [p.__name__ for p in available_plugins]
    plugin_by_name = dict(zip(name_by_index, available_plugins))

    logger = logging.getLogger(__name__ + " with pid: " + str(os.getpid()))

    update_recording_to_recent(rec_dir)

    video_path = [f for f in glob(os.path.join(rec_dir, "world.*")) if f[-3:] in ("mp4", "mkv", "avi")][0]
    timestamps_path = os.path.join(rec_dir, "world_timestamps.npy")
    pupil_data_path = os.path.join(rec_dir, "pupil_data")

    meta_info = load_meta_info(rec_dir)
    rec_version = read_rec_version(meta_info)

    g_pool = Global_Container()
    g_pool.app = "exporter"
    g_pool.min_data_confidence = min_data_confidence
    timestamps = np.load(timestamps_path)
    cap = File_Source(g_pool, video_path, timestamps=timestamps)

    # Out file path verification, we do this before but if one uses a seperate tool, this will kick in.
    if out_file_path is None:
        out_file_path = os.path.join(rec_dir, "world_viz.mp4")
    else:
        file_name = os.path.basename(out_file_path)
        dir_name = os.path.dirname(out_file_path)
        if not dir_name:
            dir_name = rec_dir
        if not file_name:
            file_name = "world_viz.mp4"
        out_file_path = os.path.expanduser(os.path.join(dir_name, file_name))

    if os.path.isfile(out_file_path):
        logger.warning("Video out file already exsists. I will overwrite!")
        os.remove(out_file_path)
    logger.debug("Saving Video to {}".format(out_file_path))

    # Trim mark verification
    # make sure the trim marks (start frame, endframe) make sense:
    # We define them like python list slices, thus we can test them like such.
    trimmed_timestamps = timestamps[start_frame:end_frame]
    if len(trimmed_timestamps) == 0:
        logger.warn("Start and end frames are set such that no video will be exported.")
        return False

    if start_frame is None:
        start_frame = 0

    # these two vars are shared with the lauching process and give a job length and progress report.
    frames_to_export.value = len(trimmed_timestamps)
    current_frame.value = 0
    exp_info = "Will export from frame {} to frame {}. This means I will export {} frames."
    logger.debug(exp_info.format(start_frame, start_frame + frames_to_export.value, frames_to_export.value))

    # setup of writer
    writer = AV_Writer(out_file_path, fps=cap.frame_rate, use_timestamps=True)

    cap.seek_to_frame(start_frame)

    start_time = time()

    g_pool.capture = cap
    g_pool.rec_dir = rec_dir
    g_pool.user_dir = user_dir
    g_pool.rec_version = rec_version
    g_pool.timestamps = timestamps
    g_pool.delayed_notifications = {}
    g_pool.notifications = []

    # load pupil_positions, gaze_positions
    pupil_data = load_object(pupil_data_path)
    pupil_list = pupil_data["pupil_positions"]
    gaze_list = pupil_data["gaze_positions"]
    g_pool.pupil_positions_by_frame = correlate_data(pupil_list, g_pool.timestamps)
    g_pool.gaze_positions_by_frame = correlate_data(gaze_list, g_pool.timestamps)
    g_pool.fixations_by_frame = [[] for x in g_pool.timestamps]  # populated by the fixation detector plugin

    # add plugins
    g_pool.plugins = Plugin_List(g_pool, plugin_by_name, plugin_initializers)

    while frames_to_export.value > current_frame.value:

        if should_terminate.value:
            logger.warning("User aborted export. Exported {} frames to {}.".format(current_frame.value, out_file_path))

            # explicit release of VideoWriter
            writer.close()
            writer = None
            return False

        try:
            frame = cap.get_frame_nowait()
        except EndofVideoFileError:
            break

        events = {}
        # new positons and events
        events["gaze_positions"] = g_pool.gaze_positions_by_frame[frame.index]
        events["pupil_positions"] = g_pool.pupil_positions_by_frame[frame.index]

        # publish delayed notifiactions when their time has come.
        for n in list(g_pool.delayed_notifications.values()):
            if n["_notify_time_"] < time():
                del n["_notify_time_"]
                del g_pool.delayed_notifications[n["subject"]]
                g_pool.notifications.append(n)

        # notify each plugin if there are new notifactions:
        while g_pool.notifications:
            n = g_pool.notifications.pop(0)
            for p in g_pool.plugins:
                p.on_notify(n)

        # allow each Plugin to do its work.
        for p in g_pool.plugins:
            p.update(frame, events)

        writer.write_video_frame(frame)
        current_frame.value += 1

    writer.close()
    writer = None

    duration = time() - start_time
    effective_fps = float(current_frame.value) / duration

    result = "Export done: Exported {} frames to {}. This took {} seconds. Exporter ran at {} frames per second."
    logger.info(result.format(current_frame.value, out_file_path, duration, effective_fps))
    return True
Exemple #3
0
def session(rec_dir):
    system_plugins = [Log_Display, Seek_Bar, Trim_Marks]
    vis_plugins = sorted([Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross,
                          Vis_Watermark, Vis_Eye_Video_Overlay, Vis_Scan_Path], key=lambda x: x.__name__)

    analysis_plugins = sorted([Gaze_Position_2D_Fixation_Detector, Pupil_Angle_3D_Fixation_Detector,
                               Manual_Gaze_Correction, Video_Export_Launcher, Offline_Surface_Tracker,
                               Raw_Data_Exporter, Batch_Exporter, Annotation_Player], key=lambda x: x.__name__)

    other_plugins = sorted([Show_Calibration, Log_History, Marker_Auto_Trim_Marks], key=lambda x: x.__name__)
    user_plugins = sorted(import_runtime_plugins(os.path.join(user_dir, 'plugins')), key=lambda x: x.__name__)

    user_launchable_plugins = vis_plugins + analysis_plugins + other_plugins + user_plugins
    available_plugins = system_plugins + user_launchable_plugins
    name_by_index = [p.__name__ for p in available_plugins]
    plugin_by_name = dict(zip(name_by_index, available_plugins))

    # Callback functions
    def on_resize(window, w, h):
        g_pool.gui.update_window(w, h)
        g_pool.gui.collect_menus()
        graph.adjust_size(w, h)
        gl_utils.adjust_gl_view(w, h)
        for p in g_pool.plugins:
            p.on_window_resize(window, w, h)

    def on_key(window, key, scancode, action, mods):
        g_pool.gui.update_key(key, scancode, action, mods)

    def on_char(window, char):
        g_pool.gui.update_char(char)

    def on_button(window, button, action, mods):
        g_pool.gui.update_button(button, action, mods)
        pos = glfwGetCursorPos(window)
        pos = normalize(pos, glfwGetWindowSize(window))
        pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0]))  # Position in img pixels
        for p in g_pool.plugins:
            p.on_click(pos, button, action)

    def on_pos(window, x, y):
        hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0])
        g_pool.gui.update_mouse(x*hdpi_factor, y*hdpi_factor)

    def on_scroll(window, x, y):
        g_pool.gui.update_scroll(x, y*scroll_factor)

    def on_drop(window, count, paths):
        for x in range(count):
            new_rec_dir = paths[x].decode('utf-8')
            if is_pupil_rec_dir(new_rec_dir):
                logger.debug("Starting new session with '{}'".format(new_rec_dir))
                global rec_dir
                rec_dir = new_rec_dir
                glfwSetWindowShouldClose(window, True)
            else:
                logger.error("'{}' is not a valid pupil recording".format(new_rec_dir))

    tick = delta_t()

    def get_dt():
        return next(tick)

    update_recording_to_recent(rec_dir)

    video_path = [f for f in glob(os.path.join(rec_dir, "world.*")) if f[-3:] in ('mp4', 'mkv', 'avi')][0]
    timestamps_path = os.path.join(rec_dir, "world_timestamps.npy")
    pupil_data_path = os.path.join(rec_dir, "pupil_data")

    meta_info = load_meta_info(rec_dir)
    rec_version = read_rec_version(meta_info)
    app_version = get_version(version_file)

    # log info about Pupil Platform and Platform in player.log
    logger.info('Application Version: {}'.format(app_version))
    logger.info('System Info: {}'.format(get_system_info()))

    timestamps = np.load(timestamps_path)

    # create container for globally scoped vars
    g_pool = Global_Container()
    g_pool.app = 'player'

    # Initialize capture
    cap = File_Source(g_pool, video_path, timestamps=list(timestamps))

    # load session persistent settings
    session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings"))
    if session_settings.get("version", VersionFormat('0.0')) < get_version(version_file):
        logger.info("Session setting are from older version of this app. I will not use those.")
        session_settings.clear()

    width, height = session_settings.get('window_size', cap.frame_size)
    window_pos = session_settings.get('window_position', window_position_default)
    main_window = glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - "
                                   + rec_dir.split(os.path.sep)[-1], None, None)
    glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
    glfwMakeContextCurrent(main_window)
    cygl.utils.init()

    # load pupil_positions, gaze_positions
    pupil_data = load_object(pupil_data_path)
    pupil_list = pupil_data['pupil_positions']
    gaze_list = pupil_data['gaze_positions']

    g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular'
    g_pool.version = app_version
    g_pool.capture = cap
    g_pool.timestamps = timestamps
    g_pool.play = False
    g_pool.new_seek = True
    g_pool.user_dir = user_dir
    g_pool.rec_dir = rec_dir
    g_pool.rec_version = rec_version
    g_pool.meta_info = meta_info
    g_pool.min_data_confidence = session_settings.get('min_data_confidence', 0.6)
    g_pool.pupil_positions_by_frame = correlate_data(pupil_list, g_pool.timestamps)
    g_pool.gaze_positions_by_frame = correlate_data(gaze_list, g_pool.timestamps)
    g_pool.fixations_by_frame = [[] for x in g_pool.timestamps]  # populated by the fixation detector plugin

    def next_frame(_):
        try:
            cap.seek_to_frame(cap.get_frame_index())
        except(FileSeekError):
            logger.warning("Could not seek to next frame.")
        else:
            g_pool.new_seek = True

    def prev_frame(_):
        try:
            cap.seek_to_frame(cap.get_frame_index()-2)
        except(FileSeekError):
            logger.warning("Could not seek to previous frame.")
        else:
            g_pool.new_seek = True

    def toggle_play(new_state):
        if cap.get_frame_index() >= cap.get_frame_count()-5:
            cap.seek_to_frame(1)  # avoid pause set by hitting trimmark pause.
            logger.warning("End of video - restart at beginning.")
        g_pool.play = new_state

    def set_scale(new_scale):
        g_pool.gui.scale = new_scale
        g_pool.gui.collect_menus()

    def set_data_confidence(new_confidence):
        g_pool.min_data_confidence = new_confidence
        notification = {'subject': 'min_data_confidence_changed'}
        notification['_notify_time_'] = time()+.8
        g_pool.delayed_notifications[notification['subject']] = notification

    def open_plugin(plugin):
        if plugin == "Select to load":
            return
        g_pool.plugins.add(plugin)

    def purge_plugins():
        for p in g_pool.plugins:
            if p.__class__ in user_launchable_plugins:
                p.alive = False
        g_pool.plugins.clean()

    def do_export(_):
        export_range = slice(g_pool.trim_marks.in_mark, g_pool.trim_marks.out_mark)
        export_dir = os.path.join(g_pool.rec_dir, 'exports', '{}-{}'.format(export_range.start, export_range.stop))
        try:
            os.makedirs(export_dir)
        except OSError as e:
            if e.errno != errno.EEXIST:
                logger.error("Could not create export dir")
                raise e
            else:
                overwrite_warning = "Previous export for range [{}-{}] already exsits - overwriting."
                logger.warning(overwrite_warning.format(export_range.start, export_range.stop))
        else:
            logger.info('Created export dir at "{}"'.format(export_dir))

        notification = {'subject': 'should_export', 'range': export_range, 'export_dir': export_dir}
        g_pool.notifications.append(notification)

    g_pool.gui = ui.UI()
    g_pool.gui.scale = session_settings.get('gui_scale', 1)
    g_pool.main_menu = ui.Scrolling_Menu("Settings", pos=(-350, 20), size=(300, 500))
    g_pool.main_menu.append(ui.Button("Close Pupil Player", lambda: glfwSetWindowShouldClose(main_window, True)))
    g_pool.main_menu.append(ui.Slider('scale', g_pool.gui, setter=set_scale, step=.05, min=0.75, max=2.5, label='Interface Size'))
    g_pool.main_menu.append(ui.Info_Text('Player Version: {}'.format(g_pool.version)))
    g_pool.main_menu.append(ui.Info_Text('Recording Version: {}'.format(rec_version)))
    g_pool.main_menu.append(ui.Slider('min_data_confidence', g_pool, setter=set_data_confidence,
                                      step=.05, min=0.0, max=1.0, label='Confidence threshold'))

    selector_label = "Select to load"

    vis_labels = ["   " + p.__name__.replace('_', ' ') for p in vis_plugins]
    analysis_labels = ["   " + p.__name__.replace('_', ' ') for p in analysis_plugins]
    other_labels = ["   " + p.__name__.replace('_', ' ') for p in other_plugins]
    user_labels = ["   " + p.__name__.replace('_', ' ') for p in user_plugins]

    plugins = ([selector_label, selector_label] + vis_plugins + [selector_label] + analysis_plugins + [selector_label]
               + other_plugins + [selector_label] + user_plugins)
    labels = ([selector_label, "Visualization"] + vis_labels + ["Analysis"] + analysis_labels + ["Other"]
              + other_labels + ["User added"] + user_labels)

    g_pool.main_menu.append(ui.Selector('Open plugin:',
                                        selection=plugins,
                                        labels=labels,
                                        setter=open_plugin,
                                        getter=lambda: selector_label))

    g_pool.main_menu.append(ui.Button('Close all plugins', purge_plugins))
    g_pool.main_menu.append(ui.Button('Reset window size',
                                      lambda: glfwSetWindowSize(main_window, cap.frame_size[0], cap.frame_size[1])))
    g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))
    g_pool.play_button = ui.Thumb('play',
                                  g_pool,
                                  label=chr(0xf04b),
                                  setter=toggle_play,
                                  hotkey=GLFW_KEY_SPACE,
                                  label_font='fontawesome',
                                  label_offset_x=5,
                                  label_offset_y=0,
                                  label_offset_size=-24)
    g_pool.play_button.on_color[:] = (0, 1., .0, .8)
    g_pool.forward_button = ui.Thumb('forward',
                                     label=chr(0xf04e),
                                     getter=lambda: False,
                                     setter=next_frame,
                                     hotkey=GLFW_KEY_RIGHT,
                                     label_font='fontawesome',
                                     label_offset_x=5,
                                     label_offset_y=0,
                                     label_offset_size=-24)
    g_pool.backward_button = ui.Thumb('backward',
                                      label=chr(0xf04a),
                                      getter=lambda: False,
                                      setter=prev_frame,
                                      hotkey=GLFW_KEY_LEFT,
                                      label_font='fontawesome',
                                      label_offset_x=-5,
                                      label_offset_y=0,
                                      label_offset_size=-24)
    g_pool.export_button = ui.Thumb('export',
                                    label=chr(0xf063),
                                    getter=lambda: False,
                                    setter=do_export,
                                    hotkey='e',
                                    label_font='fontawesome',
                                    label_offset_x=0,
                                    label_offset_y=2,
                                    label_offset_size=-24)
    g_pool.quickbar.extend([g_pool.play_button, g_pool.forward_button, g_pool.backward_button, g_pool.export_button])
    g_pool.gui.append(g_pool.quickbar)
    g_pool.gui.append(g_pool.main_menu)

    # we always load these plugins
    system_plugins = [('Trim_Marks', {}), ('Seek_Bar', {})]
    default_plugins = [('Log_Display', {}), ('Vis_Scan_Path', {}), ('Vis_Polyline', {}), ('Vis_Circle', {}), ('Video_Export_Launcher', {})]
    previous_plugins = session_settings.get('loaded_plugins', default_plugins)
    g_pool.notifications = []
    g_pool.delayed_notifications = {}
    g_pool.plugins = Plugin_List(g_pool, plugin_by_name, system_plugins+previous_plugins)


    # Register callbacks main_window
    glfwSetFramebufferSizeCallback(main_window, on_resize)
    glfwSetKeyCallback(main_window, on_key)
    glfwSetCharCallback(main_window, on_char)
    glfwSetMouseButtonCallback(main_window, on_button)
    glfwSetCursorPosCallback(main_window, on_pos)
    glfwSetScrollCallback(main_window, on_scroll)
    glfwSetDropCallback(main_window, on_drop)
    # trigger on_resize
    on_resize(main_window, *glfwGetFramebufferSize(main_window))

    g_pool.gui.configuration = session_settings.get('ui_config', {})

    # gl_state settings
    gl_utils.basic_gl_setup()
    g_pool.image_tex = Named_Texture()

    # set up performace graphs:
    pid = os.getpid()
    ps = psutil.Process(pid)
    ts = None

    cpu_graph = graph.Bar_Graph()
    cpu_graph.pos = (20, 110)
    cpu_graph.update_fn = ps.cpu_percent
    cpu_graph.update_rate = 5
    cpu_graph.label = 'CPU %0.1f'

    fps_graph = graph.Bar_Graph()
    fps_graph.pos = (140, 110)
    fps_graph.update_rate = 5
    fps_graph.label = "%0.0f REC FPS"

    pupil_graph = graph.Bar_Graph(max_val=1.0)
    pupil_graph.pos = (260, 110)
    pupil_graph.update_rate = 5
    pupil_graph.label = "Confidence: %0.2f"

    while not glfwWindowShouldClose(main_window):
        # grab new frame
        if g_pool.play or g_pool.new_seek:
            g_pool.new_seek = False
            try:
                new_frame = cap.get_frame_nowait()
            except EndofVideoFileError:
                # end of video logic: pause at last frame.
                g_pool.play = False
                logger.warning("end of video")
            update_graph = True
        else:
            update_graph = False

        frame = new_frame.copy()
        events = {}
        # report time between now and the last loop interation
        events['dt'] = get_dt()
        # new positons we make a deepcopy just like the image is a copy.
        events['gaze_positions'] = deepcopy(g_pool.gaze_positions_by_frame[frame.index])
        events['pupil_positions'] = deepcopy(g_pool.pupil_positions_by_frame[frame.index])

        if update_graph:
            # update performace graphs
            for p in events['pupil_positions']:
                pupil_graph.add(p['confidence'])

            t = new_frame.timestamp
            if ts and ts != t:
                dt, ts = t-ts, t
                fps_graph.add(1./dt)
            else:
               ts = new_frame.timestamp

            g_pool.play_button.status_text = str(frame.index)
        # always update the CPU graph
        cpu_graph.update()

        # publish delayed notifiactions when their time has come.
        for n in list(g_pool.delayed_notifications.values()):
            if n['_notify_time_'] < time():
                del n['_notify_time_']
                del g_pool.delayed_notifications[n['subject']]
                g_pool.notifications.append(n)

        # notify each plugin if there are new notifactions:
        while g_pool.notifications:
            n = g_pool.notifications.pop(0)
            for p in g_pool.plugins:
                p.on_notify(n)

        # allow each Plugin to do its work.
        for p in g_pool.plugins:
            p.update(frame, events)

        # check if a plugin need to be destroyed
        g_pool.plugins.clean()

        # render camera image
        glfwMakeContextCurrent(main_window)
        gl_utils.make_coord_system_norm_based()
        g_pool.image_tex.update_from_frame(frame)
        g_pool.image_tex.draw()
        gl_utils.make_coord_system_pixel_based(frame.img.shape)
        # render visual feedback from loaded plugins
        for p in g_pool.plugins:
            p.gl_display()

        graph.push_view()
        fps_graph.draw()
        cpu_graph.draw()
        pupil_graph.draw()
        graph.pop_view()
        g_pool.gui.update()

        # present frames at appropriate speed
        cap.wait(frame)

        glfwSwapBuffers(main_window)
        glfwPollEvents()

    session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
    session_settings['min_data_confidence'] = g_pool.min_data_confidence
    session_settings['gui_scale'] = g_pool.gui.scale
    session_settings['ui_config'] = g_pool.gui.configuration
    session_settings['window_size'] = glfwGetWindowSize(main_window)
    session_settings['window_position'] = glfwGetWindowPos(main_window)
    session_settings['version'] = g_pool.version
    session_settings.close()

    # de-init all running plugins
    for p in g_pool.plugins:
        p.alive = False
    g_pool.plugins.clean()

    cap.cleanup()
    g_pool.gui.terminate()
    glfwDestroyWindow(main_window)
Exemple #4
0
def session(rec_dir):

    system_plugins = [Log_Display, Seek_Bar, Trim_Marks]
    vis_plugins = sorted([
        Vis_Circle, Vis_Polyline, Vis_Light_Points, Vis_Cross, Vis_Watermark,
        Eye_Video_Overlay, Scan_Path
    ],
                         key=lambda x: x.__name__)
    analysis_plugins = sorted([
        Gaze_Position_2D_Fixation_Detector, Pupil_Angle_3D_Fixation_Detector,
        Manual_Gaze_Correction, Video_Export_Launcher, Offline_Surface_Tracker,
        Raw_Data_Exporter, Batch_Exporter, Annotation_Player
    ],
                              key=lambda x: x.__name__)
    other_plugins = sorted([Show_Calibration, Log_History],
                           key=lambda x: x.__name__)
    user_plugins = sorted(import_runtime_plugins(
        os.path.join(user_dir, 'plugins')),
                          key=lambda x: x.__name__)
    user_launchable_plugins = vis_plugins + analysis_plugins + other_plugins + user_plugins
    available_plugins = system_plugins + user_launchable_plugins
    name_by_index = [p.__name__ for p in available_plugins]
    index_by_name = dict(zip(name_by_index, range(len(name_by_index))))
    plugin_by_name = dict(zip(name_by_index, available_plugins))

    # Callback functions
    def on_resize(window, w, h):
        g_pool.gui.update_window(w, h)
        g_pool.gui.collect_menus()
        graph.adjust_size(w, h)
        adjust_gl_view(w, h)
        for p in g_pool.plugins:
            p.on_window_resize(window, w, h)

    def on_key(window, key, scancode, action, mods):
        g_pool.gui.update_key(key, scancode, action, mods)

    def on_char(window, char):
        g_pool.gui.update_char(char)

    def on_button(window, button, action, mods):
        g_pool.gui.update_button(button, action, mods)
        pos = glfwGetCursorPos(window)
        pos = normalize(pos, glfwGetWindowSize(window))
        pos = denormalize(
            pos,
            (frame.img.shape[1], frame.img.shape[0]))  # Position in img pixels
        for p in g_pool.plugins:
            p.on_click(pos, button, action)

    def on_pos(window, x, y):
        hdpi_factor = float(
            glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0])
        g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor)

    def on_scroll(window, x, y):
        g_pool.gui.update_scroll(x, y * y_scroll_factor)

    def on_drop(window, count, paths):
        for x in range(count):
            new_rec_dir = paths[x]
            if is_pupil_rec_dir(new_rec_dir):
                logger.debug("Starting new session with '%s'" % new_rec_dir)
                global rec_dir
                rec_dir = new_rec_dir
                glfwSetWindowShouldClose(window, True)
            else:
                logger.error("'%s' is not a valid pupil recording" %
                             new_rec_dir)

    tick = delta_t()

    def get_dt():
        return next(tick)

    update_recording_to_recent(rec_dir)

    video_path = [
        f for f in glob(os.path.join(rec_dir, "world.*"))
        if f[-3:] in ('mp4', 'mkv', 'avi')
    ][0]
    timestamps_path = os.path.join(rec_dir, "world_timestamps.npy")
    pupil_data_path = os.path.join(rec_dir, "pupil_data")

    meta_info = load_meta_info(rec_dir)
    rec_version = read_rec_version(meta_info)
    app_version = get_version(version_file)

    # log info about Pupil Platform and Platform in player.log
    logger.info('Application Version: %s' % app_version)
    logger.info('System Info: %s' % get_system_info())

    timestamps = np.load(timestamps_path)

    # create container for globally scoped vars
    g_pool = Global_Container()
    g_pool.app = 'player'

    # Initialize capture
    cap = File_Source(g_pool, video_path, timestamps=list(timestamps))

    # load session persistent settings
    session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings"))
    if session_settings.get("version",
                            VersionFormat('0.0')) < get_version(version_file):
        logger.info(
            "Session setting are from older version of this app. I will not use those."
        )
        session_settings.clear()

    width, height = session_settings.get('window_size', cap.frame_size)
    window_pos = session_settings.get('window_position', (0, 0))
    main_window = glfwCreateWindow(
        width, height, "Pupil Player: " + meta_info["Recording Name"] + " - " +
        rec_dir.split(os.path.sep)[-1], None, None)
    glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
    glfwMakeContextCurrent(main_window)
    cygl.utils.init()

    # load pupil_positions, gaze_positions
    pupil_data = load_object(pupil_data_path)
    pupil_list = pupil_data['pupil_positions']
    gaze_list = pupil_data['gaze_positions']

    g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular'
    g_pool.version = app_version
    g_pool.capture = cap
    g_pool.timestamps = timestamps
    g_pool.play = False
    g_pool.new_seek = True
    g_pool.user_dir = user_dir
    g_pool.rec_dir = rec_dir
    g_pool.rec_version = rec_version
    g_pool.meta_info = meta_info
    g_pool.min_data_confidence = session_settings.get('min_data_confidence',
                                                      0.6)
    g_pool.pupil_positions_by_frame = correlate_data(pupil_list,
                                                     g_pool.timestamps)
    g_pool.gaze_positions_by_frame = correlate_data(gaze_list,
                                                    g_pool.timestamps)
    g_pool.fixations_by_frame = [[] for x in g_pool.timestamps
                                 ]  #populated by the fixation detector plugin

    def next_frame(_):
        try:
            cap.seek_to_frame(cap.get_frame_index())
        except FileSeekError:
            logger.warning("Could not seek to next frame.")
        else:
            g_pool.new_seek = True

    def prev_frame(_):
        try:
            cap.seek_to_frame(cap.get_frame_index() - 2)
        except FileSeekError:
            logger.warning("Could not seek to previous frame.")
        else:
            g_pool.new_seek = True

    def toggle_play(new_state):
        if cap.get_frame_index() >= cap.get_frame_count() - 5:
            cap.seek_to_frame(1)  #avoid pause set by hitting trimmark pause.
            logger.warning("End of video - restart at beginning.")
        g_pool.play = new_state

    def set_scale(new_scale):
        g_pool.gui.scale = new_scale
        g_pool.gui.collect_menus()

    def set_data_confidence(new_confidence):
        g_pool.min_data_confidence = new_confidence
        notification = {'subject': 'min_data_confidence_changed'}
        notification['_notify_time_'] = time() + .8
        g_pool.delayed_notifications[notification['subject']] = notification

    def open_plugin(plugin):
        if plugin == "Select to load":
            return
        g_pool.plugins.add(plugin)

    def purge_plugins():
        for p in g_pool.plugins:
            if p.__class__ in user_launchable_plugins:
                p.alive = False
        g_pool.plugins.clean()

    def do_export(_):
        export_range = slice(g_pool.trim_marks.in_mark,
                             g_pool.trim_marks.out_mark)
        export_dir = os.path.join(
            g_pool.rec_dir, 'exports',
            '%s-%s' % (export_range.start, export_range.stop))
        try:
            os.makedirs(export_dir)
        except OSError as e:
            if e.errno != errno.EEXIST:
                logger.error("Could not create export dir")
                raise e
            else:
                logger.warning(
                    "Previous export for range [%s-%s] already exsits - overwriting."
                    % (export_range.start, export_range.stop))
        else:
            logger.info('Created export dir at "%s"' % export_dir)

        notification = {
            'subject': 'should_export',
            'range': export_range,
            'export_dir': export_dir
        }
        g_pool.notifications.append(notification)

    g_pool.gui = ui.UI()
    g_pool.gui.scale = session_settings.get('gui_scale', 1)
    g_pool.main_menu = ui.Scrolling_Menu("Settings",
                                         pos=(-350, 20),
                                         size=(300, 500))
    g_pool.main_menu.append(
        ui.Button("Close Pupil Player",
                  lambda: glfwSetWindowShouldClose(main_window, True)))
    g_pool.main_menu.append(
        ui.Slider('scale',
                  g_pool.gui,
                  setter=set_scale,
                  step=.05,
                  min=0.75,
                  max=2.5,
                  label='Interface Size'))
    g_pool.main_menu.append(ui.Info_Text('Player Version: %s' %
                                         g_pool.version))
    g_pool.main_menu.append(ui.Info_Text('Recording Version: %s' %
                                         rec_version))
    g_pool.main_menu.append(
        ui.Slider('min_data_confidence',
                  g_pool,
                  setter=set_data_confidence,
                  step=.05,
                  min=0.0,
                  max=1.0,
                  label='Confidence threshold'))

    selector_label = "Select to load"

    vis_labels = ["   " + p.__name__.replace('_', ' ') for p in vis_plugins]
    analysis_labels = [
        "   " + p.__name__.replace('_', ' ') for p in analysis_plugins
    ]
    other_labels = [
        "   " + p.__name__.replace('_', ' ') for p in other_plugins
    ]
    user_labels = ["   " + p.__name__.replace('_', ' ') for p in user_plugins]

    plugins = [selector_label, selector_label] + vis_plugins + [
        selector_label
    ] + analysis_plugins + [selector_label] + other_plugins + [selector_label
                                                               ] + user_plugins
    labels = [selector_label, "Visualization"] + vis_labels + [
        "Analysis"
    ] + analysis_labels + ["Other"] + other_labels + ["User added"
                                                      ] + user_labels

    g_pool.main_menu.append(
        ui.Selector('Open plugin:',
                    selection=plugins,
                    labels=labels,
                    setter=open_plugin,
                    getter=lambda: selector_label))

    g_pool.main_menu.append(ui.Button('Close all plugins', purge_plugins))
    g_pool.main_menu.append(
        ui.Button(
            'Reset window size', lambda: glfwSetWindowSize(
                main_window, cap.frame_size[0], cap.frame_size[1])))
    g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))
    g_pool.play_button = ui.Thumb('play',
                                  g_pool,
                                  label=unichr(0xf04b).encode('utf-8'),
                                  setter=toggle_play,
                                  hotkey=GLFW_KEY_SPACE,
                                  label_font='fontawesome',
                                  label_offset_x=5,
                                  label_offset_y=0,
                                  label_offset_size=-24)
    g_pool.play_button.on_color[:] = (0, 1., .0, .8)
    g_pool.forward_button = ui.Thumb('forward',
                                     label=unichr(0xf04e).encode('utf-8'),
                                     getter=lambda: False,
                                     setter=next_frame,
                                     hotkey=GLFW_KEY_RIGHT,
                                     label_font='fontawesome',
                                     label_offset_x=5,
                                     label_offset_y=0,
                                     label_offset_size=-24)
    g_pool.backward_button = ui.Thumb('backward',
                                      label=unichr(0xf04a).encode('utf-8'),
                                      getter=lambda: False,
                                      setter=prev_frame,
                                      hotkey=GLFW_KEY_LEFT,
                                      label_font='fontawesome',
                                      label_offset_x=-5,
                                      label_offset_y=0,
                                      label_offset_size=-24)
    g_pool.export_button = ui.Thumb('export',
                                    label=unichr(0xf063).encode('utf-8'),
                                    getter=lambda: False,
                                    setter=do_export,
                                    hotkey='e',
                                    label_font='fontawesome',
                                    label_offset_x=0,
                                    label_offset_y=2,
                                    label_offset_size=-24)
    g_pool.quickbar.extend([
        g_pool.play_button, g_pool.forward_button, g_pool.backward_button,
        g_pool.export_button
    ])
    g_pool.gui.append(g_pool.quickbar)
    g_pool.gui.append(g_pool.main_menu)

    #we always load these plugins
    system_plugins = [('Trim_Marks', {}), ('Seek_Bar', {})]
    default_plugins = [('Log_Display', {}), ('Scan_Path', {}),
                       ('Vis_Polyline', {}), ('Vis_Circle', {}),
                       ('Video_Export_Launcher', {})]
    previous_plugins = session_settings.get('loaded_plugins', default_plugins)
    g_pool.notifications = []
    g_pool.delayed_notifications = {}
    g_pool.plugins = Plugin_List(g_pool, plugin_by_name,
                                 system_plugins + previous_plugins)

    # Register callbacks main_window
    glfwSetFramebufferSizeCallback(main_window, on_resize)
    glfwSetKeyCallback(main_window, on_key)
    glfwSetCharCallback(main_window, on_char)
    glfwSetMouseButtonCallback(main_window, on_button)
    glfwSetCursorPosCallback(main_window, on_pos)
    glfwSetScrollCallback(main_window, on_scroll)
    glfwSetDropCallback(main_window, on_drop)
    #trigger on_resize
    on_resize(main_window, *glfwGetFramebufferSize(main_window))

    g_pool.gui.configuration = session_settings.get('ui_config', {})

    # gl_state settings
    basic_gl_setup()
    g_pool.image_tex = Named_Texture()

    #set up performace graphs:
    pid = os.getpid()
    ps = psutil.Process(pid)
    ts = None

    cpu_graph = graph.Bar_Graph()
    cpu_graph.pos = (20, 110)
    cpu_graph.update_fn = ps.cpu_percent
    cpu_graph.update_rate = 5
    cpu_graph.label = 'CPU %0.1f'

    fps_graph = graph.Bar_Graph()
    fps_graph.pos = (140, 110)
    fps_graph.update_rate = 5
    fps_graph.label = "%0.0f REC FPS"

    pupil_graph = graph.Bar_Graph(max_val=1.0)
    pupil_graph.pos = (260, 110)
    pupil_graph.update_rate = 5
    pupil_graph.label = "Confidence: %0.2f"

    while not glfwWindowShouldClose(main_window):

        #grab new frame
        if g_pool.play or g_pool.new_seek:
            g_pool.new_seek = False
            try:
                new_frame = cap.get_frame_nowait()
            except EndofVideoFileError:
                #end of video logic: pause at last frame.
                g_pool.play = False
                logger.warning("end of video")
            update_graph = True
        else:
            update_graph = False

        frame = new_frame.copy()
        events = {}
        #report time between now and the last loop interation
        events['dt'] = get_dt()
        #new positons we make a deepcopy just like the image is a copy.
        events['gaze_positions'] = deepcopy(
            g_pool.gaze_positions_by_frame[frame.index])
        events['pupil_positions'] = deepcopy(
            g_pool.pupil_positions_by_frame[frame.index])

        if update_graph:
            #update performace graphs
            for p in events['pupil_positions']:
                pupil_graph.add(p['confidence'])

            t = new_frame.timestamp
            if ts and ts != t:
                dt, ts = t - ts, t
                fps_graph.add(1. / dt)
            else:
                ts = new_frame.timestamp

            g_pool.play_button.status_text = str(frame.index)
        #always update the CPU graph
        cpu_graph.update()

        # publish delayed notifiactions when their time has come.
        for n in g_pool.delayed_notifications.values():
            if n['_notify_time_'] < time():
                del n['_notify_time_']
                del g_pool.delayed_notifications[n['subject']]
                g_pool.notifications.append(n)

        # notify each plugin if there are new notifactions:
        while g_pool.notifications:
            n = g_pool.notifications.pop(0)
            for p in g_pool.plugins:
                p.on_notify(n)

        # allow each Plugin to do its work.
        for p in g_pool.plugins:
            p.update(frame, events)

        #check if a plugin need to be destroyed
        g_pool.plugins.clean()

        # render camera image
        glfwMakeContextCurrent(main_window)
        make_coord_system_norm_based()
        g_pool.image_tex.update_from_frame(frame)
        g_pool.image_tex.draw()
        make_coord_system_pixel_based(frame.img.shape)
        # render visual feedback from loaded plugins
        for p in g_pool.plugins:
            p.gl_display()

        graph.push_view()
        fps_graph.draw()
        cpu_graph.draw()
        pupil_graph.draw()
        graph.pop_view()
        g_pool.gui.update()

        #present frames at appropriate speed
        cap.wait(frame)

        glfwSwapBuffers(main_window)
        glfwPollEvents()

    session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
    session_settings['min_data_confidence'] = g_pool.min_data_confidence
    session_settings['gui_scale'] = g_pool.gui.scale
    session_settings['ui_config'] = g_pool.gui.configuration
    session_settings['window_size'] = glfwGetWindowSize(main_window)
    session_settings['window_position'] = glfwGetWindowPos(main_window)
    session_settings['version'] = g_pool.version
    session_settings.close()

    # de-init all running plugins
    for p in g_pool.plugins:
        p.alive = False
    g_pool.plugins.clean()

    cap.cleanup()
    g_pool.gui.terminate()
    glfwDestroyWindow(main_window)