Exemplo n.º 1
0
    def start(self):
        self.timestamps = []
        self.data = {'pupil_positions':[],'gaze_positions':[]}
        self.pupil_pos_list = []
        self.gaze_pos_list = []

        self.frame_count = 0
        self.running = True
        self.menu.read_only = True
        self.start_time = time()

        session = os.path.join(self.rec_dir, self.session_name)
        try:
            os.makedirs(session)
            logger.debug("Created new recordings session dir %s"%session)

        except:
            logger.debug("Recordings session dir %s already exists, using it." %session)

        # set up self incrementing folder within session folder
        counter = 0
        while True:
            self.rec_path = os.path.join(session, "%03d/" % counter)
            try:
                os.mkdir(self.rec_path)
                logger.debug("Created new recording dir %s"%self.rec_path)
                break
            except:
                logger.debug("We dont want to overwrite data, incrementing counter & trying to make new data folder")
                counter += 1

        self.meta_info_path = os.path.join(self.rec_path, "info.csv")

        with open(self.meta_info_path, 'w') as f:
            f.write("Recording Name\t"+self.session_name+ "\n")
            f.write("Start Date\t"+ strftime("%d.%m.%Y", localtime(self.start_time))+ "\n")
            f.write("Start Time\t"+ strftime("%H:%M:%S", localtime(self.start_time))+ "\n")


        if self.audio_src != 'No Audio':
            audio_path = os.path.join(self.rec_path, "world.wav")
            self.audio_writer = Audio_Capture(self.audio_devices_dict[self.audio_src],audio_path)
        else:
            self.audio_writer = None

        if self.raw_jpeg  and "uvc_capture" in str(self.g_pool.capture.__class__):
            self.video_path = os.path.join(self.rec_path, "world.mp4")
            self.writer = JPEG_Writer(self.video_path,int(self.g_pool.capture.frame_rate))
        # elif 1:
        #     self.writer = av_writer.AV_Writer(self.video_path)
        else:
            self.video_path = os.path.join(self.rec_path, "world.mkv")
            self.writer = CV_Writer(self.video_path, float(self.g_pool.capture.frame_rate), self.g_pool.capture.frame_size)
        # positions path to eye process
        if self.record_eye:
            for tx in self.g_pool.eye_tx:
                tx.send((self.rec_path,self.raw_jpeg))

        if self.show_info_menu:
            self.open_info_menu()
Exemplo n.º 2
0
    def start(self):
        self.timestamps = []
        self.data = {'pupil_positions':[],'gaze_positions':[]}
        self.pupil_pos_list = []
        self.gaze_pos_list = []

        self.frame_count = 0
        self.running = True
        self.menu.read_only = True
        self.start_time = time()

        session = os.path.join(self.rec_dir, self.session_name)
        try:
            os.makedirs(session)
            logger.debug("Created new recordings session dir %s"%session)

        except:
            logger.debug("Recordings session dir %s already exists, using it." %session)

        # set up self incrementing folder within session folder
        counter = 0
        while True:
            self.rec_path = os.path.join(session, "%03d/" % counter)
            try:
                os.mkdir(self.rec_path)
                logger.debug("Created new recording dir %s"%self.rec_path)
                break
            except:
                logger.debug("We dont want to overwrite data, incrementing counter & trying to make new data folder")
                counter += 1

        self.meta_info_path = os.path.join(self.rec_path, "info.csv")

        with open(self.meta_info_path, 'w') as f:
            f.write("Recording Name\t"+self.session_name+ "\n")
            f.write("Start Date\t"+ strftime("%d.%m.%Y", localtime(self.start_time))+ "\n")
            f.write("Start Time\t"+ strftime("%H:%M:%S", localtime(self.start_time))+ "\n")


        if self.audio_src != 'No Audio':
            audio_path = os.path.join(self.rec_path, "world.wav")
            self.audio_writer = Audio_Capture(self.audio_devices_dict[self.audio_src],audio_path)
        else:
            self.audio_writer = None

        self.video_path = os.path.join(self.rec_path, "world.mkv")
        if self.raw_jpeg  and "uvc_capture" in str(self.g_pool.capture.__class__):
            self.writer = JPEG_Dumper(self.video_path)
        # elif 1:
        #     self.writer = av_writer.AV_Writer(self.video_path)
        else:
            self.writer = CV_Writer(self.video_path, float(self.g_pool.capture.frame_rate), self.g_pool.capture.frame_size)
        # positions path to eye process
        if self.record_eye:
            for tx in self.g_pool.eye_tx:
                tx.send((self.rec_path,self.raw_jpeg))

        if self.show_info_menu:
            self.open_info_menu()
Exemplo n.º 3
0
def eye(g_pool, cap_src, cap_size, pipe_to_world, eye_id=0):
    """
    Creates a window, gl context.
    Grabs images from a capture.
    Streams Pupil coordinates into g_pool.pupil_queue
    """

    # modify the root logger for this process
    logger = logging.getLogger()
    # remove inherited handlers
    logger.handlers = []
    # create file handler which logs even debug messages
    fh = logging.FileHandler(os.path.join(g_pool.user_dir,
                                          'eye%s.log' % eye_id),
                             mode='w')
    # fh.setLevel(logging.DEBUG)
    # create console handler with a higher log level
    ch = logging.StreamHandler()
    ch.setLevel(logger.level + 10)
    # create formatter and add it to the handlers
    formatter = logging.Formatter(
        'Eye' + str(eye_id) +
        ' Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s')
    fh.setFormatter(formatter)
    formatter = logging.Formatter(
        'EYE' + str(eye_id) +
        ' Process [%(levelname)s] %(name)s : %(message)s')
    ch.setFormatter(formatter)
    # add the handlers to the logger
    logger.addHandler(fh)
    logger.addHandler(ch)
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    #UI Platform tweaks
    if platform.system() == 'Linux':
        scroll_factor = 10.0
        window_position_default = (600, 300 * eye_id)
    elif platform.system() == 'Windows':
        scroll_factor = 1.0
        window_position_default = (600, 31 + 300 * eye_id)
    else:
        scroll_factor = 1.0
        window_position_default = (600, 300 * eye_id)

    # Callback functions
    def on_resize(window, w, h):
        if not g_pool.iconified:
            active_window = glfwGetCurrentContext()
            glfwMakeContextCurrent(window)
            g_pool.gui.update_window(w, h)
            graph.adjust_size(w, h)
            adjust_gl_view(w, h)
            glfwMakeContextCurrent(active_window)

    def on_key(window, key, scancode, action, mods):
        g_pool.gui.update_key(key, scancode, action, mods)

    def on_char(window, char):
        g_pool.gui.update_char(char)

    def on_iconify(window, iconified):
        g_pool.iconified = iconified

    def on_button(window, button, action, mods):
        if g_pool.display_mode == 'roi':
            if action == GLFW_RELEASE and u_r.active_edit_pt:
                u_r.active_edit_pt = False
                return  # if the roi interacts we dont what the gui to interact as well
            elif action == GLFW_PRESS:
                pos = glfwGetCursorPos(window)
                pos = normalize(pos, glfwGetWindowSize(main_window))
                if g_pool.flip:
                    pos = 1 - pos[0], 1 - pos[1]
                pos = denormalize(
                    pos, (frame.width, frame.height))  # Position in img pixels
                if u_r.mouse_over_edit_pt(pos, u_r.handle_size + 40,
                                          u_r.handle_size + 40):
                    return  # if the roi interacts we dont what the gui to interact as well

        g_pool.gui.update_button(button, action, mods)

    def on_pos(window, x, y):
        hdpi_factor = float(
            glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0])
        g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor)

        if u_r.active_edit_pt:
            pos = normalize((x, y), glfwGetWindowSize(main_window))
            if g_pool.flip:
                pos = 1 - pos[0], 1 - pos[1]
            pos = denormalize(pos, (frame.width, frame.height))
            u_r.move_vertex(u_r.active_pt_idx, pos)

    def on_scroll(window, x, y):
        g_pool.gui.update_scroll(x, y * scroll_factor)

    def on_close(window):
        g_pool.quit.value = True
        logger.info('Process closing from window')

    # load session persistent settings
    session_settings = Persistent_Dict(
        os.path.join(g_pool.user_dir, 'user_settings_eye%s' % eye_id))
    if session_settings.get("version", VersionFormat('0.0')) < g_pool.version:
        logger.info(
            "Session setting are from older version of this app. I will not use those."
        )
        session_settings.clear()
    # Initialize capture
    cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
    default_settings = {'frame_size': cap_size, 'frame_rate': 30}
    previous_settings = session_settings.get('capture_settings', None)
    if previous_settings and previous_settings['name'] == cap.name:
        cap.settings = previous_settings
    else:
        cap.settings = default_settings

    # Test capture
    try:
        frame = cap.get_frame()
    except CameraCaptureError:
        logger.error("Could not retrieve image from capture")
        cap.close()
        return

    #signal world that we are ready to go
    pipe_to_world.send('eye%s process ready' % eye_id)

    # any object we attach to the g_pool object *from now on* will only be visible to this process!
    # vars should be declared here to make them visible to the code reader.
    g_pool.iconified = False
    g_pool.capture = cap
    g_pool.flip = session_settings.get('flip', False)
    g_pool.display_mode = session_settings.get('display_mode', 'camera_image')
    g_pool.display_mode_info_text = {
        'camera_image':
        "Raw eye camera image. This uses the least amount of CPU power",
        'roi':
        "Click and drag on the blue circles to adjust the region of interest. The region should be a small as possible but big enough to capture to pupil in its movements",
        'algorithm':
        "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters with in the Pupil Detection menu below."
    }
    # g_pool.draw_pupil = session_settings.get('draw_pupil',True)

    u_r = UIRoi(frame.img.shape)
    u_r.set(session_settings.get('roi', u_r.get()))

    writer = None

    pupil_detector = Canny_Detector(g_pool)

    # UI callback functions
    def set_scale(new_scale):
        g_pool.gui.scale = new_scale
        g_pool.gui.collect_menus()

    def set_display_mode_info(val):
        g_pool.display_mode = val
        g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]

    # Initialize glfw
    glfwInit()
    if g_pool.binocular:
        title = "Binocular eye %s" % eye_id
    else:
        title = 'Eye'
    width, height = session_settings.get('window_size',
                                         (frame.width, frame.height))
    main_window = glfwCreateWindow(width, height, title, None, None)
    window_pos = session_settings.get('window_position',
                                      window_position_default)
    glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
    glfwMakeContextCurrent(main_window)
    cygl_init()

    # gl_state settings
    basic_gl_setup()
    g_pool.image_tex = Named_Texture()
    g_pool.image_tex.update_from_frame(frame)
    glfwSwapInterval(0)

    #setup GUI
    g_pool.gui = ui.UI()
    g_pool.gui.scale = session_settings.get('gui_scale', 1)
    g_pool.sidebar = ui.Scrolling_Menu("Settings",
                                       pos=(-300, 0),
                                       size=(0, 0),
                                       header_pos='left')
    general_settings = ui.Growing_Menu('General')
    general_settings.append(
        ui.Slider('scale',
                  g_pool.gui,
                  setter=set_scale,
                  step=.05,
                  min=1.,
                  max=2.5,
                  label='Interface Size'))
    general_settings.append(
        ui.Button(
            'Reset window size',
            lambda: glfwSetWindowSize(main_window, frame.width, frame.height)))
    general_settings.append(
        ui.Selector('display_mode',
                    g_pool,
                    setter=set_display_mode_info,
                    selection=['camera_image', 'roi', 'algorithm'],
                    labels=['Camera Image', 'ROI', 'Algorithm'],
                    label="Mode"))
    general_settings.append(
        ui.Switch('flip', g_pool, label='Flip image display'))
    g_pool.display_mode_info = ui.Info_Text(
        g_pool.display_mode_info_text[g_pool.display_mode])
    general_settings.append(g_pool.display_mode_info)
    g_pool.sidebar.append(general_settings)
    g_pool.gui.append(g_pool.sidebar)
    g_pool.gui.append(
        ui.Hot_Key("quit",
                   setter=on_close,
                   getter=lambda: True,
                   label="X",
                   hotkey=GLFW_KEY_ESCAPE))
    # let the camera add its GUI
    g_pool.capture.init_gui(g_pool.sidebar)
    # let detector add its GUI
    pupil_detector.init_gui(g_pool.sidebar)

    # Register callbacks main_window
    glfwSetFramebufferSizeCallback(main_window, on_resize)
    glfwSetWindowCloseCallback(main_window, on_close)
    glfwSetWindowIconifyCallback(main_window, on_iconify)
    glfwSetKeyCallback(main_window, on_key)
    glfwSetCharCallback(main_window, on_char)
    glfwSetMouseButtonCallback(main_window, on_button)
    glfwSetCursorPosCallback(main_window, on_pos)
    glfwSetScrollCallback(main_window, on_scroll)

    #set the last saved window size
    on_resize(main_window, *glfwGetWindowSize(main_window))

    # load last gui configuration
    g_pool.gui.configuration = session_settings.get('ui_config', {})

    #set up performance graphs
    pid = os.getpid()
    ps = psutil.Process(pid)
    ts = frame.timestamp

    cpu_graph = graph.Bar_Graph()
    cpu_graph.pos = (20, 130)
    cpu_graph.update_fn = ps.cpu_percent
    cpu_graph.update_rate = 5
    cpu_graph.label = 'CPU %0.1f'

    fps_graph = graph.Bar_Graph()
    fps_graph.pos = (140, 130)
    fps_graph.update_rate = 5
    fps_graph.label = "%0.0f FPS"

    #create a timer to control window update frequency
    window_update_timer = timer(1 / 60.)

    def window_should_update():
        return next(window_update_timer)

    # Event loop
    while not g_pool.quit.value:
        # Get an image from the grabber
        try:
            frame = cap.get_frame()
        except CameraCaptureError:
            logger.error("Capture from Camera Failed. Stopping.")
            break
        except EndofVideoFileError:
            logger.warning("Video File is done. Stopping")
            break

        #update performace graphs
        t = frame.timestamp
        dt, ts = t - ts, t
        try:
            fps_graph.add(1. / dt)
        except ZeroDivisionError:
            pass
        cpu_graph.update()

        ###  RECORDING of Eye Video (on demand) ###
        # Setup variables and lists for recording
        if pipe_to_world.poll():
            command, raw_mode = pipe_to_world.recv()
            if command is not None:
                record_path = command
                logger.info("Will save eye video to: %s" % record_path)
                timestamps_path = os.path.join(record_path,
                                               "eye%s_timestamps.npy" % eye_id)
                if raw_mode and hasattr(frame, 'jpeg_buffer'):
                    video_path = os.path.join(record_path,
                                              "eye%s.mp4" % eye_id)
                    writer = JPEG_Writer(video_path, cap.frame_rate)
                else:
                    video_path = os.path.join(record_path,
                                              "eye%s.mkv" % eye_id)
                    writer = CV_Writer(video_path, float(cap.frame_rate),
                                       cap.frame_size)
                timestamps = []
            else:
                logger.info("Done recording.")
                writer.release()
                writer = None
                np.save(timestamps_path, np.asarray(timestamps))
                del timestamps

        if writer:
            writer.write_video_frame(frame)
            timestamps.append(frame.timestamp)

        # pupil ellipse detection
        result = pupil_detector.detect(
            frame, user_roi=u_r, visualize=g_pool.display_mode == 'algorithm')
        result['id'] = eye_id
        # stream the result
        g_pool.pupil_queue.put(result)

        # GL drawing
        if window_should_update():
            if not g_pool.iconified:
                glfwMakeContextCurrent(main_window)
                clear_gl_screen()

                # switch to work in normalized coordinate space
                if g_pool.display_mode == 'algorithm':
                    g_pool.image_tex.update_from_ndarray(frame.img)
                elif g_pool.display_mode in ('camera_image', 'roi'):
                    g_pool.image_tex.update_from_ndarray(frame.gray)
                else:
                    pass

                make_coord_system_norm_based(g_pool.flip)
                g_pool.image_tex.draw()
                # switch to work in pixel space
                make_coord_system_pixel_based((frame.height, frame.width, 3),
                                              g_pool.flip)

                if result['confidence'] > 0:
                    if result.has_key('axes'):
                        pts = cv2.ellipse2Poly((int(
                            result['center'][0]), int(result['center'][1])),
                                               (int(result['axes'][0] / 2),
                                                int(result['axes'][1] / 2)),
                                               int(result['angle']), 0, 360,
                                               15)
                        cygl_draw_polyline(pts, 1, cygl_rgba(1., 0, 0, .5))
                    cygl_draw_points([result['center']],
                                     size=20,
                                     color=cygl_rgba(1., 0., 0., .5),
                                     sharpness=1.)

                # render graphs
                graph.push_view()
                fps_graph.draw()
                cpu_graph.draw()
                graph.pop_view()

                # render GUI
                g_pool.gui.update()

                #render the ROI
                if g_pool.display_mode == 'roi':
                    u_r.draw(g_pool.gui.scale)

                #update screen
                glfwSwapBuffers(main_window)
            glfwPollEvents()

    # END while running

    # in case eye recording was still runnnig: Save&close
    if writer:
        logger.info("Done recording eye.")
        writer = None
        np.save(timestamps_path, np.asarray(timestamps))

    glfwRestoreWindow(main_window)  #need to do this for windows os
    # save session persistent settings
    session_settings['gui_scale'] = g_pool.gui.scale
    session_settings['roi'] = u_r.get()
    session_settings['flip'] = g_pool.flip
    session_settings['display_mode'] = g_pool.display_mode
    session_settings['ui_config'] = g_pool.gui.configuration
    session_settings['capture_settings'] = g_pool.capture.settings
    session_settings['window_size'] = glfwGetWindowSize(main_window)
    session_settings['window_position'] = glfwGetWindowPos(main_window)
    session_settings['version'] = g_pool.version
    session_settings.close()

    pupil_detector.cleanup()
    g_pool.gui.terminate()
    glfwDestroyWindow(main_window)
    glfwTerminate()
    cap.close()

    #flushing queue in case world process did not exit gracefully
    while not g_pool.pupil_queue.empty():
        g_pool.pupil_queue.get()
    g_pool.pupil_queue.close()

    logger.debug("Process done")
Exemplo n.º 4
0
class Recorder(Plugin):
    """Capture Recorder"""

    def __init__(
        self,
        g_pool,
        session_name=get_auto_name(),
        rec_dir=None,
        user_info={"name": "", "additional_field": "change_me"},
        info_menu_conf={},
        show_info_menu=False,
        record_eye=False,
        audio_src="No Audio",
        raw_jpeg=False,
    ):
        super(Recorder, self).__init__(g_pool)
        # update name if it was autogenerated.
        if session_name.startswith("20") and len(session_name) == 10:
            session_name = get_auto_name()

        if rec_dir:
            self.set_rec_dir(rec_dir)
        else:
            # lets make a rec dir next to the user dir
            base_dir = self.g_pool.user_dir.rsplit(os.path.sep, 1)[0]
            self.rec_dir = os.path.join(base_dir, "recordings")
            if not os.path.isdir(self.rec_dir):
                os.mkdir(self.rec_dir)

        self.raw_jpeg = raw_jpeg
        self.order = 0.9
        self.record_eye = record_eye
        self.session_name = session_name
        self.audio_devices_dict = Audio_Input_Dict()
        if audio_src in self.audio_devices_dict.keys():
            self.audio_src = audio_src
        else:
            self.audio_src = "No Audio"
        self.running = False
        self.menu = None
        self.button = None

        self.user_info = user_info
        self.show_info_menu = show_info_menu
        self.info_menu = None
        self.info_menu_conf = info_menu_conf
        self.height, self.width = self.g_pool.capture.frame_size

    def get_init_dict(self):
        d = {}
        d["record_eye"] = self.record_eye
        d["audio_src"] = self.audio_src
        d["session_name"] = self.session_name
        d["user_info"] = self.user_info
        d["info_menu_conf"] = self.info_menu_conf
        d["show_info_menu"] = self.show_info_menu
        d["rec_dir"] = self.rec_dir
        d["raw_jpeg"] = self.raw_jpeg
        return d

    def init_gui(self):
        self.menu = ui.Growing_Menu("Recorder")
        self.g_pool.sidebar.insert(3, self.menu)
        self.menu.append(
            ui.Info_Text(
                'Pupil recordings are saved like this: "path_to_recordings/recording_session_name/nnn" where "nnn" is an increasing number to avoid overwrites. You can use "/" in your session name to create subdirectories.'
            )
        )
        self.menu.append(
            ui.Info_Text(
                'Recordings are saved to "~/pupil_recordings". You can change the path here but note that invalid input will be ignored.'
            )
        )
        self.menu.append(ui.Text_Input("rec_dir", self, setter=self.set_rec_dir, label="Path to recordings"))
        self.menu.append(
            ui.Text_Input("session_name", self, setter=self.set_session_name, label="Recording session name")
        )
        self.menu.append(
            ui.Switch("show_info_menu", self, on_val=True, off_val=False, label="Request additional user info")
        )
        self.menu.append(
            ui.Selector(
                "raw_jpeg",
                self,
                selection=[True, False],
                labels=["bigger file, less CPU", "smaller file, more CPU"],
                label="compression",
            )
        )
        self.menu.append(ui.Info_Text("Recording the raw eye video is optional. We use it for debugging."))
        self.menu.append(ui.Switch("record_eye", self, on_val=True, off_val=False, label="Record eye"))
        self.menu.append(ui.Selector("audio_src", self, selection=self.audio_devices_dict.keys()))

        self.button = ui.Thumb("running", self, setter=self.toggle, label="Record", hotkey="r")
        self.button.on_color[:] = (1, 0.0, 0.0, 0.8)
        self.g_pool.quickbar.insert(1, self.button)

    def deinit_gui(self):
        if self.menu:
            self.g_pool.sidebar.remove(self.menu)
            self.menu = None
        if self.button:
            self.g_pool.quickbar.remove(self.button)
            self.button = None

    def toggle(self, _=None):
        if self.running:
            self.stop()
        else:
            self.start()

    def get_rec_time_str(self):
        rec_time = gmtime(time() - self.start_time)
        return strftime("%H:%M:%S", rec_time)

    def start(self):
        self.timestamps = []
        self.data = {"pupil_positions": [], "gaze_positions": []}
        self.pupil_pos_list = []
        self.gaze_pos_list = []

        self.frame_count = 0
        self.running = True
        self.menu.read_only = True
        self.start_time = time()

        session = os.path.join(self.rec_dir, self.session_name)
        try:
            os.makedirs(session)
            logger.debug("Created new recordings session dir %s" % session)

        except:
            logger.debug("Recordings session dir %s already exists, using it." % session)

        # set up self incrementing folder within session folder
        counter = 0
        while True:
            self.rec_path = os.path.join(session, "%03d/" % counter)
            try:
                os.mkdir(self.rec_path)
                logger.debug("Created new recording dir %s" % self.rec_path)
                break
            except:
                logger.debug("We dont want to overwrite data, incrementing counter & trying to make new data folder")
                counter += 1

        self.meta_info_path = os.path.join(self.rec_path, "info.csv")

        with open(self.meta_info_path, "w") as f:
            f.write("Recording Name\t" + self.session_name + "\n")
            f.write("Start Date\t" + strftime("%d.%m.%Y", localtime(self.start_time)) + "\n")
            f.write("Start Time\t" + strftime("%H:%M:%S", localtime(self.start_time)) + "\n")

        if self.audio_src != "No Audio":
            audio_path = os.path.join(self.rec_path, "world.wav")
            self.audio_writer = Audio_Capture(self.audio_devices_dict[self.audio_src], audio_path)
        else:
            self.audio_writer = None

        if self.raw_jpeg and "uvc_capture" in str(self.g_pool.capture.__class__):
            self.video_path = os.path.join(self.rec_path, "world.mp4")
            self.writer = JPEG_Writer(self.video_path, int(self.g_pool.capture.frame_rate))
        # elif 1:
        #     self.writer = av_writer.AV_Writer(self.video_path)
        else:
            self.video_path = os.path.join(self.rec_path, "world.mkv")
            self.writer = CV_Writer(
                self.video_path, float(self.g_pool.capture.frame_rate), self.g_pool.capture.frame_size
            )
        # positions path to eye process
        if self.record_eye:
            for tx in self.g_pool.eye_tx:
                tx.send((self.rec_path, self.raw_jpeg))

        if self.show_info_menu:
            self.open_info_menu()

    def open_info_menu(self):
        self.info_menu = ui.Growing_Menu("additional Recording Info", size=(300, 300), pos=(300, 300))
        self.info_menu.configuration = self.info_menu_conf

        def populate_info_menu():
            self.info_menu.elements[:-2] = []
            for name in self.user_info.iterkeys():
                self.info_menu.insert(0, ui.Text_Input(name, self.user_info))

        def set_user_info(new_string):
            self.user_info = new_string
            populate_info_menu()

        populate_info_menu()
        self.info_menu.append(
            ui.Info_Text(
                'Use the *user info* field to add/remove additional fields and their values. The format must be a valid Python dictionary. For example -- {"key":"value"}. You can add as many fields as you require. Your custom fields will be saved for your next session.'
            )
        )
        self.info_menu.append(ui.Text_Input("user_info", self, setter=set_user_info, label="User info"))
        self.g_pool.gui.append(self.info_menu)

    def close_info_menu(self):
        if self.info_menu:
            self.info_menu_conf = self.info_menu.configuration
            self.g_pool.gui.remove(self.info_menu)
            self.info_menu = None

    def update(self, frame, events):
        if self.running:
            self.data["pupil_positions"] += events["pupil_positions"]
            self.data["gaze_positions"] += events["gaze_positions"]
            self.timestamps.append(frame.timestamp)
            self.writer.write_video_frame(frame)
            # self.writer.write_video_frame_yuv422(frame)
            self.frame_count += 1

            # cv2.putText(frame.img, "Frame %s"%self.frame_count,(200,200), cv2.FONT_HERSHEY_SIMPLEX,1,(255,100,100))
            for p in events["pupil_positions"]:
                pupil_pos = p["timestamp"], p["confidence"], p["id"], p["norm_pos"][0], p["norm_pos"][1], p["diameter"]
                self.pupil_pos_list.append(pupil_pos)

            for g in events.get("gaze_positions", []):
                gaze_pos = g["timestamp"], g["confidence"], g["norm_pos"][0], g["norm_pos"][1]
                self.gaze_pos_list.append(gaze_pos)

            self.button.status_text = self.get_rec_time_str()

    def stop(self):
        # explicit release of VideoWriter
        self.writer.release()
        self.writer = None

        if self.record_eye:
            for tx in self.g_pool.eye_tx:
                try:
                    tx.send((None, None))
                except:
                    logger.warning("Could not stop eye-recording. Please report this bug!")

        save_object(self.data, os.path.join(self.rec_path, "pupil_data"))

        gaze_list_path = os.path.join(self.rec_path, "gaze_positions.npy")
        np.save(gaze_list_path, np.asarray(self.gaze_pos_list))

        pupil_list_path = os.path.join(self.rec_path, "pupil_positions.npy")
        np.save(pupil_list_path, np.asarray(self.pupil_pos_list))

        timestamps_path = os.path.join(self.rec_path, "world_timestamps.npy")
        ts = sanitize_timestamps(np.array(self.timestamps))
        np.save(timestamps_path, ts)

        try:
            copy2(
                os.path.join(self.g_pool.user_dir, "surface_definitions"),
                os.path.join(self.rec_path, "surface_definitions"),
            )
        except:
            logger.info("No surface_definitions data found. You may want this if you do marker tracking.")

        try:
            copy2(
                os.path.join(self.g_pool.user_dir, "cal_pt_cloud.npy"), os.path.join(self.rec_path, "cal_pt_cloud.npy")
            )
        except:
            logger.warning("No calibration data found. Please calibrate first.")

        try:
            copy2(
                os.path.join(self.g_pool.user_dir, "camera_matrix.npy"),
                os.path.join(self.rec_path, "camera_matrix.npy"),
            )
            copy2(os.path.join(self.g_pool.user_dir, "dist_coefs.npy"), os.path.join(self.rec_path, "dist_coefs.npy"))
        except:
            logger.info("No camera intrinsics found.")

        try:
            with open(self.meta_info_path, "a") as f:
                f.write("Duration Time\t" + self.get_rec_time_str() + "\n")
                if self.g_pool.binocular:
                    f.write("Eye Mode\tbinocular\n")
                else:
                    f.write("Eye Mode\tmonocular\n")
                f.write("Duration Time\t" + self.get_rec_time_str() + "\n")
                f.write("World Camera Frames\t" + str(self.frame_count) + "\n")
                f.write("World Camera Resolution\t" + str(self.width) + "x" + str(self.height) + "\n")
                f.write("Capture Software Version\t%s\n" % self.g_pool.version)
                if platform.system() == "Windows":
                    username = os.environ["USERNAME"]
                    sysname, nodename, release, version, machine, _ = platform.uname()
                else:
                    username = os.getlogin()
                    try:
                        sysname, nodename, release, version, machine = os.uname()
                    except:
                        sysname, nodename, release, version, machine = sys.platform, None, None, None, None
                f.write("User\t" + username + "\n")
                f.write("Platform\t" + sysname + "\n")
                f.write("Machine\t" + nodename + "\n")
                f.write("Release\t" + release + "\n")
                f.write("Version\t" + version + "\n")
        except Exception:
            logger.exception("Could not save metadata. Please report this bug!")

        try:
            with open(os.path.join(self.rec_path, "user_info.csv"), "w") as f:
                for name, val in self.user_info.iteritems():
                    f.write("%s\t%s\n" % (name, val))
        except Exception:
            logger.exception("Could not save userdata. Please report this bug!")

        self.close_info_menu()

        if self.audio_writer:
            self.audio_writer = None

        self.running = False
        self.menu.read_only = False
        self.button.status_text = ""

    def cleanup(self):
        """gets called when the plugin get terminated.
           either volunatily or forced.
        """
        if self.running:
            self.stop()
        self.deinit_gui()

    def set_rec_dir(self, val):
        try:
            n_path = os.path.expanduser(val)
            logger.debug("Expanded user path.")
        except:
            n_path = val
        if not n_path:
            logger.warning("Please specify a path.")
        elif not os.path.isdir(n_path):
            logger.warning("This is not a valid path.")
        # elif not os.access(n_path, os.W_OK):
        elif not writable_dir(n_path):
            logger.warning("Do not have write access to '%s'." % n_path)
        else:
            self.rec_dir = n_path

    def set_session_name(self, val):
        if not val:
            self.session_name = get_auto_name()
        else:
            if "/" in val:
                logger.warning("You session name with create one or more subdirectories")
            self.session_name = val
Exemplo n.º 5
0
class Recorder(Plugin):
    """Capture Recorder"""
    def __init__(self,g_pool,session_name = get_auto_name(),rec_dir=None, user_info={'name':'','additional_field':'change_me'},info_menu_conf={},show_info_menu=False, record_eye = False, audio_src = 'No Audio',raw_jpeg=False):
        super(Recorder, self).__init__(g_pool)
        #update name if it was autogenerated.
        if session_name.startswith('20') and len(session_name)==10:
            session_name = get_auto_name()

        if rec_dir:
            self.set_rec_dir(rec_dir)
        else:
            #lets make a rec dir next to the user dir
            base_dir = self.g_pool.user_dir.rsplit(os.path.sep,1)[0]
            self.rec_dir = os.path.join(base_dir,'recordings')
            if not os.path.isdir(self.rec_dir):
                os.mkdir(self.rec_dir)


        self.raw_jpeg = raw_jpeg
        self.order = .9
        self.record_eye = record_eye
        self.session_name = session_name
        self.audio_devices_dict = Audio_Input_Dict()
        if audio_src in self.audio_devices_dict.keys():
            self.audio_src = audio_src
        else:
            self.audio_src = 'No Audio'
        self.running = False
        self.menu = None
        self.button = None

        self.user_info = user_info
        self.show_info_menu = show_info_menu
        self.info_menu = None
        self.info_menu_conf = info_menu_conf
        self.height, self.width = self.g_pool.capture.frame_size


    def get_init_dict(self):
        d = {}
        d['record_eye'] = self.record_eye
        d['audio_src'] = self.audio_src
        d['session_name'] = self.session_name
        d['user_info'] = self.user_info
        d['info_menu_conf'] = self.info_menu_conf
        d['show_info_menu'] = self.show_info_menu
        d['rec_dir'] = self.rec_dir
        d['raw_jpeg'] = self.raw_jpeg
        return d


    def init_gui(self):
        self.menu = ui.Growing_Menu('Recorder')
        self.g_pool.sidebar.insert(3,self.menu)
        self.menu.append(ui.Info_Text('Pupil recordings are saved like this: "path_to_recordings/recording_session_name/nnn" where "nnn" is an increasing number to avoid overwrites. You can use "/" in your session name to create subdirectories.'))
        self.menu.append(ui.Info_Text('Recordings are saved to "~/pupil_recordings". You can change the path here but note that invalid input will be ignored.'))
        self.menu.append(ui.Text_Input('rec_dir',self,setter=self.set_rec_dir,label='Path to recordings'))
        self.menu.append(ui.Text_Input('session_name',self,setter=self.set_session_name,label='Recording session name'))
        self.menu.append(ui.Switch('show_info_menu',self,on_val=True,off_val=False,label='Request additional user info'))
        self.menu.append(ui.Selector('raw_jpeg',self,selection = [True,False], labels=["bigger file, less CPU", "smaller file, more CPU"],label='compression'))
        self.menu.append(ui.Info_Text('Recording the raw eye video is optional. We use it for debugging.'))
        self.menu.append(ui.Switch('record_eye',self,on_val=True,off_val=False,label='Record eye'))
        self.menu.append(ui.Selector('audio_src',self, selection=self.audio_devices_dict.keys()))

        self.button = ui.Thumb('running',self,setter=self.toggle,label='Record',hotkey='r')
        self.button.on_color[:] = (1,.0,.0,.8)
        self.g_pool.quickbar.insert(1,self.button)


    def deinit_gui(self):
        if self.menu:
            self.g_pool.sidebar.remove(self.menu)
            self.menu = None
        if self.button:
            self.g_pool.quickbar.remove(self.button)
            self.button = None



    def toggle(self, _=None):
        if self.running:
            self.stop()
        else:
            self.start()


    def get_rec_time_str(self):
        rec_time = gmtime(time()-self.start_time)
        return strftime("%H:%M:%S", rec_time)

    def start(self):
        self.timestamps = []
        self.data = {'pupil_positions':[],'gaze_positions':[]}
        self.pupil_pos_list = []
        self.gaze_pos_list = []

        self.frame_count = 0
        self.running = True
        self.menu.read_only = True
        self.start_time = time()

        session = os.path.join(self.rec_dir, self.session_name)
        try:
            os.makedirs(session)
            logger.debug("Created new recordings session dir %s"%session)

        except:
            logger.debug("Recordings session dir %s already exists, using it." %session)

        # set up self incrementing folder within session folder
        counter = 0
        while True:
            self.rec_path = os.path.join(session, "%03d/" % counter)
            try:
                os.mkdir(self.rec_path)
                logger.debug("Created new recording dir %s"%self.rec_path)
                break
            except:
                logger.debug("We dont want to overwrite data, incrementing counter & trying to make new data folder")
                counter += 1

        self.meta_info_path = os.path.join(self.rec_path, "info.csv")

        with open(self.meta_info_path, 'w') as f:
            f.write("Recording Name\t"+self.session_name+ "\n")
            f.write("Start Date\t"+ strftime("%d.%m.%Y", localtime(self.start_time))+ "\n")
            f.write("Start Time\t"+ strftime("%H:%M:%S", localtime(self.start_time))+ "\n")


        if self.audio_src != 'No Audio':
            audio_path = os.path.join(self.rec_path, "world.wav")
            self.audio_writer = Audio_Capture(self.audio_devices_dict[self.audio_src],audio_path)
        else:
            self.audio_writer = None

        if self.raw_jpeg  and "uvc_capture" in str(self.g_pool.capture.__class__):
            self.video_path = os.path.join(self.rec_path, "world.mp4")
            self.writer = JPEG_Writer(self.video_path,int(self.g_pool.capture.frame_rate))
        # elif 1:
        #     self.writer = av_writer.AV_Writer(self.video_path)
        else:
            self.video_path = os.path.join(self.rec_path, "world.mkv")
            self.writer = CV_Writer(self.video_path, float(self.g_pool.capture.frame_rate), self.g_pool.capture.frame_size)
        # positions path to eye process
        if self.record_eye:
            for tx in self.g_pool.eye_tx:
                tx.send((self.rec_path,self.raw_jpeg))

        if self.show_info_menu:
            self.open_info_menu()

    def open_info_menu(self):
        self.info_menu = ui.Growing_Menu('additional Recording Info',size=(300,300),pos=(300,300))
        self.info_menu.configuration = self.info_menu_conf

        def populate_info_menu():
            self.info_menu.elements[:-2] = []
            for name in self.user_info.iterkeys():
                self.info_menu.insert(0,ui.Text_Input(name,self.user_info))

        def set_user_info(new_string):
            self.user_info = new_string
            populate_info_menu()

        populate_info_menu()
        self.info_menu.append(ui.Info_Text('Use the *user info* field to add/remove additional fields and their values. The format must be a valid Python dictionary. For example -- {"key":"value"}. You can add as many fields as you require. Your custom fields will be saved for your next session.'))
        self.info_menu.append(ui.Text_Input('user_info',self,setter=set_user_info,label="User info"))
        self.g_pool.gui.append(self.info_menu)

    def close_info_menu(self):
        if self.info_menu:
            self.info_menu_conf = self.info_menu.configuration
            self.g_pool.gui.remove(self.info_menu)
            self.info_menu = None

    def update(self,frame,events):
        if self.running:
            self.data['pupil_positions'] += events['pupil_positions']
            self.data['gaze_positions'] += events['gaze_positions']
            self.timestamps.append(frame.timestamp)
            self.writer.write_video_frame(frame)
            # self.writer.write_video_frame_yuv422(frame)
            self.frame_count += 1

            # cv2.putText(frame.img, "Frame %s"%self.frame_count,(200,200), cv2.FONT_HERSHEY_SIMPLEX,1,(255,100,100))
            for p in events['pupil_positions']:
                pupil_pos = p['timestamp'],p['confidence'],p['id'],p['norm_pos'][0],p['norm_pos'][1],p['diameter']
                self.pupil_pos_list.append(pupil_pos)

            for g in events.get('gaze_positions',[]):
                gaze_pos = g['timestamp'],g['confidence'],g['norm_pos'][0],g['norm_pos'][1]
                self.gaze_pos_list.append(gaze_pos)

            self.button.status_text = self.get_rec_time_str()

    def stop(self):
        #explicit release of VideoWriter
        self.writer.release()
        self.writer = None

        if self.record_eye:
            for tx in self.g_pool.eye_tx:
                try:
                    tx.send((None,None))
                except:
                    logger.warning("Could not stop eye-recording. Please report this bug!")

        save_object(self.data,os.path.join(self.rec_path, "pupil_data"))

        gaze_list_path = os.path.join(self.rec_path, "gaze_positions.npy")
        np.save(gaze_list_path,np.asarray(self.gaze_pos_list))

        pupil_list_path = os.path.join(self.rec_path, "pupil_positions.npy")
        np.save(pupil_list_path,np.asarray(self.pupil_pos_list))

        timestamps_path = os.path.join(self.rec_path, "world_timestamps.npy")
        ts = sanitize_timestamps(np.array(self.timestamps))
        np.save(timestamps_path,ts)

        try:
            copy2(os.path.join(self.g_pool.user_dir,"surface_definitions"),os.path.join(self.rec_path,"surface_definitions"))
        except:
            logger.info("No surface_definitions data found. You may want this if you do marker tracking.")

        try:
            copy2(os.path.join(self.g_pool.user_dir,"cal_pt_cloud.npy"),os.path.join(self.rec_path,"cal_pt_cloud.npy"))
        except:
            logger.warning("No calibration data found. Please calibrate first.")

        try:
            copy2(os.path.join(self.g_pool.user_dir,"camera_matrix.npy"),os.path.join(self.rec_path,"camera_matrix.npy"))
            copy2(os.path.join(self.g_pool.user_dir,"dist_coefs.npy"),os.path.join(self.rec_path,"dist_coefs.npy"))
        except:
            logger.info("No camera intrinsics found.")

        try:
            with open(self.meta_info_path, 'a') as f:
                f.write("Duration Time\t"+ self.get_rec_time_str()+ "\n")
                if self.g_pool.binocular:
                    f.write("Eye Mode\tbinocular\n")
                else:
                    f.write("Eye Mode\tmonocular\n")
                f.write("Duration Time\t"+ self.get_rec_time_str()+ "\n")
                f.write("World Camera Frames\t"+ str(self.frame_count)+ "\n")
                f.write("World Camera Resolution\t"+ str(self.width)+"x"+str(self.height)+"\n")
                f.write("Capture Software Version\t%s\n"%self.g_pool.version)
                if platform.system() == "Windows":
                    username = os.environ["USERNAME"]
                    sysname, nodename, release, version, machine, _ = platform.uname()
                else:
                    username = os.getlogin()
                    try:
                        sysname, nodename, release, version, machine = os.uname()
                    except:
                        sysname, nodename, release, version, machine = sys.platform,None,None,None,None
                f.write("User\t"+username+"\n")
                f.write("Platform\t"+sysname+"\n")
                f.write("Machine\t"+nodename+"\n")
                f.write("Release\t"+release+"\n")
                f.write("Version\t"+version+"\n")
        except Exception:
            logger.exception("Could not save metadata. Please report this bug!")

        try:
            with open(os.path.join(self.rec_path, "user_info.csv"), 'w') as f:
                for name,val in self.user_info.iteritems():
                    f.write("%s\t%s\n"%(name,val))
        except Exception:
            logger.exception("Could not save userdata. Please report this bug!")


        self.close_info_menu()

        if self.audio_writer:
            self.audio_writer = None

        self.running = False
        self.menu.read_only = False
        self.button.status_text = ''



    def cleanup(self):
        """gets called when the plugin get terminated.
           either volunatily or forced.
        """
        if self.running:
            self.stop()
        self.deinit_gui()



    def set_rec_dir(self,val):
        try:
            n_path = os.path.expanduser(val)
            logger.debug("Expanded user path.")
        except:
            n_path = val
        if not n_path:
            logger.warning("Please specify a path.")
        elif not os.path.isdir(n_path):
            logger.warning("This is not a valid path.")
        # elif not os.access(n_path, os.W_OK):
        elif not writable_dir(n_path):
            logger.warning("Do not have write access to '%s'."%n_path)
        else:
            self.rec_dir = n_path

    def set_session_name(self, val):
        if not val:
            self.session_name = get_auto_name()
        else:
            if '/' in val:
                logger.warning('You session name with create one or more subdirectories')
            self.session_name = val
Exemplo n.º 6
0
class Recorder(Plugin):
    """Capture Recorder"""
    def __init__(self,g_pool,session_name = get_auto_name(),rec_dir=None, user_info={'name':'','additional_field':'change_me'},info_menu_conf={},show_info_menu=False, record_eye = False, audio_src = 'No Audio', raw_jpeg=False):
        super(Recorder, self).__init__(g_pool)
        
        self.states = ['start', 'want', 'pick']
        self.all_qr = {'termo1' : 75, 'light1' : False, 'light2' : False, 'light3': False}
        self.this_state = 'start'
        self.start_state = time()
        self.qr_codes = {}
        self.gaze_x = -1
        self.gaze_y = -1
        self.avg_col = 0
        self.choice = ""

        #update name if it was autogenerated.
        if session_name.startswith('20') and len(session_name)==10:
            session_name = get_auto_name()

        base_dir = self.g_pool.user_dir.rsplit(os.path.sep,1)[0]
        default_rec_dir = os.path.join(base_dir, 'recordings')

        if rec_dir and rec_dir != default_rec_dir and self.verify_path(rec_dir):
            self.rec_dir = rec_dir
        else:
            try:
                os.makedirs(default_rec_dir)
            except OSError as e:
                if 'File exists' in '%s'%e:
                    pass
                else:
                    logger.error("Could not create Rec dir")
                    raise e
            else:
                logger.info('Created standard Rec dir at "%s"'%default_rec_dir)
            self.rec_dir = default_rec_dir
        # if rec_dir and rec_dir != default_rec_dir and self.verify_path(rec_dir):
        #     self.rec_dir = rec_dir
        # else:
        #     #lets make a rec dir next to the user dir
        #     base_dir = self.g_pool.user_dir.rsplit(os.path.sep,1)[0]
        #     self.rec_dir = os.path.join(base_dir,'recordings')
        #     if not os.path.isdir(self.rec_dir):
        #         os.mkdir(self.rec_dir)

        self.raw_jpeg = raw_jpeg
        self.order = .9
        self.record_eye = record_eye
        self.session_name = session_name
        self.audio_devices_dict = Audio_Input_Dict()
        if audio_src in self.audio_devices_dict.keys():
            self.audio_src = audio_src
        else:
            self.audio_src = 'No Audio'
        self.running = False
        self.menu = None
        self.button = None

        self.user_info = user_info
        self.show_info_menu = show_info_menu
        self.info_menu = None
        self.info_menu_conf = info_menu_conf
        self.height, self.width = self.g_pool.capture.frame_size


    def get_init_dict(self):
        d = {}
        d['record_eye'] = self.record_eye
        d['audio_src'] = self.audio_src
        d['session_name'] = self.session_name
        d['user_info'] = self.user_info
        d['info_menu_conf'] = self.info_menu_conf
        d['show_info_menu'] = self.show_info_menu
        d['rec_dir'] = self.rec_dir
        d['raw_jpeg'] = self.raw_jpeg
        # if self.menu:
        #     d['menu_conf'] = self.menu.configuration
        # else:
        #     d['menu_conf'] = self.menu_conf
        return d


    def init_gui(self):
        self.menu = ui.Growing_Menu('Recorder')
        # self.menu.configuration = self.menu_conf
        self.g_pool.sidebar.insert(3,self.menu)
        self.menu.append(ui.Info_Text('Pupil recordings are saved like this: "path_to_recordings/recording_session_name/nnn" where "nnn" is an increasing number to avoid overwrites. You can use "/" in your session name to create subdirectories.'))
        self.menu.append(ui.Info_Text('Recordings are saved to "~/pupil_recordings". You can change the path here but note that invalid input will be ignored.'))
        self.menu.append(ui.Text_Input('rec_dir',self,setter=self.set_rec_dir,label='Path to recordings'))
        self.menu.append(ui.Text_Input('session_name',self,setter=self.set_session_name,label='Recording session name'))
        self.menu.append(ui.Switch('show_info_menu',self,on_val=True,off_val=False,label='Request additional user info'))
        self.menu.append(ui.Selector('raw_jpeg',self,selection = [True,False], labels=["bigger file, less CPU", "smaller file, more CPU"],label='compression'))
        self.menu.append(ui.Info_Text('Recording the raw eye video is optional. We use it for debugging.'))
        self.menu.append(ui.Switch('record_eye',self,on_val=True,off_val=False,label='Record eye'))
        self.menu.append(ui.Selector('audio_src',self, selection=self.audio_devices_dict.keys()))

        self.button = ui.Thumb('running',self,setter=self.toggle,label='QR',hotkey='r')
        self.button.on_color[:] = (1,.0,.0,.8)
        self.g_pool.quickbar.insert(1,self.button)


    def deinit_gui(self):
        if self.menu:
            # self.menu_conf = self.menu.configuration
            self.g_pool.sidebar.remove(self.menu)
            self.menu = None
        if self.button:
            self.g_pool.quickbar.remove(self.button)
            self.button = None

    def toggle(self, _=None):
        if self.running:
            self.stop()
        else:
            self.start()


    def get_rec_time_str(self):
        rec_time = gmtime(time()-self.start_time)
        return strftime("%H:%M:%S", rec_time)

    def start(self):
        self.timestamps = []
        # add self.data
        self.data = {'pupil_positions':[], 'gaze_positions': []}
        self.pupil_list = []
        self.gaze_list = []
        self.frame_count = 0
        self.running = True
        self.menu.read_only = True
        self.start_time = time()
        self.start_state = time()

        session = os.path.join(self.rec_dir, self.session_name)
        try:
            os.makedirs(session)
            logger.debug("Created new recordings session dir %s"%session)

        except:
            logger.debug("Recordings session dir %s already exists, using it." %session)

        # set up self incrementing folder within session folder
        counter = 0
        while True:
            self.rec_path = os.path.join(session, "%03d/" % counter)
            try:
                os.mkdir(self.rec_path)
                logger.debug("Created new recording dir %s"%self.rec_path)
                break
            except:
                logger.debug("We dont want to overwrite data, incrementing counter & trying to make new data folder")
                counter += 1

        self.meta_info_path = os.path.join(self.rec_path, "info.csv")

        with open(self.meta_info_path, 'w') as f:
            f.write("Recording Name\t"+self.session_name+ "\n")
            f.write("Start Date\t"+ strftime("%d.%m.%Y", localtime(self.start_time))+ "\n")
            f.write("Start Time\t"+ strftime("%H:%M:%S", localtime(self.start_time))+ "\n")


        if self.audio_src != 'No Audio':
            audio_path = os.path.join(self.rec_path, "world.wav")
            self.audio_writer = Audio_Capture(self.audio_devices_dict[self.audio_src],audio_path)
        else:
            self.audio_writer = None

        if self.raw_jpeg  and "uvc_capture" in str(self.g_pool.capture.__class__):
            self.video_path = os.path.join(self.rec_path, "world.mp4")
            self.writer = JPEG_Writer(self.video_path,int(self.g_pool.capture.frame_rate))
        # elif 1:
        #     self.writer = av_writer.AV_Writer(self.video_path)
        else:
            self.video_path = os.path.join(self.rec_path, "world.mkv")
            self.writer = CV_Writer(self.video_path, float(self.g_pool.capture.frame_rate), self.g_pool.capture.frame_size)
        # positions path to eye process
        if self.record_eye:
            for tx in self.g_pool.eye_tx:
                tx.send((self.rec_path,self.raw_jpeg))

        if self.show_info_menu:
            self.open_info_menu()
        # self.video_path = os.path.join(self.rec_path, "world.mkv")
        # self.writer = cv2.VideoWriter(self.video_path, int(cv2.cv.CV_FOURCC(*'DIVX')), float(self.g_pool.capture.frame_rate), (1280,720))
        # # positions path to eye process
        # if self.record_eye:
        #     for tx in self.g_pool.eye_tx:
        #         tx.send(self.rec_path)

        # if self.show_info_menu:
        #     self.open_info_menu()
###############
    def open_info_menu(self):
        self.info_menu = ui.Growing_Menu('additional Recording Info',size=(300,300),pos=(300,300))
        self.info_menu.configuration = self.info_menu_conf

        def populate_info_menu():
            self.info_menu.elements[:-2] = []
            for name in self.user_info.iterkeys():
                self.info_menu.insert(0,ui.Text_Input(name,self.user_info))

        def set_user_info(new_string):
            self.user_info = new_string
            populate_info_menu()

        populate_info_menu()
        self.info_menu.append(ui.Info_Text('Use the *user info* field to add/remove additional fields and their values. The format must be a valid Python dictionary. For example -- {"key":"value"}. You can add as many fields as you require. Your custom fields will be saved for your next session.'))
        self.info_menu.append(ui.Text_Input('user_info',self,setter=set_user_info,label="User info"))
        self.g_pool.gui.append(self.info_menu)

    def close_info_menu(self):
        if self.info_menu:
            self.info_menu_conf = self.info_menu.configuration
            self.g_pool.gui.remove(self.info_menu)
            self.info_menu = None

    def update(self,frame,events):
        if self.running:
            # update data
            self.data['pupil_positions'] += events['pupil_positions']
            self.data['gaze_positions'] += events['gaze_positions']

            #cv2.putText(frame.img, "Frame %s"%self.frame_count,(200,200), cv2.FONT_HERSHEY_SIMPLEX,1,(255,100,100)
            cv2.putText(frame.img, self.this_state,(200,100), cv2.FONT_HERSHEY_SIMPLEX,1,(255,100,100))
            if self.this_state == 'pick':
                cv2.putText(frame.img, self.choice,(300,100), cv2.FONT_HERSHEY_SIMPLEX,1,(100,100,255))
            QRs = qr_detect(frame)              
            for p in events['pupil_positions']:
                pupil_pos = p['timestamp'],p['confidence'],p['id'],p['norm_pos'][0],p['norm_pos'][1],p['diameter']

                self.pupil_list.append(pupil_pos)

            c = 0
            avg_x = -1
            avg_y = -1
            for g in events.get('gaze_positions',[]):
                gaze_pos = g['timestamp'],g['confidence'],g['norm_pos'][0],g['norm_pos'][1]
                avg_x = (avg_x * c + g['norm_pos'][0]) / (c+1)
                avg_y = (avg_y * c + g['norm_pos'][0]) / (c+1)
                c = c + 1
                self.gaze_list.append(gaze_pos)
                
            self.timestamps.append(frame.timestamp)

            avg_col = 0    

            self.stateT(frame, QRs, avg_x, avg_y, avg_col) 

            self.writer.write_video_frame(frame)
            self.frame_count += 1

            self.button.status_text = self.get_rec_time_str()

    def stop(self):
        #explicit release of VideoWriter
        self.writer.release()
        self.writer = None

        if self.record_eye:
            for tx in self.g_pool.eye_tx:
                try:
                    tx.send(None)
                except:
                    logger.warning("Could not stop eye-recording. Please report this bug!")

        save_object(self.data, os.path.join(self.rec_path, "pupil_data"))
        
        gaze_list_path = os.path.join(self.rec_path, "gaze_positions.npy")
        np.save(gaze_list_path,np.asarray(self.gaze_list))

        pupil_list_path = os.path.join(self.rec_path, "pupil_positions.npy")
        np.save(pupil_list_path,np.asarray(self.pupil_list))

        timestamps_path = os.path.join(self.rec_path, "world_timestamps.npy")
        ts = sanitize_timestamps(np.array(self.timestamps))
        np.save(timestamps_path,ts)

        try:
            copy2(os.path.join(self.g_pool.user_dir,"surface_definitions"),os.path.join(self.rec_path,"surface_definitions"))
        except:
            logger.info("No surface_definitions data found. You may want this if you do marker tracking.")

        try:
            copy2(os.path.join(self.g_pool.user_dir,"cal_pt_cloud.npy"),os.path.join(self.rec_path,"cal_pt_cloud.npy"))
        except:
            logger.warning("No calibration data found. Please calibrate first.")

        try:
            copy2(os.path.join(self.g_pool.user_dir,"camera_matrix.npy"),os.path.join(self.rec_path,"camera_matrix.npy"))
            copy2(os.path.join(self.g_pool.user_dir,"dist_coefs.npy"),os.path.join(self.rec_path,"dist_coefs.npy"))
        except:
            logger.info("No camera intrinsics found.")

        try:
            with open(self.meta_info_path, 'a') as f:
                f.write("Duration Time\t"+ self.get_rec_time_str()+ "\n")
                if self.g_pool.binocular:
                    f.write("Eye Mode\tbinocular\n")
                else:
                    f.write("Eye Mode\tmonocular\n")
                f.write("Duration Time\t"+ self.get_rec_time_str()+ "\n")
                f.write("World Camera Frames\t"+ str(self.frame_count)+ "\n")
                f.write("World Camera Resolution\t"+ str(self.width)+"x"+str(self.height)+"\n")
                f.write("Capture Software Version\t%s\n"%self.g_pool.version)
                if platform.system() == "Windows":
                    username = os.environ["USERNAME"]
                    sysname, nodename, release, version, machine, _ = platform.uname()
                else:
                    username = os.getlogin()
                    try:
                        sysname, nodename, release, version, machine = os.uname()
                    except:
                        sysname, nodename, release, version, machine = sys.platform,None,None,None,None
                f.write("User\t"+username+"\n")
                f.write("Platform\t"+sysname+"\n")
                f.write("Machine\t"+nodename+"\n")
                f.write("Release\t"+release+"\n")
                f.write("Version\t"+version+"\n")
        except Exception:
            logger.exception("Could not save metadata. Please report this bug!")

        try:
            with open(os.path.join(self.rec_path, "user_info.csv"), 'w') as f:
                for name,val in self.user_info.iteritems():
                    f.write("%s\t%s\n"%(name,val))
        except Exception:
            logger.exception("Could not save userdata. Please report this bug!")


        self.close_info_menu()

        if self.audio_writer:
            self.audio_writer = None

        self.running = False
        self.menu.read_only = False
        self.button.status_text = ''



    def cleanup(self):
        """gets called when the plugin get terminated.
           either volunatily or forced.
        """
        if self.running:
            self.stop()
        self.deinit_gui()

    def verify_path(self, val):
        try:
            n_path = os.path.expanduser(val)
            logger.debug("Expanded user path.")
        except:
            n_path = val
        if not n_path:
            logger.waring("Please specify a path.")
            return False
        elif not os.path.isdir(n_path):
            logger.warning("This is not a valid path.")
            return False
        elif not writable_dir(n_path):
            logger.warning("Do not have write access to '%s'." %n_path)
            return False
        else:
            return n_path


    def set_rec_dir(self,val):
        # try:
        #     n_path = os.path.expanduser(val)
        #     logger.debug("Expanded user path.")
        # except:
        #     n_path = val
        # if not n_path:
        #     logger.warning("Please specify a path.")
        # elif not os.path.isdir(n_path):
        #     logger.warning("This is not a valid path.")
        # # elif not os.access(n_path, os.W_OK):
        # elif not writable_dir(n_path):
        #     logger.warning("Do not have write access to '%s'."%n_path)
        # else:
        #     self.rec_dir = n_path
        n_path = self.verify_path(val)
        if n_path:
            self.rec_dir = n_path

    def set_session_name(self, val):
        if not val:
            self.session_name = get_auto_name()
        else:
            if '/' in val:
                logger.warning('You session name with create one or more subdirectories')
            self.session_name = val

    def stateT(self, frame, QRs, agaze_x, agaze_y, avg_col):
        if time() - self.start_time < 1:
            self.gaze_x = agaze_x
            self.gaze_y = agaze_y
        
        if self.this_state == 'start':
            if time() - self.start_state < 2:
                self.qr_codes = {}
                return True
            if self.Nochange_gaze(agaze_x, agaze_y, avg_col):
                    self.this_state = 'want'
            
            self.start_state = time()        
            self.gaze_x = agaze_x
            self.gaze_y = agaze_y
            return True
        
        if self.this_state == 'want':
            if time() - self.start_state < 2:
                self.qr_codes.update(QRs)
                return True    
            if self.Nochange_gaze(agaze_x, agaze_y, avg_col) and self.qr_codes:
                self.this_state = 'pick'
                self.choice = self.choose(frame)
                #print self.qr_codes, self.choice
            else: 
                
                self.this_state = 'start'        
            self.start_state = time()
            self.gaze_x = agaze_x
            self.gaze_y = agaze_y
            return True
        if self.this_state == 'pick':
            if time() - self.start_state < 4:
                return True
            if self.Nochange_gaze(agaze_x, agaze_y, avg_col):
                    del self.qr_codes[self.choice]
                    if not self.qr_codes:
                        self.this_state = 'start'
                    else:
                        self.choice = self.choose(frame)
                        #print self.choice
            else:
                print self.choice, " chosen"
                
                self.this_state = 'start'    
            self.start_state = time()
            self.gaze_x = agaze_x
            self.gaze_y = agaze_y
        
        return True

    def Nochange_gaze(self, agaze_x, agaze_y, avg_col):
        #print self.gaze_x, self.gaze_y, agaze_x, agaze_y
        return (abs(self.gaze_x - agaze_x) < .1 and abs(self.gaze_y - agaze_y) < .1)

    def choose(self, frame):
        h = frame.img.shape[0]
        w = frame.img.shape[1]
        gaze_x = self.gaze_x
        gaze_y = self.gaze_y
        if (gaze_x < 0 or gaze_y < 0):
            gaze_x = .5
            gaz_y = .5
        k = self.qr_codes.keys()
        chosen = k[0]
        min_dist = 1000000
        for name, mc in self.qr_codes.items():
            dis = (mc[0] - gaze_x * w) * (mc[0] - gaze_x * w) + (mc[1] - gaze_y * h) * (mc[1] - gaze_y * h)
            if dis < min_dist:
                min_dist = dis
                chosen = name
        return chosen