def __init__(self,g_pool,session_name = get_auto_name(),rec_dir=None, user_info={'name':'','additional_field':'change_me'},info_menu_conf={},show_info_menu=False, record_eye = False, audio_src = 'No Audio',raw_jpeg=False): super(Recorder, self).__init__(g_pool) #update name if it was autogenerated. if session_name.startswith('20') and len(session_name)==10: session_name = get_auto_name() if rec_dir: self.set_rec_dir(rec_dir) else: #lets make a rec dir next to the user dir base_dir = self.g_pool.user_dir.rsplit(os.path.sep,1)[0] self.rec_dir = os.path.join(base_dir,'recordings') if not os.path.isdir(self.rec_dir): os.mkdir(self.rec_dir) self.raw_jpeg = raw_jpeg self.order = .9 self.record_eye = record_eye self.session_name = session_name self.audio_devices_dict = Audio_Input_Dict() if audio_src in self.audio_devices_dict.keys(): self.audio_src = audio_src else: self.audio_src = 'No Audio' self.running = False self.menu = None self.button = None self.user_info = user_info self.show_info_menu = show_info_menu self.info_menu = None self.info_menu_conf = info_menu_conf self.height, self.width = self.g_pool.capture.frame_size
def audio_dev_getter(): # fetch list of currently available audio_src_val = None if self.audio_devices_dict is not None: if self.audio_src in self.audio_devices_dict.keys(): audio_src_val = self.audio_devices_dict[self.audio_src] self.audio_devices_dict = Audio_Input_Dict() if audio_src_val is not None: self.audio_devices_dict[self.audio_src] = audio_src_val devices = list(self.audio_devices_dict.keys()) return devices, devices
def __init__(self, g_pool, audio_src='No Audio'): super().__init__(g_pool) self.audio_devices_dict = Audio_Input_Dict() if audio_src in list(self.audio_devices_dict.keys()): self.audio_src = audio_src else: self.audio_src = 'No Audio' self.thread = None self.running = Event() self.audio_container = None self.audio_out_stream = None self.queue = queue.Queue() self.start_capture(self.audio_src)
def __init__(self,g_pool,session_name = get_auto_name(),rec_dir=None, user_info={'name':'','additional_field':'change_me'},info_menu_conf={},show_info_menu=False, record_eye = False, audio_src = 'No Audio',raw_jpeg=True): super(Recorder, self).__init__(g_pool) #update name if it was autogenerated. if session_name.startswith('20') and len(session_name)==10: session_name = get_auto_name() base_dir = self.g_pool.user_dir.rsplit(os.path.sep,1)[0] default_rec_dir = os.path.join(base_dir,'recordings') if rec_dir and rec_dir != default_rec_dir and self.verify_path(rec_dir): self.rec_dir = rec_dir else: try: os.makedirs(default_rec_dir) except OSError as e: # posix windows if 'File exists' in '%s'%e or 'file already exists' in '%s'%e: pass else: logger.error("Could not create Rec dir") raise e else: logger.info('Created standard Rec dir at "%s"'%default_rec_dir) self.rec_dir = default_rec_dir self.raw_jpeg = raw_jpeg self.order = .9 self.record_eye = record_eye self.session_name = session_name self.audio_devices_dict = Audio_Input_Dict() if audio_src in self.audio_devices_dict.keys(): self.audio_src = audio_src else: self.audio_src = 'No Audio' self.running = False self.menu = None self.button = None self.user_info = user_info self.show_info_menu = show_info_menu self.info_menu = None self.info_menu_conf = info_menu_conf self.height, self.width = self.g_pool.capture.frame_size
def __init__(self,g_pool,session_name = get_auto_name(),rec_dir=None, user_info={},info_menu_conf={},show_info_menu=False, record_eye = True, audio_src = 'No Audio',raw_jpeg=True): super(Recorder, self).__init__(g_pool) #update name if it was autogenerated. if session_name.startswith('20') and len(session_name)==10: session_name = get_auto_name() base_dir = self.g_pool.user_dir.rsplit(os.path.sep,1)[0] default_rec_dir = os.path.join(base_dir,'recordings') if rec_dir and rec_dir != default_rec_dir and self.verify_path(rec_dir): self.rec_dir = rec_dir else: try: os.makedirs(default_rec_dir) except OSError as e: if e.errno != errno.EEXIST: logger.error("Could not create Rec dir") raise e else: logger.info('Created standard Rec dir at "%s"'%default_rec_dir) self.rec_dir = default_rec_dir self.raw_jpeg = raw_jpeg self.order = .9 self.record_eye = record_eye self.session_name = session_name self.audio_devices_dict = Audio_Input_Dict() if audio_src in self.audio_devices_dict.keys(): self.audio_src = audio_src else: self.audio_src = 'No Audio' self.running = False self.menu = None self.button = None self.user_info = user_info self.show_info_menu = show_info_menu self.info_menu = None self.info_menu_conf = info_menu_conf self.calibration_start_at = None self.calibration_end_at = None
def __init__(self,g_pool,session_name = get_auto_name(),user_info={'name':'','additional_field':'change_me'},info_menu_conf={},show_info_menu=False, record_eye = False, audio_src = 'No Audio', menu_conf = {}): super(Recorder, self).__init__(g_pool) #update name if it was autogenerated. if session_name.startswith('20') and len(session_name)==10: session_name = get_auto_name() self.order = .9 self.record_eye = record_eye self.session_name = session_name self.audio_devices_dict = Audio_Input_Dict() if audio_src in self.audio_devices_dict.keys(): self.audio_src = audio_src else: self.audio_src = 'No Audio' self.running = False self.menu = None self.button = None self.menu_conf = menu_conf self.user_info = user_info self.show_info_menu = show_info_menu self.info_menu = None self.info_menu_conf = info_menu_conf self.height, self.width = self.g_pool.capture.frame_size
def audio_dev_getter(): # fetch list of currently available self.audio_devices_dict = Audio_Input_Dict() devices = list(self.audio_devices_dict.keys()) return devices, devices
class Audio_Capture(Plugin): """docstring for Audio_Capture""" def __init__(self, g_pool, audio_src='No Audio'): super().__init__(g_pool) self.audio_devices_dict = Audio_Input_Dict() if audio_src in list(self.audio_devices_dict.keys()): self.audio_src = audio_src else: self.audio_src = 'No Audio' self.thread = None self.running = Event() self.recording = Event() self.recording.clear() self.audio_container = None self.audio_out_stream = None self.queue = queue.Queue() self.start_capture(self.audio_src) def init_gui(self): self.menu = ui.Growing_Menu('Audio Capture') self.menu.collapsed = True self.g_pool.sidebar.append(self.menu) def close(): self.alive = False help_str = 'Creates events for audio input.' self.menu.append(ui.Button('Close', close)) self.menu.append(ui.Info_Text(help_str)) def audio_dev_getter(): # fetch list of currently available self.audio_devices_dict = Audio_Input_Dict() devices = list(self.audio_devices_dict.keys()) return devices, devices self.menu.append(ui.Selector('audio_src', self, selection_getter=audio_dev_getter, label='Audio Source', setter=self.start_capture)) self.menu.append(ui.Info_Text(NOT_REC_STR)) def get_init_dict(self): return {'audio_src': self.audio_src} def deinit_gui(self): if self.menu: self.g_pool.sidebar.remove(self.menu) self.menu = None def cleanup(self): if self.audio_container is not None: self.close_audio_recording() self.running.clear() self.deinit_gui() if self.thread and self.thread.is_alive(): self.thread.join(timeout=1) def on_notify(self, notification): if notification['subject'] == 'recording.started': self.rec_dir = notification['rec_path'] self.recording.set() if self.running.is_set(): # and self.audio_container is None: self.menu[-2].read_only = True del self.menu[-1] self.menu.append(ui.Info_Text(REC_STR)) elif not self.running.is_set(): logger.warning('Recording was started without an active audio capture') elif notification['subject'] == 'recording.stopped': self.recording.clear() self.close_audio_recording() def close_audio_recording(self): self.menu[-2].read_only = False del self.menu[-1] self.menu.append(ui.Info_Text(NOT_REC_STR)) def write_audio_packet(self, audio_frame): # Test if audio outstream has been initialized if self.audio_out_stream is None: try: self.audio_out_stream = self.audio_container.add_stream('aac') except ValueError as e: # packet.stream codec is not supported in target container. logger.error('Failed to create audio stream. Aborting recording.') logger.debug('Reason: {}'.format(e)) self.close_audio_recording() self.timestamps.append(audio_frame.timestamp) packet = self.audio_out_stream.encode(audio_frame) if packet is not None: self.audio_container.mux(packet) def start_capture(self, audio_src): if self.thread and self.thread.is_alive(): if self.audio_src == audio_src: return # capture is already running for our selected source # else stop current capture gracefully self.running.clear() logger.debug('Closing capture for "{}"'.format(self.audio_src)) self.thread.join(timeout=1) self.thread = None if audio_src not in self.audio_devices_dict: logger.warning('Selected audio source is not available anymore') return self.audio_src = audio_src if audio_src == 'No Audio': return self.running.set() self.thread = Thread(target=self.capture_thread, args=(self.audio_devices_dict[audio_src], self.running, self.recording)) self.thread.start() def capture_thread(self, audio_src, running, recording): try: if platform.system() == "Darwin": in_container = av.open('none:{}'.format(audio_src), format="avfoundation") elif platform.system() == "Linux": in_container = av.open('hw:{}'.format(audio_src), format="alsa") else: raise av.AVError('Platform does not support audio capture.') except av.AVError as err: running.clear() self.audio_src = 'No Audio' logger.warning('Error starting audio capture: {}'.format(err)) return in_stream = None try: in_stream = in_container.streams.audio[0] except IndexError: logger.warning('No audio stream found for selected device.') running.clear() self.audio_src = 'No Audio' return out_container = None out_stream = None timestamps = None out_frame_num = 0 in_frame_size = 0 stream_epoch = in_stream.start_time * in_stream.time_base uvc_clock_dif = abs(stream_epoch - self.g_pool.get_now()) pyt_clock_dif = abs(stream_epoch - time()) max_clock_dif = 4. if uvc_clock_dif > max_clock_dif and pyt_clock_dif > max_clock_dif: logger.error('Could not identify audio stream clock.') running.clear() self.audio_src = 'No Audio' return elif uvc_clock_dif > pyt_clock_dif: logger.info('Audio stream uses time.time() as clock (Δ {}s)'.format(pyt_clock_dif)) clock_differences = self.g_pool.get_now() - time() else: logger.info('Audio stream uses uvc.get_time_monotonic() as clock') clock_differences = 0 def close_recording(): # Bind nonlocal variables, https://www.python.org/dev/peps/pep-3104/ nonlocal out_container, out_stream, in_stream, in_frame_size, timestamps, out_frame_num if out_container is not None: packet = out_stream.encode(audio_frame) while packet is not None: out_frame_num += 1 out_container.mux(packet) packet = out_stream.encode(audio_frame) out_container.close() in_frame_rate = in_stream.rate # in_stream.frame_size does not return the correct value. out_frame_size = out_stream.frame_size out_frame_rate = out_stream.rate old_ts_idx = np.arange(0, len(timestamps) * in_frame_size, in_frame_size) / in_frame_rate new_ts_idx = np.arange(0, out_frame_num * out_frame_size, out_frame_size) / out_frame_rate interpolate = interp1d(old_ts_idx, timestamps, bounds_error=False, fill_value='extrapolate') new_ts = interpolate(new_ts_idx) ts_loc = os.path.join(self.rec_dir, 'audio_timestamps.npy') np.save(ts_loc, new_ts) out_container = None out_stream = None timestamps = None for packet in in_container.demux(in_stream): # ffmpeg timestamps - in_stream.startime = packte pts relative to startime # multiply with stream_timebase to get seconds # add start time of this stream in pupil time unadjusted # finally add pupil timebase offset to adjust for settable timebase. for audio_frame in packet.decode(): timestamp = audio_frame.pts * in_stream.time_base + clock_differences - self.g_pool.timebase.value if recording.is_set(): if out_container is None: rec_file = os.path.join(self.rec_dir, 'audio.mp4') out_container = av.open(rec_file, 'w') out_stream = out_container.add_stream('aac') out_frame_num = 0 in_frame_size = audio_frame.samples # set here to make sure full packet size is used timestamps = [] timestamps.append(timestamp) packet = out_stream.encode(audio_frame) if packet is not None: out_frame_num += 1 out_container.mux(packet) elif out_container is not None: # recording stopped close_recording() if not running.is_set(): close_recording() return close_recording() self.audio_src = 'No Audio' running.clear() # in_stream stopped yielding packets
class Audio_Capture(Plugin): """docstring for Audio_Capture""" icon_chr = chr(0xe029) icon_font = 'pupil_icons' def __init__(self, g_pool, audio_src='No Audio'): super().__init__(g_pool) self.audio_devices_dict = Audio_Input_Dict() if audio_src in list(self.audio_devices_dict.keys()): self.audio_src = audio_src else: self.audio_src = 'No Audio' self.thread = None self.running = Event() self.recording = Event() self.recording.clear() self.audio_container = None self.audio_out_stream = None self.queue = queue.Queue() self.start_capture(self.audio_src) def init_ui(self): self.add_menu() self.menu.label = 'Audio Capture' help_str = 'Creates events for audio input.' self.menu.append(ui.Info_Text(help_str)) def audio_dev_getter(): # fetch list of currently available self.audio_devices_dict = Audio_Input_Dict() devices = list(self.audio_devices_dict.keys()) return devices, devices self.menu.append(ui.Selector('audio_src', self, selection_getter=audio_dev_getter, label='Audio Source', setter=self.start_capture)) self.menu.append(ui.Info_Text(NOT_REC_STR)) def deinit_ui(self): self.remove_menu() def get_init_dict(self): return {'audio_src': self.audio_src} def cleanup(self): if self.audio_container is not None: self.close_audio_recording() self.running.clear() if self.thread and self.thread.is_alive(): self.thread.join(timeout=1) def on_notify(self, notification): if notification['subject'] == 'recording.started': self.rec_dir = notification['rec_path'] self.recording.set() if self.running.is_set(): # and self.audio_container is None: self.menu[-2].read_only = True del self.menu[-1] self.menu.append(ui.Info_Text(REC_STR)) elif not self.running.is_set(): logger.warning('Recording was started without an active audio capture') elif notification['subject'] == 'recording.stopped': self.recording.clear() self.close_audio_recording() def close_audio_recording(self): self.menu[-2].read_only = False del self.menu[-1] self.menu.append(ui.Info_Text(NOT_REC_STR)) def write_audio_packet(self, audio_frame): # Test if audio outstream has been initialized if self.audio_out_stream is None: try: self.audio_out_stream = self.audio_container.add_stream('aac') except ValueError as e: # packet.stream codec is not supported in target container. logger.error('Failed to create audio stream. Aborting recording.') logger.debug('Reason: {}'.format(e)) self.close_audio_recording() self.timestamps.append(audio_frame.timestamp) packet = self.audio_out_stream.encode(audio_frame) if packet is not None: self.audio_container.mux(packet) def start_capture(self, audio_src): if self.thread and self.thread.is_alive(): if self.audio_src == audio_src: return # capture is already running for our selected source # else stop current capture gracefully self.running.clear() logger.debug('Closing capture for "{}"'.format(self.audio_src)) self.thread.join(timeout=1) self.thread = None if audio_src not in self.audio_devices_dict: logger.warning('Selected audio source is not available anymore') return self.audio_src = audio_src if audio_src == 'No Audio': return self.running.set() self.thread = Thread(target=self.capture_thread, args=(self.audio_devices_dict[audio_src], self.running, self.recording)) self.thread.start() def capture_thread(self, audio_src, running, recording): try: if platform.system() == "Darwin": in_container = av.open('none:{}'.format(audio_src), format="avfoundation") elif platform.system() == "Linux": in_container = av.open('hw:{}'.format(audio_src), format="alsa") else: raise av.AVError('Platform does not support audio capture.') except av.AVError as err: running.clear() self.audio_src = 'No Audio' logger.warning('Error starting audio capture: {}'.format(err)) return in_stream = None try: in_stream = in_container.streams.audio[0] except IndexError: logger.warning('No audio stream found for selected device.') running.clear() self.audio_src = 'No Audio' return out_container = None out_stream = None timestamps = None out_frame_num = 0 in_frame_size = 0 stream_epoch = in_stream.start_time * in_stream.time_base uvc_clock_dif = abs(stream_epoch - self.g_pool.get_now()) pyt_clock_dif = abs(stream_epoch - time()) max_clock_dif = 4. if uvc_clock_dif > max_clock_dif and pyt_clock_dif > max_clock_dif: logger.error('Could not identify audio stream clock.') running.clear() self.audio_src = 'No Audio' return elif uvc_clock_dif > pyt_clock_dif: logger.info('Audio stream uses time.time() as clock (Δ {}s)'.format(pyt_clock_dif)) clock_differences = self.g_pool.get_now() - time() else: logger.info('Audio stream uses uvc.get_time_monotonic() as clock') clock_differences = 0 def close_recording(): # Bind nonlocal variables, https://www.python.org/dev/peps/pep-3104/ nonlocal out_container, out_stream, in_stream, in_frame_size, timestamps, out_frame_num if out_container is not None: packet = out_stream.encode(audio_frame) while packet is not None: out_frame_num += 1 out_container.mux(packet) packet = out_stream.encode(audio_frame) out_container.close() in_frame_rate = in_stream.rate # in_stream.frame_size does not return the correct value. out_frame_size = out_stream.frame_size out_frame_rate = out_stream.rate old_ts_idx = np.arange(0, len(timestamps) * in_frame_size, in_frame_size) / in_frame_rate new_ts_idx = np.arange(0, out_frame_num * out_frame_size, out_frame_size) / out_frame_rate interpolate = interp1d(old_ts_idx, timestamps, bounds_error=False, fill_value='extrapolate') new_ts = interpolate(new_ts_idx) ts_loc = os.path.join(self.rec_dir, 'audio_timestamps.npy') np.save(ts_loc, new_ts) out_container = None out_stream = None timestamps = None for packet in in_container.demux(in_stream): # ffmpeg timestamps - in_stream.startime = packte pts relative to startime # multiply with stream_timebase to get seconds # add start time of this stream in pupil time unadjusted # finally add pupil timebase offset to adjust for settable timebase. for audio_frame in packet.decode(): timestamp = audio_frame.pts * in_stream.time_base + clock_differences - self.g_pool.timebase.value if recording.is_set(): if out_container is None: rec_file = os.path.join(self.rec_dir, 'audio.mp4') out_container = av.open(rec_file, 'w') out_stream = out_container.add_stream('aac') out_frame_num = 0 in_frame_size = audio_frame.samples # set here to make sure full packet size is used timestamps = [] timestamps.append(timestamp) packet = out_stream.encode(audio_frame) if packet is not None: out_frame_num += 1 out_container.mux(packet) elif out_container is not None: # recording stopped close_recording() if not running.is_set(): close_recording() return close_recording() self.audio_src = 'No Audio' running.clear() # in_stream stopped yielding packets
class Recorder(Plugin): """Capture Recorder""" def __init__( self, g_pool, session_name=get_auto_name(), rec_dir=None, user_info={"name": "", "additional_field": "change_me"}, info_menu_conf={}, show_info_menu=False, record_eye=False, audio_src="No Audio", raw_jpeg=False, ): super(Recorder, self).__init__(g_pool) # update name if it was autogenerated. if session_name.startswith("20") and len(session_name) == 10: session_name = get_auto_name() if rec_dir: self.set_rec_dir(rec_dir) else: # lets make a rec dir next to the user dir base_dir = self.g_pool.user_dir.rsplit(os.path.sep, 1)[0] self.rec_dir = os.path.join(base_dir, "recordings") if not os.path.isdir(self.rec_dir): os.mkdir(self.rec_dir) self.raw_jpeg = raw_jpeg self.order = 0.9 self.record_eye = record_eye self.session_name = session_name self.audio_devices_dict = Audio_Input_Dict() if audio_src in self.audio_devices_dict.keys(): self.audio_src = audio_src else: self.audio_src = "No Audio" self.running = False self.menu = None self.button = None self.user_info = user_info self.show_info_menu = show_info_menu self.info_menu = None self.info_menu_conf = info_menu_conf self.height, self.width = self.g_pool.capture.frame_size def get_init_dict(self): d = {} d["record_eye"] = self.record_eye d["audio_src"] = self.audio_src d["session_name"] = self.session_name d["user_info"] = self.user_info d["info_menu_conf"] = self.info_menu_conf d["show_info_menu"] = self.show_info_menu d["rec_dir"] = self.rec_dir d["raw_jpeg"] = self.raw_jpeg return d def init_gui(self): self.menu = ui.Growing_Menu("Recorder") self.g_pool.sidebar.insert(3, self.menu) self.menu.append( ui.Info_Text( 'Pupil recordings are saved like this: "path_to_recordings/recording_session_name/nnn" where "nnn" is an increasing number to avoid overwrites. You can use "/" in your session name to create subdirectories.' ) ) self.menu.append( ui.Info_Text( 'Recordings are saved to "~/pupil_recordings". You can change the path here but note that invalid input will be ignored.' ) ) self.menu.append(ui.Text_Input("rec_dir", self, setter=self.set_rec_dir, label="Path to recordings")) self.menu.append( ui.Text_Input("session_name", self, setter=self.set_session_name, label="Recording session name") ) self.menu.append( ui.Switch("show_info_menu", self, on_val=True, off_val=False, label="Request additional user info") ) self.menu.append( ui.Selector( "raw_jpeg", self, selection=[True, False], labels=["bigger file, less CPU", "smaller file, more CPU"], label="compression", ) ) self.menu.append(ui.Info_Text("Recording the raw eye video is optional. We use it for debugging.")) self.menu.append(ui.Switch("record_eye", self, on_val=True, off_val=False, label="Record eye")) self.menu.append(ui.Selector("audio_src", self, selection=self.audio_devices_dict.keys())) self.button = ui.Thumb("running", self, setter=self.toggle, label="Record", hotkey="r") self.button.on_color[:] = (1, 0.0, 0.0, 0.8) self.g_pool.quickbar.insert(1, self.button) def deinit_gui(self): if self.menu: self.g_pool.sidebar.remove(self.menu) self.menu = None if self.button: self.g_pool.quickbar.remove(self.button) self.button = None def toggle(self, _=None): if self.running: self.stop() else: self.start() def get_rec_time_str(self): rec_time = gmtime(time() - self.start_time) return strftime("%H:%M:%S", rec_time) def start(self): self.timestamps = [] self.data = {"pupil_positions": [], "gaze_positions": []} self.pupil_pos_list = [] self.gaze_pos_list = [] self.frame_count = 0 self.running = True self.menu.read_only = True self.start_time = time() session = os.path.join(self.rec_dir, self.session_name) try: os.makedirs(session) logger.debug("Created new recordings session dir %s" % session) except: logger.debug("Recordings session dir %s already exists, using it." % session) # set up self incrementing folder within session folder counter = 0 while True: self.rec_path = os.path.join(session, "%03d/" % counter) try: os.mkdir(self.rec_path) logger.debug("Created new recording dir %s" % self.rec_path) break except: logger.debug("We dont want to overwrite data, incrementing counter & trying to make new data folder") counter += 1 self.meta_info_path = os.path.join(self.rec_path, "info.csv") with open(self.meta_info_path, "w") as f: f.write("Recording Name\t" + self.session_name + "\n") f.write("Start Date\t" + strftime("%d.%m.%Y", localtime(self.start_time)) + "\n") f.write("Start Time\t" + strftime("%H:%M:%S", localtime(self.start_time)) + "\n") if self.audio_src != "No Audio": audio_path = os.path.join(self.rec_path, "world.wav") self.audio_writer = Audio_Capture(self.audio_devices_dict[self.audio_src], audio_path) else: self.audio_writer = None if self.raw_jpeg and "uvc_capture" in str(self.g_pool.capture.__class__): self.video_path = os.path.join(self.rec_path, "world.mp4") self.writer = JPEG_Writer(self.video_path, int(self.g_pool.capture.frame_rate)) # elif 1: # self.writer = av_writer.AV_Writer(self.video_path) else: self.video_path = os.path.join(self.rec_path, "world.mkv") self.writer = CV_Writer( self.video_path, float(self.g_pool.capture.frame_rate), self.g_pool.capture.frame_size ) # positions path to eye process if self.record_eye: for tx in self.g_pool.eye_tx: tx.send((self.rec_path, self.raw_jpeg)) if self.show_info_menu: self.open_info_menu() def open_info_menu(self): self.info_menu = ui.Growing_Menu("additional Recording Info", size=(300, 300), pos=(300, 300)) self.info_menu.configuration = self.info_menu_conf def populate_info_menu(): self.info_menu.elements[:-2] = [] for name in self.user_info.iterkeys(): self.info_menu.insert(0, ui.Text_Input(name, self.user_info)) def set_user_info(new_string): self.user_info = new_string populate_info_menu() populate_info_menu() self.info_menu.append( ui.Info_Text( 'Use the *user info* field to add/remove additional fields and their values. The format must be a valid Python dictionary. For example -- {"key":"value"}. You can add as many fields as you require. Your custom fields will be saved for your next session.' ) ) self.info_menu.append(ui.Text_Input("user_info", self, setter=set_user_info, label="User info")) self.g_pool.gui.append(self.info_menu) def close_info_menu(self): if self.info_menu: self.info_menu_conf = self.info_menu.configuration self.g_pool.gui.remove(self.info_menu) self.info_menu = None def update(self, frame, events): if self.running: self.data["pupil_positions"] += events["pupil_positions"] self.data["gaze_positions"] += events["gaze_positions"] self.timestamps.append(frame.timestamp) self.writer.write_video_frame(frame) # self.writer.write_video_frame_yuv422(frame) self.frame_count += 1 # cv2.putText(frame.img, "Frame %s"%self.frame_count,(200,200), cv2.FONT_HERSHEY_SIMPLEX,1,(255,100,100)) for p in events["pupil_positions"]: pupil_pos = p["timestamp"], p["confidence"], p["id"], p["norm_pos"][0], p["norm_pos"][1], p["diameter"] self.pupil_pos_list.append(pupil_pos) for g in events.get("gaze_positions", []): gaze_pos = g["timestamp"], g["confidence"], g["norm_pos"][0], g["norm_pos"][1] self.gaze_pos_list.append(gaze_pos) self.button.status_text = self.get_rec_time_str() def stop(self): # explicit release of VideoWriter self.writer.release() self.writer = None if self.record_eye: for tx in self.g_pool.eye_tx: try: tx.send((None, None)) except: logger.warning("Could not stop eye-recording. Please report this bug!") save_object(self.data, os.path.join(self.rec_path, "pupil_data")) gaze_list_path = os.path.join(self.rec_path, "gaze_positions.npy") np.save(gaze_list_path, np.asarray(self.gaze_pos_list)) pupil_list_path = os.path.join(self.rec_path, "pupil_positions.npy") np.save(pupil_list_path, np.asarray(self.pupil_pos_list)) timestamps_path = os.path.join(self.rec_path, "world_timestamps.npy") ts = sanitize_timestamps(np.array(self.timestamps)) np.save(timestamps_path, ts) try: copy2( os.path.join(self.g_pool.user_dir, "surface_definitions"), os.path.join(self.rec_path, "surface_definitions"), ) except: logger.info("No surface_definitions data found. You may want this if you do marker tracking.") try: copy2( os.path.join(self.g_pool.user_dir, "cal_pt_cloud.npy"), os.path.join(self.rec_path, "cal_pt_cloud.npy") ) except: logger.warning("No calibration data found. Please calibrate first.") try: copy2( os.path.join(self.g_pool.user_dir, "camera_matrix.npy"), os.path.join(self.rec_path, "camera_matrix.npy"), ) copy2(os.path.join(self.g_pool.user_dir, "dist_coefs.npy"), os.path.join(self.rec_path, "dist_coefs.npy")) except: logger.info("No camera intrinsics found.") try: with open(self.meta_info_path, "a") as f: f.write("Duration Time\t" + self.get_rec_time_str() + "\n") if self.g_pool.binocular: f.write("Eye Mode\tbinocular\n") else: f.write("Eye Mode\tmonocular\n") f.write("Duration Time\t" + self.get_rec_time_str() + "\n") f.write("World Camera Frames\t" + str(self.frame_count) + "\n") f.write("World Camera Resolution\t" + str(self.width) + "x" + str(self.height) + "\n") f.write("Capture Software Version\t%s\n" % self.g_pool.version) if platform.system() == "Windows": username = os.environ["USERNAME"] sysname, nodename, release, version, machine, _ = platform.uname() else: username = os.getlogin() try: sysname, nodename, release, version, machine = os.uname() except: sysname, nodename, release, version, machine = sys.platform, None, None, None, None f.write("User\t" + username + "\n") f.write("Platform\t" + sysname + "\n") f.write("Machine\t" + nodename + "\n") f.write("Release\t" + release + "\n") f.write("Version\t" + version + "\n") except Exception: logger.exception("Could not save metadata. Please report this bug!") try: with open(os.path.join(self.rec_path, "user_info.csv"), "w") as f: for name, val in self.user_info.iteritems(): f.write("%s\t%s\n" % (name, val)) except Exception: logger.exception("Could not save userdata. Please report this bug!") self.close_info_menu() if self.audio_writer: self.audio_writer = None self.running = False self.menu.read_only = False self.button.status_text = "" def cleanup(self): """gets called when the plugin get terminated. either volunatily or forced. """ if self.running: self.stop() self.deinit_gui() def set_rec_dir(self, val): try: n_path = os.path.expanduser(val) logger.debug("Expanded user path.") except: n_path = val if not n_path: logger.warning("Please specify a path.") elif not os.path.isdir(n_path): logger.warning("This is not a valid path.") # elif not os.access(n_path, os.W_OK): elif not writable_dir(n_path): logger.warning("Do not have write access to '%s'." % n_path) else: self.rec_dir = n_path def set_session_name(self, val): if not val: self.session_name = get_auto_name() else: if "/" in val: logger.warning("You session name with create one or more subdirectories") self.session_name = val
def __init__(self,g_pool,session_name = get_auto_name(),rec_dir=None, user_info={'name':'','additional_field':'change_me'},info_menu_conf={},show_info_menu=False, record_eye = False, audio_src = 'No Audio', raw_jpeg=False): super(Recorder, self).__init__(g_pool) self.states = ['start', 'want', 'pick'] self.all_qr = {'termo1' : 75, 'light1' : False, 'light2' : False, 'light3': False} self.this_state = 'start' self.start_state = time() self.qr_codes = {} self.gaze_x = -1 self.gaze_y = -1 self.avg_col = 0 self.choice = "" #update name if it was autogenerated. if session_name.startswith('20') and len(session_name)==10: session_name = get_auto_name() base_dir = self.g_pool.user_dir.rsplit(os.path.sep,1)[0] default_rec_dir = os.path.join(base_dir, 'recordings') if rec_dir and rec_dir != default_rec_dir and self.verify_path(rec_dir): self.rec_dir = rec_dir else: try: os.makedirs(default_rec_dir) except OSError as e: if 'File exists' in '%s'%e: pass else: logger.error("Could not create Rec dir") raise e else: logger.info('Created standard Rec dir at "%s"'%default_rec_dir) self.rec_dir = default_rec_dir # if rec_dir and rec_dir != default_rec_dir and self.verify_path(rec_dir): # self.rec_dir = rec_dir # else: # #lets make a rec dir next to the user dir # base_dir = self.g_pool.user_dir.rsplit(os.path.sep,1)[0] # self.rec_dir = os.path.join(base_dir,'recordings') # if not os.path.isdir(self.rec_dir): # os.mkdir(self.rec_dir) self.raw_jpeg = raw_jpeg self.order = .9 self.record_eye = record_eye self.session_name = session_name self.audio_devices_dict = Audio_Input_Dict() if audio_src in self.audio_devices_dict.keys(): self.audio_src = audio_src else: self.audio_src = 'No Audio' self.running = False self.menu = None self.button = None self.user_info = user_info self.show_info_menu = show_info_menu self.info_menu = None self.info_menu_conf = info_menu_conf self.height, self.width = self.g_pool.capture.frame_size
class Recorder(Plugin): """Capture Recorder""" def __init__(self, g_pool, session_name=get_auto_name(), rec_dir=None, user_info={ 'name': '', 'additional_field': 'change_me' }, info_menu_conf={}, show_info_menu=False, record_eye=False, audio_src='No Audio', raw_jpeg=True): super(Recorder, self).__init__(g_pool) #update name if it was autogenerated. if session_name.startswith('20') and len(session_name) == 10: session_name = get_auto_name() base_dir = self.g_pool.user_dir.rsplit(os.path.sep, 1)[0] default_rec_dir = os.path.join(base_dir, 'recordings') if rec_dir and rec_dir != default_rec_dir and self.verify_path( rec_dir): self.rec_dir = rec_dir else: try: os.makedirs(default_rec_dir) except OSError as e: if e.errno != errno.EEXIST: logger.error("Could not create Rec dir") raise e else: logger.info('Created standard Rec dir at "%s"' % default_rec_dir) self.rec_dir = default_rec_dir self.raw_jpeg = raw_jpeg self.order = .9 self.record_eye = record_eye self.session_name = session_name self.audio_devices_dict = Audio_Input_Dict() if audio_src in self.audio_devices_dict.keys(): self.audio_src = audio_src else: self.audio_src = 'No Audio' self.running = False self.menu = None self.button = None self.user_info = user_info self.show_info_menu = show_info_menu self.info_menu = None self.info_menu_conf = info_menu_conf def get_init_dict(self): d = {} d['record_eye'] = self.record_eye d['audio_src'] = self.audio_src d['session_name'] = self.session_name d['user_info'] = self.user_info d['info_menu_conf'] = self.info_menu_conf d['show_info_menu'] = self.show_info_menu d['rec_dir'] = self.rec_dir d['raw_jpeg'] = self.raw_jpeg return d def init_gui(self): self.menu = ui.Growing_Menu('Recorder') self.g_pool.sidebar.insert(3, self.menu) self.menu.append( ui.Info_Text( 'Pupil recordings are saved like this: "path_to_recordings/recording_session_name/nnn" where "nnn" is an increasing number to avoid overwrites. You can use "/" in your session name to create subdirectories.' )) self.menu.append( ui.Info_Text( 'Recordings are saved to "~/pupil_recordings". You can change the path here but note that invalid input will be ignored.' )) self.menu.append( ui.Text_Input('rec_dir', self, setter=self.set_rec_dir, label='Path to recordings')) self.menu.append( ui.Text_Input('session_name', self, setter=self.set_session_name, label='Recording session name')) self.menu.append( ui.Switch('show_info_menu', self, on_val=True, off_val=False, label='Request additional user info')) self.menu.append( ui.Selector( 'raw_jpeg', self, selection=[True, False], labels=["bigger file, less CPU", "smaller file, more CPU"], label='compression')) self.menu.append( ui.Info_Text( 'Recording the raw eye video is optional. We use it for debugging.' )) self.menu.append( ui.Switch('record_eye', self, on_val=True, off_val=False, label='Record eye')) self.menu.append( ui.Selector('audio_src', self, selection=self.audio_devices_dict.keys())) self.button = ui.Thumb('running', self, setter=self.toggle, label='Record', hotkey='r') self.button.on_color[:] = (1, .0, .0, .8) self.g_pool.quickbar.insert(1, self.button) def deinit_gui(self): if self.menu: self.g_pool.sidebar.remove(self.menu) self.menu = None if self.button: self.g_pool.quickbar.remove(self.button) self.button = None def toggle(self, _=None): if self.running: self.stop() else: self.start() def on_notify(self, notification): if notification['name'] == 'rec_should_start': if self.running: logger.warning('Recording is already running!') else: self.set_session_name(notification["session_name"]) self.start(network_propagate=notification.get( 'network_propagate', True)) elif notification['name'] == 'rec_should_stop': if self.running: self.stop(network_propagate=notification.get( 'network_propagate', True)) else: logger.warning('Recording is already stopped!') def get_rec_time_str(self): rec_time = gmtime(time() - self.start_time) return strftime("%H:%M:%S", rec_time) def start(self, network_propagate=True): self.timestamps = [] self.data = {'pupil_positions': [], 'gaze_positions': []} self.pupil_pos_list = [] self.gaze_pos_list = [] self.frame_count = 0 self.running = True self.menu.read_only = True self.start_time = time() session = os.path.join(self.rec_dir, self.session_name) try: os.makedirs(session) logger.debug("Created new recordings session dir %s" % session) except: logger.debug( "Recordings session dir %s already exists, using it." % session) # set up self incrementing folder within session folder counter = 0 while True: self.rec_path = os.path.join(session, "%03d/" % counter) try: os.mkdir(self.rec_path) logger.debug("Created new recording dir %s" % self.rec_path) break except: logger.debug( "We dont want to overwrite data, incrementing counter & trying to make new data folder" ) counter += 1 self.meta_info_path = os.path.join(self.rec_path, "info.csv") with open(self.meta_info_path, 'w') as f: f.write("Recording Name\t" + self.session_name + "\n") f.write("Start Date\t" + strftime("%d.%m.%Y", localtime(self.start_time)) + "\n") f.write("Start Time\t" + strftime("%H:%M:%S", localtime(self.start_time)) + "\n") if self.audio_src != 'No Audio': audio_path = os.path.join(self.rec_path, "world.wav") self.audio_writer = Audio_Capture( audio_path, self.audio_devices_dict[self.audio_src]) else: self.audio_writer = None if self.raw_jpeg and self.g_pool.capture.jpeg_support: self.video_path = os.path.join(self.rec_path, "world.mp4") self.writer = JPEG_Writer(self.video_path, int(self.g_pool.capture.frame_rate)) else: self.video_path = os.path.join(self.rec_path, "world.mp4") self.writer = AV_Writer(self.video_path) # positions path to eye process if self.record_eye: for tx in self.g_pool.eye_tx: tx.send((self.rec_path, self.raw_jpeg)) if self.show_info_menu: self.open_info_menu() self.notify_all({ 'name': 'rec_started', 'rec_path': self.rec_path, 'session_name': self.session_name, 'network_propagate': network_propagate }) def open_info_menu(self): self.info_menu = ui.Growing_Menu('additional Recording Info', size=(300, 300), pos=(300, 300)) self.info_menu.configuration = self.info_menu_conf def populate_info_menu(): self.info_menu.elements[:-2] = [] for name in self.user_info.iterkeys(): self.info_menu.insert(0, ui.Text_Input(name, self.user_info)) def set_user_info(new_string): self.user_info = new_string populate_info_menu() populate_info_menu() self.info_menu.append( ui.Info_Text( 'Use the *user info* field to add/remove additional fields and their values. The format must be a valid Python dictionary. For example -- {"key":"value"}. You can add as many fields as you require. Your custom fields will be saved for your next session.' )) self.info_menu.append( ui.Text_Input('user_info', self, setter=set_user_info, label="User info")) self.g_pool.gui.append(self.info_menu) def close_info_menu(self): if self.info_menu: self.info_menu_conf = self.info_menu.configuration self.g_pool.gui.remove(self.info_menu) self.info_menu = None def update(self, frame, events): if self.running: self.data['pupil_positions'] += events['pupil_positions'] self.data['gaze_positions'] += events['gaze_positions'] self.timestamps.append(frame.timestamp) if self.g_pool.capture.jpeg_support: self.writer.write_video_frame_compressed(frame) else: self.writer.write_video_frame(frame) self.frame_count += 1 # cv2.putText(frame.img, "Frame %s"%self.frame_count,(200,200), cv2.FONT_HERSHEY_SIMPLEX,1,(255,100,100)) for p in events['pupil_positions']: pupil_pos = p['timestamp'], p['confidence'], p['id'], p[ 'norm_pos'][0], p['norm_pos'][1], p['diameter'] self.pupil_pos_list.append(pupil_pos) for g in events.get('gaze_positions', []): gaze_pos = g['timestamp'], g['confidence'], g['norm_pos'][ 0], g['norm_pos'][1] self.gaze_pos_list.append(gaze_pos) self.button.status_text = self.get_rec_time_str() def stop(self, network_propagate=True): #explicit release of VideoWriter self.writer.release() self.writer = None if self.record_eye: for tx in self.g_pool.eye_tx: try: tx.send((None, None)) except: logger.warning( "Could not stop eye-recording. Please report this bug!" ) save_object(self.data, os.path.join(self.rec_path, "pupil_data")) gaze_list_path = os.path.join(self.rec_path, "gaze_positions.npy") np.save(gaze_list_path, np.asarray(self.gaze_pos_list)) pupil_list_path = os.path.join(self.rec_path, "pupil_positions.npy") np.save(pupil_list_path, np.asarray(self.pupil_pos_list)) timestamps_path = os.path.join(self.rec_path, "world_timestamps.npy") # ts = sanitize_timestamps(np.array(self.timestamps)) ts = np.array(self.timestamps) np.save(timestamps_path, ts) try: copy2(os.path.join(self.g_pool.user_dir, "surface_definitions"), os.path.join(self.rec_path, "surface_definitions")) except: logger.info( "No surface_definitions data found. You may want this if you do marker tracking." ) try: copy2(os.path.join(self.g_pool.user_dir, "cal_pt_cloud.npy"), os.path.join(self.rec_path, "cal_pt_cloud.npy")) except: logger.warning( "No calibration data found. Please calibrate first.") try: copy2(os.path.join(self.g_pool.user_dir, "camera_calibration"), os.path.join(self.rec_path, "camera_calibration")) except: logger.info("No camera calibration found.") try: with open(self.meta_info_path, 'a') as f: f.write("Duration Time\t" + self.get_rec_time_str() + "\n") if self.g_pool.binocular: f.write("Eye Mode\tbinocular\n") else: f.write("Eye Mode\tmonocular\n") f.write("Duration Time\t" + self.get_rec_time_str() + "\n") f.write("World Camera Frames\t" + str(self.frame_count) + "\n") f.write("World Camera Resolution\t" + str(self.g_pool.capture.frame_size[0]) + "x" + str(self.g_pool.capture.frame_size[1]) + "\n") f.write("Capture Software Version\t%s\n" % self.g_pool.version) if platform.system() == "Windows": username = os.environ["USERNAME"] sysname, nodename, release, version, machine, _ = platform.uname( ) else: username = getpass.getuser() try: sysname, nodename, release, version, machine = os.uname( ) except: sysname, nodename, release, version, machine = sys.platform, None, None, None, None f.write("User\t" + username + "\n") f.write("Platform\t" + sysname + "\n") f.write("Machine\t" + nodename + "\n") f.write("Release\t" + release + "\n") f.write("Version\t" + version + "\n") except Exception: logger.exception( "Could not save metadata. Please report this bug!") try: with open(os.path.join(self.rec_path, "user_info.csv"), 'w') as f: for name, val in self.user_info.iteritems(): f.write("%s\t%s\n" % (name, val)) except Exception: logger.exception( "Could not save userdata. Please report this bug!") self.close_info_menu() if self.audio_writer: self.audio_writer = None self.running = False self.menu.read_only = False self.button.status_text = '' self.timestamps = [] self.data = {'pupil_positions': [], 'gaze_positions': []} self.pupil_pos_list = [] self.gaze_pos_list = [] self.notify_all({ 'name': 'rec_stopped', 'rec_path': self.rec_path, 'network_propagate': network_propagate }) def cleanup(self): """gets called when the plugin get terminated. either volunatily or forced. """ if self.running: self.stop() self.deinit_gui() def verify_path(self, val): try: n_path = os.path.expanduser(val) logger.debug("Expanded user path.") except: n_path = val if not n_path: logger.warning("Please specify a path.") return False elif not os.path.isdir(n_path): logger.warning("This is not a valid path.") return False # elif not os.access(n_path, os.W_OK): elif not writable_dir(n_path): logger.warning("Do not have write access to '%s'." % n_path) return False else: return n_path def set_rec_dir(self, val): n_path = self.verify_path(val) if n_path: self.rec_dir = n_path def set_session_name(self, val): if not val: self.session_name = get_auto_name() else: if '/' in val: logger.warning( 'You session name will create one or more subdirectories') self.session_name = val
def __init__(self, g_pool, session_name=get_auto_name(), rec_dir=None, user_info={ 'name': '', 'additional_field': 'change_me' }, info_menu_conf={}, show_info_menu=False, record_eye=False, audio_src='No Audio', raw_jpeg=False): super(Recorder, self).__init__(g_pool) self.states = ['start', 'want', 'pick'] self.all_qr = { 'termo1': 75, 'light1': False, 'light2': False, 'light3': False } self.this_state = 'start' self.start_state = time() self.qr_codes = {} self.gaze_x = -1 self.gaze_y = -1 self.avg_col = 0 self.choice = "" #update name if it was autogenerated. if session_name.startswith('20') and len(session_name) == 10: session_name = get_auto_name() base_dir = self.g_pool.user_dir.rsplit(os.path.sep, 1)[0] default_rec_dir = os.path.join(base_dir, 'recordings') if rec_dir and rec_dir != default_rec_dir and self.verify_path( rec_dir): self.rec_dir = rec_dir else: try: os.makedirs(default_rec_dir) except OSError as e: if 'File exists' in '%s' % e: pass else: logger.error("Could not create Rec dir") raise e else: logger.info('Created standard Rec dir at "%s"' % default_rec_dir) self.rec_dir = default_rec_dir # if rec_dir and rec_dir != default_rec_dir and self.verify_path(rec_dir): # self.rec_dir = rec_dir # else: # #lets make a rec dir next to the user dir # base_dir = self.g_pool.user_dir.rsplit(os.path.sep,1)[0] # self.rec_dir = os.path.join(base_dir,'recordings') # if not os.path.isdir(self.rec_dir): # os.mkdir(self.rec_dir) self.raw_jpeg = raw_jpeg self.order = .9 self.record_eye = record_eye self.session_name = session_name self.audio_devices_dict = Audio_Input_Dict() if audio_src in self.audio_devices_dict.keys(): self.audio_src = audio_src else: self.audio_src = 'No Audio' self.running = False self.menu = None self.button = None self.user_info = user_info self.show_info_menu = show_info_menu self.info_menu = None self.info_menu_conf = info_menu_conf self.height, self.width = self.g_pool.capture.frame_size
class Audio_Capture(Plugin): """docstring for Audio_Capture""" icon_chr = chr(0xE029) icon_font = "pupil_icons" def __init__(self, g_pool, audio_src="No Audio"): super().__init__(g_pool) self.audio_devices_dict = Audio_Input_Dict() if audio_src in list(self.audio_devices_dict.keys()): self.audio_src = audio_src else: self.audio_src = "No Audio" self.thread = None self.running = Event() self.recording = Event() self.recording.clear() self.audio_container = None self.audio_out_stream = None self.queue = queue.Queue() self.start_capture(self.audio_src) def init_ui(self): self.add_menu() self.menu.label = "Audio Capture" help_str = "Creates events for audio input." self.menu.append(ui.Info_Text(help_str)) def audio_dev_getter(): # fetch list of currently available audio_src_val = None if self.audio_devices_dict is not None: if self.audio_src in self.audio_devices_dict.keys(): audio_src_val = self.audio_devices_dict[self.audio_src] self.audio_devices_dict = Audio_Input_Dict() if audio_src_val is not None: self.audio_devices_dict[self.audio_src] = audio_src_val devices = list(self.audio_devices_dict.keys()) return devices, devices self.menu.append( # TODO: potential race condition through selection_getter. Should ensure # that current selection will always be present in the list returned by the # selection_getter. Highly unlikely though as this needs to happen between # having clicked the Selector and the next redraw. # See https://github.com/pupil-labs/pyglui/pull/112/commits/587818e9556f14bfedd8ff8d093107358745c29b ui.Selector( "audio_src", self, selection_getter=audio_dev_getter, label="Audio Source", setter=self.start_capture, ) ) self.menu.append(ui.Info_Text(NOT_REC_STR)) def deinit_ui(self): self.remove_menu() def get_init_dict(self): return {"audio_src": self.audio_src} def cleanup(self): if self.audio_container is not None: self.close_audio_recording() self.running.clear() if self.thread and self.thread.is_alive(): self.thread.join(timeout=1) def on_notify(self, notification): if notification["subject"] == "recording.started": self.rec_dir = notification["rec_path"] self.recording.set() if self.running.is_set(): # and self.audio_container is None: self.menu[-2].read_only = True del self.menu[-1] self.menu.append(ui.Info_Text(REC_STR)) elif not self.running.is_set(): logger.warning("Recording was started without an active audio capture") elif notification["subject"] == "recording.stopped": self.recording.clear() self.close_audio_recording() def close_audio_recording(self): self.menu[-2].read_only = False del self.menu[-1] self.menu.append(ui.Info_Text(NOT_REC_STR)) def write_audio_packet(self, audio_frame): # Test if audio outstream has been initialized if self.audio_out_stream is None: try: self.audio_out_stream = self.audio_container.add_stream("aac") except ValueError as e: # packet.stream codec is not supported in target container. logger.error("Failed to create audio stream. Aborting recording.") logger.debug("Reason: {}".format(e)) self.close_audio_recording() self.timestamps.append(audio_frame.timestamp) packet = self.audio_out_stream.encode(audio_frame) if packet is not None: self.audio_container.mux(packet) def start_capture(self, audio_src): if self.thread and self.thread.is_alive(): if self.audio_src == audio_src: return # capture is already running for our selected source # else stop current capture gracefully self.running.clear() logger.debug('Closing capture for "{}"'.format(self.audio_src)) self.thread.join(timeout=1) self.thread = None if audio_src not in self.audio_devices_dict: logger.warning("Selected audio source is not available anymore") return self.audio_src = audio_src if audio_src == "No Audio": return self.running.set() self.thread = Thread( target=self.capture_thread, args=(self.audio_devices_dict[audio_src], self.running, self.recording), ) self.thread.start() def capture_thread(self, audio_src, running, recording): try: if platform.system() == "Darwin": in_container = av.open( "none:{}".format(audio_src), format="avfoundation" ) elif platform.system() == "Linux": print("audio src = {}".format(audio_src)) dev_str = re.search("(hw:\s*\d+,\s*\d+)", audio_src) in_container = av.open(dev_str.group(0), format="alsa") elif platform.system() == "Windows": in_container = av.open( "audio={}".format(audio_src), format="dshow", options={"audio_buffer_size": "23"}, ) else: raise av.AVError("Platform does not support audio capture.") except av.AVError as err: running.clear() self.audio_src = "No Audio" logger.warning("Error starting audio capture: {}".format(err)) return in_stream = None try: in_stream = in_container.streams.audio[0] except IndexError: logger.warning("No audio stream found for selected device.") running.clear() self.audio_src = "No Audio" return out_container = None out_stream = None timestamps = None in_frame_size = 0 stream_epoch = in_stream.start_time * in_stream.time_base uvc_clock_dif = abs(stream_epoch - self.g_pool.get_now()) pyt_clock_dif = abs(stream_epoch - time()) max_clock_dif = 4.0 if uvc_clock_dif > max_clock_dif and pyt_clock_dif > max_clock_dif: logger.error("Could not identify audio stream clock.") running.clear() self.audio_src = "No Audio" return elif uvc_clock_dif > pyt_clock_dif: logger.info( "Audio stream uses time.time() as clock (Δ {}s)".format(pyt_clock_dif) ) clock_differences = self.g_pool.get_now() - time() else: logger.info("Audio stream uses uvc.get_time_monotonic() as clock") clock_differences = 0 def close_recording(): # Bind nonlocal variables, https://www.python.org/dev/peps/pep-3104/ nonlocal out_container, out_stream, in_stream, in_frame_size, timestamps, out_frame_num if out_container is not None: timestamps.append(timestamp) audio_frame.pts = None out_packets = [out_stream.encode(audio_frame)] while out_packets[-1]: out_packets.append(out_stream.encode(None)) for out_packet in out_packets: if out_packet is not None: out_container.mux(out_packet) out_container.close() out_frame_num = out_stream.frames in_frame_rate = in_stream.rate # in_stream.frame_size does not return the correct value. out_frame_size = out_stream.frame_size out_frame_rate = out_stream.rate new_ts_idx = ( np.arange(0, out_frame_num * out_frame_size, out_frame_size) / out_frame_rate ) if in_frame_rate != out_frame_rate: old_ts_idx = ( np.arange(0, len(timestamps) * in_frame_size, in_frame_size) / in_frame_rate ) interpolate = interp1d( old_ts_idx, timestamps, bounds_error=False, fill_value="extrapolate", ) new_ts = interpolate(new_ts_idx) else: new_ts = timestamps[0] + new_ts_idx ts_loc = os.path.join(self.rec_dir, "audio_timestamps.npy") np.save(ts_loc, new_ts) out_container = None out_stream = None timestamps = None for packet in in_container.demux(in_stream): # ffmpeg timestamps - in_stream.startime = packte pts relative to startime # multiply with stream_timebase to get seconds # add start time of this stream in pupil time unadjusted # finally add pupil timebase offset to adjust for settable timebase. for audio_frame in packet.decode(): timestamp = ( audio_frame.pts * in_stream.time_base + clock_differences - self.g_pool.timebase.value ) if recording.is_set(): if out_container is None: rec_file = os.path.join(self.rec_dir, "audio.mp4") out_container = av.open(rec_file, "w") out_stream = out_container.add_stream( "aac", rate=in_stream.rate ) out_frame_num = 0 in_frame_size = ( audio_frame.samples ) # set here to make sure full packet size is used timestamps = [] timestamps.append(timestamp) audio_frame.pts = None out_packets = [out_stream.encode(audio_frame)] for out_packet in out_packets: if out_packet is not None: out_container.mux(out_packet) elif out_container is not None: # recording stopped close_recording() if not running.is_set(): close_recording() return close_recording() self.audio_src = "No Audio" running.clear() # in_stream stopped yielding packets
class Audio_Capture(Plugin): """docstring for Audio_Capture""" icon_chr = chr(0xE029) icon_font = "pupil_icons" def __init__(self, g_pool, audio_src="No Audio"): super().__init__(g_pool) self.audio_devices_dict = Audio_Input_Dict() if audio_src in list(self.audio_devices_dict.keys()): self.audio_src = audio_src else: self.audio_src = "No Audio" self.thread = None self.running = Event() self.recording = Event() self.recording.clear() self.audio_container = None self.audio_out_stream = None self.queue = queue.Queue() self.start_capture(self.audio_src) def init_ui(self): self.add_menu() self.menu.label = "Audio Capture" help_str = "Creates events for audio input." self.menu.append(ui.Info_Text(help_str)) def audio_dev_getter(): # fetch list of currently available audio_src_val = None if self.audio_devices_dict is not None: if self.audio_src in self.audio_devices_dict.keys(): audio_src_val = self.audio_devices_dict[self.audio_src] self.audio_devices_dict = Audio_Input_Dict() if audio_src_val is not None: self.audio_devices_dict[self.audio_src] = audio_src_val devices = list(self.audio_devices_dict.keys()) return devices, devices self.menu.append( ui.Selector( "audio_src", self, selection_getter=audio_dev_getter, label="Audio Source", setter=self.start_capture, ) ) self.menu.append(ui.Info_Text(NOT_REC_STR)) def deinit_ui(self): self.remove_menu() def get_init_dict(self): return {"audio_src": self.audio_src} def cleanup(self): if self.audio_container is not None: self.close_audio_recording() self.running.clear() if self.thread and self.thread.is_alive(): self.thread.join(timeout=1) def on_notify(self, notification): if notification["subject"] == "recording.started": self.rec_dir = notification["rec_path"] self.recording.set() if self.running.is_set(): # and self.audio_container is None: self.menu[-2].read_only = True del self.menu[-1] self.menu.append(ui.Info_Text(REC_STR)) elif not self.running.is_set(): logger.warning("Recording was started without an active audio capture") elif notification["subject"] == "recording.stopped": self.recording.clear() self.close_audio_recording() def close_audio_recording(self): self.menu[-2].read_only = False del self.menu[-1] self.menu.append(ui.Info_Text(NOT_REC_STR)) def write_audio_packet(self, audio_frame): # Test if audio outstream has been initialized if self.audio_out_stream is None: try: self.audio_out_stream = self.audio_container.add_stream("aac") except ValueError as e: # packet.stream codec is not supported in target container. logger.error("Failed to create audio stream. Aborting recording.") logger.debug("Reason: {}".format(e)) self.close_audio_recording() self.timestamps.append(audio_frame.timestamp) packet = self.audio_out_stream.encode(audio_frame) if packet is not None: self.audio_container.mux(packet) def start_capture(self, audio_src): if self.thread and self.thread.is_alive(): if self.audio_src == audio_src: return # capture is already running for our selected source # else stop current capture gracefully self.running.clear() logger.debug('Closing capture for "{}"'.format(self.audio_src)) self.thread.join(timeout=1) self.thread = None if audio_src not in self.audio_devices_dict: logger.warning("Selected audio source is not available anymore") return self.audio_src = audio_src if audio_src == "No Audio": return self.running.set() self.thread = Thread( target=self.capture_thread, args=(self.audio_devices_dict[audio_src], self.running, self.recording), ) self.thread.start() def capture_thread(self, audio_src, running, recording): try: if platform.system() == "Darwin": in_container = av.open( "none:{}".format(audio_src), format="avfoundation" ) elif platform.system() == "Linux": print("audio src = {}".format(audio_src)) dev_str = re.search("(hw:\s*\d+,\s*\d+)", audio_src) in_container = av.open(dev_str.group(0), format="alsa") elif platform.system() == "Windows": in_container = av.open( "audio={}".format(audio_src), format="dshow", options={"audio_buffer_size": "23"}, ) else: raise av.AVError("Platform does not support audio capture.") except av.AVError as err: running.clear() self.audio_src = "No Audio" logger.warning("Error starting audio capture: {}".format(err)) return in_stream = None try: in_stream = in_container.streams.audio[0] except IndexError: logger.warning("No audio stream found for selected device.") running.clear() self.audio_src = "No Audio" return out_container = None out_stream = None timestamps = None in_frame_size = 0 stream_epoch = in_stream.start_time * in_stream.time_base uvc_clock_dif = abs(stream_epoch - self.g_pool.get_now()) pyt_clock_dif = abs(stream_epoch - time()) max_clock_dif = 4.0 if uvc_clock_dif > max_clock_dif and pyt_clock_dif > max_clock_dif: logger.error("Could not identify audio stream clock.") running.clear() self.audio_src = "No Audio" return elif uvc_clock_dif > pyt_clock_dif: logger.info( "Audio stream uses time.time() as clock (Δ {}s)".format(pyt_clock_dif) ) clock_differences = self.g_pool.get_now() - time() else: logger.info("Audio stream uses uvc.get_time_monotonic() as clock") clock_differences = 0 def close_recording(): # Bind nonlocal variables, https://www.python.org/dev/peps/pep-3104/ nonlocal out_container, out_stream, in_stream, in_frame_size, timestamps, out_frame_num if out_container is not None: timestamps.append(timestamp) audio_frame.pts = None out_packets = [out_stream.encode(audio_frame)] while out_packets[-1]: out_packets.append(out_stream.encode(None)) for out_packet in out_packets: if out_packet is not None: out_container.mux(out_packet) out_container.close() out_frame_num = out_stream.frames in_frame_rate = in_stream.rate # in_stream.frame_size does not return the correct value. out_frame_size = out_stream.frame_size out_frame_rate = out_stream.rate new_ts_idx = ( np.arange(0, out_frame_num * out_frame_size, out_frame_size) / out_frame_rate ) if in_frame_rate != out_frame_rate: old_ts_idx = ( np.arange(0, len(timestamps) * in_frame_size, in_frame_size) / in_frame_rate ) interpolate = interp1d( old_ts_idx, timestamps, bounds_error=False, fill_value="extrapolate", ) new_ts = interpolate(new_ts_idx) else: new_ts = timestamps[0] + new_ts_idx ts_loc = os.path.join(self.rec_dir, "audio_timestamps.npy") np.save(ts_loc, new_ts) out_container = None out_stream = None timestamps = None for packet in in_container.demux(in_stream): # ffmpeg timestamps - in_stream.startime = packte pts relative to startime # multiply with stream_timebase to get seconds # add start time of this stream in pupil time unadjusted # finally add pupil timebase offset to adjust for settable timebase. for audio_frame in packet.decode(): timestamp = ( audio_frame.pts * in_stream.time_base + clock_differences - self.g_pool.timebase.value ) if recording.is_set(): if out_container is None: rec_file = os.path.join(self.rec_dir, "audio.mp4") out_container = av.open(rec_file, "w") out_stream = out_container.add_stream( "aac", rate=in_stream.rate ) out_frame_num = 0 in_frame_size = ( audio_frame.samples ) # set here to make sure full packet size is used timestamps = [] timestamps.append(timestamp) audio_frame.pts = None out_packets = [out_stream.encode(audio_frame)] for out_packet in out_packets: if out_packet is not None: out_container.mux(out_packet) elif out_container is not None: # recording stopped close_recording() if not running.is_set(): close_recording() return close_recording() self.audio_src = "No Audio" running.clear() # in_stream stopped yielding packets
class Recorder(Plugin): """Capture Recorder""" def __init__(self,g_pool,session_name = get_auto_name(),rec_dir=None, user_info={'name':'','additional_field':'change_me'},info_menu_conf={},show_info_menu=False, record_eye = False, audio_src = 'No Audio',raw_jpeg=True): super(Recorder, self).__init__(g_pool) #update name if it was autogenerated. if session_name.startswith('20') and len(session_name)==10: session_name = get_auto_name() base_dir = self.g_pool.user_dir.rsplit(os.path.sep,1)[0] default_rec_dir = os.path.join(base_dir,'recordings') if rec_dir and rec_dir != default_rec_dir and self.verify_path(rec_dir): self.rec_dir = rec_dir else: try: os.makedirs(default_rec_dir) except OSError as e: if e.errno != errno.EEXIST: logger.error("Could not create Rec dir") raise e else: logger.info('Created standard Rec dir at "%s"'%default_rec_dir) self.rec_dir = default_rec_dir self.raw_jpeg = raw_jpeg self.order = .9 self.record_eye = record_eye self.session_name = session_name self.audio_devices_dict = Audio_Input_Dict() if audio_src in self.audio_devices_dict.keys(): self.audio_src = audio_src else: self.audio_src = 'No Audio' self.running = False self.menu = None self.button = None self.user_info = user_info self.show_info_menu = show_info_menu self.info_menu = None self.info_menu_conf = info_menu_conf def get_init_dict(self): d = {} d['record_eye'] = self.record_eye d['audio_src'] = self.audio_src d['session_name'] = self.session_name d['user_info'] = self.user_info d['info_menu_conf'] = self.info_menu_conf d['show_info_menu'] = self.show_info_menu d['rec_dir'] = self.rec_dir d['raw_jpeg'] = self.raw_jpeg return d def init_gui(self): self.menu = ui.Growing_Menu('Recorder') self.g_pool.sidebar.insert(3,self.menu) self.menu.append(ui.Info_Text('Pupil recordings are saved like this: "path_to_recordings/recording_session_name/nnn" where "nnn" is an increasing number to avoid overwrites. You can use "/" in your session name to create subdirectories.')) self.menu.append(ui.Info_Text('Recordings are saved to "~/pupil_recordings". You can change the path here but note that invalid input will be ignored.')) self.menu.append(ui.Text_Input('rec_dir',self,setter=self.set_rec_dir,label='Path to recordings')) self.menu.append(ui.Text_Input('session_name',self,setter=self.set_session_name,label='Recording session name')) self.menu.append(ui.Switch('show_info_menu',self,on_val=True,off_val=False,label='Request additional user info')) self.menu.append(ui.Selector('raw_jpeg',self,selection = [True,False], labels=["bigger file, less CPU", "smaller file, more CPU"],label='Compression')) self.menu.append(ui.Info_Text('Recording the raw eye video is optional. We use it for debugging.')) self.menu.append(ui.Switch('record_eye',self,on_val=True,off_val=False,label='Record eye')) self.menu.append(ui.Selector('audio_src',self, selection=self.audio_devices_dict.keys(),label='Audio Source')) self.button = ui.Thumb('running',self,setter=self.toggle,label='Record',hotkey='r') self.button.on_color[:] = (1,.0,.0,.8) self.g_pool.quickbar.insert(1,self.button) def deinit_gui(self): if self.menu: self.g_pool.sidebar.remove(self.menu) self.menu = None if self.button: self.g_pool.quickbar.remove(self.button) self.button = None def toggle(self, _=None): if self.running: self.notify_all( {'subject':'should_stop_recording','network_propagate':True} ) else: self.notify_all( {'subject':'should_start_recording','session_name':self.session_name,'network_propagate':True} ) def on_notify(self,notification): # notification wants to be recorded if notification.get('record',False) and self.running: self.data['notifications'].append(notification) # Notificatio to start recording elif notification['subject'] == 'should_start_recording': if self.running: logger.info('Recording already running!') else: if notification.get("session_name",""): self.set_session_name(notification["session_name"]) self.start() # Remote has stopped recording, we should stop as well. elif notification['subject'] == 'should_stop_recording': if self.running: self.stop() else: logger.info('Recording already stopped!') def get_rec_time_str(self): rec_time = gmtime(time()-self.start_time) return strftime("%H:%M:%S", rec_time) def start(self): self.timestamps = [] self.data = {'pupil_positions':[],'gaze_positions':[],'notifications':[]} self.pupil_diameter = [] #addBy Changsoon self.gaze_point_z = [] #addBy Changsoon self.gaze_point_x = [] #addBy Changsoon self.gaze_point_y = [] #addBy Changsoon self.frame_count = 0 self.running = True self.menu.read_only = True self.start_time = time() session = os.path.join(self.rec_dir, self.session_name) try: os.makedirs(session) logger.debug("Created new recordings session dir %s"%session) except: logger.debug("Recordings session dir %s already exists, using it." %session) # set up self incrementing folder within session folder counter = 0 while True: self.rec_path = os.path.join(session, "%03d/" % counter) try: os.mkdir(self.rec_path) logger.debug("Created new recording dir %s"%self.rec_path) break except: logger.debug("We dont want to overwrite data, incrementing counter & trying to make new data folder") counter += 1 self.meta_info_path = os.path.join(self.rec_path, "info.csv") with open(self.meta_info_path, 'w') as f: f.write("Recording Name\t"+self.session_name+ "\n") f.write("Start Date\t"+ strftime("%d.%m.%Y", localtime(self.start_time))+ "\n") f.write("Start Time\t"+ strftime("%H:%M:%S", localtime(self.start_time))+ "\n") if self.audio_src != 'No Audio': audio_path = os.path.join(self.rec_path, "world.wav") self.audio_writer = Audio_Capture(audio_path,self.audio_devices_dict[self.audio_src]) else: self.audio_writer = None if self.raw_jpeg and self.g_pool.capture.jpeg_support: self.video_path = os.path.join(self.rec_path, "world.mp4") self.writer = JPEG_Writer(self.video_path,self.g_pool.capture.frame_rate) else: self.video_path = os.path.join(self.rec_path, "world.mp4") self.writer = AV_Writer(self.video_path,fps=self.g_pool.capture.frame_rate) # positions path to eye process if self.record_eye: for alive, pipe in zip(self.g_pool.eyes_are_alive,self.g_pool.eye_pipes): if alive.value: pipe.send( ('Rec_Start',(self.rec_path,self.raw_jpeg) ) ) if self.show_info_menu: self.open_info_menu() logger.info("Started Recording.") self.notify_all( {'subject':'rec_started','rec_path':self.rec_path,'session_name':self.session_name,'network_propagate':True} ) def open_info_menu(self): self.info_menu = ui.Growing_Menu('additional Recording Info',size=(300,300),pos=(300,300)) self.info_menu.configuration = self.info_menu_conf def populate_info_menu(): self.info_menu.elements[:-2] = [] for name in self.user_info.iterkeys(): self.info_menu.insert(0,ui.Text_Input(name,self.user_info)) def set_user_info(new_string): self.user_info = new_string populate_info_menu() populate_info_menu() self.info_menu.append(ui.Info_Text('Use the *user info* field to add/remove additional fields and their values. The format must be a valid Python dictionary. For example -- {"key":"value"}. You can add as many fields as you require. Your custom fields will be saved for your next session.')) self.info_menu.append(ui.Text_Input('user_info',self,setter=set_user_info,label="User info")) self.g_pool.gui.append(self.info_menu) def close_info_menu(self): if self.info_menu: self.info_menu_conf = self.info_menu.configuration self.g_pool.gui.remove(self.info_menu) self.info_menu = None def update(self,frame,events): if self.running: for key,data in events.iteritems(): if key not in ('dt'): try: self.data[key] += data if (key == 'pupil_positions'): try: pupilDict = data[0] self.pupil_diameter.append(pupilDict.get('diameter')) except: print('error'); elif (key == 'gaze_positions'): try: gazeDict = data[0] self.gaze_point_x.append(gazeDict.get('norm_pos')[0]) self.gaze_point_y.append(gazeDict.get('norm_pos')[1]) self.gaze_point_z.append(gazeDict.get('gaze_point_3d')[2]) except: print('error') print(self.pupil_diameter) print(self.gaze_point_x) print(self.gaze_point_y) cv2.namedWindow("window") #image process (OCR) find the condion that show use OCR except Exception as error: print(error)
class Audio_Capture(Plugin): """docstring for Audio_Capture""" def __init__(self, g_pool, audio_src='No Audio'): super().__init__(g_pool) self.audio_devices_dict = Audio_Input_Dict() if audio_src in list(self.audio_devices_dict.keys()): self.audio_src = audio_src else: self.audio_src = 'No Audio' self.thread = None self.running = Event() self.audio_container = None self.audio_out_stream = None self.queue = queue.Queue() self.start_capture(self.audio_src) def recent_events(self, events): audio_packets = [] while True: try: packet = self.queue.get_nowait() except queue.Empty: break audio_packets.append(packet) events['audio_packets'] = audio_packets if self.audio_container is not None: for packet in audio_packets: self.write_audio_packet(packet) def init_gui(self): self.menu = ui.Growing_Menu('Audio Capture') self.menu.collapsed = True self.g_pool.sidebar.append(self.menu) def close(): self.alive = False help_str = 'Creates events for audio input.' self.menu.append(ui.Button('Close', close)) self.menu.append(ui.Info_Text(help_str)) def audio_dev_getter(): # fetch list of currently available self.audio_devices_dict = Audio_Input_Dict() devices = list(self.audio_devices_dict.keys()) return devices, devices self.menu.append(ui.Selector('audio_src', self, selection_getter=audio_dev_getter, label='Audio Source', setter=self.start_capture)) self.menu.append(ui.Info_Text(NOT_REC_STR)) def get_init_dict(self): return {'audio_src': self.audio_src} def deinit_gui(self): if self.menu: self.g_pool.sidebar.remove(self.menu) self.menu = None def cleanup(self): if self.audio_container is not None: self.close_audio_recording() self.running.clear() self.deinit_gui() if self.thread and self.thread.is_alive(): self.thread.join(timeout=1) def on_notify(self, notification): if notification['subject'] == 'recording.started': if self.running.is_set() and self.audio_container is None: self.rec_dir = notification['rec_path'] rec_file = os.path.join(self.rec_dir, 'audio.wav') self.audio_container = av.open(rec_file, 'w') self.timestamps = [] self.menu[-2].read_only = True del self.menu[-1] self.menu.append(ui.Info_Text(REC_STR)) elif not self.running.is_set(): logger.warning('Recording was started without an active audio capture') else: logger.warning('Audio is already being recorded') elif notification['subject'] == 'recording.stopped': if self.audio_container is not None and self.audio_out_stream is not None: self.close_audio_recording() def close_audio_recording(self): self.audio_container.close() ts_loc = os.path.join(self.rec_dir, 'audio_timestamps.npy') np.save(ts_loc, np.asarray(self.timestamps)) self.timestamps = None self.audio_out_stream = None self.audio_container = None self.menu[-2].read_only = False del self.menu[-1] self.menu.append(ui.Info_Text(NOT_REC_STR)) def write_audio_packet(self, packet): # Test if audio outstream has been initialized if self.audio_out_stream is None: try: self.audio_out_stream = self.audio_container.add_stream(template=packet.stream) except ValueError as e: # packet.stream codec is not supported in target container. logger.error('Failed to create audio stream. Aborting recording.') logger.debug('Reason: {}'.format(e)) self.close_audio_recording() self.timestamps.append(packet.timestamp) self.audio_container.mux(packet) def start_capture(self, audio_src): if self.thread and self.thread.is_alive(): if self.audio_src == audio_src: return # capture is already running for our selected source # else stop current capture gracefully self.running.clear() logger.debug('Closing capture for "{}"'.format(self.audio_src)) self.thread.join(timeout=1) self.thread = None if audio_src not in self.audio_devices_dict: logger.warning('Selected audio source is not available anymore') return self.audio_src = audio_src if audio_src == 'No Audio': return self.running.set() self.thread = Thread(target=self.capture_thread, args=(self.audio_devices_dict[audio_src], self.running)) self.thread.start() def capture_thread(self, audio_src, running): try: if platform.system() == "Darwin": in_container = av.open('none:{}'.format(audio_src), format="avfoundation") elif platform.system() == "Linux": in_container = av.open('hw:{}'.format(audio_src), format="alsa") else: raise av.AVError('Platform does not support audio capture.') except av.AVError: running.clear() return in_stream = None for stream in in_container.streams: if stream.type == 'audio': in_stream = stream break if not in_stream: logger.warning('No audio stream found for selected device.') running.clear() return stream_start_ts = self.g_pool.get_now() for packet in in_container.demux(in_stream): try: # ffmpeg timestamps - in_stream.startime = packte pts relative to startime # multiply with stream_timebase to get seconds # add start time of this stream in pupil time unadjusted # finally add pupil timebase offset to adjust for settable timebase. packet.timestamp = (packet.pts - in_stream.start_time) * in_stream.time_base + stream_start_ts - self.g_pool.timebase.value self.queue.put_nowait(packet) except queue.Full: pass # drop packet if not running.is_set(): return self.audio_src = 'No Audio' running.clear() # in_stream stopped yielding packets
class Recorder(Plugin): """Capture Recorder""" def __init__(self,g_pool,session_name = get_auto_name(),rec_dir=None, user_info={},info_menu_conf={},show_info_menu=False, record_eye = True, audio_src = 'No Audio',raw_jpeg=True): super(Recorder, self).__init__(g_pool) #update name if it was autogenerated. if session_name.startswith('20') and len(session_name)==10: session_name = get_auto_name() base_dir = self.g_pool.user_dir.rsplit(os.path.sep,1)[0] default_rec_dir = os.path.join(base_dir,'recordings') if rec_dir and rec_dir != default_rec_dir and self.verify_path(rec_dir): self.rec_dir = rec_dir else: try: os.makedirs(default_rec_dir) except OSError as e: if e.errno != errno.EEXIST: logger.error("Could not create Rec dir") raise e else: logger.info('Created standard Rec dir at "%s"'%default_rec_dir) self.rec_dir = default_rec_dir self.raw_jpeg = raw_jpeg self.order = .9 self.record_eye = record_eye self.session_name = session_name self.audio_devices_dict = Audio_Input_Dict() if audio_src in self.audio_devices_dict.keys(): self.audio_src = audio_src else: self.audio_src = 'No Audio' self.running = False self.menu = None self.button = None self.user_info = user_info self.show_info_menu = show_info_menu self.info_menu = None self.info_menu_conf = info_menu_conf self.calibration_start_at = None self.calibration_end_at = None def get_init_dict(self): d = {} d['record_eye'] = self.record_eye d['audio_src'] = self.audio_src d['session_name'] = self.session_name d['user_info'] = self.user_info d['info_menu_conf'] = self.info_menu_conf d['show_info_menu'] = self.show_info_menu d['rec_dir'] = self.rec_dir d['raw_jpeg'] = self.raw_jpeg return d def init_gui(self): self.menu = ui.Growing_Menu('Recorder') self.g_pool.sidebar.insert(3,self.menu) self.menu.append(ui.Info_Text('Pupil recordings are saved like this: "path_to_recordings/recording_session_name/nnn" where "nnn" is an increasing number to avoid overwrites. You can use "/" in your session name to create subdirectories.')) self.menu.append(ui.Info_Text('Recordings are saved to "~/pupil_recordings". You can change the path here but note that invalid input will be ignored.')) self.menu.append(ui.Text_Input('rec_dir',self,setter=self.set_rec_dir,label='Path to recordings')) self.menu.append(ui.Text_Input('session_name',self,setter=self.set_session_name,label='Recording session name')) self.menu.append(ui.Switch('show_info_menu',self,on_val=True,off_val=False,label='Request additional user info')) self.menu.append(ui.Selector('raw_jpeg',self,selection = [True,False], labels=["bigger file, less CPU", "smaller file, more CPU"],label='Compression')) self.menu.append(ui.Info_Text('Recording the raw eye video is optional. We use it for debugging.')) self.menu.append(ui.Switch('record_eye',self,on_val=True,off_val=False,label='Record eye')) self.menu.append(ui.Selector('audio_src',self, selection=self.audio_devices_dict.keys(),label='Audio Source')) # record on enter click from the remote shutter self.button = ui.Thumb('running',self,setter=self.toggle,label='Record',hotkey=GLFW_KEY_ENTER) self.button.on_color[:] = (1,.0,.0,.8) self.g_pool.quickbar.insert(1,self.button) def deinit_gui(self): if self.menu: self.g_pool.sidebar.remove(self.menu) self.menu = None if self.button: self.g_pool.quickbar.remove(self.button) self.button = None def toggle(self, _=None): if self.running: if self.calibration_start_at and not self.calibration_end_at: # Mark the end of claibration self.calibration_end_at = self.timestamps[-1] else: self.notify_all( {'subject':'should_stop_recording','network_propagate':True} ) else: self.notify_all( {'subject':'should_start_recording','session_name':self.session_name,'network_propagate':True} ) def on_notify(self,notification): # notification wants to be recorded if notification.get('record',False) and self.running: self.data['notifications'].append(notification) # Notificatio to start recording elif notification['subject'] == 'should_start_recording': if self.running: logger.info('Recording already running!') else: if notification.get("session_name",""): self.set_session_name(notification["session_name"]) self.start() # Remote has stopped recording, we should stop as well. elif notification['subject'] == 'should_stop_recording': if self.running: self.stop(shutdown_os=True) else: logger.info('Recording already stopped!') def get_rec_time_str(self): rec_time = gmtime(time()-self.start_time) return strftime("%H:%M:%S", rec_time) def start(self): self.timestamps = [] self.data = {'pupil_positions':[],'gaze_positions':[],'notifications':[]} self.frame_count = 0 self.running = True self.menu.read_only = True self.start_time = time() session = os.path.join(self.rec_dir, self.session_name) try: os.makedirs(session) logger.debug("Created new recordings session dir %s"%session) except: logger.debug("Recordings session dir %s already exists, using it." %session) # set up self incrementing folder within session folder counter = 0 while True: self.rec_path = os.path.join(session, "%03d/" % counter) try: os.mkdir(self.rec_path) logger.debug("Created new recording dir %s"%self.rec_path) break except: logger.debug("We dont want to overwrite data, incrementing counter & trying to make new data folder") counter += 1 self.meta_info_path = os.path.join(self.rec_path, "info.csv") with open(self.meta_info_path, 'w') as f: f.write("Recording Name\t"+self.session_name+ "\n") f.write("Start Date\t"+ strftime("%d.%m.%Y", localtime(self.start_time))+ "\n") f.write("Start Time\t"+ strftime("%H:%M:%S", localtime(self.start_time))+ "\n") if self.audio_src != 'No Audio': audio_path = os.path.join(self.rec_path, "world.wav") self.audio_writer = Audio_Capture(audio_path,self.audio_devices_dict[self.audio_src]) else: self.audio_writer = None if self.raw_jpeg and self.g_pool.capture.jpeg_support: self.video_path = os.path.join(self.rec_path, "world.mp4") self.writer = JPEG_Writer(self.video_path,self.g_pool.capture.frame_rate) else: self.video_path = os.path.join(self.rec_path, "world.mp4") self.writer = AV_Writer(self.video_path,fps=self.g_pool.capture.frame_rate) # positions path to eye process if self.record_eye: for alive, pipe in zip(self.g_pool.eyes_are_alive,self.g_pool.eye_pipes): if alive.value: pipe.send( ('Rec_Start',(self.rec_path,self.raw_jpeg) ) ) if self.show_info_menu: self.open_info_menu() logger.info("Started Recording.") self.notify_all( {'subject':'rec_started','rec_path':self.rec_path,'session_name':self.session_name,'network_propagate':True} ) def open_info_menu(self): self.info_menu = ui.Growing_Menu('additional Recording Info',size=(300,300),pos=(300,300)) self.info_menu.configuration = self.info_menu_conf def populate_info_menu(): self.info_menu.elements[:-2] = [] for name in self.user_info.iterkeys(): self.info_menu.insert(0,ui.Text_Input(name,self.user_info)) def set_user_info(new_string): self.user_info = new_string populate_info_menu() populate_info_menu() self.info_menu.append(ui.Info_Text('Use the *user info* field to add/remove additional fields and their values. The format must be a valid Python dictionary. For example -- {"key":"value"}. You can add as many fields as you require. Your custom fields will be saved for your next session.')) self.info_menu.append(ui.Text_Input('user_info',self,setter=set_user_info,label="User info")) self.g_pool.gui.append(self.info_menu) def close_info_menu(self): if self.info_menu: self.info_menu_conf = self.info_menu.configuration self.g_pool.gui.remove(self.info_menu) self.info_menu = None def update(self,frame,events): ''' listen to commands from eye window ''' if self.g_pool.eye_pipes[0].poll(): cmd = self.g_pool.eye_pipes[0].recv() # toggle recording if enter key clicked if cmd == GLFW_KEY_ENTER: self.toggle() if self.running: # for key,data in events.iteritems(): # if key not in ('dt'): # try: # self.data[key] += data # except KeyError: # self.data[key] = [] # self.data[key] += data ''' Record when it first started as this will be the start of calibration. When user click Enter again this should end calibration. Click once more to end recording''' if not self.calibration_start_at: self.calibration_start_at = frame.timestamp self.timestamps.append(frame.timestamp) self.writer.write_video_frame(frame) self.frame_count += 1 # # cv2.putText(frame.img, "Frame %s"%self.frame_count,(200,200), cv2.FONT_HERSHEY_SIMPLEX,1,(255,100,100)) self.button.status_text = self.get_rec_time_str() def stop(self, shutdown_os=False): #explicit release of VideoWriter self.writer.release() self.writer = None if self.record_eye: for alive, pipe in zip(self.g_pool.eyes_are_alive,self.g_pool.eye_pipes): if alive.value: pipe.send(('Rec_Stop',None)) # Mohammad: This will now save an empty dicts list save_object(self.data,os.path.join(self.rec_path, "pupil_data")) timestamps_path = os.path.join(self.rec_path, "world_timestamps.npy") # ts = sanitize_timestamps(np.array(self.timestamps)) ts = np.array(self.timestamps) np.save(timestamps_path,ts) try: copy2(os.path.join(self.g_pool.user_dir,"surface_definitions"),os.path.join(self.rec_path,"surface_definitions")) except: logger.info("No surface_definitions data found. You may want this if you do marker tracking.") try: copy2(os.path.join(self.g_pool.user_dir,"user_calibration_data"),os.path.join(self.rec_path,"user_calibration_data")) except: logger.warning("No user calibration data found. Please calibrate first.") # camera_calibration = load_camera_calibration(self.g_pool) # if camera_calibration is not None: # save_object(camera_calibration,os.path.join(self.rec_path, "camera_calibration")) # else: # logger.info("No camera calibration found.") try: with open(self.meta_info_path, 'a') as f: f.write("Duration Time\t"+ self.get_rec_time_str()+ "\n") f.write("World Camera Frames\t"+ str(self.frame_count)+ "\n") f.write("World Camera Resolution\t"+ str(self.g_pool.capture.frame_size[0])+"x"+str(self.g_pool.capture.frame_size[1])+"\n") f.write("Capture Software Version\t%s\n"%self.g_pool.version) f.write("System Info\t%s"%get_system_info()) except Exception: logger.exception("Could not save metadata. Please report this bug!") try: with open(os.path.join(self.rec_path, "user_info.csv"), 'w') as f: self.user_info.update({'cal_start':self.calibration_start_at, 'cal_end':self.calibration_end_at}) for name,val in self.user_info.iteritems(): f.write("%s\t%s\n"%(name,val)) except Exception: logger.exception("Could not save userdata. Please report this bug!") self.calibration_start_at = None self.calibration_end_at = None self.close_info_menu() if self.audio_writer: self.audio_writer = None self.running = False self.menu.read_only = False self.button.status_text = '' self.timestamps = [] self.data = {'pupil_positions':[],'gaze_positions':[]} self.pupil_pos_list = [] self.gaze_pos_list = [] logger.info("Saved Recording.") self.notify_all( {'subject':'rec_stopped','rec_path':self.rec_path,'network_propagate':True} ) # Mohammad - shutdown OS when stop recording if shutdown_os: os.system('shutdown now') def cleanup(self): """gets called when the plugin get terminated. either volunatily or forced. """ if self.running: self.stop() self.deinit_gui() def verify_path(self,val): try: n_path = os.path.expanduser(val) logger.debug("Expanded user path.") except: n_path = val if not n_path: logger.warning("Please specify a path.") return False elif not os.path.isdir(n_path): logger.warning("This is not a valid path.") return False # elif not os.access(n_path, os.W_OK): elif not writable_dir(n_path): logger.warning("Do not have write access to '%s'."%n_path) return False else: return n_path def set_rec_dir(self,val): n_path = self.verify_path(val) if n_path: self.rec_dir = n_path def set_session_name(self, val): if not val: self.session_name = get_auto_name() else: if '/' in val: logger.warning('You session name will create one or more subdirectories') self.session_name = val
class Recorder(Plugin): """Capture Recorder""" def __init__(self, g_pool, session_name=get_auto_name(), rec_dir=None, user_info={ 'name': '', 'additional_field': 'change_me' }, info_menu_conf={}, show_info_menu=False, record_eye=False, audio_src='No Audio', raw_jpeg=True): super().__init__(g_pool) # update name if it was autogenerated. if session_name.startswith('20') and len(session_name) == 10: session_name = get_auto_name() base_dir = self.g_pool.user_dir.rsplit(os.path.sep, 1)[0] default_rec_dir = os.path.join(base_dir, 'recordings') if rec_dir and rec_dir != default_rec_dir and self.verify_path( rec_dir): self.rec_dir = rec_dir else: try: os.makedirs(default_rec_dir) except OSError as e: if e.errno != errno.EEXIST: logger.error("Could not create Rec dir") raise e else: logger.info( 'Created standard Rec dir at "{}"'.format(default_rec_dir)) self.rec_dir = default_rec_dir self.raw_jpeg = raw_jpeg self.order = .9 self.record_eye = record_eye self.session_name = session_name self.audio_devices_dict = Audio_Input_Dict() if audio_src in list(self.audio_devices_dict.keys()): self.audio_src = audio_src else: self.audio_src = 'No Audio' self.running = False self.menu = None self.button = None self.user_info = user_info self.show_info_menu = show_info_menu self.info_menu = None self.info_menu_conf = info_menu_conf def get_init_dict(self): d = {} d['record_eye'] = self.record_eye d['audio_src'] = self.audio_src d['session_name'] = self.session_name d['user_info'] = self.user_info d['info_menu_conf'] = self.info_menu_conf d['show_info_menu'] = self.show_info_menu d['rec_dir'] = self.rec_dir d['raw_jpeg'] = self.raw_jpeg return d def init_gui(self): self.menu = ui.Growing_Menu('Recorder') self.menu.collapsed = True self.g_pool.sidebar.insert(3, self.menu) self.menu.append( ui.Info_Text( 'Pupil recordings are saved like this: "path_to_recordings/recording_session_name/nnn" where "nnn" is an increasing number to avoid overwrites. You can use "/" in your session name to create subdirectories.' )) self.menu.append( ui.Info_Text( 'Recordings are saved to "~/pupil_recordings". You can change the path here but note that invalid input will be ignored.' )) self.menu.append( ui.Text_Input('rec_dir', self, setter=self.set_rec_dir, label='Path to recordings')) self.menu.append( ui.Text_Input('session_name', self, setter=self.set_session_name, label='Recording session name')) self.menu.append( ui.Switch('show_info_menu', self, on_val=True, off_val=False, label='Request additional user info')) self.menu.append( ui.Selector( 'raw_jpeg', self, selection=[True, False], labels=["bigger file, less CPU", "smaller file, more CPU"], label='Compression')) self.menu.append( ui.Info_Text( 'Recording the raw eye video is optional. We use it for debugging.' )) self.menu.append( ui.Switch('record_eye', self, on_val=True, off_val=False, label='Record eye')) def audio_dev_getter(): # fetch list of currently available self.audio_devices_dict = Audio_Input_Dict() devices = list(self.audio_devices_dict.keys()) return devices, devices self.menu.append( ui.Selector('audio_src', self, selection_getter=audio_dev_getter, label='Audio Source')) self.button = ui.Thumb('running', self, setter=self.toggle, label='R', hotkey='r') self.button.on_color[:] = (1, .0, .0, .8) self.g_pool.quickbar.insert(1, self.button) def deinit_gui(self): if self.menu: self.g_pool.sidebar.remove(self.menu) self.menu = None if self.button: self.g_pool.quickbar.remove(self.button) self.button = None def toggle(self, _=None): if self.running: self.notify_all({'subject': 'recording.should_stop'}) self.notify_all({ 'subject': 'recording.should_stop', 'remote_notify': 'all' }) else: self.notify_all({ 'subject': 'recording.should_start', 'session_name': self.session_name }) self.notify_all({ 'subject': 'recording.should_start', 'session_name': self.session_name, 'remote_notify': 'all' }) def on_notify(self, notification): """Handles recorder notifications Reacts to notifications: ``recording.should_start``: Starts a new recording session ``recording.should_stop``: Stops current recording session Emits notifications: ``recording.started``: New recording session started ``recording.stopped``: Current recording session stopped Args: notification (dictionary): Notification dictionary """ # notification wants to be recorded if notification.get('record', False) and self.running: if 'timestamp' not in notification: logger.error( "Notification without timestamp will not be saved.") else: self.data['notifications'].append(notification) elif notification['subject'] == 'recording.should_start': if self.running: logger.info('Recording already running!') elif not self.g_pool.capture.online: logger.error( "Current world capture is offline. Please reconnect or switch to fake capture" ) else: if notification.get("session_name", ""): self.set_session_name(notification["session_name"]) self.start() elif notification['subject'] == 'recording.should_stop': if self.running: self.stop() else: logger.info('Recording already stopped!') def get_rec_time_str(self): rec_time = gmtime(time() - self.start_time) return strftime("%H:%M:%S", rec_time) def start(self): self.timestamps = [] self.data = { 'pupil_positions': [], 'gaze_positions': [], 'notifications': [] } self.frame_count = 0 self.running = True self.menu.read_only = True self.start_time = time() session = os.path.join(self.rec_dir, self.session_name) try: os.makedirs(session) logger.debug( "Created new recordings session dir {}".format(session)) except: logger.debug( "Recordings session dir {} already exists, using it.".format( session)) # set up self incrementing folder within session folder counter = 0 while True: self.rec_path = os.path.join(session, "{:03d}/".format(counter)) try: os.mkdir(self.rec_path) logger.debug("Created new recording dir {}".format( self.rec_path)) break except: logger.debug( "We dont want to overwrite data, incrementing counter & trying to make new data folder" ) counter += 1 self.meta_info_path = os.path.join(self.rec_path, "info.csv") with open(self.meta_info_path, 'w', newline='') as csvfile: csv_utils.write_key_value_file( csvfile, { 'Recording Name': self.session_name, 'Start Date': strftime("%d.%m.%Y", localtime(self.start_time)), 'Start Time': strftime("%H:%M:%S", localtime(self.start_time)) }) if self.audio_src != 'No Audio': audio_path = os.path.join(self.rec_path, "world.wav") self.audio_writer = Audio_Capture( audio_path, self.audio_devices_dict[self.audio_src]) else: self.audio_writer = None if self.raw_jpeg and self.g_pool.capture.jpeg_support: self.video_path = os.path.join(self.rec_path, "world.mp4") self.writer = JPEG_Writer(self.video_path, self.g_pool.capture.frame_rate) else: self.video_path = os.path.join(self.rec_path, "world.mp4") self.writer = AV_Writer(self.video_path, fps=self.g_pool.capture.frame_rate) try: cal_pt_path = os.path.join(self.g_pool.user_dir, "user_calibration_data") cal_data = load_object(cal_pt_path) notification = { 'subject': 'calibration.calibration_data', 'record': True } notification.update(cal_data) self.data['notifications'].append(notification) except: pass if self.show_info_menu: self.open_info_menu() logger.info("Started Recording.") self.notify_all({ 'subject': 'recording.started', 'rec_path': self.rec_path, 'session_name': self.session_name, 'record_eye': self.record_eye, 'compression': self.raw_jpeg }) def open_info_menu(self): self.info_menu = ui.Growing_Menu('additional Recording Info', size=(300, 300), pos=(300, 300)) self.info_menu.configuration = self.info_menu_conf def populate_info_menu(): self.info_menu.elements[:-2] = [] for name in self.user_info.keys(): self.info_menu.insert(0, ui.Text_Input(name, self.user_info)) def set_user_info(new_string): self.user_info = new_string populate_info_menu() populate_info_menu() self.info_menu.append( ui.Info_Text( 'Use the *user info* field to add/remove additional fields and their values. The format must be a valid Python dictionary. For example -- {"key":"value"}. You can add as many fields as you require. Your custom fields will be saved for your next session.' )) self.info_menu.append( ui.Text_Input('user_info', self, setter=set_user_info, label="User info")) self.g_pool.gui.append(self.info_menu) def close_info_menu(self): if self.info_menu: self.info_menu_conf = self.info_menu.configuration self.g_pool.gui.remove(self.info_menu) self.info_menu = None def recent_events(self, events): if self.running: for key, data in events.items(): if key not in ('dt', 'frame'): try: self.data[key] += data except KeyError: self.data[key] = [] self.data[key] += data if 'frame' in events: frame = events['frame'] self.timestamps.append(frame.timestamp) self.writer.write_video_frame(frame) self.frame_count += 1 # # cv2.putText(frame.img, "Frame %s"%self.frame_count,(200,200), cv2.FONT_HERSHEY_SIMPLEX,1,(255,100,100)) self.button.status_text = self.get_rec_time_str() def stop(self): # explicit release of VideoWriter self.writer.release() self.writer = None save_object(self.data, os.path.join(self.rec_path, "pupil_data")) timestamps_path = os.path.join(self.rec_path, "world_timestamps.npy") # ts = sanitize_timestamps(np.array(self.timestamps)) ts = np.array(self.timestamps) np.save(timestamps_path, ts) try: copy2(os.path.join(self.g_pool.user_dir, "surface_definitions"), os.path.join(self.rec_path, "surface_definitions")) except: logger.info( "No surface_definitions data found. You may want this if you do marker tracking." ) camera_calibration = load_camera_calibration(self.g_pool) if camera_calibration is not None: save_object(camera_calibration, os.path.join(self.rec_path, "camera_calibration")) else: logger.info("No camera calibration found.") try: with open(self.meta_info_path, 'a', newline='') as csvfile: csv_utils.write_key_value_file(csvfile, { 'Duration Time': self.get_rec_time_str(), 'World Camera Frames': self.frame_count, 'World Camera Resolution': str(self.g_pool.capture.frame_size[0]) + "x" + str(self.g_pool.capture.frame_size[1]), 'Capture Software Version': self.g_pool.version, 'Data Format Version': self.g_pool.version, 'System Info': get_system_info() }, append=True) except Exception: logger.exception( "Could not save metadata. Please report this bug!") try: with open(os.path.join(self.rec_path, "user_info.csv"), 'w', newline='') as csvfile: csv_utils.write_key_value_file(csvfile, self.user_info) except Exception: logger.exception( "Could not save userdata. Please report this bug!") self.close_info_menu() if self.audio_writer: self.audio_writer = None self.running = False self.menu.read_only = False self.button.status_text = '' self.timestamps = [] self.data = {'pupil_positions': [], 'gaze_positions': []} self.pupil_pos_list = [] self.gaze_pos_list = [] logger.info("Saved Recording.") self.notify_all({ 'subject': 'recording.stopped', 'rec_path': self.rec_path }) def cleanup(self): """gets called when the plugin get terminated. either volunatily or forced. """ if self.running: self.stop() self.deinit_gui() def verify_path(self, val): try: n_path = os.path.expanduser(val) logger.debug("Expanded user path.") except: n_path = val if not n_path: logger.warning("Please specify a path.") return False elif not os.path.isdir(n_path): logger.warning("This is not a valid path.") return False # elif not os.access(n_path, os.W_OK): elif not writable_dir(n_path): logger.warning("Do not have write access to '{}'.".format(n_path)) return False else: return n_path def set_rec_dir(self, val): n_path = self.verify_path(val) if n_path: self.rec_dir = n_path def set_session_name(self, val): if not val: self.session_name = get_auto_name() else: if os.path.sep in val: logger.warning( 'You session name will create one or more subdirectories') self.session_name = val
class Recorder(Plugin): """Capture Recorder""" def __init__(self, g_pool, session_name=get_auto_name(), rec_dir=None, user_info={ 'name': '', 'additional_field': 'change_me' }, info_menu_conf={}, show_info_menu=False, record_eye=False, audio_src='No Audio', raw_jpeg=False): super(Recorder, self).__init__(g_pool) self.states = ['start', 'want', 'pick'] self.all_qr = { 'termo1': 75, 'light1': False, 'light2': False, 'light3': False } self.this_state = 'start' self.start_state = time() self.qr_codes = {} self.gaze_x = -1 self.gaze_y = -1 self.avg_col = 0 self.choice = "" #update name if it was autogenerated. if session_name.startswith('20') and len(session_name) == 10: session_name = get_auto_name() base_dir = self.g_pool.user_dir.rsplit(os.path.sep, 1)[0] default_rec_dir = os.path.join(base_dir, 'recordings') if rec_dir and rec_dir != default_rec_dir and self.verify_path( rec_dir): self.rec_dir = rec_dir else: try: os.makedirs(default_rec_dir) except OSError as e: if 'File exists' in '%s' % e: pass else: logger.error("Could not create Rec dir") raise e else: logger.info('Created standard Rec dir at "%s"' % default_rec_dir) self.rec_dir = default_rec_dir # if rec_dir and rec_dir != default_rec_dir and self.verify_path(rec_dir): # self.rec_dir = rec_dir # else: # #lets make a rec dir next to the user dir # base_dir = self.g_pool.user_dir.rsplit(os.path.sep,1)[0] # self.rec_dir = os.path.join(base_dir,'recordings') # if not os.path.isdir(self.rec_dir): # os.mkdir(self.rec_dir) self.raw_jpeg = raw_jpeg self.order = .9 self.record_eye = record_eye self.session_name = session_name self.audio_devices_dict = Audio_Input_Dict() if audio_src in self.audio_devices_dict.keys(): self.audio_src = audio_src else: self.audio_src = 'No Audio' self.running = False self.menu = None self.button = None self.user_info = user_info self.show_info_menu = show_info_menu self.info_menu = None self.info_menu_conf = info_menu_conf self.height, self.width = self.g_pool.capture.frame_size def get_init_dict(self): d = {} d['record_eye'] = self.record_eye d['audio_src'] = self.audio_src d['session_name'] = self.session_name d['user_info'] = self.user_info d['info_menu_conf'] = self.info_menu_conf d['show_info_menu'] = self.show_info_menu d['rec_dir'] = self.rec_dir d['raw_jpeg'] = self.raw_jpeg # if self.menu: # d['menu_conf'] = self.menu.configuration # else: # d['menu_conf'] = self.menu_conf return d def init_gui(self): self.menu = ui.Growing_Menu('Recorder') # self.menu.configuration = self.menu_conf self.g_pool.sidebar.insert(3, self.menu) self.menu.append( ui.Info_Text( 'Pupil recordings are saved like this: "path_to_recordings/recording_session_name/nnn" where "nnn" is an increasing number to avoid overwrites. You can use "/" in your session name to create subdirectories.' )) self.menu.append( ui.Info_Text( 'Recordings are saved to "~/pupil_recordings". You can change the path here but note that invalid input will be ignored.' )) self.menu.append( ui.Text_Input('rec_dir', self, setter=self.set_rec_dir, label='Path to recordings')) self.menu.append( ui.Text_Input('session_name', self, setter=self.set_session_name, label='Recording session name')) self.menu.append( ui.Switch('show_info_menu', self, on_val=True, off_val=False, label='Request additional user info')) self.menu.append( ui.Selector( 'raw_jpeg', self, selection=[True, False], labels=["bigger file, less CPU", "smaller file, more CPU"], label='compression')) self.menu.append( ui.Info_Text( 'Recording the raw eye video is optional. We use it for debugging.' )) self.menu.append( ui.Switch('record_eye', self, on_val=True, off_val=False, label='Record eye')) self.menu.append( ui.Selector('audio_src', self, selection=self.audio_devices_dict.keys())) self.button = ui.Thumb('running', self, setter=self.toggle, label='QR', hotkey='r') self.button.on_color[:] = (1, .0, .0, .8) self.g_pool.quickbar.insert(1, self.button) def deinit_gui(self): if self.menu: # self.menu_conf = self.menu.configuration self.g_pool.sidebar.remove(self.menu) self.menu = None if self.button: self.g_pool.quickbar.remove(self.button) self.button = None def toggle(self, _=None): if self.running: self.stop() else: self.start() def get_rec_time_str(self): rec_time = gmtime(time() - self.start_time) return strftime("%H:%M:%S", rec_time) def start(self): self.timestamps = [] # add self.data self.data = {'pupil_positions': [], 'gaze_positions': []} self.pupil_list = [] self.gaze_list = [] self.frame_count = 0 self.running = True self.menu.read_only = True self.start_time = time() self.start_state = time() session = os.path.join(self.rec_dir, self.session_name) try: os.makedirs(session) logger.debug("Created new recordings session dir %s" % session) except: logger.debug( "Recordings session dir %s already exists, using it." % session) # set up self incrementing folder within session folder counter = 0 while True: self.rec_path = os.path.join(session, "%03d/" % counter) try: os.mkdir(self.rec_path) logger.debug("Created new recording dir %s" % self.rec_path) break except: logger.debug( "We dont want to overwrite data, incrementing counter & trying to make new data folder" ) counter += 1 self.meta_info_path = os.path.join(self.rec_path, "info.csv") with open(self.meta_info_path, 'w') as f: f.write("Recording Name\t" + self.session_name + "\n") f.write("Start Date\t" + strftime("%d.%m.%Y", localtime(self.start_time)) + "\n") f.write("Start Time\t" + strftime("%H:%M:%S", localtime(self.start_time)) + "\n") if self.audio_src != 'No Audio': audio_path = os.path.join(self.rec_path, "world.wav") self.audio_writer = Audio_Capture( self.audio_devices_dict[self.audio_src], audio_path) else: self.audio_writer = None if self.raw_jpeg and "uvc_capture" in str( self.g_pool.capture.__class__): self.video_path = os.path.join(self.rec_path, "world.mp4") self.writer = JPEG_Writer(self.video_path, int(self.g_pool.capture.frame_rate)) # elif 1: # self.writer = av_writer.AV_Writer(self.video_path) else: self.video_path = os.path.join(self.rec_path, "world.mkv") self.writer = CV_Writer(self.video_path, float(self.g_pool.capture.frame_rate), self.g_pool.capture.frame_size) # positions path to eye process if self.record_eye: for tx in self.g_pool.eye_tx: tx.send((self.rec_path, self.raw_jpeg)) if self.show_info_menu: self.open_info_menu() # self.video_path = os.path.join(self.rec_path, "world.mkv") # self.writer = cv2.VideoWriter(self.video_path, int(cv2.cv.CV_FOURCC(*'DIVX')), float(self.g_pool.capture.frame_rate), (1280,720)) # # positions path to eye process # if self.record_eye: # for tx in self.g_pool.eye_tx: # tx.send(self.rec_path) # if self.show_info_menu: # self.open_info_menu() ############### def open_info_menu(self): self.info_menu = ui.Growing_Menu('additional Recording Info', size=(300, 300), pos=(300, 300)) self.info_menu.configuration = self.info_menu_conf def populate_info_menu(): self.info_menu.elements[:-2] = [] for name in self.user_info.iterkeys(): self.info_menu.insert(0, ui.Text_Input(name, self.user_info)) def set_user_info(new_string): self.user_info = new_string populate_info_menu() populate_info_menu() self.info_menu.append( ui.Info_Text( 'Use the *user info* field to add/remove additional fields and their values. The format must be a valid Python dictionary. For example -- {"key":"value"}. You can add as many fields as you require. Your custom fields will be saved for your next session.' )) self.info_menu.append( ui.Text_Input('user_info', self, setter=set_user_info, label="User info")) self.g_pool.gui.append(self.info_menu) def close_info_menu(self): if self.info_menu: self.info_menu_conf = self.info_menu.configuration self.g_pool.gui.remove(self.info_menu) self.info_menu = None def update(self, frame, events): if self.running: # update data self.data['pupil_positions'] += events['pupil_positions'] self.data['gaze_positions'] += events['gaze_positions'] #cv2.putText(frame.img, "Frame %s"%self.frame_count,(200,200), cv2.FONT_HERSHEY_SIMPLEX,1,(255,100,100) cv2.putText(frame.img, self.this_state, (200, 100), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 100, 100)) if self.this_state == 'pick': cv2.putText(frame.img, self.choice, (300, 100), cv2.FONT_HERSHEY_SIMPLEX, 1, (100, 100, 255)) QRs = qr_detect(frame) for p in events['pupil_positions']: pupil_pos = p['timestamp'], p['confidence'], p['id'], p[ 'norm_pos'][0], p['norm_pos'][1], p['diameter'] self.pupil_list.append(pupil_pos) c = 0 avg_x = -1 avg_y = -1 for g in events.get('gaze_positions', []): gaze_pos = g['timestamp'], g['confidence'], g['norm_pos'][ 0], g['norm_pos'][1] avg_x = (avg_x * c + g['norm_pos'][0]) / (c + 1) avg_y = (avg_y * c + g['norm_pos'][0]) / (c + 1) c = c + 1 self.gaze_list.append(gaze_pos) self.timestamps.append(frame.timestamp) avg_col = 0 self.stateT(frame, QRs, avg_x, avg_y, avg_col) self.writer.write_video_frame(frame) self.frame_count += 1 self.button.status_text = self.get_rec_time_str() def stop(self): #explicit release of VideoWriter self.writer.release() self.writer = None if self.record_eye: for tx in self.g_pool.eye_tx: try: tx.send(None) except: logger.warning( "Could not stop eye-recording. Please report this bug!" ) save_object(self.data, os.path.join(self.rec_path, "pupil_data")) gaze_list_path = os.path.join(self.rec_path, "gaze_positions.npy") np.save(gaze_list_path, np.asarray(self.gaze_list)) pupil_list_path = os.path.join(self.rec_path, "pupil_positions.npy") np.save(pupil_list_path, np.asarray(self.pupil_list)) timestamps_path = os.path.join(self.rec_path, "world_timestamps.npy") ts = sanitize_timestamps(np.array(self.timestamps)) np.save(timestamps_path, ts) try: copy2(os.path.join(self.g_pool.user_dir, "surface_definitions"), os.path.join(self.rec_path, "surface_definitions")) except: logger.info( "No surface_definitions data found. You may want this if you do marker tracking." ) try: copy2(os.path.join(self.g_pool.user_dir, "cal_pt_cloud.npy"), os.path.join(self.rec_path, "cal_pt_cloud.npy")) except: logger.warning( "No calibration data found. Please calibrate first.") try: copy2(os.path.join(self.g_pool.user_dir, "camera_matrix.npy"), os.path.join(self.rec_path, "camera_matrix.npy")) copy2(os.path.join(self.g_pool.user_dir, "dist_coefs.npy"), os.path.join(self.rec_path, "dist_coefs.npy")) except: logger.info("No camera intrinsics found.") try: with open(self.meta_info_path, 'a') as f: f.write("Duration Time\t" + self.get_rec_time_str() + "\n") if self.g_pool.binocular: f.write("Eye Mode\tbinocular\n") else: f.write("Eye Mode\tmonocular\n") f.write("Duration Time\t" + self.get_rec_time_str() + "\n") f.write("World Camera Frames\t" + str(self.frame_count) + "\n") f.write("World Camera Resolution\t" + str(self.width) + "x" + str(self.height) + "\n") f.write("Capture Software Version\t%s\n" % self.g_pool.version) if platform.system() == "Windows": username = os.environ["USERNAME"] sysname, nodename, release, version, machine, _ = platform.uname( ) else: username = os.getlogin() try: sysname, nodename, release, version, machine = os.uname( ) except: sysname, nodename, release, version, machine = sys.platform, None, None, None, None f.write("User\t" + username + "\n") f.write("Platform\t" + sysname + "\n") f.write("Machine\t" + nodename + "\n") f.write("Release\t" + release + "\n") f.write("Version\t" + version + "\n") except Exception: logger.exception( "Could not save metadata. Please report this bug!") try: with open(os.path.join(self.rec_path, "user_info.csv"), 'w') as f: for name, val in self.user_info.iteritems(): f.write("%s\t%s\n" % (name, val)) except Exception: logger.exception( "Could not save userdata. Please report this bug!") self.close_info_menu() if self.audio_writer: self.audio_writer = None self.running = False self.menu.read_only = False self.button.status_text = '' def cleanup(self): """gets called when the plugin get terminated. either volunatily or forced. """ if self.running: self.stop() self.deinit_gui() def verify_path(self, val): try: n_path = os.path.expanduser(val) logger.debug("Expanded user path.") except: n_path = val if not n_path: logger.waring("Please specify a path.") return False elif not os.path.isdir(n_path): logger.warning("This is not a valid path.") return False elif not writable_dir(n_path): logger.warning("Do not have write access to '%s'." % n_path) return False else: return n_path def set_rec_dir(self, val): # try: # n_path = os.path.expanduser(val) # logger.debug("Expanded user path.") # except: # n_path = val # if not n_path: # logger.warning("Please specify a path.") # elif not os.path.isdir(n_path): # logger.warning("This is not a valid path.") # # elif not os.access(n_path, os.W_OK): # elif not writable_dir(n_path): # logger.warning("Do not have write access to '%s'."%n_path) # else: # self.rec_dir = n_path n_path = self.verify_path(val) if n_path: self.rec_dir = n_path def set_session_name(self, val): if not val: self.session_name = get_auto_name() else: if '/' in val: logger.warning( 'You session name with create one or more subdirectories') self.session_name = val def stateT(self, frame, QRs, agaze_x, agaze_y, avg_col): if time() - self.start_time < 1: self.gaze_x = agaze_x self.gaze_y = agaze_y if self.this_state == 'start': if time() - self.start_state < 2: self.qr_codes = {} return True if self.Nochange_gaze(agaze_x, agaze_y, avg_col): self.this_state = 'want' self.start_state = time() self.gaze_x = agaze_x self.gaze_y = agaze_y return True if self.this_state == 'want': if time() - self.start_state < 2: self.qr_codes.update(QRs) return True if self.Nochange_gaze(agaze_x, agaze_y, avg_col) and self.qr_codes: self.this_state = 'pick' self.choice = self.choose(frame) #print self.qr_codes, self.choice else: self.this_state = 'start' self.start_state = time() self.gaze_x = agaze_x self.gaze_y = agaze_y return True if self.this_state == 'pick': if time() - self.start_state < 4: return True if self.Nochange_gaze(agaze_x, agaze_y, avg_col): del self.qr_codes[self.choice] if not self.qr_codes: self.this_state = 'start' else: self.choice = self.choose(frame) #print self.choice else: print self.choice, " chosen" self.this_state = 'start' self.start_state = time() self.gaze_x = agaze_x self.gaze_y = agaze_y return True def Nochange_gaze(self, agaze_x, agaze_y, avg_col): #print self.gaze_x, self.gaze_y, agaze_x, agaze_y return (abs(self.gaze_x - agaze_x) < .1 and abs(self.gaze_y - agaze_y) < .1) def choose(self, frame): h = frame.img.shape[0] w = frame.img.shape[1] gaze_x = self.gaze_x gaze_y = self.gaze_y if (gaze_x < 0 or gaze_y < 0): gaze_x = .5 gaz_y = .5 k = self.qr_codes.keys() chosen = k[0] min_dist = 1000000 for name, mc in self.qr_codes.items(): dis = (mc[0] - gaze_x * w) * (mc[0] - gaze_x * w) + ( mc[1] - gaze_y * h) * (mc[1] - gaze_y * h) if dis < min_dist: min_dist = dis chosen = name return chosen
class Recorder(Plugin): """Capture Recorder""" def __init__(self,g_pool,session_name = get_auto_name(),rec_dir=None, user_info={'name':'','additional_field':'change_me'},info_menu_conf={},show_info_menu=False, record_eye = True, audio_src = 'No Audio',raw_jpeg=True): super(Recorder, self).__init__(g_pool) #update name if it was autogenerated. if session_name.startswith('20') and len(session_name)==10: session_name = get_auto_name() base_dir = self.g_pool.user_dir.rsplit(os.path.sep,1)[0] default_rec_dir = os.path.join(base_dir,'recordings') if rec_dir and rec_dir != default_rec_dir and self.verify_path(rec_dir): self.rec_dir = rec_dir else: try: os.makedirs(default_rec_dir) except OSError as e: if e.errno != errno.EEXIST: logger.error("Could not create Rec dir") raise e else: logger.info('Created standard Rec dir at "%s"'%default_rec_dir) self.rec_dir = default_rec_dir self.raw_jpeg = raw_jpeg self.order = .9 self.record_eye = record_eye self.session_name = session_name self.audio_devices_dict = Audio_Input_Dict() if audio_src in self.audio_devices_dict.keys(): self.audio_src = audio_src else: self.audio_src = 'No Audio' self.running = False self.menu = None self.button = None self.user_info = user_info self.show_info_menu = show_info_menu self.info_menu = None self.info_menu_conf = info_menu_conf def get_init_dict(self): d = {} d['record_eye'] = self.record_eye d['audio_src'] = self.audio_src d['session_name'] = self.session_name d['user_info'] = self.user_info d['info_menu_conf'] = self.info_menu_conf d['show_info_menu'] = self.show_info_menu d['rec_dir'] = self.rec_dir d['raw_jpeg'] = self.raw_jpeg return d def init_gui(self): self.menu = ui.Growing_Menu('Recorder') self.g_pool.sidebar.insert(3,self.menu) self.menu.append(ui.Info_Text('Pupil recordings are saved like this: "path_to_recordings/recording_session_name/nnn" where "nnn" is an increasing number to avoid overwrites. You can use "/" in your session name to create subdirectories.')) self.menu.append(ui.Info_Text('Recordings are saved to "~/pupil_recordings". You can change the path here but note that invalid input will be ignored.')) self.menu.append(ui.Text_Input('rec_dir',self,setter=self.set_rec_dir,label='Path to recordings')) self.menu.append(ui.Text_Input('session_name',self,setter=self.set_session_name,label='Recording session name')) self.menu.append(ui.Switch('show_info_menu',self,on_val=True,off_val=False,label='Request additional user info')) self.menu.append(ui.Selector('raw_jpeg',self,selection = [True,False], labels=["bigger file, less CPU", "smaller file, more CPU"],label='Compression')) self.menu.append(ui.Info_Text('Recording the raw eye video is optional. We use it for debugging.')) self.menu.append(ui.Switch('record_eye',self,on_val=True,off_val=False,label='Record eye')) self.menu.append(ui.Selector('audio_src',self, selection=self.audio_devices_dict.keys(),label='Audio Source')) self.button = ui.Thumb('running',self,setter=self.toggle,label='Record',hotkey='r') self.button.on_color[:] = (1,.0,.0,.8) self.g_pool.quickbar.insert(1,self.button) def deinit_gui(self): if self.menu: self.g_pool.sidebar.remove(self.menu) self.menu = None if self.button: self.g_pool.quickbar.remove(self.button) self.button = None def toggle(self, _=None): if self.running: self.notify_all( {'subject':'recording.should_stop'} ) self.notify_all( {'subject':'recording.should_stop', 'remote_notify':'all'} ) else: self.notify_all( {'subject':'recording.should_start','session_name':self.session_name} ) self.notify_all( {'subject':'recording.should_start','session_name':self.session_name,'remote_notify':'all'} ) def on_notify(self,notification): """Handles recorder notifications Reacts to notifications: ``recording.should_start``: Starts a new recording session ``recording.should_stop``: Stops current recording session Emits notifications: ``recording.started``: New recording session started ``recording.stopped``: Current recording session stopped Args: notification (dictionary): Notification dictionary """ # notification wants to be recorded if notification.get('record',False) and self.running: self.data['notifications'].append(notification) elif notification['subject'] == 'recording.should_start': if self.running: logger.info('Recording already running!') else: if notification.get("session_name",""): self.set_session_name(notification["session_name"]) self.start() elif notification['subject'] == 'recording.should_stop': if self.running: self.stop() else: logger.info('Recording already stopped!') def get_rec_time_str(self): rec_time = gmtime(time()-self.start_time) return strftime("%H:%M:%S", rec_time) def start(self): self.timestamps = [] self.timestampsUnix = [] self.glint_pos_list = [] self.pupil_pos_list = [] self.gaze_pos_list = [] self.data = {'pupil_positions':[],'gaze_positions':[],'notifications':[]} self.frame_count = 0 self.running = True self.menu.read_only = True self.start_time = time() session = os.path.join(self.rec_dir, self.session_name) try: os.makedirs(session) logger.debug("Created new recordings session dir %s"%session) except: logger.debug("Recordings session dir %s already exists, using it." %session) # set up self incrementing folder within session folder counter = 0 while True: self.rec_path = os.path.join(session, "%03d/" % counter) try: os.mkdir(self.rec_path) logger.debug("Created new recording dir %s"%self.rec_path) break except: logger.debug("We dont want to overwrite data, incrementing counter & trying to make new data folder") counter += 1 self.meta_info_path = os.path.join(self.rec_path, "info.csv") with open(self.meta_info_path, 'w') as csvfile: csv_utils.write_key_value_file(csvfile,{ 'Recording Name': self.session_name, 'Start Date': strftime("%d.%m.%Y", localtime(self.start_time)), 'Start Time': strftime("%H:%M:%S", localtime(self.start_time)), 'Start Time (seconds since epoch)': str(self.start_time) }) if self.audio_src != 'No Audio': audio_path = os.path.join(self.rec_path, "world.wav") self.audio_writer = Audio_Capture(audio_path,self.audio_devices_dict[self.audio_src]) else: self.audio_writer = None if self.raw_jpeg and self.g_pool.capture.jpeg_support: self.video_path = os.path.join(self.rec_path, "world.mp4") self.writer = JPEG_Writer(self.video_path,self.g_pool.capture.frame_rate) else: self.video_path = os.path.join(self.rec_path, "world.mp4") self.writer = AV_Writer(self.video_path,fps=self.g_pool.capture.frame_rate) if self.show_info_menu: self.open_info_menu() logger.info("Started Recording.") self.notify_all( {'subject':'recording.started','rec_path':self.rec_path,'session_name':self.session_name,'record_eye':self.record_eye,'compression':self.raw_jpeg} ) def open_info_menu(self): self.info_menu = ui.Growing_Menu('additional Recording Info',size=(300,300),pos=(300,300)) self.info_menu.configuration = self.info_menu_conf def populate_info_menu(): self.info_menu.elements[:-2] = [] for name in self.user_info.iterkeys(): self.info_menu.insert(0,ui.Text_Input(name,self.user_info)) def set_user_info(new_string): self.user_info = new_string populate_info_menu() populate_info_menu() self.info_menu.append(ui.Info_Text('Use the *user info* field to add/remove additional fields and their values. The format must be a valid Python dictionary. For example -- {"key":"value"}. You can add as many fields as you require. Your custom fields will be saved for your next session.')) self.info_menu.append(ui.Text_Input('user_info',self,setter=set_user_info,label="User info")) self.g_pool.gui.append(self.info_menu) def close_info_menu(self): if self.info_menu: self.info_menu_conf = self.info_menu.configuration self.g_pool.gui.remove(self.info_menu) self.info_menu = None def update(self,frame,events): if self.running: for key,data in events.iteritems(): if key not in ('dt') and key != 'timestamp_unix': try: self.data[key] += data except KeyError: self.data[key] = [] self.data[key] += data self.timestamps.append(frame.timestamp) self.writer.write_video_frame(frame) self.frame_count += 1 for glint in events['glint_positions']: self.glint_pos_list += glint self.timestampsUnix.append(events['timestamp_unix']) # # cv2.putText(frame.img, "Frame %s"%self.frame_count,(200,200), cv2.FONT_HERSHEY_SIMPLEX,1,(255,100,100)) self.button.status_text = self.get_rec_time_str() def stop(self): #explicit release of VideoWriter self.writer.release() self.writer = None save_object(self.data,os.path.join(self.rec_path, "pupil_data")) self.glint_pos_list = np.array(self.glint_pos_list) glint_list_path = os.path.join(self.rec_path, "glint_positions.npy") np.save(glint_list_path,self.glint_pos_list) timestamps_path = os.path.join(self.rec_path, "world_timestamps.npy") # ts = sanitize_timestamps(np.array(self.timestamps)) ts = np.array(self.timestamps) np.save(timestamps_path,ts) timestampsUnix_path = os.path.join(self.rec_path, "world_timestamps_unix.npy") tsUnix = np.array(self.timestampsUnix) np.save(timestampsUnix_path,tsUnix) try: copy2(os.path.join(self.g_pool.user_dir,"surface_definitions"),os.path.join(self.rec_path,"surface_definitions")) except: logger.info("No surface_definitions data found. You may want this if you do marker tracking.") try: copy2(os.path.join(self.g_pool.user_dir,"user_calibration_data"),os.path.join(self.rec_path,"user_calibration_data")) except: logger.warning("No user calibration data found. Please calibrate first.") try: copy2(os.path.join(self.g_pool.user_dir,"cal_pt_cloud_glint.npy"),os.path.join(self.rec_path,"cal_pt_cloud_glint.npy")) except: logger.warning("No pupil-glint-vector calibration data found. Please calibrate first.") try: copy2(os.path.join(self.g_pool.user_dir,"cal_ref_list.npy"),os.path.join(self.rec_path,"cal_ref_list.npy")) except: logger.warning("No calibration reference list found.") try: copy2(os.path.join(self.g_pool.user_dir,"accuracy_test_pt_cloud.npy"),os.path.join(self.rec_path,"accuracy_test_pt_cloud.npy")) copy2(os.path.join(self.g_pool.user_dir,"accuracy_test_ref_list.npy"),os.path.join(self.rec_path,"accuracy_test_ref_list.npy")) except: logger.warning("No accuracy test found.") try: copy2(os.path.join(self.g_pool.user_dir,"accuracy_test_pt_cloud_previous.npy"),os.path.join(self.rec_path,"accuracy_test_pt_cloud_previous.npy")) copy2(os.path.join(self.g_pool.user_dir,"accuracy_test_ref_list_previous.npy"),os.path.join(self.rec_path,"accuracy_test_ref_list_previous.npy")) except: logger.warning("No previous accuracy test results.") camera_calibration = load_camera_calibration(self.g_pool) if camera_calibration is not None: save_object(camera_calibration,os.path.join(self.rec_path, "camera_calibration")) else: logger.info("No camera calibration found.") try: with open(self.meta_info_path, 'a') as csvfile: csv_utils.write_key_value_file(csvfile, { 'Duration Time': self.get_rec_time_str(), 'World Camera Frames': self.frame_count, 'World Camera Resolution': str(self.g_pool.capture.frame_size[0])+"x"+str(self.g_pool.capture.frame_size[1]), 'Capture Software Version': self.g_pool.version, 'System Info': get_system_info() }, append=True) except Exception: logger.exception("Could not save metadata. Please report this bug!") try: with open(os.path.join(self.rec_path, "user_info.csv"), 'w') as csvfile: csv_utils.write_key_value_file(csvfile, self.user_info) except Exception: logger.exception("Could not save userdata. Please report this bug!") self.close_info_menu() if self.audio_writer: self.audio_writer = None self.running = False self.menu.read_only = False self.button.status_text = '' self.timestamps = [] self.data = {'pupil_positions':[],'gaze_positions':[]} self.pupil_pos_list = [] self.gaze_pos_list = [] logger.info("Saved Recording.") self.notify_all( {'subject':'rec_stopped','rec_path':self.rec_path,'network_propagate':True} ) self.notify_all( {'subject':'recording.stopped','rec_path':self.rec_path} ) copyfile(os.path.join(self.g_pool.user_dir,'capture.log'), os.path.join(self.rec_path,"capture.log")) def cleanup(self): """gets called when the plugin get terminated. either volunatily or forced. """ if self.running: self.stop() self.deinit_gui() def verify_path(self,val): try: n_path = os.path.expanduser(val) logger.debug("Expanded user path.") except: n_path = val if not n_path: logger.warning("Please specify a path.") return False elif not os.path.isdir(n_path): logger.warning("This is not a valid path.") return False # elif not os.access(n_path, os.W_OK): elif not writable_dir(n_path): logger.warning("Do not have write access to '%s'."%n_path) return False else: return n_path def set_rec_dir(self,val): n_path = self.verify_path(val) if n_path: self.rec_dir = n_path def set_session_name(self, val): if not val: self.session_name = get_auto_name() else: if '/' in val: logger.warning('You session name will create one or more subdirectories') self.session_name = val
class Recorder(Plugin): """Capture Recorder""" def __init__(self,g_pool,session_name = get_auto_name(),rec_dir=None, user_info={'name':'','additional_field':'change_me'},info_menu_conf={},show_info_menu=False, record_eye = False, audio_src = 'No Audio',raw_jpeg=False): super(Recorder, self).__init__(g_pool) #update name if it was autogenerated. if session_name.startswith('20') and len(session_name)==10: session_name = get_auto_name() base_dir = self.g_pool.user_dir.rsplit(os.path.sep,1)[0] default_rec_dir = os.path.join(base_dir,'recordings') if rec_dir and rec_dir != default_rec_dir and self.verify_path(rec_dir): self.rec_dir = rec_dir else: try: os.makedirs(default_rec_dir) except OSError as e: if e.errno != errno.EEXIST: logger.error("Could not create Rec dir") raise e else: logger.info('Created standard Rec dir at "%s"'%default_rec_dir) self.rec_dir = default_rec_dir self.raw_jpeg = raw_jpeg self.order = .9 self.record_eye = record_eye self.session_name = session_name self.audio_devices_dict = Audio_Input_Dict() if audio_src in self.audio_devices_dict.keys(): self.audio_src = audio_src else: self.audio_src = 'No Audio' self.running = False self.menu = None self.button = None self.user_info = user_info self.show_info_menu = show_info_menu self.info_menu = None self.info_menu_conf = info_menu_conf def get_init_dict(self): d = {} d['record_eye'] = self.record_eye d['audio_src'] = self.audio_src d['session_name'] = self.session_name d['user_info'] = self.user_info d['info_menu_conf'] = self.info_menu_conf d['show_info_menu'] = self.show_info_menu d['rec_dir'] = self.rec_dir d['raw_jpeg'] = self.raw_jpeg return d def init_gui(self): self.menu = ui.Growing_Menu('Recorder') self.g_pool.sidebar.insert(3,self.menu) self.menu.append(ui.Info_Text('Pupil recordings are saved like this: "path_to_recordings/recording_session_name/nnn" where "nnn" is an increasing number to avoid overwrites. You can use "/" in your session name to create subdirectories.')) self.menu.append(ui.Info_Text('Recordings are saved to "~/pupil_recordings". You can change the path here but note that invalid input will be ignored.')) self.menu.append(ui.Text_Input('rec_dir',self,setter=self.set_rec_dir,label='Path to recordings')) self.menu.append(ui.Text_Input('session_name',self,setter=self.set_session_name,label='Recording session name')) self.menu.append(ui.Switch('show_info_menu',self,on_val=True,off_val=False,label='Request additional user info')) self.menu.append(ui.Selector('raw_jpeg',self,selection = [True,False], labels=["bigger file, less CPU", "smaller file, more CPU"],label='compression')) self.menu.append(ui.Info_Text('Recording the raw eye video is optional. We use it for debugging.')) self.menu.append(ui.Switch('record_eye',self,on_val=True,off_val=False,label='Record eye')) self.menu.append(ui.Selector('audio_src',self, selection=self.audio_devices_dict.keys())) self.button = ui.Thumb('running',self,setter=self.toggle,label='Record',hotkey='r') self.button.on_color[:] = (1,.0,.0,.8) self.g_pool.quickbar.insert(1,self.button) def deinit_gui(self): if self.menu: self.g_pool.sidebar.remove(self.menu) self.menu = None if self.button: self.g_pool.quickbar.remove(self.button) self.button = None def toggle(self, _=None): if self.running: self.stop() else: self.start() def on_notify(self,notification): if notification['subject'] == 'rec_should_start': if self.running: logger.warning('Recording is already running!') else: self.set_session_name(notification["session_name"]) self.start(network_propagate=notification.get('network_propagate',True)) elif notification['subject'] == 'rec_should_stop': if self.running: self.stop(network_propagate=notification.get('network_propagate',True)) else: logger.warning('Recording is already stopped!') def get_rec_time_str(self): rec_time = gmtime(time()-self.start_time) return strftime("%H:%M:%S", rec_time) def start(self,network_propagate=True): self.timestamps = [] self.data = {'pupil_positions':[],'gaze_positions':[]} self.pupil_pos_list = [] self.gaze_pos_list = [] self.frame_count = 0 self.running = True self.menu.read_only = True self.start_time = time() session = os.path.join(self.rec_dir, self.session_name) try: os.makedirs(session) logger.debug("Created new recordings session dir %s"%session) except: logger.debug("Recordings session dir %s already exists, using it." %session) # set up self incrementing folder within session folder counter = 0 while True: self.rec_path = os.path.join(session, "%03d/" % counter) try: os.mkdir(self.rec_path) logger.debug("Created new recording dir %s"%self.rec_path) break except: logger.debug("We dont want to overwrite data, incrementing counter & trying to make new data folder") counter += 1 self.meta_info_path = os.path.join(self.rec_path, "info.csv") with open(self.meta_info_path, 'w') as f: f.write("Recording Name\t"+self.session_name+ "\n") f.write("Start Date\t"+ strftime("%d.%m.%Y", localtime(self.start_time))+ "\n") f.write("Start Time\t"+ strftime("%H:%M:%S", localtime(self.start_time))+ "\n") if self.audio_src != 'No Audio': audio_path = os.path.join(self.rec_path, "world.wav") self.audio_writer = Audio_Capture(audio_path,self.audio_devices_dict[self.audio_src]) else: self.audio_writer = None if self.raw_jpeg and self.g_pool.capture.jpeg_support: self.video_path = os.path.join(self.rec_path, "world.mp4") self.writer = JPEG_Writer(self.video_path,self.g_pool.capture.frame_rate) else: self.video_path = os.path.join(self.rec_path, "world.mp4") self.writer = AV_Writer(self.video_path,fps=self.g_pool.capture.frame_rate) # positions path to eye process if self.record_eye: for tx in self.g_pool.eye_tx: tx.send((self.rec_path,self.raw_jpeg)) if self.show_info_menu: self.open_info_menu() self.notify_all( {'subject':'rec_started','rec_path':self.rec_path,'session_name':self.session_name,'network_propagate':network_propagate} ) def open_info_menu(self): self.info_menu = ui.Growing_Menu('additional Recording Info',size=(300,300),pos=(300,300)) self.info_menu.configuration = self.info_menu_conf def populate_info_menu(): self.info_menu.elements[:-2] = [] for name in self.user_info.iterkeys(): self.info_menu.insert(0,ui.Text_Input(name,self.user_info)) def set_user_info(new_string): self.user_info = new_string populate_info_menu() populate_info_menu() self.info_menu.append(ui.Info_Text('Use the *user info* field to add/remove additional fields and their values. The format must be a valid Python dictionary. For example -- {"key":"value"}. You can add as many fields as you require. Your custom fields will be saved for your next session.')) self.info_menu.append(ui.Text_Input('user_info',self,setter=set_user_info,label="User info")) self.g_pool.gui.append(self.info_menu) def close_info_menu(self): if self.info_menu: self.info_menu_conf = self.info_menu.configuration self.g_pool.gui.remove(self.info_menu) self.info_menu = None def update(self,frame,events): if self.running: self.data['pupil_positions'] += events['pupil_positions'] self.data['gaze_positions'] += events['gaze_positions'] self.timestamps.append(frame.timestamp) self.writer.write_video_frame(frame) self.frame_count += 1 # cv2.putText(frame.img, "Frame %s"%self.frame_count,(200,200), cv2.FONT_HERSHEY_SIMPLEX,1,(255,100,100)) for p in events['pupil_positions']: pupil_pos = p['timestamp'],p['confidence'],p['id'],p['norm_pos'][0],p['norm_pos'][1],p['diameter'] self.pupil_pos_list.append(pupil_pos) for g in events.get('gaze_positions',[]): gaze_pos = g['timestamp'],g['confidence'],g['norm_pos'][0],g['norm_pos'][1] self.gaze_pos_list.append(gaze_pos) self.button.status_text = self.get_rec_time_str() def stop(self,network_propagate=True): #explicit release of VideoWriter self.writer.release() self.writer = None if self.record_eye: for tx in self.g_pool.eye_tx: try: tx.send((None,None)) except: logger.warning("Could not stop eye-recording. Please report this bug!") save_object(self.data,os.path.join(self.rec_path, "pupil_data")) gaze_list_path = os.path.join(self.rec_path, "gaze_positions.npy") np.save(gaze_list_path,np.asarray(self.gaze_pos_list)) pupil_list_path = os.path.join(self.rec_path, "pupil_positions.npy") np.save(pupil_list_path,np.asarray(self.pupil_pos_list)) timestamps_path = os.path.join(self.rec_path, "world_timestamps.npy") # ts = sanitize_timestamps(np.array(self.timestamps)) ts = np.array(self.timestamps) np.save(timestamps_path,ts) try: copy2(os.path.join(self.g_pool.user_dir,"surface_definitions"),os.path.join(self.rec_path,"surface_definitions")) except: logger.info("No surface_definitions data found. You may want this if you do marker tracking.") try: copy2(os.path.join(self.g_pool.user_dir,"cal_pt_cloud.npy"),os.path.join(self.rec_path,"cal_pt_cloud.npy")) except: logger.warning("No calibration data found. Please calibrate first.") try: copy2(os.path.join(self.g_pool.user_dir,"camera_calibration"),os.path.join(self.rec_path,"camera_calibration")) except: logger.info("No camera calibration found.") try: with open(self.meta_info_path, 'a') as f: f.write("Duration Time\t"+ self.get_rec_time_str()+ "\n") if self.g_pool.binocular: f.write("Eye Mode\tbinocular\n") else: f.write("Eye Mode\tmonocular\n") f.write("Duration Time\t"+ self.get_rec_time_str()+ "\n") f.write("World Camera Frames\t"+ str(self.frame_count)+ "\n") f.write("World Camera Resolution\t"+ str(self.g_pool.capture.frame_size[0])+"x"+str(self.g_pool.capture.frame_size[1])+"\n") f.write("Capture Software Version\t%s\n"%self.g_pool.version) if platform.system() == "Windows": username = os.environ["USERNAME"] sysname, nodename, release, version, machine, _ = platform.uname() else: username = getpass.getuser() try: sysname, nodename, release, version, machine = os.uname() except: sysname, nodename, release, version, machine = sys.platform,None,None,None,None f.write("User\t"+username+"\n") f.write("Platform\t"+sysname+"\n") f.write("Machine\t"+nodename+"\n") f.write("Release\t"+release+"\n") f.write("Version\t"+version+"\n") except Exception: logger.exception("Could not save metadata. Please report this bug!") try: with open(os.path.join(self.rec_path, "user_info.csv"), 'w') as f: for name,val in self.user_info.iteritems(): f.write("%s\t%s\n"%(name,val)) except Exception: logger.exception("Could not save userdata. Please report this bug!") self.close_info_menu() if self.audio_writer: self.audio_writer = None self.running = False self.menu.read_only = False self.button.status_text = '' self.timestamps = [] self.data = {'pupil_positions':[],'gaze_positions':[]} self.pupil_pos_list = [] self.gaze_pos_list = [] self.notify_all( {'subject':'rec_stopped','rec_path':self.rec_path,'network_propagate':network_propagate} ) def cleanup(self): """gets called when the plugin get terminated. either volunatily or forced. """ if self.running: self.stop() self.deinit_gui() def verify_path(self,val): try: n_path = os.path.expanduser(val) logger.debug("Expanded user path.") except: n_path = val if not n_path: logger.warning("Please specify a path.") return False elif not os.path.isdir(n_path): logger.warning("This is not a valid path.") return False # elif not os.access(n_path, os.W_OK): elif not writable_dir(n_path): logger.warning("Do not have write access to '%s'."%n_path) return False else: return n_path def set_rec_dir(self,val): n_path = self.verify_path(val) if n_path: self.rec_dir = n_path def set_session_name(self, val): if not val: self.session_name = get_auto_name() else: if '/' in val: logger.warning('You session name will create one or more subdirectories') self.session_name = val
class Audio_Capture(Plugin): """docstring for Audio_Capture""" def __init__(self, g_pool, audio_src='No Audio'): super().__init__(g_pool) self.audio_devices_dict = Audio_Input_Dict() if audio_src in list(self.audio_devices_dict.keys()): self.audio_src = audio_src else: self.audio_src = 'No Audio' self.thread = None self.running = Event() self.audio_container = None self.audio_out_stream = None self.queue = queue.Queue() self.start_capture(self.audio_src) def recent_events(self, events): audio_packets = [] while True: try: packet = self.queue.get_nowait() except queue.Empty: break audio_packets.append(packet) events['audio_packets'] = audio_packets if self.audio_container is not None: for packet in audio_packets: self.write_audio_packet(packet) def init_gui(self): self.menu = ui.Growing_Menu('Audio Capture') self.menu.collapsed = True self.g_pool.sidebar.append(self.menu) def close(): self.alive = False help_str = 'Creates events for audio input.' self.menu.append(ui.Button('Close', close)) self.menu.append(ui.Info_Text(help_str)) def audio_dev_getter(): # fetch list of currently available self.audio_devices_dict = Audio_Input_Dict() devices = list(self.audio_devices_dict.keys()) return devices, devices self.menu.append( ui.Selector('audio_src', self, selection_getter=audio_dev_getter, label='Audio Source', setter=self.start_capture)) self.menu.append(ui.Info_Text(NOT_REC_STR)) def get_init_dict(self): return {'audio_src': self.audio_src} def deinit_gui(self): if self.menu: self.g_pool.sidebar.remove(self.menu) self.menu = None def cleanup(self): if self.audio_container is not None: self.close_audio_recording() self.running.clear() self.deinit_gui() if self.thread and self.thread.is_alive(): self.thread.join(timeout=1) def on_notify(self, notification): if notification['subject'] == 'recording.started': if self.running.is_set() and self.audio_container is None: self.rec_dir = notification['rec_path'] rec_file = os.path.join(self.rec_dir, 'audio.wav') self.audio_container = av.open(rec_file, 'w') self.timestamps = [] self.menu[-2].read_only = True del self.menu[-1] self.menu.append(ui.Info_Text(REC_STR)) elif not self.running.is_set(): logger.warning( 'Recording was started without an active audio capture') else: logger.warning('Audio is already being recorded') elif notification['subject'] == 'recording.stopped': if self.audio_container is not None and self.audio_out_stream is not None: self.close_audio_recording() def close_audio_recording(self): self.audio_container.close() ts_loc = os.path.join(self.rec_dir, 'audio_timestamps.npy') np.save(ts_loc, np.asarray(self.timestamps)) self.timestamps = None self.audio_out_stream = None self.audio_container = None self.menu[-2].read_only = False del self.menu[-1] self.menu.append(ui.Info_Text(NOT_REC_STR)) def write_audio_packet(self, packet): # Test if audio outstream has been initialized if self.audio_out_stream is None: try: self.audio_out_stream = self.audio_container.add_stream( template=packet.stream) except ValueError as e: # packet.stream codec is not supported in target container. logger.error( 'Failed to create audio stream. Aborting recording.') logger.debug('Reason: {}'.format(e)) self.close_audio_recording() self.timestamps.append(packet.timestamp) self.audio_container.mux(packet) def start_capture(self, audio_src): if self.thread and self.thread.is_alive(): if self.audio_src == audio_src: return # capture is already running for our selected source # else stop current capture gracefully self.running.clear() logger.debug('Closing capture for "{}"'.format(self.audio_src)) self.thread.join(timeout=1) self.thread = None if audio_src not in self.audio_devices_dict: logger.warning('Selected audio source is not available anymore') return self.audio_src = audio_src if audio_src == 'No Audio': return self.running.set() self.thread = Thread(target=self.capture_thread, args=(self.audio_devices_dict[audio_src], self.running)) self.thread.start() def capture_thread(self, audio_src, running): try: if platform.system() == "Darwin": in_container = av.open('none:{}'.format(audio_src), format="avfoundation") elif platform.system() == "Linux": in_container = av.open('hw:{}'.format(audio_src), format="alsa") else: raise av.AVError('Platform does not support audio capture.') except av.AVError: running.clear() return in_stream = None for stream in in_container.streams: if stream.type == 'audio': in_stream = stream break if not in_stream: logger.warning('No audio stream found for selected device.') running.clear() return stream_start_ts = self.g_pool.get_now() for packet in in_container.demux(in_stream): try: # ffmpeg timestamps - in_stream.startime = packte pts relative to startime # multiply with stream_timebase to get seconds # add start time of this stream in pupil time unadjusted # finally add pupil timebase offset to adjust for settable timebase. packet.timestamp = ( packet.pts - in_stream.start_time ) * in_stream.time_base + stream_start_ts - self.g_pool.timebase.value self.queue.put_nowait(packet) except queue.Full: pass # drop packet if not running.is_set(): return self.audio_src = 'No Audio' running.clear() # in_stream stopped yielding packets
class Recorder(Plugin): """Capture Recorder""" def __init__(self,g_pool,session_name = get_auto_name(),rec_dir=None, user_info={'name':'','additional_field':'change_me'},info_menu_conf={},show_info_menu=False, record_eye = False, audio_src = 'No Audio',raw_jpeg=True): super(Recorder, self).__init__(g_pool) #update name if it was autogenerated. if session_name.startswith('20') and len(session_name)==10: session_name = get_auto_name() base_dir = self.g_pool.user_dir.rsplit(os.path.sep,1)[0] default_rec_dir = os.path.join(base_dir,'recordings') if rec_dir and rec_dir != default_rec_dir and self.verify_path(rec_dir): self.rec_dir = rec_dir else: try: os.makedirs(default_rec_dir) except OSError as e: if e.errno != errno.EEXIST: logger.error("Could not create Rec dir") raise e else: logger.info('Created standard Rec dir at "%s"'%default_rec_dir) self.rec_dir = default_rec_dir self.raw_jpeg = raw_jpeg self.order = .9 self.record_eye = record_eye self.session_name = session_name self.audio_devices_dict = Audio_Input_Dict() if audio_src in self.audio_devices_dict.keys(): self.audio_src = audio_src else: self.audio_src = 'No Audio' self.running = False self.menu = None self.button = None self.user_info = user_info self.show_info_menu = show_info_menu self.info_menu = None self.info_menu_conf = info_menu_conf def get_init_dict(self): d = {} d['record_eye'] = self.record_eye d['audio_src'] = self.audio_src d['session_name'] = self.session_name d['user_info'] = self.user_info d['info_menu_conf'] = self.info_menu_conf d['show_info_menu'] = self.show_info_menu d['rec_dir'] = self.rec_dir d['raw_jpeg'] = self.raw_jpeg return d def init_gui(self): self.menu = ui.Growing_Menu('Recorder') self.g_pool.sidebar.insert(3,self.menu) self.menu.append(ui.Info_Text('Pupil recordings are saved like this: "path_to_recordings/recording_session_name/nnn" where "nnn" is an increasing number to avoid overwrites. You can use "/" in your session name to create subdirectories.')) self.menu.append(ui.Info_Text('Recordings are saved to "~/pupil_recordings". You can change the path here but note that invalid input will be ignored.')) self.menu.append(ui.Text_Input('rec_dir',self,setter=self.set_rec_dir,label='Path to recordings')) self.menu.append(ui.Text_Input('session_name',self,setter=self.set_session_name,label='Recording session name')) self.menu.append(ui.Switch('show_info_menu',self,on_val=True,off_val=False,label='Request additional user info')) self.menu.append(ui.Selector('raw_jpeg',self,selection = [True,False], labels=["bigger file, less CPU", "smaller file, more CPU"],label='Compression')) self.menu.append(ui.Info_Text('Recording the raw eye video is optional. We use it for debugging.')) self.menu.append(ui.Switch('record_eye',self,on_val=True,off_val=False,label='Record eye')) self.menu.append(ui.Selector('audio_src',self, selection=self.audio_devices_dict.keys(),label='Audio Source')) self.button = ui.Thumb('running',self,setter=self.toggle,label='Record',hotkey='r') self.button.on_color[:] = (1,.0,.0,.8) self.g_pool.quickbar.insert(1,self.button) def deinit_gui(self): if self.menu: self.g_pool.sidebar.remove(self.menu) self.menu = None if self.button: self.g_pool.quickbar.remove(self.button) self.button = None def toggle(self, _=None): if self.running: self.notify_all( {'subject':'should_stop_recording','network_propagate':True} ) else: self.notify_all( {'subject':'should_start_recording','session_name':self.session_name,'network_propagate':True} ) def on_notify(self,notification): # notification wants to be recorded if notification.get('record',False) and self.running: self.data['notifications'].append(notification) # Notificatio to start recording elif notification['subject'] == 'should_start_recording': if self.running: logger.info('Recording already running!') else: if notification.get("session_name",""): self.set_session_name(notification["session_name"]) self.start() # Remote has stopped recording, we should stop as well. elif notification['subject'] == 'should_stop_recording': if self.running: self.stop() else: logger.info('Recording already stopped!') def get_rec_time_str(self): rec_time = gmtime(time()-self.start_time) return strftime("%H:%M:%S", rec_time) def start(self): self.timestamps = [] self.data = {'pupil_positions':[],'gaze_positions':[],'notifications':[]} self.pupil_diameter = [] #addBy Changsoon self.gaze_point_z = [] #addBy Changsoon self.gaze_point_x = [] #addBy Changsoon self.gaze_point_y = [] #addBy Changsoon self.cnum = 0 self.frame_count = 0 self.running = True self.menu.read_only = True self.start_time = time() session = os.path.join(self.rec_dir, self.session_name) try: os.makedirs(session) logger.debug("Created new recordings session dir %s"%session) except: logger.debug("Recordings session dir %s already exists, using it." %session) # set up self incrementing folder within session folder counter = 0 while True: self.rec_path = os.path.join(session, "%03d/" % counter) try: os.mkdir(self.rec_path) logger.debug("Created new recording dir %s"%self.rec_path) break except: logger.debug("We dont want to overwrite data, incrementing counter & trying to make new data folder") counter += 1 self.meta_info_path = os.path.join(self.rec_path, "info.csv") with open(self.meta_info_path, 'w') as f: f.write("Recording Name\t"+self.session_name+ "\n") f.write("Start Date\t"+ strftime("%d.%m.%Y", localtime(self.start_time))+ "\n") f.write("Start Time\t"+ strftime("%H:%M:%S", localtime(self.start_time))+ "\n") if self.audio_src != 'No Audio': audio_path = os.path.join(self.rec_path, "world.wav") self.audio_writer = Audio_Capture(audio_path,self.audio_devices_dict[self.audio_src]) else: self.audio_writer = None if self.raw_jpeg and self.g_pool.capture.jpeg_support: self.video_path = os.path.join(self.rec_path, "world.mp4") self.writer = JPEG_Writer(self.video_path,self.g_pool.capture.frame_rate) else: self.video_path = os.path.join(self.rec_path, "world.mp4") self.writer = AV_Writer(self.video_path,fps=self.g_pool.capture.frame_rate) # positions path to eye process if self.record_eye: for alive, pipe in zip(self.g_pool.eyes_are_alive,self.g_pool.eye_pipes): if alive.value: pipe.send( ('Rec_Start',(self.rec_path,self.raw_jpeg) ) ) if self.show_info_menu: self.open_info_menu() logger.info("Started Recording.") self.notify_all( {'subject':'rec_started','rec_path':self.rec_path,'session_name':self.session_name,'network_propagate':True} ) def open_info_menu(self): self.info_menu = ui.Growing_Menu('additional Recording Info',size=(300,300),pos=(300,300)) self.info_menu.configuration = self.info_menu_conf def populate_info_menu(): self.info_menu.elements[:-2] = [] for name in self.user_info.iterkeys(): self.info_menu.insert(0,ui.Text_Input(name,self.user_info)) def set_user_info(new_string): self.user_info = new_string populate_info_menu() populate_info_menu() self.info_menu.append(ui.Info_Text('Use the *user info* field to add/remove additional fields and their values. The format must be a valid Python dictionary. For example -- {"key":"value"}. You can add as many fields as you require. Your custom fields will be saved for your next session.')) self.info_menu.append(ui.Text_Input('user_info',self,setter=set_user_info,label="User info")) self.g_pool.gui.append(self.info_menu) def close_info_menu(self): if self.info_menu: self.info_menu_conf = self.info_menu.configuration self.g_pool.gui.remove(self.info_menu) self.info_menu = None def update(self,frame,events): if self.running: for key,data in events.iteritems(): if key not in ('dt'): try: self.data[key] += data if (key == 'pupil_positions'): try: pupilDict = data[0] self.pupil_diameter.append(pupilDict.get('diameter')) except: print('error'); elif (key == 'gaze_positions'): try: gazeDict = data[0] self.gaze_point_x.append(gazeDict.get('norm_pos')[0]) self.gaze_point_y.append(gazeDict.get('norm_pos')[1]) self.gaze_point_z.append(gazeDict.get('gaze_point_3d')[2]) except: print('error') #image process (OCR) find the condion that show use OCR except KeyError: self.data[key] = [] self.data[key] += data print str(self.cnum) self.cnum = self.cnum + 1 if len(self.gaze_point_x) > 0 and self.cnum == 30: print('Enter TO the block') height = (1 - self.gaze_point_y[-1]) * 720 height = int(height) image = frame.img if (height < 100): image = frame.img[0:height+200, 300:1000] elif (height > 520): image = frame.img[height-100:720, 300:1000] else: image = frame.img[height-100:height+200, 300:1000] cv2.imwrite('/home/eyetracking/pupil/'+"frame%d.jpg" %self.cnum,image) image=Image.open('/home/eyetracking/pupil/'+"frame%d.jpg" %self.cnum) enhancer = ImageEnhance.Contrast(image) image = enhancer.enhance(4) totemp = sys.stdout sys.stdout = codecs.open('/home/eyetracking/pupil/ocr.txt', encoding='utf-8', mode='w+') #save the OCR print image_to_string(image) sys.stdout.close() sys.stdout = totemp try: i=300 for line in open('/home/eyetracking/pupil/ocr.txt'): i=i+20 cv2.putText(dst, line,(0,i), cv2.FONT_HERSHEY_COMPLEX,0.6,(0,0,255)) cv2.imshow("window",dst) cv2.waitKey(33) except Exception as error: print(error) self.timestamps.append(frame.timestamp) self.writer.write_video_frame(frame) self.frame_count += 1 # # cv2.putText(frame.img, "Frame %s"%self.frame_count,(200,200), cv2.FONT_HERSHEY_SIMPLEX,1,(255,100,100)) self.button.status_text = self.get_rec_time_str() def stop(self): #explicit release of VideoWriter self.writer.release() self.writer = None if self.record_eye: for alive, pipe in zip(self.g_pool.eyes_are_alive,self.g_pool.eye_pipes): if alive.value: pipe.send(('Rec_Stop',None)) save_object(self.data,os.path.join(self.rec_path, "pupil_data")) timestamps_path = os.path.join(self.rec_path, "world_timestamps.npy") # ts = sanitize_timestamps(np.array(self.timestamps)) ts = np.array(self.timestamps) np.save(timestamps_path,ts) try: copy2(os.path.join(self.g_pool.user_dir,"surface_definitions"),os.path.join(self.rec_path,"surface_definitions")) except: logger.info("No surface_definitions data found. You may want this if you do marker tracking.") try: copy2(os.path.join(self.g_pool.user_dir,"user_calibration_data"),os.path.join(self.rec_path,"user_calibration_data")) except: logger.warning("No user calibration data found. Please calibrate first.") camera_calibration = load_camera_calibration(self.g_pool) if camera_calibration is not None: save_object(camera_calibration,os.path.join(self.rec_path, "camera_calibration")) else: logger.info("No camera calibration found.") try: with open(self.meta_info_path, 'a') as f: f.write("Duration Time\t"+ self.get_rec_time_str()+ "\n") f.write("World Camera Frames\t"+ str(self.frame_count)+ "\n") f.write("World Camera Resolution\t"+ str(self.g_pool.capture.frame_size[0])+"x"+str(self.g_pool.capture.frame_size[1])+"\n") f.write("Capture Software Version\t%s\n"%self.g_pool.version) f.write("System Info\t%s"%get_system_info()) except Exception: logger.exception("Could not save metadata. Please report this bug!") try: with open(os.path.join(self.rec_path, "user_info.csv"), 'w') as f: for name,val in self.user_info.iteritems(): f.write("%s\t%s\n"%(name,val)) except Exception: logger.exception("Could not save userdata. Please report this bug!") self.close_info_menu() if self.audio_writer: self.audio_writer = None self.running = False self.menu.read_only = False self.button.status_text = '' self.timestamps = [] self.data = {'pupil_positions':[],'gaze_positions':[]} self.pupil_pos_list = [] self.gaze_pos_list = [] logger.info("Saved Recording.") self.notify_all( {'subject':'rec_stopped','rec_path':self.rec_path,'network_propagate':True} ) def cleanup(self): """gets called when the plugin get terminated. either volunatily or forced. """ if self.running: self.stop() self.deinit_gui() def verify_path(self,val): try: n_path = os.path.expanduser(val) logger.debug("Expanded user path.") except: n_path = val if not n_path: logger.warning("Please specify a path.") return False elif not os.path.isdir(n_path): logger.warning("This is not a valid path.") return False # elif not os.access(n_path, os.W_OK): elif not writable_dir(n_path): logger.warning("Do not have write access to '%s'."%n_path) return False else: return n_path def set_rec_dir(self,val): n_path = self.verify_path(val) if n_path: self.rec_dir = n_path def set_session_name(self, val): if not val: self.session_name = get_auto_name() else: if '/' in val: logger.warning('You session name will create one or more subdirectories') self.session_name = val
class Recorder(Plugin): """Capture Recorder""" def __init__(self,g_pool,session_name = get_auto_name(),rec_dir=None, user_info={'name':'','additional_field':'change_me'},info_menu_conf={},show_info_menu=False, record_eye = False, audio_src = 'No Audio', raw_jpeg=False): super(Recorder, self).__init__(g_pool) self.states = ['start', 'want', 'pick'] self.all_qr = {'termo1' : 75, 'light1' : False, 'light2' : False, 'light3': False} self.this_state = 'start' self.start_state = time() self.qr_codes = {} self.gaze_x = -1 self.gaze_y = -1 self.avg_col = 0 self.choice = "" #update name if it was autogenerated. if session_name.startswith('20') and len(session_name)==10: session_name = get_auto_name() base_dir = self.g_pool.user_dir.rsplit(os.path.sep,1)[0] default_rec_dir = os.path.join(base_dir, 'recordings') if rec_dir and rec_dir != default_rec_dir and self.verify_path(rec_dir): self.rec_dir = rec_dir else: try: os.makedirs(default_rec_dir) except OSError as e: if 'File exists' in '%s'%e: pass else: logger.error("Could not create Rec dir") raise e else: logger.info('Created standard Rec dir at "%s"'%default_rec_dir) self.rec_dir = default_rec_dir # if rec_dir and rec_dir != default_rec_dir and self.verify_path(rec_dir): # self.rec_dir = rec_dir # else: # #lets make a rec dir next to the user dir # base_dir = self.g_pool.user_dir.rsplit(os.path.sep,1)[0] # self.rec_dir = os.path.join(base_dir,'recordings') # if not os.path.isdir(self.rec_dir): # os.mkdir(self.rec_dir) self.raw_jpeg = raw_jpeg self.order = .9 self.record_eye = record_eye self.session_name = session_name self.audio_devices_dict = Audio_Input_Dict() if audio_src in self.audio_devices_dict.keys(): self.audio_src = audio_src else: self.audio_src = 'No Audio' self.running = False self.menu = None self.button = None self.user_info = user_info self.show_info_menu = show_info_menu self.info_menu = None self.info_menu_conf = info_menu_conf self.height, self.width = self.g_pool.capture.frame_size def get_init_dict(self): d = {} d['record_eye'] = self.record_eye d['audio_src'] = self.audio_src d['session_name'] = self.session_name d['user_info'] = self.user_info d['info_menu_conf'] = self.info_menu_conf d['show_info_menu'] = self.show_info_menu d['rec_dir'] = self.rec_dir d['raw_jpeg'] = self.raw_jpeg # if self.menu: # d['menu_conf'] = self.menu.configuration # else: # d['menu_conf'] = self.menu_conf return d def init_gui(self): self.menu = ui.Growing_Menu('Recorder') # self.menu.configuration = self.menu_conf self.g_pool.sidebar.insert(3,self.menu) self.menu.append(ui.Info_Text('Pupil recordings are saved like this: "path_to_recordings/recording_session_name/nnn" where "nnn" is an increasing number to avoid overwrites. You can use "/" in your session name to create subdirectories.')) self.menu.append(ui.Info_Text('Recordings are saved to "~/pupil_recordings". You can change the path here but note that invalid input will be ignored.')) self.menu.append(ui.Text_Input('rec_dir',self,setter=self.set_rec_dir,label='Path to recordings')) self.menu.append(ui.Text_Input('session_name',self,setter=self.set_session_name,label='Recording session name')) self.menu.append(ui.Switch('show_info_menu',self,on_val=True,off_val=False,label='Request additional user info')) self.menu.append(ui.Selector('raw_jpeg',self,selection = [True,False], labels=["bigger file, less CPU", "smaller file, more CPU"],label='compression')) self.menu.append(ui.Info_Text('Recording the raw eye video is optional. We use it for debugging.')) self.menu.append(ui.Switch('record_eye',self,on_val=True,off_val=False,label='Record eye')) self.menu.append(ui.Selector('audio_src',self, selection=self.audio_devices_dict.keys())) self.button = ui.Thumb('running',self,setter=self.toggle,label='QR',hotkey='r') self.button.on_color[:] = (1,.0,.0,.8) self.g_pool.quickbar.insert(1,self.button) def deinit_gui(self): if self.menu: # self.menu_conf = self.menu.configuration self.g_pool.sidebar.remove(self.menu) self.menu = None if self.button: self.g_pool.quickbar.remove(self.button) self.button = None def toggle(self, _=None): if self.running: self.stop() else: self.start() def get_rec_time_str(self): rec_time = gmtime(time()-self.start_time) return strftime("%H:%M:%S", rec_time) def start(self): self.timestamps = [] # add self.data self.data = {'pupil_positions':[], 'gaze_positions': []} self.pupil_list = [] self.gaze_list = [] self.frame_count = 0 self.running = True self.menu.read_only = True self.start_time = time() self.start_state = time() session = os.path.join(self.rec_dir, self.session_name) try: os.makedirs(session) logger.debug("Created new recordings session dir %s"%session) except: logger.debug("Recordings session dir %s already exists, using it." %session) # set up self incrementing folder within session folder counter = 0 while True: self.rec_path = os.path.join(session, "%03d/" % counter) try: os.mkdir(self.rec_path) logger.debug("Created new recording dir %s"%self.rec_path) break except: logger.debug("We dont want to overwrite data, incrementing counter & trying to make new data folder") counter += 1 self.meta_info_path = os.path.join(self.rec_path, "info.csv") with open(self.meta_info_path, 'w') as f: f.write("Recording Name\t"+self.session_name+ "\n") f.write("Start Date\t"+ strftime("%d.%m.%Y", localtime(self.start_time))+ "\n") f.write("Start Time\t"+ strftime("%H:%M:%S", localtime(self.start_time))+ "\n") if self.audio_src != 'No Audio': audio_path = os.path.join(self.rec_path, "world.wav") self.audio_writer = Audio_Capture(self.audio_devices_dict[self.audio_src],audio_path) else: self.audio_writer = None if self.raw_jpeg and "uvc_capture" in str(self.g_pool.capture.__class__): self.video_path = os.path.join(self.rec_path, "world.mp4") self.writer = JPEG_Writer(self.video_path,int(self.g_pool.capture.frame_rate)) # elif 1: # self.writer = av_writer.AV_Writer(self.video_path) else: self.video_path = os.path.join(self.rec_path, "world.mkv") self.writer = CV_Writer(self.video_path, float(self.g_pool.capture.frame_rate), self.g_pool.capture.frame_size) # positions path to eye process if self.record_eye: for tx in self.g_pool.eye_tx: tx.send((self.rec_path,self.raw_jpeg)) if self.show_info_menu: self.open_info_menu() # self.video_path = os.path.join(self.rec_path, "world.mkv") # self.writer = cv2.VideoWriter(self.video_path, int(cv2.cv.CV_FOURCC(*'DIVX')), float(self.g_pool.capture.frame_rate), (1280,720)) # # positions path to eye process # if self.record_eye: # for tx in self.g_pool.eye_tx: # tx.send(self.rec_path) # if self.show_info_menu: # self.open_info_menu() ############### def open_info_menu(self): self.info_menu = ui.Growing_Menu('additional Recording Info',size=(300,300),pos=(300,300)) self.info_menu.configuration = self.info_menu_conf def populate_info_menu(): self.info_menu.elements[:-2] = [] for name in self.user_info.iterkeys(): self.info_menu.insert(0,ui.Text_Input(name,self.user_info)) def set_user_info(new_string): self.user_info = new_string populate_info_menu() populate_info_menu() self.info_menu.append(ui.Info_Text('Use the *user info* field to add/remove additional fields and their values. The format must be a valid Python dictionary. For example -- {"key":"value"}. You can add as many fields as you require. Your custom fields will be saved for your next session.')) self.info_menu.append(ui.Text_Input('user_info',self,setter=set_user_info,label="User info")) self.g_pool.gui.append(self.info_menu) def close_info_menu(self): if self.info_menu: self.info_menu_conf = self.info_menu.configuration self.g_pool.gui.remove(self.info_menu) self.info_menu = None def update(self,frame,events): if self.running: # update data self.data['pupil_positions'] += events['pupil_positions'] self.data['gaze_positions'] += events['gaze_positions'] #cv2.putText(frame.img, "Frame %s"%self.frame_count,(200,200), cv2.FONT_HERSHEY_SIMPLEX,1,(255,100,100) cv2.putText(frame.img, self.this_state,(200,100), cv2.FONT_HERSHEY_SIMPLEX,1,(255,100,100)) if self.this_state == 'pick': cv2.putText(frame.img, self.choice,(300,100), cv2.FONT_HERSHEY_SIMPLEX,1,(100,100,255)) QRs = qr_detect(frame) for p in events['pupil_positions']: pupil_pos = p['timestamp'],p['confidence'],p['id'],p['norm_pos'][0],p['norm_pos'][1],p['diameter'] self.pupil_list.append(pupil_pos) c = 0 avg_x = -1 avg_y = -1 for g in events.get('gaze_positions',[]): gaze_pos = g['timestamp'],g['confidence'],g['norm_pos'][0],g['norm_pos'][1] avg_x = (avg_x * c + g['norm_pos'][0]) / (c+1) avg_y = (avg_y * c + g['norm_pos'][0]) / (c+1) c = c + 1 self.gaze_list.append(gaze_pos) self.timestamps.append(frame.timestamp) avg_col = 0 self.stateT(frame, QRs, avg_x, avg_y, avg_col) self.writer.write_video_frame(frame) self.frame_count += 1 self.button.status_text = self.get_rec_time_str() def stop(self): #explicit release of VideoWriter self.writer.release() self.writer = None if self.record_eye: for tx in self.g_pool.eye_tx: try: tx.send(None) except: logger.warning("Could not stop eye-recording. Please report this bug!") save_object(self.data, os.path.join(self.rec_path, "pupil_data")) gaze_list_path = os.path.join(self.rec_path, "gaze_positions.npy") np.save(gaze_list_path,np.asarray(self.gaze_list)) pupil_list_path = os.path.join(self.rec_path, "pupil_positions.npy") np.save(pupil_list_path,np.asarray(self.pupil_list)) timestamps_path = os.path.join(self.rec_path, "world_timestamps.npy") ts = sanitize_timestamps(np.array(self.timestamps)) np.save(timestamps_path,ts) try: copy2(os.path.join(self.g_pool.user_dir,"surface_definitions"),os.path.join(self.rec_path,"surface_definitions")) except: logger.info("No surface_definitions data found. You may want this if you do marker tracking.") try: copy2(os.path.join(self.g_pool.user_dir,"cal_pt_cloud.npy"),os.path.join(self.rec_path,"cal_pt_cloud.npy")) except: logger.warning("No calibration data found. Please calibrate first.") try: copy2(os.path.join(self.g_pool.user_dir,"camera_matrix.npy"),os.path.join(self.rec_path,"camera_matrix.npy")) copy2(os.path.join(self.g_pool.user_dir,"dist_coefs.npy"),os.path.join(self.rec_path,"dist_coefs.npy")) except: logger.info("No camera intrinsics found.") try: with open(self.meta_info_path, 'a') as f: f.write("Duration Time\t"+ self.get_rec_time_str()+ "\n") if self.g_pool.binocular: f.write("Eye Mode\tbinocular\n") else: f.write("Eye Mode\tmonocular\n") f.write("Duration Time\t"+ self.get_rec_time_str()+ "\n") f.write("World Camera Frames\t"+ str(self.frame_count)+ "\n") f.write("World Camera Resolution\t"+ str(self.width)+"x"+str(self.height)+"\n") f.write("Capture Software Version\t%s\n"%self.g_pool.version) if platform.system() == "Windows": username = os.environ["USERNAME"] sysname, nodename, release, version, machine, _ = platform.uname() else: username = os.getlogin() try: sysname, nodename, release, version, machine = os.uname() except: sysname, nodename, release, version, machine = sys.platform,None,None,None,None f.write("User\t"+username+"\n") f.write("Platform\t"+sysname+"\n") f.write("Machine\t"+nodename+"\n") f.write("Release\t"+release+"\n") f.write("Version\t"+version+"\n") except Exception: logger.exception("Could not save metadata. Please report this bug!") try: with open(os.path.join(self.rec_path, "user_info.csv"), 'w') as f: for name,val in self.user_info.iteritems(): f.write("%s\t%s\n"%(name,val)) except Exception: logger.exception("Could not save userdata. Please report this bug!") self.close_info_menu() if self.audio_writer: self.audio_writer = None self.running = False self.menu.read_only = False self.button.status_text = '' def cleanup(self): """gets called when the plugin get terminated. either volunatily or forced. """ if self.running: self.stop() self.deinit_gui() def verify_path(self, val): try: n_path = os.path.expanduser(val) logger.debug("Expanded user path.") except: n_path = val if not n_path: logger.waring("Please specify a path.") return False elif not os.path.isdir(n_path): logger.warning("This is not a valid path.") return False elif not writable_dir(n_path): logger.warning("Do not have write access to '%s'." %n_path) return False else: return n_path def set_rec_dir(self,val): # try: # n_path = os.path.expanduser(val) # logger.debug("Expanded user path.") # except: # n_path = val # if not n_path: # logger.warning("Please specify a path.") # elif not os.path.isdir(n_path): # logger.warning("This is not a valid path.") # # elif not os.access(n_path, os.W_OK): # elif not writable_dir(n_path): # logger.warning("Do not have write access to '%s'."%n_path) # else: # self.rec_dir = n_path n_path = self.verify_path(val) if n_path: self.rec_dir = n_path def set_session_name(self, val): if not val: self.session_name = get_auto_name() else: if '/' in val: logger.warning('You session name with create one or more subdirectories') self.session_name = val def stateT(self, frame, QRs, agaze_x, agaze_y, avg_col): if time() - self.start_time < 1: self.gaze_x = agaze_x self.gaze_y = agaze_y if self.this_state == 'start': if time() - self.start_state < 2: self.qr_codes = {} return True if self.Nochange_gaze(agaze_x, agaze_y, avg_col): self.this_state = 'want' self.start_state = time() self.gaze_x = agaze_x self.gaze_y = agaze_y return True if self.this_state == 'want': if time() - self.start_state < 2: self.qr_codes.update(QRs) return True if self.Nochange_gaze(agaze_x, agaze_y, avg_col) and self.qr_codes: self.this_state = 'pick' self.choice = self.choose(frame) #print self.qr_codes, self.choice else: self.this_state = 'start' self.start_state = time() self.gaze_x = agaze_x self.gaze_y = agaze_y return True if self.this_state == 'pick': if time() - self.start_state < 4: return True if self.Nochange_gaze(agaze_x, agaze_y, avg_col): del self.qr_codes[self.choice] if not self.qr_codes: self.this_state = 'start' else: self.choice = self.choose(frame) #print self.choice else: print self.choice, " chosen" self.this_state = 'start' self.start_state = time() self.gaze_x = agaze_x self.gaze_y = agaze_y return True def Nochange_gaze(self, agaze_x, agaze_y, avg_col): #print self.gaze_x, self.gaze_y, agaze_x, agaze_y return (abs(self.gaze_x - agaze_x) < .1 and abs(self.gaze_y - agaze_y) < .1) def choose(self, frame): h = frame.img.shape[0] w = frame.img.shape[1] gaze_x = self.gaze_x gaze_y = self.gaze_y if (gaze_x < 0 or gaze_y < 0): gaze_x = .5 gaz_y = .5 k = self.qr_codes.keys() chosen = k[0] min_dist = 1000000 for name, mc in self.qr_codes.items(): dis = (mc[0] - gaze_x * w) * (mc[0] - gaze_x * w) + (mc[1] - gaze_y * h) * (mc[1] - gaze_y * h) if dis < min_dist: min_dist = dis chosen = name return chosen