class Recorder(Plugin):
"""Capture Recorder"""
def __init__(self,
g_pool,
session_name=get_auto_name(),
rec_dir=None,
user_info={
'name': '',
'additional_field': 'change_me'
},
info_menu_conf={},
show_info_menu=False,
record_eye=False,
audio_src='No Audio',
menu_conf={}):
super(Recorder, self).__init__(g_pool)
#update name if it was autogenerated.
if session_name.startswith('20') and len(session_name) == 10:
session_name = get_auto_name()
if rec_dir:
self.set_rec_dir(rec_dir)
else:
#lets make a rec dir next to the user dir
base_dir = self.g_pool.user_dir.rsplit(os.path.sep, 1)[0]
self.rec_dir = os.path.join(base_dir, 'recordings')
if not os.path.isdir(self.rec_dir):
os.mkdir(self.rec_dir)
self.order = .9
self.record_eye = record_eye
self.session_name = session_name
self.audio_devices_dict = Audio_Input_Dict()
if audio_src in self.audio_devices_dict.keys():
self.audio_src = audio_src
else:
self.audio_src = 'No Audio'
self.running = False
self.menu = None
self.button = None
self.menu_conf = menu_conf
self.user_info = user_info
self.show_info_menu = show_info_menu
self.info_menu = None
self.info_menu_conf = info_menu_conf
self.height, self.width = self.g_pool.capture.frame_size
def get_init_dict(self):
d = {}
d['record_eye'] = self.record_eye
d['audio_src'] = self.audio_src
d['session_name'] = self.session_name
d['user_info'] = self.user_info
d['info_menu_conf'] = self.info_menu_conf
d['show_info_menu'] = self.show_info_menu
d['rec_dir'] = self.rec_dir
if self.menu:
d['menu_conf'] = self.menu.configuration
else:
d['menu_conf'] = self.menu_conf
return d
def init_gui(self):
self.menu = ui.Growing_Menu('Recorder')
self.menu.configuration = self.menu_conf
self.g_pool.sidebar.insert(3, self.menu)
self.menu.append(
ui.Info_Text(
'Pupil recordings are saved like this: "path_to_recordings/recording_session_name/nnn" where "nnn" is an increasing number to avoid overwrites. You can use "/" in your session name to create subdirectories.'
))
self.menu.append(
ui.Info_Text(
'Recordings are saved to "~/pupil_recordings". You can change the path here but note that invalid input will be ignored.'
))
self.menu.append(
ui.Text_Input('rec_dir',
self,
setter=self.set_rec_dir,
label='Path to recordings'))
self.menu.append(
ui.Text_Input('session_name',
self,
setter=self.set_session_name,
label='Recording session name'))
self.menu.append(
ui.Switch('show_info_menu',
self,
on_val=True,
off_val=False,
label='Request additional user info'))
self.menu.append(
ui.Info_Text(
'Recording the raw eye video is optional. We use it for debugging.'
))
self.menu.append(
ui.Switch('record_eye',
self,
on_val=True,
off_val=False,
label='Record eye'))
self.menu.append(
ui.Selector('audio_src',
self,
selection=self.audio_devices_dict.keys()))
self.button = ui.Thumb('running',
self,
setter=self.toggle,
label='Record',
hotkey='r')
self.button.on_color[:] = (1, .0, .0, .8)
self.g_pool.quickbar.insert(1, self.button)
def deinit_gui(self):
if self.menu:
self.menu_conf = self.menu.configuration
self.g_pool.sidebar.remove(self.menu)
self.menu = None
if self.button:
self.g_pool.quickbar.remove(self.button)
self.button = None
def toggle(self, _=None):
if self.running:
self.stop()
else:
self.start()
def get_rec_time_str(self):
rec_time = gmtime(time() - self.start_time)
return strftime("%H:%M:%S", rec_time)
def start(self):
self.timestamps = []
self.pupil_list = []
self.gaze_list = []
self.frame_count = 0
self.running = True
self.menu.read_only = True
self.start_time = time()
session = os.path.join(self.rec_dir, self.session_name)
try:
os.makedirs(session)
logger.debug("Created new recordings session dir %s" % session)
except:
logger.debug(
"Recordings session dir %s already exists, using it." %
session)
# set up self incrementing folder within session folder
counter = 0
while True:
self.rec_path = os.path.join(session, "%03d/" % counter)
try:
os.mkdir(self.rec_path)
logger.debug("Created new recording dir %s" % self.rec_path)
break
except:
logger.debug(
"We dont want to overwrite data, incrementing counter & trying to make new data folder"
)
counter += 1
self.meta_info_path = os.path.join(self.rec_path, "info.csv")
with open(self.meta_info_path, 'w') as f:
f.write("Recording Name\t" + self.session_name + "\n")
f.write("Start Date\t" +
strftime("%d.%m.%Y", localtime(self.start_time)) + "\n")
f.write("Start Time\t" +
strftime("%H:%M:%S", localtime(self.start_time)) + "\n")
if self.audio_src != 'No Audio':
audio_path = os.path.join(self.rec_path, "world.wav")
self.audio_writer = Audio_Capture(
self.audio_devices_dict[self.audio_src], audio_path)
else:
self.audio_writer = None
self.video_path = os.path.join(self.rec_path, "world.mkv")
self.writer = cv2.VideoWriter(self.video_path,
cv2.cv.CV_FOURCC(*'DIVX'),
float(self.g_pool.capture.frame_rate),
self.g_pool.capture.frame_size)
# positions path to eye process
if self.record_eye:
for tx in self.g_pool.eye_tx:
tx.send(self.rec_path)
if self.show_info_menu:
self.open_info_menu()
def open_info_menu(self):
self.info_menu = ui.Growing_Menu('additional Recording Info',
size=(300, 300),
pos=(300, 300))
self.info_menu.configuration = self.info_menu_conf
def populate_info_menu():
self.info_menu.elements[:-2] = []
for name in self.user_info.iterkeys():
self.info_menu.insert(0, ui.Text_Input(name, self.user_info))
def set_user_info(new_string):
self.user_info = new_string
populate_info_menu()
populate_info_menu()
self.info_menu.append(
ui.Info_Text(
'Use the *user info* field to add/remove additional fields and their values. The format must be a valid Python dictionary. For example -- {"key":"value"}. You can add as many fields as you require. Your custom fields will be saved for your next session.'
))
self.info_menu.append(
ui.Text_Input('user_info',
self,
setter=set_user_info,
label="User info"))
self.g_pool.gui.append(self.info_menu)
def close_info_menu(self):
if self.info_menu:
self.info_menu_conf = self.info_menu.configuration
self.g_pool.gui.remove(self.info_menu)
self.info_menu = None
def update(self, frame, events):
if self.running:
# cv2.putText(frame.img, "Frame %s"%self.frame_count,(200,200), cv2.FONT_HERSHEY_SIMPLEX,1,(255,100,100))
for p in events['pupil_positions']:
pupil_pos = p['timestamp'], p['confidence'], p['id'], p[
'norm_pos'][0], p['norm_pos'][1], p['diameter']
self.pupil_list.append(pupil_pos)
for g in events.get('gaze', []):
gaze_pos = g['timestamp'], g['confidence'], g['norm_pos'][
0], g['norm_pos'][1]
self.gaze_list.append(gaze_pos)
self.timestamps.append(frame.timestamp)
self.writer.write(frame.img)
self.frame_count += 1
self.button.status_text = self.get_rec_time_str()
def stop(self):
#explicit release of VideoWriter
self.writer.release()
self.writer = None
if self.record_eye:
for tx in self.g_pool.eye_tx:
try:
tx.send(None)
except:
logger.warning(
"Could not stop eye-recording. Please report this bug!"
)
gaze_list_path = os.path.join(self.rec_path, "gaze_positions.npy")
np.save(gaze_list_path, np.asarray(self.gaze_list))
pupil_list_path = os.path.join(self.rec_path, "pupil_positions.npy")
np.save(pupil_list_path, np.asarray(self.pupil_list))
timestamps_path = os.path.join(self.rec_path, "world_timestamps.npy")
ts = sanitize_timestamps(np.array(self.timestamps))
np.save(timestamps_path, ts)
try:
copy2(os.path.join(self.g_pool.user_dir, "surface_definitions"),
os.path.join(self.rec_path, "surface_definitions"))
except:
logger.info(
"No surface_definitions data found. You may want this if you do marker tracking."
)
try:
copy2(os.path.join(self.g_pool.user_dir, "cal_pt_cloud.npy"),
os.path.join(self.rec_path, "cal_pt_cloud.npy"))
except:
logger.warning(
"No calibration data found. Please calibrate first.")
try:
copy2(os.path.join(self.g_pool.user_dir, "camera_matrix.npy"),
os.path.join(self.rec_path, "camera_matrix.npy"))
copy2(os.path.join(self.g_pool.user_dir, "dist_coefs.npy"),
os.path.join(self.rec_path, "dist_coefs.npy"))
except:
logger.info("No camera intrinsics found.")
try:
with open(self.meta_info_path, 'a') as f:
f.write("Duration Time\t" + self.get_rec_time_str() + "\n")
if self.g_pool.binocular:
f.write("Eye Mode\tbinocular\n")
else:
f.write("Eye Mode\tmonocular\n")
f.write("Duration Time\t" + self.get_rec_time_str() + "\n")
f.write("World Camera Frames\t" + str(self.frame_count) + "\n")
f.write("World Camera Resolution\t" + str(self.width) + "x" +
str(self.height) + "\n")
f.write("Capture Software Version\t%s\n" % self.g_pool.version)
if platform.system() == "Windows":
username = os.environ["USERNAME"]
sysname, nodename, release, version, machine, _ = platform.uname(
)
else:
username = os.getlogin()
try:
sysname, nodename, release, version, machine = os.uname(
)
except:
sysname, nodename, release, version, machine = sys.platform, None, None, None, None
f.write("User\t" + username + "\n")
f.write("Platform\t" + sysname + "\n")
f.write("Machine\t" + nodename + "\n")
f.write("Release\t" + release + "\n")
f.write("Version\t" + version + "\n")
except Exception:
logger.exception(
"Could not save metadata. Please report this bug!")
try:
with open(os.path.join(self.rec_path, "user_info.csv"), 'w') as f:
for name, val in self.user_info.iteritems():
f.write("%s\t%s\n" % (name, val))
except Exception:
logger.exception(
"Could not save userdata. Please report this bug!")
self.close_info_menu()
if self.audio_writer:
self.audio_writer = None
self.running = False
self.menu.read_only = False
self.button.status_text = ''
def cleanup(self):
"""gets called when the plugin get terminated.
either volunatily or forced.
"""
if self.running:
self.stop()
self.deinit_gui()
def set_rec_dir(self, val):
try:
n_path = os.path.expanduser(val)
logger.debug("Expanded user path.")
except:
n_path = val
if not n_path:
logger.warning("Please specify a path.")
elif not os.path.isdir(n_path):
logger.warning("This is not a valid path.")
# elif not os.access(n_path, os.W_OK):
elif not writable_dir(n_path):
logger.warning("Do not have write access to '%s'." % n_path)
else:
self.rec_dir = n_path
def set_session_name(self, val):
if not val:
self.session_name = get_auto_name()
else:
if '/' in val:
logger.warning(
'You session name with create one or more subdirectories')
self.session_name = val
class Recorder(Plugin):
"""Capture Recorder"""
def __init__(self,g_pool,session_name = get_auto_name(),rec_dir=None, user_info={'name':'','additional_field':'change_me'},info_menu_conf={},show_info_menu=False, record_eye = False, audio_src = 'No Audio',raw_jpeg=False):
super(Recorder, self).__init__(g_pool)
#update name if it was autogenerated.
if session_name.startswith('20') and len(session_name)==10:
session_name = get_auto_name()
base_dir = self.g_pool.user_dir.rsplit(os.path.sep,1)[0]
default_rec_dir = os.path.join(base_dir,'recordings')
if rec_dir and rec_dir != default_rec_dir and self.verify_path(rec_dir):
self.rec_dir = rec_dir
else:
try:
os.makedirs(default_rec_dir)
except OSError as e:
if e.errno != errno.EEXIST:
logger.error("Could not create Rec dir")
raise e
else:
logger.info('Created standard Rec dir at "%s"'%default_rec_dir)
self.rec_dir = default_rec_dir
self.raw_jpeg = raw_jpeg
self.order = .9
self.record_eye = record_eye
self.session_name = session_name
self.audio_devices_dict = Audio_Input_Dict()
if audio_src in self.audio_devices_dict.keys():
self.audio_src = audio_src
else:
self.audio_src = 'No Audio'
self.running = False
self.menu = None
self.button = None
self.user_info = user_info
self.show_info_menu = show_info_menu
self.info_menu = None
self.info_menu_conf = info_menu_conf
def get_init_dict(self):
d = {}
d['record_eye'] = self.record_eye
d['audio_src'] = self.audio_src
d['session_name'] = self.session_name
d['user_info'] = self.user_info
d['info_menu_conf'] = self.info_menu_conf
d['show_info_menu'] = self.show_info_menu
d['rec_dir'] = self.rec_dir
d['raw_jpeg'] = self.raw_jpeg
return d
def init_gui(self):
self.menu = ui.Growing_Menu('Recorder')
self.g_pool.sidebar.insert(3,self.menu)
self.menu.append(ui.Info_Text('Pupil recordings are saved like this: "path_to_recordings/recording_session_name/nnn" where "nnn" is an increasing number to avoid overwrites. You can use "/" in your session name to create subdirectories.'))
self.menu.append(ui.Info_Text('Recordings are saved to "~/pupil_recordings". You can change the path here but note that invalid input will be ignored.'))
self.menu.append(ui.Text_Input('rec_dir',self,setter=self.set_rec_dir,label='Path to recordings'))
self.menu.append(ui.Text_Input('session_name',self,setter=self.set_session_name,label='Recording session name'))
self.menu.append(ui.Switch('show_info_menu',self,on_val=True,off_val=False,label='Request additional user info'))
self.menu.append(ui.Selector('raw_jpeg',self,selection = [True,False], labels=["bigger file, less CPU", "smaller file, more CPU"],label='compression'))
self.menu.append(ui.Info_Text('Recording the raw eye video is optional. We use it for debugging.'))
self.menu.append(ui.Switch('record_eye',self,on_val=True,off_val=False,label='Record eye'))
self.menu.append(ui.Selector('audio_src',self, selection=self.audio_devices_dict.keys()))
self.button = ui.Thumb('running',self,setter=self.toggle,label='Record',hotkey='r')
self.button.on_color[:] = (1,.0,.0,.8)
self.g_pool.quickbar.insert(1,self.button)
def deinit_gui(self):
if self.menu:
self.g_pool.sidebar.remove(self.menu)
self.menu = None
if self.button:
self.g_pool.quickbar.remove(self.button)
self.button = None
def toggle(self, _=None):
if self.running:
self.stop()
else:
self.start()
def on_notify(self,notification):
if notification['subject'] == 'rec_should_start':
if self.running:
logger.warning('Recording is already running!')
else:
self.set_session_name(notification["session_name"])
self.start(network_propagate=notification.get('network_propagate',True))
elif notification['subject'] == 'rec_should_stop':
if self.running:
self.stop(network_propagate=notification.get('network_propagate',True))
else:
logger.warning('Recording is already stopped!')
def get_rec_time_str(self):
rec_time = gmtime(time()-self.start_time)
return strftime("%H:%M:%S", rec_time)
def start(self,network_propagate=True):
self.timestamps = []
self.data = {'pupil_positions':[],'gaze_positions':[]}
self.pupil_pos_list = []
self.gaze_pos_list = []
self.frame_count = 0
self.running = True
self.menu.read_only = True
self.start_time = time()
session = os.path.join(self.rec_dir, self.session_name)
try:
os.makedirs(session)
logger.debug("Created new recordings session dir %s"%session)
except:
logger.debug("Recordings session dir %s already exists, using it." %session)
# set up self incrementing folder within session folder
counter = 0
while True:
self.rec_path = os.path.join(session, "%03d/" % counter)
try:
os.mkdir(self.rec_path)
logger.debug("Created new recording dir %s"%self.rec_path)
break
except:
logger.debug("We dont want to overwrite data, incrementing counter & trying to make new data folder")
counter += 1
self.meta_info_path = os.path.join(self.rec_path, "info.csv")
with open(self.meta_info_path, 'w') as f:
f.write("Recording Name\t"+self.session_name+ "\n")
f.write("Start Date\t"+ strftime("%d.%m.%Y", localtime(self.start_time))+ "\n")
f.write("Start Time\t"+ strftime("%H:%M:%S", localtime(self.start_time))+ "\n")
if self.audio_src != 'No Audio':
audio_path = os.path.join(self.rec_path, "world.wav")
self.audio_writer = Audio_Capture(audio_path,self.audio_devices_dict[self.audio_src])
else:
self.audio_writer = None
if self.raw_jpeg and self.g_pool.capture.jpeg_support:
self.video_path = os.path.join(self.rec_path, "world.mp4")
self.writer = JPEG_Writer(self.video_path,self.g_pool.capture.frame_rate)
else:
self.video_path = os.path.join(self.rec_path, "world.mp4")
self.writer = AV_Writer(self.video_path,fps=self.g_pool.capture.frame_rate)
# positions path to eye process
if self.record_eye:
for tx in self.g_pool.eye_tx:
tx.send((self.rec_path,self.raw_jpeg))
if self.show_info_menu:
self.open_info_menu()
self.notify_all( {'subject':'rec_started','rec_path':self.rec_path,'session_name':self.session_name,'network_propagate':network_propagate} )
def open_info_menu(self):
self.info_menu = ui.Growing_Menu('additional Recording Info',size=(300,300),pos=(300,300))
self.info_menu.configuration = self.info_menu_conf
def populate_info_menu():
self.info_menu.elements[:-2] = []
for name in self.user_info.iterkeys():
self.info_menu.insert(0,ui.Text_Input(name,self.user_info))
def set_user_info(new_string):
self.user_info = new_string
populate_info_menu()
populate_info_menu()
self.info_menu.append(ui.Info_Text('Use the *user info* field to add/remove additional fields and their values. The format must be a valid Python dictionary. For example -- {"key":"value"}. You can add as many fields as you require. Your custom fields will be saved for your next session.'))
self.info_menu.append(ui.Text_Input('user_info',self,setter=set_user_info,label="User info"))
self.g_pool.gui.append(self.info_menu)
def close_info_menu(self):
if self.info_menu:
self.info_menu_conf = self.info_menu.configuration
self.g_pool.gui.remove(self.info_menu)
self.info_menu = None
def update(self,frame,events):
if self.running:
self.data['pupil_positions'] += events['pupil_positions']
self.data['gaze_positions'] += events['gaze_positions']
self.timestamps.append(frame.timestamp)
self.writer.write_video_frame(frame)
self.frame_count += 1
# cv2.putText(frame.img, "Frame %s"%self.frame_count,(200,200), cv2.FONT_HERSHEY_SIMPLEX,1,(255,100,100))
for p in events['pupil_positions']:
pupil_pos = p['timestamp'],p['confidence'],p['id'],p['norm_pos'][0],p['norm_pos'][1],p['diameter']
self.pupil_pos_list.append(pupil_pos)
for g in events.get('gaze_positions',[]):
gaze_pos = g['timestamp'],g['confidence'],g['norm_pos'][0],g['norm_pos'][1]
self.gaze_pos_list.append(gaze_pos)
self.button.status_text = self.get_rec_time_str()
def stop(self,network_propagate=True):
#explicit release of VideoWriter
self.writer.release()
self.writer = None
if self.record_eye:
for tx in self.g_pool.eye_tx:
try:
tx.send((None,None))
except:
logger.warning("Could not stop eye-recording. Please report this bug!")
save_object(self.data,os.path.join(self.rec_path, "pupil_data"))
gaze_list_path = os.path.join(self.rec_path, "gaze_positions.npy")
np.save(gaze_list_path,np.asarray(self.gaze_pos_list))
pupil_list_path = os.path.join(self.rec_path, "pupil_positions.npy")
np.save(pupil_list_path,np.asarray(self.pupil_pos_list))
timestamps_path = os.path.join(self.rec_path, "world_timestamps.npy")
# ts = sanitize_timestamps(np.array(self.timestamps))
ts = np.array(self.timestamps)
np.save(timestamps_path,ts)
try:
copy2(os.path.join(self.g_pool.user_dir,"surface_definitions"),os.path.join(self.rec_path,"surface_definitions"))
except:
logger.info("No surface_definitions data found. You may want this if you do marker tracking.")
try:
copy2(os.path.join(self.g_pool.user_dir,"cal_pt_cloud.npy"),os.path.join(self.rec_path,"cal_pt_cloud.npy"))
except:
logger.warning("No calibration data found. Please calibrate first.")
try:
copy2(os.path.join(self.g_pool.user_dir,"camera_calibration"),os.path.join(self.rec_path,"camera_calibration"))
except:
logger.info("No camera calibration found.")
try:
with open(self.meta_info_path, 'a') as f:
f.write("Duration Time\t"+ self.get_rec_time_str()+ "\n")
if self.g_pool.binocular:
f.write("Eye Mode\tbinocular\n")
else:
f.write("Eye Mode\tmonocular\n")
f.write("Duration Time\t"+ self.get_rec_time_str()+ "\n")
f.write("World Camera Frames\t"+ str(self.frame_count)+ "\n")
f.write("World Camera Resolution\t"+ str(self.g_pool.capture.frame_size[0])+"x"+str(self.g_pool.capture.frame_size[1])+"\n")
f.write("Capture Software Version\t%s\n"%self.g_pool.version)
if platform.system() == "Windows":
username = os.environ["USERNAME"]
sysname, nodename, release, version, machine, _ = platform.uname()
else:
username = getpass.getuser()
try:
sysname, nodename, release, version, machine = os.uname()
except:
sysname, nodename, release, version, machine = sys.platform,None,None,None,None
f.write("User\t"+username+"\n")
f.write("Platform\t"+sysname+"\n")
f.write("Machine\t"+nodename+"\n")
f.write("Release\t"+release+"\n")
f.write("Version\t"+version+"\n")
except Exception:
logger.exception("Could not save metadata. Please report this bug!")
try:
with open(os.path.join(self.rec_path, "user_info.csv"), 'w') as f:
for name,val in self.user_info.iteritems():
f.write("%s\t%s\n"%(name,val))
except Exception:
logger.exception("Could not save userdata. Please report this bug!")
self.close_info_menu()
if self.audio_writer:
self.audio_writer = None
self.running = False
self.menu.read_only = False
self.button.status_text = ''
self.timestamps = []
self.data = {'pupil_positions':[],'gaze_positions':[]}
self.pupil_pos_list = []
self.gaze_pos_list = []
self.notify_all( {'subject':'rec_stopped','rec_path':self.rec_path,'network_propagate':network_propagate} )
def cleanup(self):
"""gets called when the plugin get terminated.
either volunatily or forced.
"""
if self.running:
self.stop()
self.deinit_gui()
def verify_path(self,val):
try:
n_path = os.path.expanduser(val)
logger.debug("Expanded user path.")
except:
n_path = val
if not n_path:
logger.warning("Please specify a path.")
return False
elif not os.path.isdir(n_path):
logger.warning("This is not a valid path.")
return False
# elif not os.access(n_path, os.W_OK):
elif not writable_dir(n_path):
logger.warning("Do not have write access to '%s'."%n_path)
return False
else:
return n_path
def set_rec_dir(self,val):
n_path = self.verify_path(val)
if n_path:
self.rec_dir = n_path
def set_session_name(self, val):
if not val:
self.session_name = get_auto_name()
else:
if '/' in val:
logger.warning('You session name will create one or more subdirectories')
self.session_name = val
class Recorder(Plugin):
"""Capture Recorder"""
def __init__(self,g_pool,session_name = get_auto_name(),rec_dir=None, user_info={'name':'','additional_field':'change_me'},info_menu_conf={},show_info_menu=False, record_eye = False, audio_src = 'No Audio',raw_jpeg=True):
super(Recorder, self).__init__(g_pool)
#update name if it was autogenerated.
if session_name.startswith('20') and len(session_name)==10:
session_name = get_auto_name()
base_dir = self.g_pool.user_dir.rsplit(os.path.sep,1)[0]
default_rec_dir = os.path.join(base_dir,'recordings')
if rec_dir and rec_dir != default_rec_dir and self.verify_path(rec_dir):
self.rec_dir = rec_dir
else:
try:
os.makedirs(default_rec_dir)
except OSError as e:
if e.errno != errno.EEXIST:
logger.error("Could not create Rec dir")
raise e
else:
logger.info('Created standard Rec dir at "%s"'%default_rec_dir)
self.rec_dir = default_rec_dir
self.raw_jpeg = raw_jpeg
self.order = .9
self.record_eye = record_eye
self.session_name = session_name
self.audio_devices_dict = Audio_Input_Dict()
if audio_src in self.audio_devices_dict.keys():
self.audio_src = audio_src
else:
self.audio_src = 'No Audio'
self.running = False
self.menu = None
self.button = None
self.user_info = user_info
self.show_info_menu = show_info_menu
self.info_menu = None
self.info_menu_conf = info_menu_conf
def get_init_dict(self):
d = {}
d['record_eye'] = self.record_eye
d['audio_src'] = self.audio_src
d['session_name'] = self.session_name
d['user_info'] = self.user_info
d['info_menu_conf'] = self.info_menu_conf
d['show_info_menu'] = self.show_info_menu
d['rec_dir'] = self.rec_dir
d['raw_jpeg'] = self.raw_jpeg
return d
def init_gui(self):
self.menu = ui.Growing_Menu('Recorder')
self.g_pool.sidebar.insert(3,self.menu)
self.menu.append(ui.Info_Text('Pupil recordings are saved like this: "path_to_recordings/recording_session_name/nnn" where "nnn" is an increasing number to avoid overwrites. You can use "/" in your session name to create subdirectories.'))
self.menu.append(ui.Info_Text('Recordings are saved to "~/pupil_recordings". You can change the path here but note that invalid input will be ignored.'))
self.menu.append(ui.Text_Input('rec_dir',self,setter=self.set_rec_dir,label='Path to recordings'))
self.menu.append(ui.Text_Input('session_name',self,setter=self.set_session_name,label='Recording session name'))
self.menu.append(ui.Switch('show_info_menu',self,on_val=True,off_val=False,label='Request additional user info'))
self.menu.append(ui.Selector('raw_jpeg',self,selection = [True,False], labels=["bigger file, less CPU", "smaller file, more CPU"],label='Compression'))
self.menu.append(ui.Info_Text('Recording the raw eye video is optional. We use it for debugging.'))
self.menu.append(ui.Switch('record_eye',self,on_val=True,off_val=False,label='Record eye'))
self.menu.append(ui.Selector('audio_src',self, selection=self.audio_devices_dict.keys(),label='Audio Source'))
self.button = ui.Thumb('running',self,setter=self.toggle,label='Record',hotkey='r')
self.button.on_color[:] = (1,.0,.0,.8)
self.g_pool.quickbar.insert(1,self.button)
def deinit_gui(self):
if self.menu:
self.g_pool.sidebar.remove(self.menu)
self.menu = None
if self.button:
self.g_pool.quickbar.remove(self.button)
self.button = None
def toggle(self, _=None):
if self.running:
self.notify_all( {'subject':'should_stop_recording','network_propagate':True} )
else:
self.notify_all( {'subject':'should_start_recording','session_name':self.session_name,'network_propagate':True} )
def on_notify(self,notification):
# notification wants to be recorded
if notification.get('record',False) and self.running:
self.data['notifications'].append(notification)
# Notificatio to start recording
elif notification['subject'] == 'should_start_recording':
if self.running:
logger.info('Recording already running!')
else:
if notification.get("session_name",""):
self.set_session_name(notification["session_name"])
self.start()
# Remote has stopped recording, we should stop as well.
elif notification['subject'] == 'should_stop_recording':
if self.running:
self.stop()
else:
logger.info('Recording already stopped!')
def get_rec_time_str(self):
rec_time = gmtime(time()-self.start_time)
return strftime("%H:%M:%S", rec_time)
def start(self):
self.timestamps = []
self.data = {'pupil_positions':[],'gaze_positions':[],'notifications':[]}
self.pupil_diameter = [] #addBy Changsoon
self.gaze_point_z = [] #addBy Changsoon
self.gaze_point_x = [] #addBy Changsoon
self.gaze_point_y = [] #addBy Changsoon
self.cnum = 0
self.frame_count = 0
self.running = True
self.menu.read_only = True
self.start_time = time()
session = os.path.join(self.rec_dir, self.session_name)
try:
os.makedirs(session)
logger.debug("Created new recordings session dir %s"%session)
except:
logger.debug("Recordings session dir %s already exists, using it." %session)
# set up self incrementing folder within session folder
counter = 0
while True:
self.rec_path = os.path.join(session, "%03d/" % counter)
try:
os.mkdir(self.rec_path)
logger.debug("Created new recording dir %s"%self.rec_path)
break
except:
logger.debug("We dont want to overwrite data, incrementing counter & trying to make new data folder")
counter += 1
self.meta_info_path = os.path.join(self.rec_path, "info.csv")
with open(self.meta_info_path, 'w') as f:
f.write("Recording Name\t"+self.session_name+ "\n")
f.write("Start Date\t"+ strftime("%d.%m.%Y", localtime(self.start_time))+ "\n")
f.write("Start Time\t"+ strftime("%H:%M:%S", localtime(self.start_time))+ "\n")
if self.audio_src != 'No Audio':
audio_path = os.path.join(self.rec_path, "world.wav")
self.audio_writer = Audio_Capture(audio_path,self.audio_devices_dict[self.audio_src])
else:
self.audio_writer = None
if self.raw_jpeg and self.g_pool.capture.jpeg_support:
self.video_path = os.path.join(self.rec_path, "world.mp4")
self.writer = JPEG_Writer(self.video_path,self.g_pool.capture.frame_rate)
else:
self.video_path = os.path.join(self.rec_path, "world.mp4")
self.writer = AV_Writer(self.video_path,fps=self.g_pool.capture.frame_rate)
# positions path to eye process
if self.record_eye:
for alive, pipe in zip(self.g_pool.eyes_are_alive,self.g_pool.eye_pipes):
if alive.value:
pipe.send( ('Rec_Start',(self.rec_path,self.raw_jpeg) ) )
if self.show_info_menu:
self.open_info_menu()
logger.info("Started Recording.")
self.notify_all( {'subject':'rec_started','rec_path':self.rec_path,'session_name':self.session_name,'network_propagate':True} )
def open_info_menu(self):
self.info_menu = ui.Growing_Menu('additional Recording Info',size=(300,300),pos=(300,300))
self.info_menu.configuration = self.info_menu_conf
def populate_info_menu():
self.info_menu.elements[:-2] = []
for name in self.user_info.iterkeys():
self.info_menu.insert(0,ui.Text_Input(name,self.user_info))
def set_user_info(new_string):
self.user_info = new_string
populate_info_menu()
populate_info_menu()
self.info_menu.append(ui.Info_Text('Use the *user info* field to add/remove additional fields and their values. The format must be a valid Python dictionary. For example -- {"key":"value"}. You can add as many fields as you require. Your custom fields will be saved for your next session.'))
self.info_menu.append(ui.Text_Input('user_info',self,setter=set_user_info,label="User info"))
self.g_pool.gui.append(self.info_menu)
def close_info_menu(self):
if self.info_menu:
self.info_menu_conf = self.info_menu.configuration
self.g_pool.gui.remove(self.info_menu)
self.info_menu = None
def update(self,frame,events):
if self.running:
for key,data in events.iteritems():
if key not in ('dt'):
try:
self.data[key] += data
if (key == 'pupil_positions'):
try:
pupilDict = data[0]
self.pupil_diameter.append(pupilDict.get('diameter'))
except:
print('error');
elif (key == 'gaze_positions'):
try:
gazeDict = data[0]
self.gaze_point_x.append(gazeDict.get('norm_pos')[0])
self.gaze_point_y.append(gazeDict.get('norm_pos')[1])
self.gaze_point_z.append(gazeDict.get('gaze_point_3d')[2])
except:
print('error')
#image process (OCR) find the condion that show use OCR
except KeyError:
self.data[key] = []
self.data[key] += data
print str(self.cnum)
self.cnum = self.cnum + 1
if len(self.gaze_point_x) > 0 and self.cnum == 30:
print('Enter TO the block')
height = (1 - self.gaze_point_y[-1]) * 720
height = int(height)
image = frame.img
if (height < 100):
image = frame.img[0:height+200, 300:1000]
elif (height > 520):
image = frame.img[height-100:720, 300:1000]
else:
image = frame.img[height-100:height+200, 300:1000]
cv2.imwrite('/home/eyetracking/pupil/'+"frame%d.jpg" %self.cnum,image)
image=Image.open('/home/eyetracking/pupil/'+"frame%d.jpg" %self.cnum)
enhancer = ImageEnhance.Contrast(image)
image = enhancer.enhance(4)
totemp = sys.stdout
sys.stdout = codecs.open('/home/eyetracking/pupil/ocr.txt', encoding='utf-8', mode='w+') #save the OCR
print image_to_string(image)
sys.stdout.close()
sys.stdout = totemp
try:
i=300
for line in open('/home/eyetracking/pupil/ocr.txt'):
i=i+20
cv2.putText(dst, line,(0,i), cv2.FONT_HERSHEY_COMPLEX,0.6,(0,0,255))
cv2.imshow("window",dst)
cv2.waitKey(33)
except Exception as error:
print(error)
self.timestamps.append(frame.timestamp)
self.writer.write_video_frame(frame)
self.frame_count += 1
# # cv2.putText(frame.img, "Frame %s"%self.frame_count,(200,200), cv2.FONT_HERSHEY_SIMPLEX,1,(255,100,100))
self.button.status_text = self.get_rec_time_str()
def stop(self):
#explicit release of VideoWriter
self.writer.release()
self.writer = None
if self.record_eye:
for alive, pipe in zip(self.g_pool.eyes_are_alive,self.g_pool.eye_pipes):
if alive.value:
pipe.send(('Rec_Stop',None))
save_object(self.data,os.path.join(self.rec_path, "pupil_data"))
timestamps_path = os.path.join(self.rec_path, "world_timestamps.npy")
# ts = sanitize_timestamps(np.array(self.timestamps))
ts = np.array(self.timestamps)
np.save(timestamps_path,ts)
try:
copy2(os.path.join(self.g_pool.user_dir,"surface_definitions"),os.path.join(self.rec_path,"surface_definitions"))
except:
logger.info("No surface_definitions data found. You may want this if you do marker tracking.")
try:
copy2(os.path.join(self.g_pool.user_dir,"user_calibration_data"),os.path.join(self.rec_path,"user_calibration_data"))
except:
logger.warning("No user calibration data found. Please calibrate first.")
camera_calibration = load_camera_calibration(self.g_pool)
if camera_calibration is not None:
save_object(camera_calibration,os.path.join(self.rec_path, "camera_calibration"))
else:
logger.info("No camera calibration found.")
try:
with open(self.meta_info_path, 'a') as f:
f.write("Duration Time\t"+ self.get_rec_time_str()+ "\n")
f.write("World Camera Frames\t"+ str(self.frame_count)+ "\n")
f.write("World Camera Resolution\t"+ str(self.g_pool.capture.frame_size[0])+"x"+str(self.g_pool.capture.frame_size[1])+"\n")
f.write("Capture Software Version\t%s\n"%self.g_pool.version)
f.write("System Info\t%s"%get_system_info())
except Exception:
logger.exception("Could not save metadata. Please report this bug!")
try:
with open(os.path.join(self.rec_path, "user_info.csv"), 'w') as f:
for name,val in self.user_info.iteritems():
f.write("%s\t%s\n"%(name,val))
except Exception:
logger.exception("Could not save userdata. Please report this bug!")
self.close_info_menu()
if self.audio_writer:
self.audio_writer = None
self.running = False
self.menu.read_only = False
self.button.status_text = ''
self.timestamps = []
self.data = {'pupil_positions':[],'gaze_positions':[]}
self.pupil_pos_list = []
self.gaze_pos_list = []
logger.info("Saved Recording.")
self.notify_all( {'subject':'rec_stopped','rec_path':self.rec_path,'network_propagate':True} )
def cleanup(self):
"""gets called when the plugin get terminated.
either volunatily or forced.
"""
if self.running:
self.stop()
self.deinit_gui()
def verify_path(self,val):
try:
n_path = os.path.expanduser(val)
logger.debug("Expanded user path.")
except:
n_path = val
if not n_path:
logger.warning("Please specify a path.")
return False
elif not os.path.isdir(n_path):
logger.warning("This is not a valid path.")
return False
# elif not os.access(n_path, os.W_OK):
elif not writable_dir(n_path):
logger.warning("Do not have write access to '%s'."%n_path)
return False
else:
return n_path
def set_rec_dir(self,val):
n_path = self.verify_path(val)
if n_path:
self.rec_dir = n_path
def set_session_name(self, val):
if not val:
self.session_name = get_auto_name()
else:
if '/' in val:
logger.warning('You session name will create one or more subdirectories')
self.session_name = val
class Audio_Capture(Plugin):
"""docstring for Audio_Capture"""
icon_chr = chr(0xE029)
icon_font = "pupil_icons"
def __init__(self, g_pool, audio_src="No Audio"):
super().__init__(g_pool)
self.audio_devices_dict = Audio_Input_Dict()
if audio_src in list(self.audio_devices_dict.keys()):
self.audio_src = audio_src
else:
self.audio_src = "No Audio"
self.thread = None
self.running = Event()
self.recording = Event()
self.recording.clear()
self.audio_container = None
self.audio_out_stream = None
self.queue = queue.Queue()
self.start_capture(self.audio_src)
def init_ui(self):
self.add_menu()
self.menu.label = "Audio Capture"
help_str = "Creates events for audio input."
self.menu.append(ui.Info_Text(help_str))
def audio_dev_getter():
# fetch list of currently available
audio_src_val = None
if self.audio_devices_dict is not None:
if self.audio_src in self.audio_devices_dict.keys():
audio_src_val = self.audio_devices_dict[self.audio_src]
self.audio_devices_dict = Audio_Input_Dict()
if audio_src_val is not None:
self.audio_devices_dict[self.audio_src] = audio_src_val
devices = list(self.audio_devices_dict.keys())
return devices, devices
self.menu.append(
# TODO: potential race condition through selection_getter. Should ensure
# that current selection will always be present in the list returned by the
# selection_getter. Highly unlikely though as this needs to happen between
# having clicked the Selector and the next redraw.
# See https://github.com/pupil-labs/pyglui/pull/112/commits/587818e9556f14bfedd8ff8d093107358745c29b
ui.Selector(
"audio_src",
self,
selection_getter=audio_dev_getter,
label="Audio Source",
setter=self.start_capture,
)
)
self.menu.append(ui.Info_Text(NOT_REC_STR))
def deinit_ui(self):
self.remove_menu()
def get_init_dict(self):
return {"audio_src": self.audio_src}
def cleanup(self):
if self.audio_container is not None:
self.close_audio_recording()
self.running.clear()
if self.thread and self.thread.is_alive():
self.thread.join(timeout=1)
def on_notify(self, notification):
if notification["subject"] == "recording.started":
self.rec_dir = notification["rec_path"]
self.recording.set()
if self.running.is_set(): # and self.audio_container is None:
self.menu[-2].read_only = True
del self.menu[-1]
self.menu.append(ui.Info_Text(REC_STR))
elif not self.running.is_set():
logger.warning("Recording was started without an active audio capture")
elif notification["subject"] == "recording.stopped":
self.recording.clear()
self.close_audio_recording()
def close_audio_recording(self):
self.menu[-2].read_only = False
del self.menu[-1]
self.menu.append(ui.Info_Text(NOT_REC_STR))
def write_audio_packet(self, audio_frame):
# Test if audio outstream has been initialized
if self.audio_out_stream is None:
try:
self.audio_out_stream = self.audio_container.add_stream("aac")
except ValueError as e:
# packet.stream codec is not supported in target container.
logger.error("Failed to create audio stream. Aborting recording.")
logger.debug("Reason: {}".format(e))
self.close_audio_recording()
self.timestamps.append(audio_frame.timestamp)
packet = self.audio_out_stream.encode(audio_frame)
if packet is not None:
self.audio_container.mux(packet)
def start_capture(self, audio_src):
if self.thread and self.thread.is_alive():
if self.audio_src == audio_src:
return # capture is already running for our selected source
# else stop current capture gracefully
self.running.clear()
logger.debug('Closing capture for "{}"'.format(self.audio_src))
self.thread.join(timeout=1)
self.thread = None
if audio_src not in self.audio_devices_dict:
logger.warning("Selected audio source is not available anymore")
return
self.audio_src = audio_src
if audio_src == "No Audio":
return
self.running.set()
self.thread = Thread(
target=self.capture_thread,
args=(self.audio_devices_dict[audio_src], self.running, self.recording),
)
self.thread.start()
def capture_thread(self, audio_src, running, recording):
try:
if platform.system() == "Darwin":
in_container = av.open(
"none:{}".format(audio_src), format="avfoundation"
)
elif platform.system() == "Linux":
print("audio src = {}".format(audio_src))
dev_str = re.search("(hw:\s*\d+,\s*\d+)", audio_src)
in_container = av.open(dev_str.group(0), format="alsa")
elif platform.system() == "Windows":
in_container = av.open(
"audio={}".format(audio_src),
format="dshow",
options={"audio_buffer_size": "23"},
)
else:
raise av.AVError("Platform does not support audio capture.")
except av.AVError as err:
running.clear()
self.audio_src = "No Audio"
logger.warning("Error starting audio capture: {}".format(err))
return
in_stream = None
try:
in_stream = in_container.streams.audio[0]
except IndexError:
logger.warning("No audio stream found for selected device.")
running.clear()
self.audio_src = "No Audio"
return
out_container = None
out_stream = None
timestamps = None
in_frame_size = 0
stream_epoch = in_stream.start_time * in_stream.time_base
uvc_clock_dif = abs(stream_epoch - self.g_pool.get_now())
pyt_clock_dif = abs(stream_epoch - time())
max_clock_dif = 4.0
if uvc_clock_dif > max_clock_dif and pyt_clock_dif > max_clock_dif:
logger.error("Could not identify audio stream clock.")
running.clear()
self.audio_src = "No Audio"
return
elif uvc_clock_dif > pyt_clock_dif:
logger.info(
"Audio stream uses time.time() as clock (Δ {}s)".format(pyt_clock_dif)
)
clock_differences = self.g_pool.get_now() - time()
else:
logger.info("Audio stream uses uvc.get_time_monotonic() as clock")
clock_differences = 0
def close_recording():
# Bind nonlocal variables, https://www.python.org/dev/peps/pep-3104/
nonlocal out_container, out_stream, in_stream, in_frame_size, timestamps, out_frame_num
if out_container is not None:
timestamps.append(timestamp)
audio_frame.pts = None
out_packets = [out_stream.encode(audio_frame)]
while out_packets[-1]:
out_packets.append(out_stream.encode(None))
for out_packet in out_packets:
if out_packet is not None:
out_container.mux(out_packet)
out_container.close()
out_frame_num = out_stream.frames
in_frame_rate = in_stream.rate
# in_stream.frame_size does not return the correct value.
out_frame_size = out_stream.frame_size
out_frame_rate = out_stream.rate
new_ts_idx = (
np.arange(0, out_frame_num * out_frame_size, out_frame_size)
/ out_frame_rate
)
if in_frame_rate != out_frame_rate:
old_ts_idx = (
np.arange(0, len(timestamps) * in_frame_size, in_frame_size)
/ in_frame_rate
)
interpolate = interp1d(
old_ts_idx,
timestamps,
bounds_error=False,
fill_value="extrapolate",
)
new_ts = interpolate(new_ts_idx)
else:
new_ts = timestamps[0] + new_ts_idx
ts_loc = os.path.join(self.rec_dir, "audio_timestamps.npy")
np.save(ts_loc, new_ts)
out_container = None
out_stream = None
timestamps = None
for packet in in_container.demux(in_stream):
# ffmpeg timestamps - in_stream.startime = packte pts relative to startime
# multiply with stream_timebase to get seconds
# add start time of this stream in pupil time unadjusted
# finally add pupil timebase offset to adjust for settable timebase.
for audio_frame in packet.decode():
timestamp = (
audio_frame.pts * in_stream.time_base
+ clock_differences
- self.g_pool.timebase.value
)
if recording.is_set():
if out_container is None:
rec_file = os.path.join(self.rec_dir, "audio.mp4")
out_container = av.open(rec_file, "w")
out_stream = out_container.add_stream(
"aac", rate=in_stream.rate
)
out_frame_num = 0
in_frame_size = (
audio_frame.samples
) # set here to make sure full packet size is used
timestamps = []
timestamps.append(timestamp)
audio_frame.pts = None
out_packets = [out_stream.encode(audio_frame)]
for out_packet in out_packets:
if out_packet is not None:
out_container.mux(out_packet)
elif out_container is not None:
# recording stopped
close_recording()
if not running.is_set():
close_recording()
return
close_recording()
self.audio_src = "No Audio"
running.clear() # in_stream stopped yielding packets
class Recorder(Plugin):
"""Capture Recorder"""
def __init__(self,
g_pool,
session_name=get_auto_name(),
rec_dir=None,
user_info={
'name': '',
'additional_field': 'change_me'
},
info_menu_conf={},
show_info_menu=False,
record_eye=False,
audio_src='No Audio',
raw_jpeg=True):
super().__init__(g_pool)
# update name if it was autogenerated.
if session_name.startswith('20') and len(session_name) == 10:
session_name = get_auto_name()
base_dir = self.g_pool.user_dir.rsplit(os.path.sep, 1)[0]
default_rec_dir = os.path.join(base_dir, 'recordings')
if rec_dir and rec_dir != default_rec_dir and self.verify_path(
rec_dir):
self.rec_dir = rec_dir
else:
try:
os.makedirs(default_rec_dir)
except OSError as e:
if e.errno != errno.EEXIST:
logger.error("Could not create Rec dir")
raise e
else:
logger.info(
'Created standard Rec dir at "{}"'.format(default_rec_dir))
self.rec_dir = default_rec_dir
self.raw_jpeg = raw_jpeg
self.order = .9
self.record_eye = record_eye
self.session_name = session_name
self.audio_devices_dict = Audio_Input_Dict()
if audio_src in list(self.audio_devices_dict.keys()):
self.audio_src = audio_src
else:
self.audio_src = 'No Audio'
self.running = False
self.menu = None
self.button = None
self.user_info = user_info
self.show_info_menu = show_info_menu
self.info_menu = None
self.info_menu_conf = info_menu_conf
def get_init_dict(self):
d = {}
d['record_eye'] = self.record_eye
d['audio_src'] = self.audio_src
d['session_name'] = self.session_name
d['user_info'] = self.user_info
d['info_menu_conf'] = self.info_menu_conf
d['show_info_menu'] = self.show_info_menu
d['rec_dir'] = self.rec_dir
d['raw_jpeg'] = self.raw_jpeg
return d
def init_gui(self):
self.menu = ui.Growing_Menu('Recorder')
self.menu.collapsed = True
self.g_pool.sidebar.insert(3, self.menu)
self.menu.append(
ui.Info_Text(
'Pupil recordings are saved like this: "path_to_recordings/recording_session_name/nnn" where "nnn" is an increasing number to avoid overwrites. You can use "/" in your session name to create subdirectories.'
))
self.menu.append(
ui.Info_Text(
'Recordings are saved to "~/pupil_recordings". You can change the path here but note that invalid input will be ignored.'
))
self.menu.append(
ui.Text_Input('rec_dir',
self,
setter=self.set_rec_dir,
label='Path to recordings'))
self.menu.append(
ui.Text_Input('session_name',
self,
setter=self.set_session_name,
label='Recording session name'))
self.menu.append(
ui.Switch('show_info_menu',
self,
on_val=True,
off_val=False,
label='Request additional user info'))
self.menu.append(
ui.Selector(
'raw_jpeg',
self,
selection=[True, False],
labels=["bigger file, less CPU", "smaller file, more CPU"],
label='Compression'))
self.menu.append(
ui.Info_Text(
'Recording the raw eye video is optional. We use it for debugging.'
))
self.menu.append(
ui.Switch('record_eye',
self,
on_val=True,
off_val=False,
label='Record eye'))
def audio_dev_getter():
# fetch list of currently available
self.audio_devices_dict = Audio_Input_Dict()
devices = list(self.audio_devices_dict.keys())
return devices, devices
self.menu.append(
ui.Selector('audio_src',
self,
selection_getter=audio_dev_getter,
label='Audio Source'))
self.button = ui.Thumb('running',
self,
setter=self.toggle,
label='R',
hotkey='r')
self.button.on_color[:] = (1, .0, .0, .8)
self.g_pool.quickbar.insert(1, self.button)
def deinit_gui(self):
if self.menu:
self.g_pool.sidebar.remove(self.menu)
self.menu = None
if self.button:
self.g_pool.quickbar.remove(self.button)
self.button = None
def toggle(self, _=None):
if self.running:
self.notify_all({'subject': 'recording.should_stop'})
self.notify_all({
'subject': 'recording.should_stop',
'remote_notify': 'all'
})
else:
self.notify_all({
'subject': 'recording.should_start',
'session_name': self.session_name
})
self.notify_all({
'subject': 'recording.should_start',
'session_name': self.session_name,
'remote_notify': 'all'
})
def on_notify(self, notification):
"""Handles recorder notifications
Reacts to notifications:
``recording.should_start``: Starts a new recording session
``recording.should_stop``: Stops current recording session
Emits notifications:
``recording.started``: New recording session started
``recording.stopped``: Current recording session stopped
Args:
notification (dictionary): Notification dictionary
"""
# notification wants to be recorded
if notification.get('record', False) and self.running:
if 'timestamp' not in notification:
logger.error(
"Notification without timestamp will not be saved.")
else:
self.data['notifications'].append(notification)
elif notification['subject'] == 'recording.should_start':
if self.running:
logger.info('Recording already running!')
elif not self.g_pool.capture.online:
logger.error(
"Current world capture is offline. Please reconnect or switch to fake capture"
)
else:
if notification.get("session_name", ""):
self.set_session_name(notification["session_name"])
self.start()
elif notification['subject'] == 'recording.should_stop':
if self.running:
self.stop()
else:
logger.info('Recording already stopped!')
def get_rec_time_str(self):
rec_time = gmtime(time() - self.start_time)
return strftime("%H:%M:%S", rec_time)
def start(self):
self.timestamps = []
self.data = {
'pupil_positions': [],
'gaze_positions': [],
'notifications': []
}
self.frame_count = 0
self.running = True
self.menu.read_only = True
self.start_time = time()
session = os.path.join(self.rec_dir, self.session_name)
try:
os.makedirs(session)
logger.debug(
"Created new recordings session dir {}".format(session))
except:
logger.debug(
"Recordings session dir {} already exists, using it.".format(
session))
# set up self incrementing folder within session folder
counter = 0
while True:
self.rec_path = os.path.join(session, "{:03d}/".format(counter))
try:
os.mkdir(self.rec_path)
logger.debug("Created new recording dir {}".format(
self.rec_path))
break
except:
logger.debug(
"We dont want to overwrite data, incrementing counter & trying to make new data folder"
)
counter += 1
self.meta_info_path = os.path.join(self.rec_path, "info.csv")
with open(self.meta_info_path, 'w', newline='') as csvfile:
csv_utils.write_key_value_file(
csvfile, {
'Recording Name': self.session_name,
'Start Date': strftime("%d.%m.%Y",
localtime(self.start_time)),
'Start Time': strftime("%H:%M:%S",
localtime(self.start_time))
})
if self.audio_src != 'No Audio':
audio_path = os.path.join(self.rec_path, "world.wav")
self.audio_writer = Audio_Capture(
audio_path, self.audio_devices_dict[self.audio_src])
else:
self.audio_writer = None
if self.raw_jpeg and self.g_pool.capture.jpeg_support:
self.video_path = os.path.join(self.rec_path, "world.mp4")
self.writer = JPEG_Writer(self.video_path,
self.g_pool.capture.frame_rate)
else:
self.video_path = os.path.join(self.rec_path, "world.mp4")
self.writer = AV_Writer(self.video_path,
fps=self.g_pool.capture.frame_rate)
try:
cal_pt_path = os.path.join(self.g_pool.user_dir,
"user_calibration_data")
cal_data = load_object(cal_pt_path)
notification = {
'subject': 'calibration.calibration_data',
'record': True
}
notification.update(cal_data)
self.data['notifications'].append(notification)
except:
pass
if self.show_info_menu:
self.open_info_menu()
logger.info("Started Recording.")
self.notify_all({
'subject': 'recording.started',
'rec_path': self.rec_path,
'session_name': self.session_name,
'record_eye': self.record_eye,
'compression': self.raw_jpeg
})
def open_info_menu(self):
self.info_menu = ui.Growing_Menu('additional Recording Info',
size=(300, 300),
pos=(300, 300))
self.info_menu.configuration = self.info_menu_conf
def populate_info_menu():
self.info_menu.elements[:-2] = []
for name in self.user_info.keys():
self.info_menu.insert(0, ui.Text_Input(name, self.user_info))
def set_user_info(new_string):
self.user_info = new_string
populate_info_menu()
populate_info_menu()
self.info_menu.append(
ui.Info_Text(
'Use the *user info* field to add/remove additional fields and their values. The format must be a valid Python dictionary. For example -- {"key":"value"}. You can add as many fields as you require. Your custom fields will be saved for your next session.'
))
self.info_menu.append(
ui.Text_Input('user_info',
self,
setter=set_user_info,
label="User info"))
self.g_pool.gui.append(self.info_menu)
def close_info_menu(self):
if self.info_menu:
self.info_menu_conf = self.info_menu.configuration
self.g_pool.gui.remove(self.info_menu)
self.info_menu = None
def recent_events(self, events):
if self.running:
for key, data in events.items():
if key not in ('dt', 'frame'):
try:
self.data[key] += data
except KeyError:
self.data[key] = []
self.data[key] += data
if 'frame' in events:
frame = events['frame']
self.timestamps.append(frame.timestamp)
self.writer.write_video_frame(frame)
self.frame_count += 1
# # cv2.putText(frame.img, "Frame %s"%self.frame_count,(200,200), cv2.FONT_HERSHEY_SIMPLEX,1,(255,100,100))
self.button.status_text = self.get_rec_time_str()
def stop(self):
# explicit release of VideoWriter
self.writer.release()
self.writer = None
save_object(self.data, os.path.join(self.rec_path, "pupil_data"))
timestamps_path = os.path.join(self.rec_path, "world_timestamps.npy")
# ts = sanitize_timestamps(np.array(self.timestamps))
ts = np.array(self.timestamps)
np.save(timestamps_path, ts)
try:
copy2(os.path.join(self.g_pool.user_dir, "surface_definitions"),
os.path.join(self.rec_path, "surface_definitions"))
except:
logger.info(
"No surface_definitions data found. You may want this if you do marker tracking."
)
camera_calibration = load_camera_calibration(self.g_pool)
if camera_calibration is not None:
save_object(camera_calibration,
os.path.join(self.rec_path, "camera_calibration"))
else:
logger.info("No camera calibration found.")
try:
with open(self.meta_info_path, 'a', newline='') as csvfile:
csv_utils.write_key_value_file(csvfile, {
'Duration Time':
self.get_rec_time_str(),
'World Camera Frames':
self.frame_count,
'World Camera Resolution':
str(self.g_pool.capture.frame_size[0]) + "x" +
str(self.g_pool.capture.frame_size[1]),
'Capture Software Version':
self.g_pool.version,
'Data Format Version':
self.g_pool.version,
'System Info':
get_system_info()
},
append=True)
except Exception:
logger.exception(
"Could not save metadata. Please report this bug!")
try:
with open(os.path.join(self.rec_path, "user_info.csv"),
'w',
newline='') as csvfile:
csv_utils.write_key_value_file(csvfile, self.user_info)
except Exception:
logger.exception(
"Could not save userdata. Please report this bug!")
self.close_info_menu()
if self.audio_writer:
self.audio_writer = None
self.running = False
self.menu.read_only = False
self.button.status_text = ''
self.timestamps = []
self.data = {'pupil_positions': [], 'gaze_positions': []}
self.pupil_pos_list = []
self.gaze_pos_list = []
logger.info("Saved Recording.")
self.notify_all({
'subject': 'recording.stopped',
'rec_path': self.rec_path
})
def cleanup(self):
"""gets called when the plugin get terminated.
either volunatily or forced.
"""
if self.running:
self.stop()
self.deinit_gui()
def verify_path(self, val):
try:
n_path = os.path.expanduser(val)
logger.debug("Expanded user path.")
except:
n_path = val
if not n_path:
logger.warning("Please specify a path.")
return False
elif not os.path.isdir(n_path):
logger.warning("This is not a valid path.")
return False
# elif not os.access(n_path, os.W_OK):
elif not writable_dir(n_path):
logger.warning("Do not have write access to '{}'.".format(n_path))
return False
else:
return n_path
def set_rec_dir(self, val):
n_path = self.verify_path(val)
if n_path:
self.rec_dir = n_path
def set_session_name(self, val):
if not val:
self.session_name = get_auto_name()
else:
if os.path.sep in val:
logger.warning(
'You session name will create one or more subdirectories')
self.session_name = val
class Audio_Capture(Plugin):
"""docstring for Audio_Capture"""
def __init__(self, g_pool, audio_src='No Audio'):
super().__init__(g_pool)
self.audio_devices_dict = Audio_Input_Dict()
if audio_src in list(self.audio_devices_dict.keys()):
self.audio_src = audio_src
else:
self.audio_src = 'No Audio'
self.thread = None
self.running = Event()
self.recording = Event()
self.recording.clear()
self.audio_container = None
self.audio_out_stream = None
self.queue = queue.Queue()
self.start_capture(self.audio_src)
def init_gui(self):
self.menu = ui.Growing_Menu('Audio Capture')
self.menu.collapsed = True
self.g_pool.sidebar.append(self.menu)
def close():
self.alive = False
help_str = 'Creates events for audio input.'
self.menu.append(ui.Button('Close', close))
self.menu.append(ui.Info_Text(help_str))
def audio_dev_getter():
# fetch list of currently available
self.audio_devices_dict = Audio_Input_Dict()
devices = list(self.audio_devices_dict.keys())
return devices, devices
self.menu.append(ui.Selector('audio_src', self,
selection_getter=audio_dev_getter,
label='Audio Source',
setter=self.start_capture))
self.menu.append(ui.Info_Text(NOT_REC_STR))
def get_init_dict(self):
return {'audio_src': self.audio_src}
def deinit_gui(self):
if self.menu:
self.g_pool.sidebar.remove(self.menu)
self.menu = None
def cleanup(self):
if self.audio_container is not None:
self.close_audio_recording()
self.running.clear()
self.deinit_gui()
if self.thread and self.thread.is_alive():
self.thread.join(timeout=1)
def on_notify(self, notification):
if notification['subject'] == 'recording.started':
self.rec_dir = notification['rec_path']
self.recording.set()
if self.running.is_set(): # and self.audio_container is None:
self.menu[-2].read_only = True
del self.menu[-1]
self.menu.append(ui.Info_Text(REC_STR))
elif not self.running.is_set():
logger.warning('Recording was started without an active audio capture')
elif notification['subject'] == 'recording.stopped':
self.recording.clear()
self.close_audio_recording()
def close_audio_recording(self):
self.menu[-2].read_only = False
del self.menu[-1]
self.menu.append(ui.Info_Text(NOT_REC_STR))
def write_audio_packet(self, audio_frame):
# Test if audio outstream has been initialized
if self.audio_out_stream is None:
try:
self.audio_out_stream = self.audio_container.add_stream('aac')
except ValueError as e:
# packet.stream codec is not supported in target container.
logger.error('Failed to create audio stream. Aborting recording.')
logger.debug('Reason: {}'.format(e))
self.close_audio_recording()
self.timestamps.append(audio_frame.timestamp)
packet = self.audio_out_stream.encode(audio_frame)
if packet is not None:
self.audio_container.mux(packet)
def start_capture(self, audio_src):
if self.thread and self.thread.is_alive():
if self.audio_src == audio_src:
return # capture is already running for our selected source
# else stop current capture gracefully
self.running.clear()
logger.debug('Closing capture for "{}"'.format(self.audio_src))
self.thread.join(timeout=1)
self.thread = None
if audio_src not in self.audio_devices_dict:
logger.warning('Selected audio source is not available anymore')
return
self.audio_src = audio_src
if audio_src == 'No Audio':
return
self.running.set()
self.thread = Thread(target=self.capture_thread,
args=(self.audio_devices_dict[audio_src], self.running, self.recording))
self.thread.start()
def capture_thread(self, audio_src, running, recording):
try:
if platform.system() == "Darwin":
in_container = av.open('none:{}'.format(audio_src), format="avfoundation")
elif platform.system() == "Linux":
in_container = av.open('hw:{}'.format(audio_src), format="alsa")
else:
raise av.AVError('Platform does not support audio capture.')
except av.AVError as err:
running.clear()
self.audio_src = 'No Audio'
logger.warning('Error starting audio capture: {}'.format(err))
return
in_stream = None
try:
in_stream = in_container.streams.audio[0]
except IndexError:
logger.warning('No audio stream found for selected device.')
running.clear()
self.audio_src = 'No Audio'
return
out_container = None
out_stream = None
timestamps = None
out_frame_num = 0
in_frame_size = 0
stream_epoch = in_stream.start_time * in_stream.time_base
uvc_clock_dif = abs(stream_epoch - self.g_pool.get_now())
pyt_clock_dif = abs(stream_epoch - time())
max_clock_dif = 4.
if uvc_clock_dif > max_clock_dif and pyt_clock_dif > max_clock_dif:
logger.error('Could not identify audio stream clock.')
running.clear()
self.audio_src = 'No Audio'
return
elif uvc_clock_dif > pyt_clock_dif:
logger.info('Audio stream uses time.time() as clock (Δ {}s)'.format(pyt_clock_dif))
clock_differences = self.g_pool.get_now() - time()
else:
logger.info('Audio stream uses uvc.get_time_monotonic() as clock')
clock_differences = 0
def close_recording():
# Bind nonlocal variables, https://www.python.org/dev/peps/pep-3104/
nonlocal out_container, out_stream, in_stream, in_frame_size, timestamps, out_frame_num
if out_container is not None:
packet = out_stream.encode(audio_frame)
while packet is not None:
out_frame_num += 1
out_container.mux(packet)
packet = out_stream.encode(audio_frame)
out_container.close()
in_frame_rate = in_stream.rate
# in_stream.frame_size does not return the correct value.
out_frame_size = out_stream.frame_size
out_frame_rate = out_stream.rate
old_ts_idx = np.arange(0, len(timestamps) * in_frame_size, in_frame_size) / in_frame_rate
new_ts_idx = np.arange(0, out_frame_num * out_frame_size, out_frame_size) / out_frame_rate
interpolate = interp1d(old_ts_idx, timestamps, bounds_error=False, fill_value='extrapolate')
new_ts = interpolate(new_ts_idx)
ts_loc = os.path.join(self.rec_dir, 'audio_timestamps.npy')
np.save(ts_loc, new_ts)
out_container = None
out_stream = None
timestamps = None
for packet in in_container.demux(in_stream):
# ffmpeg timestamps - in_stream.startime = packte pts relative to startime
# multiply with stream_timebase to get seconds
# add start time of this stream in pupil time unadjusted
# finally add pupil timebase offset to adjust for settable timebase.
for audio_frame in packet.decode():
timestamp = audio_frame.pts * in_stream.time_base + clock_differences - self.g_pool.timebase.value
if recording.is_set():
if out_container is None:
rec_file = os.path.join(self.rec_dir, 'audio.mp4')
out_container = av.open(rec_file, 'w')
out_stream = out_container.add_stream('aac')
out_frame_num = 0
in_frame_size = audio_frame.samples # set here to make sure full packet size is used
timestamps = []
timestamps.append(timestamp)
packet = out_stream.encode(audio_frame)
if packet is not None:
out_frame_num += 1
out_container.mux(packet)
elif out_container is not None:
# recording stopped
close_recording()
if not running.is_set():
close_recording()
return
close_recording()
self.audio_src = 'No Audio'
running.clear() # in_stream stopped yielding packets
class Recorder(Plugin):
"""Capture Recorder"""
def __init__(self,g_pool,session_name = get_auto_name(),rec_dir=None, user_info={},info_menu_conf={},show_info_menu=False, record_eye = True, audio_src = 'No Audio',raw_jpeg=True):
super(Recorder, self).__init__(g_pool)
#update name if it was autogenerated.
if session_name.startswith('20') and len(session_name)==10:
session_name = get_auto_name()
base_dir = self.g_pool.user_dir.rsplit(os.path.sep,1)[0]
default_rec_dir = os.path.join(base_dir,'recordings')
if rec_dir and rec_dir != default_rec_dir and self.verify_path(rec_dir):
self.rec_dir = rec_dir
else:
try:
os.makedirs(default_rec_dir)
except OSError as e:
if e.errno != errno.EEXIST:
logger.error("Could not create Rec dir")
raise e
else:
logger.info('Created standard Rec dir at "%s"'%default_rec_dir)
self.rec_dir = default_rec_dir
self.raw_jpeg = raw_jpeg
self.order = .9
self.record_eye = record_eye
self.session_name = session_name
self.audio_devices_dict = Audio_Input_Dict()
if audio_src in self.audio_devices_dict.keys():
self.audio_src = audio_src
else:
self.audio_src = 'No Audio'
self.running = False
self.menu = None
self.button = None
self.user_info = user_info
self.show_info_menu = show_info_menu
self.info_menu = None
self.info_menu_conf = info_menu_conf
self.calibration_start_at = None
self.calibration_end_at = None
def get_init_dict(self):
d = {}
d['record_eye'] = self.record_eye
d['audio_src'] = self.audio_src
d['session_name'] = self.session_name
d['user_info'] = self.user_info
d['info_menu_conf'] = self.info_menu_conf
d['show_info_menu'] = self.show_info_menu
d['rec_dir'] = self.rec_dir
d['raw_jpeg'] = self.raw_jpeg
return d
def init_gui(self):
self.menu = ui.Growing_Menu('Recorder')
self.g_pool.sidebar.insert(3,self.menu)
self.menu.append(ui.Info_Text('Pupil recordings are saved like this: "path_to_recordings/recording_session_name/nnn" where "nnn" is an increasing number to avoid overwrites. You can use "/" in your session name to create subdirectories.'))
self.menu.append(ui.Info_Text('Recordings are saved to "~/pupil_recordings". You can change the path here but note that invalid input will be ignored.'))
self.menu.append(ui.Text_Input('rec_dir',self,setter=self.set_rec_dir,label='Path to recordings'))
self.menu.append(ui.Text_Input('session_name',self,setter=self.set_session_name,label='Recording session name'))
self.menu.append(ui.Switch('show_info_menu',self,on_val=True,off_val=False,label='Request additional user info'))
self.menu.append(ui.Selector('raw_jpeg',self,selection = [True,False], labels=["bigger file, less CPU", "smaller file, more CPU"],label='Compression'))
self.menu.append(ui.Info_Text('Recording the raw eye video is optional. We use it for debugging.'))
self.menu.append(ui.Switch('record_eye',self,on_val=True,off_val=False,label='Record eye'))
self.menu.append(ui.Selector('audio_src',self, selection=self.audio_devices_dict.keys(),label='Audio Source'))
# record on enter click from the remote shutter
self.button = ui.Thumb('running',self,setter=self.toggle,label='Record',hotkey=GLFW_KEY_ENTER)
self.button.on_color[:] = (1,.0,.0,.8)
self.g_pool.quickbar.insert(1,self.button)
def deinit_gui(self):
if self.menu:
self.g_pool.sidebar.remove(self.menu)
self.menu = None
if self.button:
self.g_pool.quickbar.remove(self.button)
self.button = None
def toggle(self, _=None):
if self.running:
if self.calibration_start_at and not self.calibration_end_at: # Mark the end of claibration
self.calibration_end_at = self.timestamps[-1]
else:
self.notify_all( {'subject':'should_stop_recording','network_propagate':True} )
else:
self.notify_all( {'subject':'should_start_recording','session_name':self.session_name,'network_propagate':True} )
def on_notify(self,notification):
# notification wants to be recorded
if notification.get('record',False) and self.running:
self.data['notifications'].append(notification)
# Notificatio to start recording
elif notification['subject'] == 'should_start_recording':
if self.running:
logger.info('Recording already running!')
else:
if notification.get("session_name",""):
self.set_session_name(notification["session_name"])
self.start()
# Remote has stopped recording, we should stop as well.
elif notification['subject'] == 'should_stop_recording':
if self.running:
self.stop(shutdown_os=True)
else:
logger.info('Recording already stopped!')
def get_rec_time_str(self):
rec_time = gmtime(time()-self.start_time)
return strftime("%H:%M:%S", rec_time)
def start(self):
self.timestamps = []
self.data = {'pupil_positions':[],'gaze_positions':[],'notifications':[]}
self.frame_count = 0
self.running = True
self.menu.read_only = True
self.start_time = time()
session = os.path.join(self.rec_dir, self.session_name)
try:
os.makedirs(session)
logger.debug("Created new recordings session dir %s"%session)
except:
logger.debug("Recordings session dir %s already exists, using it." %session)
# set up self incrementing folder within session folder
counter = 0
while True:
self.rec_path = os.path.join(session, "%03d/" % counter)
try:
os.mkdir(self.rec_path)
logger.debug("Created new recording dir %s"%self.rec_path)
break
except:
logger.debug("We dont want to overwrite data, incrementing counter & trying to make new data folder")
counter += 1
self.meta_info_path = os.path.join(self.rec_path, "info.csv")
with open(self.meta_info_path, 'w') as f:
f.write("Recording Name\t"+self.session_name+ "\n")
f.write("Start Date\t"+ strftime("%d.%m.%Y", localtime(self.start_time))+ "\n")
f.write("Start Time\t"+ strftime("%H:%M:%S", localtime(self.start_time))+ "\n")
if self.audio_src != 'No Audio':
audio_path = os.path.join(self.rec_path, "world.wav")
self.audio_writer = Audio_Capture(audio_path,self.audio_devices_dict[self.audio_src])
else:
self.audio_writer = None
if self.raw_jpeg and self.g_pool.capture.jpeg_support:
self.video_path = os.path.join(self.rec_path, "world.mp4")
self.writer = JPEG_Writer(self.video_path,self.g_pool.capture.frame_rate)
else:
self.video_path = os.path.join(self.rec_path, "world.mp4")
self.writer = AV_Writer(self.video_path,fps=self.g_pool.capture.frame_rate)
# positions path to eye process
if self.record_eye:
for alive, pipe in zip(self.g_pool.eyes_are_alive,self.g_pool.eye_pipes):
if alive.value:
pipe.send( ('Rec_Start',(self.rec_path,self.raw_jpeg) ) )
if self.show_info_menu:
self.open_info_menu()
logger.info("Started Recording.")
self.notify_all( {'subject':'rec_started','rec_path':self.rec_path,'session_name':self.session_name,'network_propagate':True} )
def open_info_menu(self):
self.info_menu = ui.Growing_Menu('additional Recording Info',size=(300,300),pos=(300,300))
self.info_menu.configuration = self.info_menu_conf
def populate_info_menu():
self.info_menu.elements[:-2] = []
for name in self.user_info.iterkeys():
self.info_menu.insert(0,ui.Text_Input(name,self.user_info))
def set_user_info(new_string):
self.user_info = new_string
populate_info_menu()
populate_info_menu()
self.info_menu.append(ui.Info_Text('Use the *user info* field to add/remove additional fields and their values. The format must be a valid Python dictionary. For example -- {"key":"value"}. You can add as many fields as you require. Your custom fields will be saved for your next session.'))
self.info_menu.append(ui.Text_Input('user_info',self,setter=set_user_info,label="User info"))
self.g_pool.gui.append(self.info_menu)
def close_info_menu(self):
if self.info_menu:
self.info_menu_conf = self.info_menu.configuration
self.g_pool.gui.remove(self.info_menu)
self.info_menu = None
def update(self,frame,events):
''' listen to commands from eye window '''
if self.g_pool.eye_pipes[0].poll():
cmd = self.g_pool.eye_pipes[0].recv()
# toggle recording if enter key clicked
if cmd == GLFW_KEY_ENTER:
self.toggle()
if self.running:
# for key,data in events.iteritems():
# if key not in ('dt'):
# try:
# self.data[key] += data
# except KeyError:
# self.data[key] = []
# self.data[key] += data
''' Record when it first started as this will be the start of calibration. When user click Enter again this
should end calibration. Click once more to end recording'''
if not self.calibration_start_at:
self.calibration_start_at = frame.timestamp
self.timestamps.append(frame.timestamp)
self.writer.write_video_frame(frame)
self.frame_count += 1
# # cv2.putText(frame.img, "Frame %s"%self.frame_count,(200,200), cv2.FONT_HERSHEY_SIMPLEX,1,(255,100,100))
self.button.status_text = self.get_rec_time_str()
def stop(self, shutdown_os=False):
#explicit release of VideoWriter
self.writer.release()
self.writer = None
if self.record_eye:
for alive, pipe in zip(self.g_pool.eyes_are_alive,self.g_pool.eye_pipes):
if alive.value:
pipe.send(('Rec_Stop',None))
# Mohammad: This will now save an empty dicts list
save_object(self.data,os.path.join(self.rec_path, "pupil_data"))
timestamps_path = os.path.join(self.rec_path, "world_timestamps.npy")
# ts = sanitize_timestamps(np.array(self.timestamps))
ts = np.array(self.timestamps)
np.save(timestamps_path,ts)
try:
copy2(os.path.join(self.g_pool.user_dir,"surface_definitions"),os.path.join(self.rec_path,"surface_definitions"))
except:
logger.info("No surface_definitions data found. You may want this if you do marker tracking.")
try:
copy2(os.path.join(self.g_pool.user_dir,"user_calibration_data"),os.path.join(self.rec_path,"user_calibration_data"))
except:
logger.warning("No user calibration data found. Please calibrate first.")
# camera_calibration = load_camera_calibration(self.g_pool)
# if camera_calibration is not None:
# save_object(camera_calibration,os.path.join(self.rec_path, "camera_calibration"))
# else:
# logger.info("No camera calibration found.")
try:
with open(self.meta_info_path, 'a') as f:
f.write("Duration Time\t"+ self.get_rec_time_str()+ "\n")
f.write("World Camera Frames\t"+ str(self.frame_count)+ "\n")
f.write("World Camera Resolution\t"+ str(self.g_pool.capture.frame_size[0])+"x"+str(self.g_pool.capture.frame_size[1])+"\n")
f.write("Capture Software Version\t%s\n"%self.g_pool.version)
f.write("System Info\t%s"%get_system_info())
except Exception:
logger.exception("Could not save metadata. Please report this bug!")
try:
with open(os.path.join(self.rec_path, "user_info.csv"), 'w') as f:
self.user_info.update({'cal_start':self.calibration_start_at, 'cal_end':self.calibration_end_at})
for name,val in self.user_info.iteritems():
f.write("%s\t%s\n"%(name,val))
except Exception:
logger.exception("Could not save userdata. Please report this bug!")
self.calibration_start_at = None
self.calibration_end_at = None
self.close_info_menu()
if self.audio_writer:
self.audio_writer = None
self.running = False
self.menu.read_only = False
self.button.status_text = ''
self.timestamps = []
self.data = {'pupil_positions':[],'gaze_positions':[]}
self.pupil_pos_list = []
self.gaze_pos_list = []
logger.info("Saved Recording.")
self.notify_all( {'subject':'rec_stopped','rec_path':self.rec_path,'network_propagate':True} )
# Mohammad - shutdown OS when stop recording
if shutdown_os:
os.system('shutdown now')
def cleanup(self):
"""gets called when the plugin get terminated.
either volunatily or forced.
"""
if self.running:
self.stop()
self.deinit_gui()
def verify_path(self,val):
try:
n_path = os.path.expanduser(val)
logger.debug("Expanded user path.")
except:
n_path = val
if not n_path:
logger.warning("Please specify a path.")
return False
elif not os.path.isdir(n_path):
logger.warning("This is not a valid path.")
return False
# elif not os.access(n_path, os.W_OK):
elif not writable_dir(n_path):
logger.warning("Do not have write access to '%s'."%n_path)
return False
else:
return n_path
def set_rec_dir(self,val):
n_path = self.verify_path(val)
if n_path:
self.rec_dir = n_path
def set_session_name(self, val):
if not val:
self.session_name = get_auto_name()
else:
if '/' in val:
logger.warning('You session name will create one or more subdirectories')
self.session_name = val
class Recorder(Plugin):
"""Capture Recorder"""
def __init__(
self,
g_pool,
session_name=get_auto_name(),
rec_dir=None,
user_info={"name": "", "additional_field": "change_me"},
info_menu_conf={},
show_info_menu=False,
record_eye=False,
audio_src="No Audio",
raw_jpeg=False,
):
super(Recorder, self).__init__(g_pool)
# update name if it was autogenerated.
if session_name.startswith("20") and len(session_name) == 10:
session_name = get_auto_name()
if rec_dir:
self.set_rec_dir(rec_dir)
else:
# lets make a rec dir next to the user dir
base_dir = self.g_pool.user_dir.rsplit(os.path.sep, 1)[0]
self.rec_dir = os.path.join(base_dir, "recordings")
if not os.path.isdir(self.rec_dir):
os.mkdir(self.rec_dir)
self.raw_jpeg = raw_jpeg
self.order = 0.9
self.record_eye = record_eye
self.session_name = session_name
self.audio_devices_dict = Audio_Input_Dict()
if audio_src in self.audio_devices_dict.keys():
self.audio_src = audio_src
else:
self.audio_src = "No Audio"
self.running = False
self.menu = None
self.button = None
self.user_info = user_info
self.show_info_menu = show_info_menu
self.info_menu = None
self.info_menu_conf = info_menu_conf
self.height, self.width = self.g_pool.capture.frame_size
def get_init_dict(self):
d = {}
d["record_eye"] = self.record_eye
d["audio_src"] = self.audio_src
d["session_name"] = self.session_name
d["user_info"] = self.user_info
d["info_menu_conf"] = self.info_menu_conf
d["show_info_menu"] = self.show_info_menu
d["rec_dir"] = self.rec_dir
d["raw_jpeg"] = self.raw_jpeg
return d
def init_gui(self):
self.menu = ui.Growing_Menu("Recorder")
self.g_pool.sidebar.insert(3, self.menu)
self.menu.append(
ui.Info_Text(
'Pupil recordings are saved like this: "path_to_recordings/recording_session_name/nnn" where "nnn" is an increasing number to avoid overwrites. You can use "/" in your session name to create subdirectories.'
)
)
self.menu.append(
ui.Info_Text(
'Recordings are saved to "~/pupil_recordings". You can change the path here but note that invalid input will be ignored.'
)
)
self.menu.append(ui.Text_Input("rec_dir", self, setter=self.set_rec_dir, label="Path to recordings"))
self.menu.append(
ui.Text_Input("session_name", self, setter=self.set_session_name, label="Recording session name")
)
self.menu.append(
ui.Switch("show_info_menu", self, on_val=True, off_val=False, label="Request additional user info")
)
self.menu.append(
ui.Selector(
"raw_jpeg",
self,
selection=[True, False],
labels=["bigger file, less CPU", "smaller file, more CPU"],
label="compression",
)
)
self.menu.append(ui.Info_Text("Recording the raw eye video is optional. We use it for debugging."))
self.menu.append(ui.Switch("record_eye", self, on_val=True, off_val=False, label="Record eye"))
self.menu.append(ui.Selector("audio_src", self, selection=self.audio_devices_dict.keys()))
self.button = ui.Thumb("running", self, setter=self.toggle, label="Record", hotkey="r")
self.button.on_color[:] = (1, 0.0, 0.0, 0.8)
self.g_pool.quickbar.insert(1, self.button)
def deinit_gui(self):
if self.menu:
self.g_pool.sidebar.remove(self.menu)
self.menu = None
if self.button:
self.g_pool.quickbar.remove(self.button)
self.button = None
def toggle(self, _=None):
if self.running:
self.stop()
else:
self.start()
def get_rec_time_str(self):
rec_time = gmtime(time() - self.start_time)
return strftime("%H:%M:%S", rec_time)
def start(self):
self.timestamps = []
self.data = {"pupil_positions": [], "gaze_positions": []}
self.pupil_pos_list = []
self.gaze_pos_list = []
self.frame_count = 0
self.running = True
self.menu.read_only = True
self.start_time = time()
session = os.path.join(self.rec_dir, self.session_name)
try:
os.makedirs(session)
logger.debug("Created new recordings session dir %s" % session)
except:
logger.debug("Recordings session dir %s already exists, using it." % session)
# set up self incrementing folder within session folder
counter = 0
while True:
self.rec_path = os.path.join(session, "%03d/" % counter)
try:
os.mkdir(self.rec_path)
logger.debug("Created new recording dir %s" % self.rec_path)
break
except:
logger.debug("We dont want to overwrite data, incrementing counter & trying to make new data folder")
counter += 1
self.meta_info_path = os.path.join(self.rec_path, "info.csv")
with open(self.meta_info_path, "w") as f:
f.write("Recording Name\t" + self.session_name + "\n")
f.write("Start Date\t" + strftime("%d.%m.%Y", localtime(self.start_time)) + "\n")
f.write("Start Time\t" + strftime("%H:%M:%S", localtime(self.start_time)) + "\n")
if self.audio_src != "No Audio":
audio_path = os.path.join(self.rec_path, "world.wav")
self.audio_writer = Audio_Capture(self.audio_devices_dict[self.audio_src], audio_path)
else:
self.audio_writer = None
if self.raw_jpeg and "uvc_capture" in str(self.g_pool.capture.__class__):
self.video_path = os.path.join(self.rec_path, "world.mp4")
self.writer = JPEG_Writer(self.video_path, int(self.g_pool.capture.frame_rate))
# elif 1:
# self.writer = av_writer.AV_Writer(self.video_path)
else:
self.video_path = os.path.join(self.rec_path, "world.mkv")
self.writer = CV_Writer(
self.video_path, float(self.g_pool.capture.frame_rate), self.g_pool.capture.frame_size
)
# positions path to eye process
if self.record_eye:
for tx in self.g_pool.eye_tx:
tx.send((self.rec_path, self.raw_jpeg))
if self.show_info_menu:
self.open_info_menu()
def open_info_menu(self):
self.info_menu = ui.Growing_Menu("additional Recording Info", size=(300, 300), pos=(300, 300))
self.info_menu.configuration = self.info_menu_conf
def populate_info_menu():
self.info_menu.elements[:-2] = []
for name in self.user_info.iterkeys():
self.info_menu.insert(0, ui.Text_Input(name, self.user_info))
def set_user_info(new_string):
self.user_info = new_string
populate_info_menu()
populate_info_menu()
self.info_menu.append(
ui.Info_Text(
'Use the *user info* field to add/remove additional fields and their values. The format must be a valid Python dictionary. For example -- {"key":"value"}. You can add as many fields as you require. Your custom fields will be saved for your next session.'
)
)
self.info_menu.append(ui.Text_Input("user_info", self, setter=set_user_info, label="User info"))
self.g_pool.gui.append(self.info_menu)
def close_info_menu(self):
if self.info_menu:
self.info_menu_conf = self.info_menu.configuration
self.g_pool.gui.remove(self.info_menu)
self.info_menu = None
def update(self, frame, events):
if self.running:
self.data["pupil_positions"] += events["pupil_positions"]
self.data["gaze_positions"] += events["gaze_positions"]
self.timestamps.append(frame.timestamp)
self.writer.write_video_frame(frame)
# self.writer.write_video_frame_yuv422(frame)
self.frame_count += 1
# cv2.putText(frame.img, "Frame %s"%self.frame_count,(200,200), cv2.FONT_HERSHEY_SIMPLEX,1,(255,100,100))
for p in events["pupil_positions"]:
pupil_pos = p["timestamp"], p["confidence"], p["id"], p["norm_pos"][0], p["norm_pos"][1], p["diameter"]
self.pupil_pos_list.append(pupil_pos)
for g in events.get("gaze_positions", []):
gaze_pos = g["timestamp"], g["confidence"], g["norm_pos"][0], g["norm_pos"][1]
self.gaze_pos_list.append(gaze_pos)
self.button.status_text = self.get_rec_time_str()
def stop(self):
# explicit release of VideoWriter
self.writer.release()
self.writer = None
if self.record_eye:
for tx in self.g_pool.eye_tx:
try:
tx.send((None, None))
except:
logger.warning("Could not stop eye-recording. Please report this bug!")
save_object(self.data, os.path.join(self.rec_path, "pupil_data"))
gaze_list_path = os.path.join(self.rec_path, "gaze_positions.npy")
np.save(gaze_list_path, np.asarray(self.gaze_pos_list))
pupil_list_path = os.path.join(self.rec_path, "pupil_positions.npy")
np.save(pupil_list_path, np.asarray(self.pupil_pos_list))
timestamps_path = os.path.join(self.rec_path, "world_timestamps.npy")
ts = sanitize_timestamps(np.array(self.timestamps))
np.save(timestamps_path, ts)
try:
copy2(
os.path.join(self.g_pool.user_dir, "surface_definitions"),
os.path.join(self.rec_path, "surface_definitions"),
)
except:
logger.info("No surface_definitions data found. You may want this if you do marker tracking.")
try:
copy2(
os.path.join(self.g_pool.user_dir, "cal_pt_cloud.npy"), os.path.join(self.rec_path, "cal_pt_cloud.npy")
)
except:
logger.warning("No calibration data found. Please calibrate first.")
try:
copy2(
os.path.join(self.g_pool.user_dir, "camera_matrix.npy"),
os.path.join(self.rec_path, "camera_matrix.npy"),
)
copy2(os.path.join(self.g_pool.user_dir, "dist_coefs.npy"), os.path.join(self.rec_path, "dist_coefs.npy"))
except:
logger.info("No camera intrinsics found.")
try:
with open(self.meta_info_path, "a") as f:
f.write("Duration Time\t" + self.get_rec_time_str() + "\n")
if self.g_pool.binocular:
f.write("Eye Mode\tbinocular\n")
else:
f.write("Eye Mode\tmonocular\n")
f.write("Duration Time\t" + self.get_rec_time_str() + "\n")
f.write("World Camera Frames\t" + str(self.frame_count) + "\n")
f.write("World Camera Resolution\t" + str(self.width) + "x" + str(self.height) + "\n")
f.write("Capture Software Version\t%s\n" % self.g_pool.version)
if platform.system() == "Windows":
username = os.environ["USERNAME"]
sysname, nodename, release, version, machine, _ = platform.uname()
else:
username = os.getlogin()
try:
sysname, nodename, release, version, machine = os.uname()
except:
sysname, nodename, release, version, machine = sys.platform, None, None, None, None
f.write("User\t" + username + "\n")
f.write("Platform\t" + sysname + "\n")
f.write("Machine\t" + nodename + "\n")
f.write("Release\t" + release + "\n")
f.write("Version\t" + version + "\n")
except Exception:
logger.exception("Could not save metadata. Please report this bug!")
try:
with open(os.path.join(self.rec_path, "user_info.csv"), "w") as f:
for name, val in self.user_info.iteritems():
f.write("%s\t%s\n" % (name, val))
except Exception:
logger.exception("Could not save userdata. Please report this bug!")
self.close_info_menu()
if self.audio_writer:
self.audio_writer = None
self.running = False
self.menu.read_only = False
self.button.status_text = ""
def cleanup(self):
"""gets called when the plugin get terminated.
either volunatily or forced.
"""
if self.running:
self.stop()
self.deinit_gui()
def set_rec_dir(self, val):
try:
n_path = os.path.expanduser(val)
logger.debug("Expanded user path.")
except:
n_path = val
if not n_path:
logger.warning("Please specify a path.")
elif not os.path.isdir(n_path):
logger.warning("This is not a valid path.")
# elif not os.access(n_path, os.W_OK):
elif not writable_dir(n_path):
logger.warning("Do not have write access to '%s'." % n_path)
else:
self.rec_dir = n_path
def set_session_name(self, val):
if not val:
self.session_name = get_auto_name()
else:
if "/" in val:
logger.warning("You session name with create one or more subdirectories")
self.session_name = val
class Audio_Capture(Plugin):
"""docstring for Audio_Capture"""
icon_chr = chr(0xe029)
icon_font = 'pupil_icons'
def __init__(self, g_pool, audio_src='No Audio'):
super().__init__(g_pool)
self.audio_devices_dict = Audio_Input_Dict()
if audio_src in list(self.audio_devices_dict.keys()):
self.audio_src = audio_src
else:
self.audio_src = 'No Audio'
self.thread = None
self.running = Event()
self.recording = Event()
self.recording.clear()
self.audio_container = None
self.audio_out_stream = None
self.queue = queue.Queue()
self.start_capture(self.audio_src)
def init_ui(self):
self.add_menu()
self.menu.label = 'Audio Capture'
help_str = 'Creates events for audio input.'
self.menu.append(ui.Info_Text(help_str))
def audio_dev_getter():
# fetch list of currently available
self.audio_devices_dict = Audio_Input_Dict()
devices = list(self.audio_devices_dict.keys())
return devices, devices
self.menu.append(ui.Selector('audio_src', self,
selection_getter=audio_dev_getter,
label='Audio Source',
setter=self.start_capture))
self.menu.append(ui.Info_Text(NOT_REC_STR))
def deinit_ui(self):
self.remove_menu()
def get_init_dict(self):
return {'audio_src': self.audio_src}
def cleanup(self):
if self.audio_container is not None:
self.close_audio_recording()
self.running.clear()
if self.thread and self.thread.is_alive():
self.thread.join(timeout=1)
def on_notify(self, notification):
if notification['subject'] == 'recording.started':
self.rec_dir = notification['rec_path']
self.recording.set()
if self.running.is_set(): # and self.audio_container is None:
self.menu[-2].read_only = True
del self.menu[-1]
self.menu.append(ui.Info_Text(REC_STR))
elif not self.running.is_set():
logger.warning('Recording was started without an active audio capture')
elif notification['subject'] == 'recording.stopped':
self.recording.clear()
self.close_audio_recording()
def close_audio_recording(self):
self.menu[-2].read_only = False
del self.menu[-1]
self.menu.append(ui.Info_Text(NOT_REC_STR))
def write_audio_packet(self, audio_frame):
# Test if audio outstream has been initialized
if self.audio_out_stream is None:
try:
self.audio_out_stream = self.audio_container.add_stream('aac')
except ValueError as e:
# packet.stream codec is not supported in target container.
logger.error('Failed to create audio stream. Aborting recording.')
logger.debug('Reason: {}'.format(e))
self.close_audio_recording()
self.timestamps.append(audio_frame.timestamp)
packet = self.audio_out_stream.encode(audio_frame)
if packet is not None:
self.audio_container.mux(packet)
def start_capture(self, audio_src):
if self.thread and self.thread.is_alive():
if self.audio_src == audio_src:
return # capture is already running for our selected source
# else stop current capture gracefully
self.running.clear()
logger.debug('Closing capture for "{}"'.format(self.audio_src))
self.thread.join(timeout=1)
self.thread = None
if audio_src not in self.audio_devices_dict:
logger.warning('Selected audio source is not available anymore')
return
self.audio_src = audio_src
if audio_src == 'No Audio':
return
self.running.set()
self.thread = Thread(target=self.capture_thread,
args=(self.audio_devices_dict[audio_src], self.running, self.recording))
self.thread.start()
def capture_thread(self, audio_src, running, recording):
try:
if platform.system() == "Darwin":
in_container = av.open('none:{}'.format(audio_src), format="avfoundation")
elif platform.system() == "Linux":
in_container = av.open('hw:{}'.format(audio_src), format="alsa")
else:
raise av.AVError('Platform does not support audio capture.')
except av.AVError as err:
running.clear()
self.audio_src = 'No Audio'
logger.warning('Error starting audio capture: {}'.format(err))
return
in_stream = None
try:
in_stream = in_container.streams.audio[0]
except IndexError:
logger.warning('No audio stream found for selected device.')
running.clear()
self.audio_src = 'No Audio'
return
out_container = None
out_stream = None
timestamps = None
out_frame_num = 0
in_frame_size = 0
stream_epoch = in_stream.start_time * in_stream.time_base
uvc_clock_dif = abs(stream_epoch - self.g_pool.get_now())
pyt_clock_dif = abs(stream_epoch - time())
max_clock_dif = 4.
if uvc_clock_dif > max_clock_dif and pyt_clock_dif > max_clock_dif:
logger.error('Could not identify audio stream clock.')
running.clear()
self.audio_src = 'No Audio'
return
elif uvc_clock_dif > pyt_clock_dif:
logger.info('Audio stream uses time.time() as clock (Δ {}s)'.format(pyt_clock_dif))
clock_differences = self.g_pool.get_now() - time()
else:
logger.info('Audio stream uses uvc.get_time_monotonic() as clock')
clock_differences = 0
def close_recording():
# Bind nonlocal variables, https://www.python.org/dev/peps/pep-3104/
nonlocal out_container, out_stream, in_stream, in_frame_size, timestamps, out_frame_num
if out_container is not None:
packet = out_stream.encode(audio_frame)
while packet is not None:
out_frame_num += 1
out_container.mux(packet)
packet = out_stream.encode(audio_frame)
out_container.close()
in_frame_rate = in_stream.rate
# in_stream.frame_size does not return the correct value.
out_frame_size = out_stream.frame_size
out_frame_rate = out_stream.rate
old_ts_idx = np.arange(0, len(timestamps) * in_frame_size, in_frame_size) / in_frame_rate
new_ts_idx = np.arange(0, out_frame_num * out_frame_size, out_frame_size) / out_frame_rate
interpolate = interp1d(old_ts_idx, timestamps, bounds_error=False, fill_value='extrapolate')
new_ts = interpolate(new_ts_idx)
ts_loc = os.path.join(self.rec_dir, 'audio_timestamps.npy')
np.save(ts_loc, new_ts)
out_container = None
out_stream = None
timestamps = None
for packet in in_container.demux(in_stream):
# ffmpeg timestamps - in_stream.startime = packte pts relative to startime
# multiply with stream_timebase to get seconds
# add start time of this stream in pupil time unadjusted
# finally add pupil timebase offset to adjust for settable timebase.
for audio_frame in packet.decode():
timestamp = audio_frame.pts * in_stream.time_base + clock_differences - self.g_pool.timebase.value
if recording.is_set():
if out_container is None:
rec_file = os.path.join(self.rec_dir, 'audio.mp4')
out_container = av.open(rec_file, 'w')
out_stream = out_container.add_stream('aac')
out_frame_num = 0
in_frame_size = audio_frame.samples # set here to make sure full packet size is used
timestamps = []
timestamps.append(timestamp)
packet = out_stream.encode(audio_frame)
if packet is not None:
out_frame_num += 1
out_container.mux(packet)
elif out_container is not None:
# recording stopped
close_recording()
if not running.is_set():
close_recording()
return
close_recording()
self.audio_src = 'No Audio'
running.clear() # in_stream stopped yielding packets
class Recorder(Plugin):
"""Capture Recorder"""
def __init__(self,g_pool,session_name = get_auto_name(),rec_dir=None, user_info={'name':'','additional_field':'change_me'},info_menu_conf={},show_info_menu=False, record_eye = True, audio_src = 'No Audio',raw_jpeg=True):
super(Recorder, self).__init__(g_pool)
#update name if it was autogenerated.
if session_name.startswith('20') and len(session_name)==10:
session_name = get_auto_name()
base_dir = self.g_pool.user_dir.rsplit(os.path.sep,1)[0]
default_rec_dir = os.path.join(base_dir,'recordings')
if rec_dir and rec_dir != default_rec_dir and self.verify_path(rec_dir):
self.rec_dir = rec_dir
else:
try:
os.makedirs(default_rec_dir)
except OSError as e:
if e.errno != errno.EEXIST:
logger.error("Could not create Rec dir")
raise e
else:
logger.info('Created standard Rec dir at "%s"'%default_rec_dir)
self.rec_dir = default_rec_dir
self.raw_jpeg = raw_jpeg
self.order = .9
self.record_eye = record_eye
self.session_name = session_name
self.audio_devices_dict = Audio_Input_Dict()
if audio_src in self.audio_devices_dict.keys():
self.audio_src = audio_src
else:
self.audio_src = 'No Audio'
self.running = False
self.menu = None
self.button = None
self.user_info = user_info
self.show_info_menu = show_info_menu
self.info_menu = None
self.info_menu_conf = info_menu_conf
def get_init_dict(self):
d = {}
d['record_eye'] = self.record_eye
d['audio_src'] = self.audio_src
d['session_name'] = self.session_name
d['user_info'] = self.user_info
d['info_menu_conf'] = self.info_menu_conf
d['show_info_menu'] = self.show_info_menu
d['rec_dir'] = self.rec_dir
d['raw_jpeg'] = self.raw_jpeg
return d
def init_gui(self):
self.menu = ui.Growing_Menu('Recorder')
self.g_pool.sidebar.insert(3,self.menu)
self.menu.append(ui.Info_Text('Pupil recordings are saved like this: "path_to_recordings/recording_session_name/nnn" where "nnn" is an increasing number to avoid overwrites. You can use "/" in your session name to create subdirectories.'))
self.menu.append(ui.Info_Text('Recordings are saved to "~/pupil_recordings". You can change the path here but note that invalid input will be ignored.'))
self.menu.append(ui.Text_Input('rec_dir',self,setter=self.set_rec_dir,label='Path to recordings'))
self.menu.append(ui.Text_Input('session_name',self,setter=self.set_session_name,label='Recording session name'))
self.menu.append(ui.Switch('show_info_menu',self,on_val=True,off_val=False,label='Request additional user info'))
self.menu.append(ui.Selector('raw_jpeg',self,selection = [True,False], labels=["bigger file, less CPU", "smaller file, more CPU"],label='Compression'))
self.menu.append(ui.Info_Text('Recording the raw eye video is optional. We use it for debugging.'))
self.menu.append(ui.Switch('record_eye',self,on_val=True,off_val=False,label='Record eye'))
self.menu.append(ui.Selector('audio_src',self, selection=self.audio_devices_dict.keys(),label='Audio Source'))
self.button = ui.Thumb('running',self,setter=self.toggle,label='Record',hotkey='r')
self.button.on_color[:] = (1,.0,.0,.8)
self.g_pool.quickbar.insert(1,self.button)
def deinit_gui(self):
if self.menu:
self.g_pool.sidebar.remove(self.menu)
self.menu = None
if self.button:
self.g_pool.quickbar.remove(self.button)
self.button = None
def toggle(self, _=None):
if self.running:
self.notify_all( {'subject':'recording.should_stop'} )
self.notify_all( {'subject':'recording.should_stop', 'remote_notify':'all'} )
else:
self.notify_all( {'subject':'recording.should_start','session_name':self.session_name} )
self.notify_all( {'subject':'recording.should_start','session_name':self.session_name,'remote_notify':'all'} )
def on_notify(self,notification):
"""Handles recorder notifications
Reacts to notifications:
``recording.should_start``: Starts a new recording session
``recording.should_stop``: Stops current recording session
Emits notifications:
``recording.started``: New recording session started
``recording.stopped``: Current recording session stopped
Args:
notification (dictionary): Notification dictionary
"""
# notification wants to be recorded
if notification.get('record',False) and self.running:
self.data['notifications'].append(notification)
elif notification['subject'] == 'recording.should_start':
if self.running:
logger.info('Recording already running!')
else:
if notification.get("session_name",""):
self.set_session_name(notification["session_name"])
self.start()
elif notification['subject'] == 'recording.should_stop':
if self.running:
self.stop()
else:
logger.info('Recording already stopped!')
def get_rec_time_str(self):
rec_time = gmtime(time()-self.start_time)
return strftime("%H:%M:%S", rec_time)
def start(self):
self.timestamps = []
self.timestampsUnix = []
self.glint_pos_list = []
self.pupil_pos_list = []
self.gaze_pos_list = []
self.data = {'pupil_positions':[],'gaze_positions':[],'notifications':[]}
self.frame_count = 0
self.running = True
self.menu.read_only = True
self.start_time = time()
session = os.path.join(self.rec_dir, self.session_name)
try:
os.makedirs(session)
logger.debug("Created new recordings session dir %s"%session)
except:
logger.debug("Recordings session dir %s already exists, using it." %session)
# set up self incrementing folder within session folder
counter = 0
while True:
self.rec_path = os.path.join(session, "%03d/" % counter)
try:
os.mkdir(self.rec_path)
logger.debug("Created new recording dir %s"%self.rec_path)
break
except:
logger.debug("We dont want to overwrite data, incrementing counter & trying to make new data folder")
counter += 1
self.meta_info_path = os.path.join(self.rec_path, "info.csv")
with open(self.meta_info_path, 'w') as csvfile:
csv_utils.write_key_value_file(csvfile,{
'Recording Name': self.session_name,
'Start Date': strftime("%d.%m.%Y", localtime(self.start_time)),
'Start Time': strftime("%H:%M:%S", localtime(self.start_time)),
'Start Time (seconds since epoch)': str(self.start_time)
})
if self.audio_src != 'No Audio':
audio_path = os.path.join(self.rec_path, "world.wav")
self.audio_writer = Audio_Capture(audio_path,self.audio_devices_dict[self.audio_src])
else:
self.audio_writer = None
if self.raw_jpeg and self.g_pool.capture.jpeg_support:
self.video_path = os.path.join(self.rec_path, "world.mp4")
self.writer = JPEG_Writer(self.video_path,self.g_pool.capture.frame_rate)
else:
self.video_path = os.path.join(self.rec_path, "world.mp4")
self.writer = AV_Writer(self.video_path,fps=self.g_pool.capture.frame_rate)
if self.show_info_menu:
self.open_info_menu()
logger.info("Started Recording.")
self.notify_all( {'subject':'recording.started','rec_path':self.rec_path,'session_name':self.session_name,'record_eye':self.record_eye,'compression':self.raw_jpeg} )
def open_info_menu(self):
self.info_menu = ui.Growing_Menu('additional Recording Info',size=(300,300),pos=(300,300))
self.info_menu.configuration = self.info_menu_conf
def populate_info_menu():
self.info_menu.elements[:-2] = []
for name in self.user_info.iterkeys():
self.info_menu.insert(0,ui.Text_Input(name,self.user_info))
def set_user_info(new_string):
self.user_info = new_string
populate_info_menu()
populate_info_menu()
self.info_menu.append(ui.Info_Text('Use the *user info* field to add/remove additional fields and their values. The format must be a valid Python dictionary. For example -- {"key":"value"}. You can add as many fields as you require. Your custom fields will be saved for your next session.'))
self.info_menu.append(ui.Text_Input('user_info',self,setter=set_user_info,label="User info"))
self.g_pool.gui.append(self.info_menu)
def close_info_menu(self):
if self.info_menu:
self.info_menu_conf = self.info_menu.configuration
self.g_pool.gui.remove(self.info_menu)
self.info_menu = None
def update(self,frame,events):
if self.running:
for key,data in events.iteritems():
if key not in ('dt') and key != 'timestamp_unix':
try:
self.data[key] += data
except KeyError:
self.data[key] = []
self.data[key] += data
self.timestamps.append(frame.timestamp)
self.writer.write_video_frame(frame)
self.frame_count += 1
for glint in events['glint_positions']:
self.glint_pos_list += glint
self.timestampsUnix.append(events['timestamp_unix'])
# # cv2.putText(frame.img, "Frame %s"%self.frame_count,(200,200), cv2.FONT_HERSHEY_SIMPLEX,1,(255,100,100))
self.button.status_text = self.get_rec_time_str()
def stop(self):
#explicit release of VideoWriter
self.writer.release()
self.writer = None
save_object(self.data,os.path.join(self.rec_path, "pupil_data"))
self.glint_pos_list = np.array(self.glint_pos_list)
glint_list_path = os.path.join(self.rec_path, "glint_positions.npy")
np.save(glint_list_path,self.glint_pos_list)
timestamps_path = os.path.join(self.rec_path, "world_timestamps.npy")
# ts = sanitize_timestamps(np.array(self.timestamps))
ts = np.array(self.timestamps)
np.save(timestamps_path,ts)
timestampsUnix_path = os.path.join(self.rec_path, "world_timestamps_unix.npy")
tsUnix = np.array(self.timestampsUnix)
np.save(timestampsUnix_path,tsUnix)
try:
copy2(os.path.join(self.g_pool.user_dir,"surface_definitions"),os.path.join(self.rec_path,"surface_definitions"))
except:
logger.info("No surface_definitions data found. You may want this if you do marker tracking.")
try:
copy2(os.path.join(self.g_pool.user_dir,"user_calibration_data"),os.path.join(self.rec_path,"user_calibration_data"))
except:
logger.warning("No user calibration data found. Please calibrate first.")
try:
copy2(os.path.join(self.g_pool.user_dir,"cal_pt_cloud_glint.npy"),os.path.join(self.rec_path,"cal_pt_cloud_glint.npy"))
except:
logger.warning("No pupil-glint-vector calibration data found. Please calibrate first.")
try:
copy2(os.path.join(self.g_pool.user_dir,"cal_ref_list.npy"),os.path.join(self.rec_path,"cal_ref_list.npy"))
except:
logger.warning("No calibration reference list found.")
try:
copy2(os.path.join(self.g_pool.user_dir,"accuracy_test_pt_cloud.npy"),os.path.join(self.rec_path,"accuracy_test_pt_cloud.npy"))
copy2(os.path.join(self.g_pool.user_dir,"accuracy_test_ref_list.npy"),os.path.join(self.rec_path,"accuracy_test_ref_list.npy"))
except:
logger.warning("No accuracy test found.")
try:
copy2(os.path.join(self.g_pool.user_dir,"accuracy_test_pt_cloud_previous.npy"),os.path.join(self.rec_path,"accuracy_test_pt_cloud_previous.npy"))
copy2(os.path.join(self.g_pool.user_dir,"accuracy_test_ref_list_previous.npy"),os.path.join(self.rec_path,"accuracy_test_ref_list_previous.npy"))
except:
logger.warning("No previous accuracy test results.")
camera_calibration = load_camera_calibration(self.g_pool)
if camera_calibration is not None:
save_object(camera_calibration,os.path.join(self.rec_path, "camera_calibration"))
else:
logger.info("No camera calibration found.")
try:
with open(self.meta_info_path, 'a') as csvfile:
csv_utils.write_key_value_file(csvfile, {
'Duration Time': self.get_rec_time_str(),
'World Camera Frames': self.frame_count,
'World Camera Resolution': str(self.g_pool.capture.frame_size[0])+"x"+str(self.g_pool.capture.frame_size[1]),
'Capture Software Version': self.g_pool.version,
'System Info': get_system_info()
}, append=True)
except Exception:
logger.exception("Could not save metadata. Please report this bug!")
try:
with open(os.path.join(self.rec_path, "user_info.csv"), 'w') as csvfile:
csv_utils.write_key_value_file(csvfile, self.user_info)
except Exception:
logger.exception("Could not save userdata. Please report this bug!")
self.close_info_menu()
if self.audio_writer:
self.audio_writer = None
self.running = False
self.menu.read_only = False
self.button.status_text = ''
self.timestamps = []
self.data = {'pupil_positions':[],'gaze_positions':[]}
self.pupil_pos_list = []
self.gaze_pos_list = []
logger.info("Saved Recording.")
self.notify_all( {'subject':'rec_stopped','rec_path':self.rec_path,'network_propagate':True} )
self.notify_all( {'subject':'recording.stopped','rec_path':self.rec_path} )
copyfile(os.path.join(self.g_pool.user_dir,'capture.log'), os.path.join(self.rec_path,"capture.log"))
def cleanup(self):
"""gets called when the plugin get terminated.
either volunatily or forced.
"""
if self.running:
self.stop()
self.deinit_gui()
def verify_path(self,val):
try:
n_path = os.path.expanduser(val)
logger.debug("Expanded user path.")
except:
n_path = val
if not n_path:
logger.warning("Please specify a path.")
return False
elif not os.path.isdir(n_path):
logger.warning("This is not a valid path.")
return False
# elif not os.access(n_path, os.W_OK):
elif not writable_dir(n_path):
logger.warning("Do not have write access to '%s'."%n_path)
return False
else:
return n_path
def set_rec_dir(self,val):
n_path = self.verify_path(val)
if n_path:
self.rec_dir = n_path
def set_session_name(self, val):
if not val:
self.session_name = get_auto_name()
else:
if '/' in val:
logger.warning('You session name will create one or more subdirectories')
self.session_name = val
class Audio_Capture(Plugin):
"""docstring for Audio_Capture"""
def __init__(self, g_pool, audio_src='No Audio'):
super().__init__(g_pool)
self.audio_devices_dict = Audio_Input_Dict()
if audio_src in list(self.audio_devices_dict.keys()):
self.audio_src = audio_src
else:
self.audio_src = 'No Audio'
self.thread = None
self.running = Event()
self.audio_container = None
self.audio_out_stream = None
self.queue = queue.Queue()
self.start_capture(self.audio_src)
def recent_events(self, events):
audio_packets = []
while True:
try:
packet = self.queue.get_nowait()
except queue.Empty:
break
audio_packets.append(packet)
events['audio_packets'] = audio_packets
if self.audio_container is not None:
for packet in audio_packets:
self.write_audio_packet(packet)
def init_gui(self):
self.menu = ui.Growing_Menu('Audio Capture')
self.menu.collapsed = True
self.g_pool.sidebar.append(self.menu)
def close():
self.alive = False
help_str = 'Creates events for audio input.'
self.menu.append(ui.Button('Close', close))
self.menu.append(ui.Info_Text(help_str))
def audio_dev_getter():
# fetch list of currently available
self.audio_devices_dict = Audio_Input_Dict()
devices = list(self.audio_devices_dict.keys())
return devices, devices
self.menu.append(
ui.Selector('audio_src',
self,
selection_getter=audio_dev_getter,
label='Audio Source',
setter=self.start_capture))
self.menu.append(ui.Info_Text(NOT_REC_STR))
def get_init_dict(self):
return {'audio_src': self.audio_src}
def deinit_gui(self):
if self.menu:
self.g_pool.sidebar.remove(self.menu)
self.menu = None
def cleanup(self):
if self.audio_container is not None:
self.close_audio_recording()
self.running.clear()
self.deinit_gui()
if self.thread and self.thread.is_alive():
self.thread.join(timeout=1)
def on_notify(self, notification):
if notification['subject'] == 'recording.started':
if self.running.is_set() and self.audio_container is None:
self.rec_dir = notification['rec_path']
rec_file = os.path.join(self.rec_dir, 'audio.wav')
self.audio_container = av.open(rec_file, 'w')
self.timestamps = []
self.menu[-2].read_only = True
del self.menu[-1]
self.menu.append(ui.Info_Text(REC_STR))
elif not self.running.is_set():
logger.warning(
'Recording was started without an active audio capture')
else:
logger.warning('Audio is already being recorded')
elif notification['subject'] == 'recording.stopped':
if self.audio_container is not None and self.audio_out_stream is not None:
self.close_audio_recording()
def close_audio_recording(self):
self.audio_container.close()
ts_loc = os.path.join(self.rec_dir, 'audio_timestamps.npy')
np.save(ts_loc, np.asarray(self.timestamps))
self.timestamps = None
self.audio_out_stream = None
self.audio_container = None
self.menu[-2].read_only = False
del self.menu[-1]
self.menu.append(ui.Info_Text(NOT_REC_STR))
def write_audio_packet(self, packet):
# Test if audio outstream has been initialized
if self.audio_out_stream is None:
try:
self.audio_out_stream = self.audio_container.add_stream(
template=packet.stream)
except ValueError as e:
# packet.stream codec is not supported in target container.
logger.error(
'Failed to create audio stream. Aborting recording.')
logger.debug('Reason: {}'.format(e))
self.close_audio_recording()
self.timestamps.append(packet.timestamp)
self.audio_container.mux(packet)
def start_capture(self, audio_src):
if self.thread and self.thread.is_alive():
if self.audio_src == audio_src:
return # capture is already running for our selected source
# else stop current capture gracefully
self.running.clear()
logger.debug('Closing capture for "{}"'.format(self.audio_src))
self.thread.join(timeout=1)
self.thread = None
if audio_src not in self.audio_devices_dict:
logger.warning('Selected audio source is not available anymore')
return
self.audio_src = audio_src
if audio_src == 'No Audio':
return
self.running.set()
self.thread = Thread(target=self.capture_thread,
args=(self.audio_devices_dict[audio_src],
self.running))
self.thread.start()
def capture_thread(self, audio_src, running):
try:
if platform.system() == "Darwin":
in_container = av.open('none:{}'.format(audio_src),
format="avfoundation")
elif platform.system() == "Linux":
in_container = av.open('hw:{}'.format(audio_src),
format="alsa")
else:
raise av.AVError('Platform does not support audio capture.')
except av.AVError:
running.clear()
return
in_stream = None
for stream in in_container.streams:
if stream.type == 'audio':
in_stream = stream
break
if not in_stream:
logger.warning('No audio stream found for selected device.')
running.clear()
return
stream_start_ts = self.g_pool.get_now()
for packet in in_container.demux(in_stream):
try:
# ffmpeg timestamps - in_stream.startime = packte pts relative to startime
# multiply with stream_timebase to get seconds
# add start time of this stream in pupil time unadjusted
# finally add pupil timebase offset to adjust for settable timebase.
packet.timestamp = (
packet.pts - in_stream.start_time
) * in_stream.time_base + stream_start_ts - self.g_pool.timebase.value
self.queue.put_nowait(packet)
except queue.Full:
pass # drop packet
if not running.is_set():
return
self.audio_src = 'No Audio'
running.clear() # in_stream stopped yielding packets
class Audio_Capture(Plugin):
"""docstring for Audio_Capture"""
icon_chr = chr(0xE029)
icon_font = "pupil_icons"
def __init__(self, g_pool, audio_src="No Audio"):
super().__init__(g_pool)
self.audio_devices_dict = Audio_Input_Dict()
if audio_src in list(self.audio_devices_dict.keys()):
self.audio_src = audio_src
else:
self.audio_src = "No Audio"
self.thread = None
self.running = Event()
self.recording = Event()
self.recording.clear()
self.audio_container = None
self.audio_out_stream = None
self.queue = queue.Queue()
self.start_capture(self.audio_src)
def init_ui(self):
self.add_menu()
self.menu.label = "Audio Capture"
help_str = "Creates events for audio input."
self.menu.append(ui.Info_Text(help_str))
def audio_dev_getter():
# fetch list of currently available
audio_src_val = None
if self.audio_devices_dict is not None:
if self.audio_src in self.audio_devices_dict.keys():
audio_src_val = self.audio_devices_dict[self.audio_src]
self.audio_devices_dict = Audio_Input_Dict()
if audio_src_val is not None:
self.audio_devices_dict[self.audio_src] = audio_src_val
devices = list(self.audio_devices_dict.keys())
return devices, devices
self.menu.append(
ui.Selector(
"audio_src",
self,
selection_getter=audio_dev_getter,
label="Audio Source",
setter=self.start_capture,
)
)
self.menu.append(ui.Info_Text(NOT_REC_STR))
def deinit_ui(self):
self.remove_menu()
def get_init_dict(self):
return {"audio_src": self.audio_src}
def cleanup(self):
if self.audio_container is not None:
self.close_audio_recording()
self.running.clear()
if self.thread and self.thread.is_alive():
self.thread.join(timeout=1)
def on_notify(self, notification):
if notification["subject"] == "recording.started":
self.rec_dir = notification["rec_path"]
self.recording.set()
if self.running.is_set(): # and self.audio_container is None:
self.menu[-2].read_only = True
del self.menu[-1]
self.menu.append(ui.Info_Text(REC_STR))
elif not self.running.is_set():
logger.warning("Recording was started without an active audio capture")
elif notification["subject"] == "recording.stopped":
self.recording.clear()
self.close_audio_recording()
def close_audio_recording(self):
self.menu[-2].read_only = False
del self.menu[-1]
self.menu.append(ui.Info_Text(NOT_REC_STR))
def write_audio_packet(self, audio_frame):
# Test if audio outstream has been initialized
if self.audio_out_stream is None:
try:
self.audio_out_stream = self.audio_container.add_stream("aac")
except ValueError as e:
# packet.stream codec is not supported in target container.
logger.error("Failed to create audio stream. Aborting recording.")
logger.debug("Reason: {}".format(e))
self.close_audio_recording()
self.timestamps.append(audio_frame.timestamp)
packet = self.audio_out_stream.encode(audio_frame)
if packet is not None:
self.audio_container.mux(packet)
def start_capture(self, audio_src):
if self.thread and self.thread.is_alive():
if self.audio_src == audio_src:
return # capture is already running for our selected source
# else stop current capture gracefully
self.running.clear()
logger.debug('Closing capture for "{}"'.format(self.audio_src))
self.thread.join(timeout=1)
self.thread = None
if audio_src not in self.audio_devices_dict:
logger.warning("Selected audio source is not available anymore")
return
self.audio_src = audio_src
if audio_src == "No Audio":
return
self.running.set()
self.thread = Thread(
target=self.capture_thread,
args=(self.audio_devices_dict[audio_src], self.running, self.recording),
)
self.thread.start()
def capture_thread(self, audio_src, running, recording):
try:
if platform.system() == "Darwin":
in_container = av.open(
"none:{}".format(audio_src), format="avfoundation"
)
elif platform.system() == "Linux":
print("audio src = {}".format(audio_src))
dev_str = re.search("(hw:\s*\d+,\s*\d+)", audio_src)
in_container = av.open(dev_str.group(0), format="alsa")
elif platform.system() == "Windows":
in_container = av.open(
"audio={}".format(audio_src),
format="dshow",
options={"audio_buffer_size": "23"},
)
else:
raise av.AVError("Platform does not support audio capture.")
except av.AVError as err:
running.clear()
self.audio_src = "No Audio"
logger.warning("Error starting audio capture: {}".format(err))
return
in_stream = None
try:
in_stream = in_container.streams.audio[0]
except IndexError:
logger.warning("No audio stream found for selected device.")
running.clear()
self.audio_src = "No Audio"
return
out_container = None
out_stream = None
timestamps = None
in_frame_size = 0
stream_epoch = in_stream.start_time * in_stream.time_base
uvc_clock_dif = abs(stream_epoch - self.g_pool.get_now())
pyt_clock_dif = abs(stream_epoch - time())
max_clock_dif = 4.0
if uvc_clock_dif > max_clock_dif and pyt_clock_dif > max_clock_dif:
logger.error("Could not identify audio stream clock.")
running.clear()
self.audio_src = "No Audio"
return
elif uvc_clock_dif > pyt_clock_dif:
logger.info(
"Audio stream uses time.time() as clock (Δ {}s)".format(pyt_clock_dif)
)
clock_differences = self.g_pool.get_now() - time()
else:
logger.info("Audio stream uses uvc.get_time_monotonic() as clock")
clock_differences = 0
def close_recording():
# Bind nonlocal variables, https://www.python.org/dev/peps/pep-3104/
nonlocal out_container, out_stream, in_stream, in_frame_size, timestamps, out_frame_num
if out_container is not None:
timestamps.append(timestamp)
audio_frame.pts = None
out_packets = [out_stream.encode(audio_frame)]
while out_packets[-1]:
out_packets.append(out_stream.encode(None))
for out_packet in out_packets:
if out_packet is not None:
out_container.mux(out_packet)
out_container.close()
out_frame_num = out_stream.frames
in_frame_rate = in_stream.rate
# in_stream.frame_size does not return the correct value.
out_frame_size = out_stream.frame_size
out_frame_rate = out_stream.rate
new_ts_idx = (
np.arange(0, out_frame_num * out_frame_size, out_frame_size)
/ out_frame_rate
)
if in_frame_rate != out_frame_rate:
old_ts_idx = (
np.arange(0, len(timestamps) * in_frame_size, in_frame_size)
/ in_frame_rate
)
interpolate = interp1d(
old_ts_idx,
timestamps,
bounds_error=False,
fill_value="extrapolate",
)
new_ts = interpolate(new_ts_idx)
else:
new_ts = timestamps[0] + new_ts_idx
ts_loc = os.path.join(self.rec_dir, "audio_timestamps.npy")
np.save(ts_loc, new_ts)
out_container = None
out_stream = None
timestamps = None
for packet in in_container.demux(in_stream):
# ffmpeg timestamps - in_stream.startime = packte pts relative to startime
# multiply with stream_timebase to get seconds
# add start time of this stream in pupil time unadjusted
# finally add pupil timebase offset to adjust for settable timebase.
for audio_frame in packet.decode():
timestamp = (
audio_frame.pts * in_stream.time_base
+ clock_differences
- self.g_pool.timebase.value
)
if recording.is_set():
if out_container is None:
rec_file = os.path.join(self.rec_dir, "audio.mp4")
out_container = av.open(rec_file, "w")
out_stream = out_container.add_stream(
"aac", rate=in_stream.rate
)
out_frame_num = 0
in_frame_size = (
audio_frame.samples
) # set here to make sure full packet size is used
timestamps = []
timestamps.append(timestamp)
audio_frame.pts = None
out_packets = [out_stream.encode(audio_frame)]
for out_packet in out_packets:
if out_packet is not None:
out_container.mux(out_packet)
elif out_container is not None:
# recording stopped
close_recording()
if not running.is_set():
close_recording()
return
close_recording()
self.audio_src = "No Audio"
running.clear() # in_stream stopped yielding packets
class Recorder(Plugin):
"""Capture Recorder"""
def __init__(self,g_pool,session_name = get_auto_name(),rec_dir=None, user_info={'name':'','additional_field':'change_me'},info_menu_conf={},show_info_menu=False, record_eye = False, audio_src = 'No Audio',raw_jpeg=True):
super(Recorder, self).__init__(g_pool)
#update name if it was autogenerated.
if session_name.startswith('20') and len(session_name)==10:
session_name = get_auto_name()
base_dir = self.g_pool.user_dir.rsplit(os.path.sep,1)[0]
default_rec_dir = os.path.join(base_dir,'recordings')
if rec_dir and rec_dir != default_rec_dir and self.verify_path(rec_dir):
self.rec_dir = rec_dir
else:
try:
os.makedirs(default_rec_dir)
except OSError as e:
if e.errno != errno.EEXIST:
logger.error("Could not create Rec dir")
raise e
else:
logger.info('Created standard Rec dir at "%s"'%default_rec_dir)
self.rec_dir = default_rec_dir
self.raw_jpeg = raw_jpeg
self.order = .9
self.record_eye = record_eye
self.session_name = session_name
self.audio_devices_dict = Audio_Input_Dict()
if audio_src in self.audio_devices_dict.keys():
self.audio_src = audio_src
else:
self.audio_src = 'No Audio'
self.running = False
self.menu = None
self.button = None
self.user_info = user_info
self.show_info_menu = show_info_menu
self.info_menu = None
self.info_menu_conf = info_menu_conf
def get_init_dict(self):
d = {}
d['record_eye'] = self.record_eye
d['audio_src'] = self.audio_src
d['session_name'] = self.session_name
d['user_info'] = self.user_info
d['info_menu_conf'] = self.info_menu_conf
d['show_info_menu'] = self.show_info_menu
d['rec_dir'] = self.rec_dir
d['raw_jpeg'] = self.raw_jpeg
return d
def init_gui(self):
self.menu = ui.Growing_Menu('Recorder')
self.g_pool.sidebar.insert(3,self.menu)
self.menu.append(ui.Info_Text('Pupil recordings are saved like this: "path_to_recordings/recording_session_name/nnn" where "nnn" is an increasing number to avoid overwrites. You can use "/" in your session name to create subdirectories.'))
self.menu.append(ui.Info_Text('Recordings are saved to "~/pupil_recordings". You can change the path here but note that invalid input will be ignored.'))
self.menu.append(ui.Text_Input('rec_dir',self,setter=self.set_rec_dir,label='Path to recordings'))
self.menu.append(ui.Text_Input('session_name',self,setter=self.set_session_name,label='Recording session name'))
self.menu.append(ui.Switch('show_info_menu',self,on_val=True,off_val=False,label='Request additional user info'))
self.menu.append(ui.Selector('raw_jpeg',self,selection = [True,False], labels=["bigger file, less CPU", "smaller file, more CPU"],label='Compression'))
self.menu.append(ui.Info_Text('Recording the raw eye video is optional. We use it for debugging.'))
self.menu.append(ui.Switch('record_eye',self,on_val=True,off_val=False,label='Record eye'))
self.menu.append(ui.Selector('audio_src',self, selection=self.audio_devices_dict.keys(),label='Audio Source'))
self.button = ui.Thumb('running',self,setter=self.toggle,label='Record',hotkey='r')
self.button.on_color[:] = (1,.0,.0,.8)
self.g_pool.quickbar.insert(1,self.button)
def deinit_gui(self):
if self.menu:
self.g_pool.sidebar.remove(self.menu)
self.menu = None
if self.button:
self.g_pool.quickbar.remove(self.button)
self.button = None
def toggle(self, _=None):
if self.running:
self.notify_all( {'subject':'should_stop_recording','network_propagate':True} )
else:
self.notify_all( {'subject':'should_start_recording','session_name':self.session_name,'network_propagate':True} )
def on_notify(self,notification):
# notification wants to be recorded
if notification.get('record',False) and self.running:
self.data['notifications'].append(notification)
# Notificatio to start recording
elif notification['subject'] == 'should_start_recording':
if self.running:
logger.info('Recording already running!')
else:
if notification.get("session_name",""):
self.set_session_name(notification["session_name"])
self.start()
# Remote has stopped recording, we should stop as well.
elif notification['subject'] == 'should_stop_recording':
if self.running:
self.stop()
else:
logger.info('Recording already stopped!')
def get_rec_time_str(self):
rec_time = gmtime(time()-self.start_time)
return strftime("%H:%M:%S", rec_time)
def start(self):
self.timestamps = []
self.data = {'pupil_positions':[],'gaze_positions':[],'notifications':[]}
self.pupil_diameter = [] #addBy Changsoon
self.gaze_point_z = [] #addBy Changsoon
self.gaze_point_x = [] #addBy Changsoon
self.gaze_point_y = [] #addBy Changsoon
self.frame_count = 0
self.running = True
self.menu.read_only = True
self.start_time = time()
session = os.path.join(self.rec_dir, self.session_name)
try:
os.makedirs(session)
logger.debug("Created new recordings session dir %s"%session)
except:
logger.debug("Recordings session dir %s already exists, using it." %session)
# set up self incrementing folder within session folder
counter = 0
while True:
self.rec_path = os.path.join(session, "%03d/" % counter)
try:
os.mkdir(self.rec_path)
logger.debug("Created new recording dir %s"%self.rec_path)
break
except:
logger.debug("We dont want to overwrite data, incrementing counter & trying to make new data folder")
counter += 1
self.meta_info_path = os.path.join(self.rec_path, "info.csv")
with open(self.meta_info_path, 'w') as f:
f.write("Recording Name\t"+self.session_name+ "\n")
f.write("Start Date\t"+ strftime("%d.%m.%Y", localtime(self.start_time))+ "\n")
f.write("Start Time\t"+ strftime("%H:%M:%S", localtime(self.start_time))+ "\n")
if self.audio_src != 'No Audio':
audio_path = os.path.join(self.rec_path, "world.wav")
self.audio_writer = Audio_Capture(audio_path,self.audio_devices_dict[self.audio_src])
else:
self.audio_writer = None
if self.raw_jpeg and self.g_pool.capture.jpeg_support:
self.video_path = os.path.join(self.rec_path, "world.mp4")
self.writer = JPEG_Writer(self.video_path,self.g_pool.capture.frame_rate)
else:
self.video_path = os.path.join(self.rec_path, "world.mp4")
self.writer = AV_Writer(self.video_path,fps=self.g_pool.capture.frame_rate)
# positions path to eye process
if self.record_eye:
for alive, pipe in zip(self.g_pool.eyes_are_alive,self.g_pool.eye_pipes):
if alive.value:
pipe.send( ('Rec_Start',(self.rec_path,self.raw_jpeg) ) )
if self.show_info_menu:
self.open_info_menu()
logger.info("Started Recording.")
self.notify_all( {'subject':'rec_started','rec_path':self.rec_path,'session_name':self.session_name,'network_propagate':True} )
def open_info_menu(self):
self.info_menu = ui.Growing_Menu('additional Recording Info',size=(300,300),pos=(300,300))
self.info_menu.configuration = self.info_menu_conf
def populate_info_menu():
self.info_menu.elements[:-2] = []
for name in self.user_info.iterkeys():
self.info_menu.insert(0,ui.Text_Input(name,self.user_info))
def set_user_info(new_string):
self.user_info = new_string
populate_info_menu()
populate_info_menu()
self.info_menu.append(ui.Info_Text('Use the *user info* field to add/remove additional fields and their values. The format must be a valid Python dictionary. For example -- {"key":"value"}. You can add as many fields as you require. Your custom fields will be saved for your next session.'))
self.info_menu.append(ui.Text_Input('user_info',self,setter=set_user_info,label="User info"))
self.g_pool.gui.append(self.info_menu)
def close_info_menu(self):
if self.info_menu:
self.info_menu_conf = self.info_menu.configuration
self.g_pool.gui.remove(self.info_menu)
self.info_menu = None
def update(self,frame,events):
if self.running:
for key,data in events.iteritems():
if key not in ('dt'):
try:
self.data[key] += data
if (key == 'pupil_positions'):
try:
pupilDict = data[0]
self.pupil_diameter.append(pupilDict.get('diameter'))
except:
print('error');
elif (key == 'gaze_positions'):
try:
gazeDict = data[0]
self.gaze_point_x.append(gazeDict.get('norm_pos')[0])
self.gaze_point_y.append(gazeDict.get('norm_pos')[1])
self.gaze_point_z.append(gazeDict.get('gaze_point_3d')[2])
except:
print('error')
print(self.pupil_diameter)
print(self.gaze_point_x)
print(self.gaze_point_y)
cv2.namedWindow("window")
#image process (OCR) find the condion that show use OCR
except Exception as error:
print(error)
class Audio_Capture(Plugin):
"""docstring for Audio_Capture"""
def __init__(self, g_pool, audio_src='No Audio'):
super().__init__(g_pool)
self.audio_devices_dict = Audio_Input_Dict()
if audio_src in list(self.audio_devices_dict.keys()):
self.audio_src = audio_src
else:
self.audio_src = 'No Audio'
self.thread = None
self.running = Event()
self.audio_container = None
self.audio_out_stream = None
self.queue = queue.Queue()
self.start_capture(self.audio_src)
def recent_events(self, events):
audio_packets = []
while True:
try:
packet = self.queue.get_nowait()
except queue.Empty:
break
audio_packets.append(packet)
events['audio_packets'] = audio_packets
if self.audio_container is not None:
for packet in audio_packets:
self.write_audio_packet(packet)
def init_gui(self):
self.menu = ui.Growing_Menu('Audio Capture')
self.menu.collapsed = True
self.g_pool.sidebar.append(self.menu)
def close():
self.alive = False
help_str = 'Creates events for audio input.'
self.menu.append(ui.Button('Close', close))
self.menu.append(ui.Info_Text(help_str))
def audio_dev_getter():
# fetch list of currently available
self.audio_devices_dict = Audio_Input_Dict()
devices = list(self.audio_devices_dict.keys())
return devices, devices
self.menu.append(ui.Selector('audio_src', self,
selection_getter=audio_dev_getter,
label='Audio Source',
setter=self.start_capture))
self.menu.append(ui.Info_Text(NOT_REC_STR))
def get_init_dict(self):
return {'audio_src': self.audio_src}
def deinit_gui(self):
if self.menu:
self.g_pool.sidebar.remove(self.menu)
self.menu = None
def cleanup(self):
if self.audio_container is not None:
self.close_audio_recording()
self.running.clear()
self.deinit_gui()
if self.thread and self.thread.is_alive():
self.thread.join(timeout=1)
def on_notify(self, notification):
if notification['subject'] == 'recording.started':
if self.running.is_set() and self.audio_container is None:
self.rec_dir = notification['rec_path']
rec_file = os.path.join(self.rec_dir, 'audio.wav')
self.audio_container = av.open(rec_file, 'w')
self.timestamps = []
self.menu[-2].read_only = True
del self.menu[-1]
self.menu.append(ui.Info_Text(REC_STR))
elif not self.running.is_set():
logger.warning('Recording was started without an active audio capture')
else:
logger.warning('Audio is already being recorded')
elif notification['subject'] == 'recording.stopped':
if self.audio_container is not None and self.audio_out_stream is not None:
self.close_audio_recording()
def close_audio_recording(self):
self.audio_container.close()
ts_loc = os.path.join(self.rec_dir, 'audio_timestamps.npy')
np.save(ts_loc, np.asarray(self.timestamps))
self.timestamps = None
self.audio_out_stream = None
self.audio_container = None
self.menu[-2].read_only = False
del self.menu[-1]
self.menu.append(ui.Info_Text(NOT_REC_STR))
def write_audio_packet(self, packet):
# Test if audio outstream has been initialized
if self.audio_out_stream is None:
try:
self.audio_out_stream = self.audio_container.add_stream(template=packet.stream)
except ValueError as e:
# packet.stream codec is not supported in target container.
logger.error('Failed to create audio stream. Aborting recording.')
logger.debug('Reason: {}'.format(e))
self.close_audio_recording()
self.timestamps.append(packet.timestamp)
self.audio_container.mux(packet)
def start_capture(self, audio_src):
if self.thread and self.thread.is_alive():
if self.audio_src == audio_src:
return # capture is already running for our selected source
# else stop current capture gracefully
self.running.clear()
logger.debug('Closing capture for "{}"'.format(self.audio_src))
self.thread.join(timeout=1)
self.thread = None
if audio_src not in self.audio_devices_dict:
logger.warning('Selected audio source is not available anymore')
return
self.audio_src = audio_src
if audio_src == 'No Audio':
return
self.running.set()
self.thread = Thread(target=self.capture_thread,
args=(self.audio_devices_dict[audio_src], self.running))
self.thread.start()
def capture_thread(self, audio_src, running):
try:
if platform.system() == "Darwin":
in_container = av.open('none:{}'.format(audio_src), format="avfoundation")
elif platform.system() == "Linux":
in_container = av.open('hw:{}'.format(audio_src), format="alsa")
else:
raise av.AVError('Platform does not support audio capture.')
except av.AVError:
running.clear()
return
in_stream = None
for stream in in_container.streams:
if stream.type == 'audio':
in_stream = stream
break
if not in_stream:
logger.warning('No audio stream found for selected device.')
running.clear()
return
stream_start_ts = self.g_pool.get_now()
for packet in in_container.demux(in_stream):
try:
# ffmpeg timestamps - in_stream.startime = packte pts relative to startime
# multiply with stream_timebase to get seconds
# add start time of this stream in pupil time unadjusted
# finally add pupil timebase offset to adjust for settable timebase.
packet.timestamp = (packet.pts - in_stream.start_time) * in_stream.time_base + stream_start_ts - self.g_pool.timebase.value
self.queue.put_nowait(packet)
except queue.Full:
pass # drop packet
if not running.is_set():
return
self.audio_src = 'No Audio'
running.clear() # in_stream stopped yielding packets
class Recorder(Plugin):
"""Capture Recorder"""
def __init__(self,
g_pool,
session_name=get_auto_name(),
rec_dir=None,
user_info={
'name': '',
'additional_field': 'change_me'
},
info_menu_conf={},
show_info_menu=False,
record_eye=False,
audio_src='No Audio',
raw_jpeg=False):
super(Recorder, self).__init__(g_pool)
self.states = ['start', 'want', 'pick']
self.all_qr = {
'termo1': 75,
'light1': False,
'light2': False,
'light3': False
}
self.this_state = 'start'
self.start_state = time()
self.qr_codes = {}
self.gaze_x = -1
self.gaze_y = -1
self.avg_col = 0
self.choice = ""
#update name if it was autogenerated.
if session_name.startswith('20') and len(session_name) == 10:
session_name = get_auto_name()
base_dir = self.g_pool.user_dir.rsplit(os.path.sep, 1)[0]
default_rec_dir = os.path.join(base_dir, 'recordings')
if rec_dir and rec_dir != default_rec_dir and self.verify_path(
rec_dir):
self.rec_dir = rec_dir
else:
try:
os.makedirs(default_rec_dir)
except OSError as e:
if 'File exists' in '%s' % e:
pass
else:
logger.error("Could not create Rec dir")
raise e
else:
logger.info('Created standard Rec dir at "%s"' %
default_rec_dir)
self.rec_dir = default_rec_dir
# if rec_dir and rec_dir != default_rec_dir and self.verify_path(rec_dir):
# self.rec_dir = rec_dir
# else:
# #lets make a rec dir next to the user dir
# base_dir = self.g_pool.user_dir.rsplit(os.path.sep,1)[0]
# self.rec_dir = os.path.join(base_dir,'recordings')
# if not os.path.isdir(self.rec_dir):
# os.mkdir(self.rec_dir)
self.raw_jpeg = raw_jpeg
self.order = .9
self.record_eye = record_eye
self.session_name = session_name
self.audio_devices_dict = Audio_Input_Dict()
if audio_src in self.audio_devices_dict.keys():
self.audio_src = audio_src
else:
self.audio_src = 'No Audio'
self.running = False
self.menu = None
self.button = None
self.user_info = user_info
self.show_info_menu = show_info_menu
self.info_menu = None
self.info_menu_conf = info_menu_conf
self.height, self.width = self.g_pool.capture.frame_size
def get_init_dict(self):
d = {}
d['record_eye'] = self.record_eye
d['audio_src'] = self.audio_src
d['session_name'] = self.session_name
d['user_info'] = self.user_info
d['info_menu_conf'] = self.info_menu_conf
d['show_info_menu'] = self.show_info_menu
d['rec_dir'] = self.rec_dir
d['raw_jpeg'] = self.raw_jpeg
# if self.menu:
# d['menu_conf'] = self.menu.configuration
# else:
# d['menu_conf'] = self.menu_conf
return d
def init_gui(self):
self.menu = ui.Growing_Menu('Recorder')
# self.menu.configuration = self.menu_conf
self.g_pool.sidebar.insert(3, self.menu)
self.menu.append(
ui.Info_Text(
'Pupil recordings are saved like this: "path_to_recordings/recording_session_name/nnn" where "nnn" is an increasing number to avoid overwrites. You can use "/" in your session name to create subdirectories.'
))
self.menu.append(
ui.Info_Text(
'Recordings are saved to "~/pupil_recordings". You can change the path here but note that invalid input will be ignored.'
))
self.menu.append(
ui.Text_Input('rec_dir',
self,
setter=self.set_rec_dir,
label='Path to recordings'))
self.menu.append(
ui.Text_Input('session_name',
self,
setter=self.set_session_name,
label='Recording session name'))
self.menu.append(
ui.Switch('show_info_menu',
self,
on_val=True,
off_val=False,
label='Request additional user info'))
self.menu.append(
ui.Selector(
'raw_jpeg',
self,
selection=[True, False],
labels=["bigger file, less CPU", "smaller file, more CPU"],
label='compression'))
self.menu.append(
ui.Info_Text(
'Recording the raw eye video is optional. We use it for debugging.'
))
self.menu.append(
ui.Switch('record_eye',
self,
on_val=True,
off_val=False,
label='Record eye'))
self.menu.append(
ui.Selector('audio_src',
self,
selection=self.audio_devices_dict.keys()))
self.button = ui.Thumb('running',
self,
setter=self.toggle,
label='QR',
hotkey='r')
self.button.on_color[:] = (1, .0, .0, .8)
self.g_pool.quickbar.insert(1, self.button)
def deinit_gui(self):
if self.menu:
# self.menu_conf = self.menu.configuration
self.g_pool.sidebar.remove(self.menu)
self.menu = None
if self.button:
self.g_pool.quickbar.remove(self.button)
self.button = None
def toggle(self, _=None):
if self.running:
self.stop()
else:
self.start()
def get_rec_time_str(self):
rec_time = gmtime(time() - self.start_time)
return strftime("%H:%M:%S", rec_time)
def start(self):
self.timestamps = []
# add self.data
self.data = {'pupil_positions': [], 'gaze_positions': []}
self.pupil_list = []
self.gaze_list = []
self.frame_count = 0
self.running = True
self.menu.read_only = True
self.start_time = time()
self.start_state = time()
session = os.path.join(self.rec_dir, self.session_name)
try:
os.makedirs(session)
logger.debug("Created new recordings session dir %s" % session)
except:
logger.debug(
"Recordings session dir %s already exists, using it." %
session)
# set up self incrementing folder within session folder
counter = 0
while True:
self.rec_path = os.path.join(session, "%03d/" % counter)
try:
os.mkdir(self.rec_path)
logger.debug("Created new recording dir %s" % self.rec_path)
break
except:
logger.debug(
"We dont want to overwrite data, incrementing counter & trying to make new data folder"
)
counter += 1
self.meta_info_path = os.path.join(self.rec_path, "info.csv")
with open(self.meta_info_path, 'w') as f:
f.write("Recording Name\t" + self.session_name + "\n")
f.write("Start Date\t" +
strftime("%d.%m.%Y", localtime(self.start_time)) + "\n")
f.write("Start Time\t" +
strftime("%H:%M:%S", localtime(self.start_time)) + "\n")
if self.audio_src != 'No Audio':
audio_path = os.path.join(self.rec_path, "world.wav")
self.audio_writer = Audio_Capture(
self.audio_devices_dict[self.audio_src], audio_path)
else:
self.audio_writer = None
if self.raw_jpeg and "uvc_capture" in str(
self.g_pool.capture.__class__):
self.video_path = os.path.join(self.rec_path, "world.mp4")
self.writer = JPEG_Writer(self.video_path,
int(self.g_pool.capture.frame_rate))
# elif 1:
# self.writer = av_writer.AV_Writer(self.video_path)
else:
self.video_path = os.path.join(self.rec_path, "world.mkv")
self.writer = CV_Writer(self.video_path,
float(self.g_pool.capture.frame_rate),
self.g_pool.capture.frame_size)
# positions path to eye process
if self.record_eye:
for tx in self.g_pool.eye_tx:
tx.send((self.rec_path, self.raw_jpeg))
if self.show_info_menu:
self.open_info_menu()
# self.video_path = os.path.join(self.rec_path, "world.mkv")
# self.writer = cv2.VideoWriter(self.video_path, int(cv2.cv.CV_FOURCC(*'DIVX')), float(self.g_pool.capture.frame_rate), (1280,720))
# # positions path to eye process
# if self.record_eye:
# for tx in self.g_pool.eye_tx:
# tx.send(self.rec_path)
# if self.show_info_menu:
# self.open_info_menu()
###############
def open_info_menu(self):
self.info_menu = ui.Growing_Menu('additional Recording Info',
size=(300, 300),
pos=(300, 300))
self.info_menu.configuration = self.info_menu_conf
def populate_info_menu():
self.info_menu.elements[:-2] = []
for name in self.user_info.iterkeys():
self.info_menu.insert(0, ui.Text_Input(name, self.user_info))
def set_user_info(new_string):
self.user_info = new_string
populate_info_menu()
populate_info_menu()
self.info_menu.append(
ui.Info_Text(
'Use the *user info* field to add/remove additional fields and their values. The format must be a valid Python dictionary. For example -- {"key":"value"}. You can add as many fields as you require. Your custom fields will be saved for your next session.'
))
self.info_menu.append(
ui.Text_Input('user_info',
self,
setter=set_user_info,
label="User info"))
self.g_pool.gui.append(self.info_menu)
def close_info_menu(self):
if self.info_menu:
self.info_menu_conf = self.info_menu.configuration
self.g_pool.gui.remove(self.info_menu)
self.info_menu = None
def update(self, frame, events):
if self.running:
# update data
self.data['pupil_positions'] += events['pupil_positions']
self.data['gaze_positions'] += events['gaze_positions']
#cv2.putText(frame.img, "Frame %s"%self.frame_count,(200,200), cv2.FONT_HERSHEY_SIMPLEX,1,(255,100,100)
cv2.putText(frame.img, self.this_state, (200, 100),
cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 100, 100))
if self.this_state == 'pick':
cv2.putText(frame.img, self.choice, (300, 100),
cv2.FONT_HERSHEY_SIMPLEX, 1, (100, 100, 255))
QRs = qr_detect(frame)
for p in events['pupil_positions']:
pupil_pos = p['timestamp'], p['confidence'], p['id'], p[
'norm_pos'][0], p['norm_pos'][1], p['diameter']
self.pupil_list.append(pupil_pos)
c = 0
avg_x = -1
avg_y = -1
for g in events.get('gaze_positions', []):
gaze_pos = g['timestamp'], g['confidence'], g['norm_pos'][
0], g['norm_pos'][1]
avg_x = (avg_x * c + g['norm_pos'][0]) / (c + 1)
avg_y = (avg_y * c + g['norm_pos'][0]) / (c + 1)
c = c + 1
self.gaze_list.append(gaze_pos)
self.timestamps.append(frame.timestamp)
avg_col = 0
self.stateT(frame, QRs, avg_x, avg_y, avg_col)
self.writer.write_video_frame(frame)
self.frame_count += 1
self.button.status_text = self.get_rec_time_str()
def stop(self):
#explicit release of VideoWriter
self.writer.release()
self.writer = None
if self.record_eye:
for tx in self.g_pool.eye_tx:
try:
tx.send(None)
except:
logger.warning(
"Could not stop eye-recording. Please report this bug!"
)
save_object(self.data, os.path.join(self.rec_path, "pupil_data"))
gaze_list_path = os.path.join(self.rec_path, "gaze_positions.npy")
np.save(gaze_list_path, np.asarray(self.gaze_list))
pupil_list_path = os.path.join(self.rec_path, "pupil_positions.npy")
np.save(pupil_list_path, np.asarray(self.pupil_list))
timestamps_path = os.path.join(self.rec_path, "world_timestamps.npy")
ts = sanitize_timestamps(np.array(self.timestamps))
np.save(timestamps_path, ts)
try:
copy2(os.path.join(self.g_pool.user_dir, "surface_definitions"),
os.path.join(self.rec_path, "surface_definitions"))
except:
logger.info(
"No surface_definitions data found. You may want this if you do marker tracking."
)
try:
copy2(os.path.join(self.g_pool.user_dir, "cal_pt_cloud.npy"),
os.path.join(self.rec_path, "cal_pt_cloud.npy"))
except:
logger.warning(
"No calibration data found. Please calibrate first.")
try:
copy2(os.path.join(self.g_pool.user_dir, "camera_matrix.npy"),
os.path.join(self.rec_path, "camera_matrix.npy"))
copy2(os.path.join(self.g_pool.user_dir, "dist_coefs.npy"),
os.path.join(self.rec_path, "dist_coefs.npy"))
except:
logger.info("No camera intrinsics found.")
try:
with open(self.meta_info_path, 'a') as f:
f.write("Duration Time\t" + self.get_rec_time_str() + "\n")
if self.g_pool.binocular:
f.write("Eye Mode\tbinocular\n")
else:
f.write("Eye Mode\tmonocular\n")
f.write("Duration Time\t" + self.get_rec_time_str() + "\n")
f.write("World Camera Frames\t" + str(self.frame_count) + "\n")
f.write("World Camera Resolution\t" + str(self.width) + "x" +
str(self.height) + "\n")
f.write("Capture Software Version\t%s\n" % self.g_pool.version)
if platform.system() == "Windows":
username = os.environ["USERNAME"]
sysname, nodename, release, version, machine, _ = platform.uname(
)
else:
username = os.getlogin()
try:
sysname, nodename, release, version, machine = os.uname(
)
except:
sysname, nodename, release, version, machine = sys.platform, None, None, None, None
f.write("User\t" + username + "\n")
f.write("Platform\t" + sysname + "\n")
f.write("Machine\t" + nodename + "\n")
f.write("Release\t" + release + "\n")
f.write("Version\t" + version + "\n")
except Exception:
logger.exception(
"Could not save metadata. Please report this bug!")
try:
with open(os.path.join(self.rec_path, "user_info.csv"), 'w') as f:
for name, val in self.user_info.iteritems():
f.write("%s\t%s\n" % (name, val))
except Exception:
logger.exception(
"Could not save userdata. Please report this bug!")
self.close_info_menu()
if self.audio_writer:
self.audio_writer = None
self.running = False
self.menu.read_only = False
self.button.status_text = ''
def cleanup(self):
"""gets called when the plugin get terminated.
either volunatily or forced.
"""
if self.running:
self.stop()
self.deinit_gui()
def verify_path(self, val):
try:
n_path = os.path.expanduser(val)
logger.debug("Expanded user path.")
except:
n_path = val
if not n_path:
logger.waring("Please specify a path.")
return False
elif not os.path.isdir(n_path):
logger.warning("This is not a valid path.")
return False
elif not writable_dir(n_path):
logger.warning("Do not have write access to '%s'." % n_path)
return False
else:
return n_path
def set_rec_dir(self, val):
# try:
# n_path = os.path.expanduser(val)
# logger.debug("Expanded user path.")
# except:
# n_path = val
# if not n_path:
# logger.warning("Please specify a path.")
# elif not os.path.isdir(n_path):
# logger.warning("This is not a valid path.")
# # elif not os.access(n_path, os.W_OK):
# elif not writable_dir(n_path):
# logger.warning("Do not have write access to '%s'."%n_path)
# else:
# self.rec_dir = n_path
n_path = self.verify_path(val)
if n_path:
self.rec_dir = n_path
def set_session_name(self, val):
if not val:
self.session_name = get_auto_name()
else:
if '/' in val:
logger.warning(
'You session name with create one or more subdirectories')
self.session_name = val
def stateT(self, frame, QRs, agaze_x, agaze_y, avg_col):
if time() - self.start_time < 1:
self.gaze_x = agaze_x
self.gaze_y = agaze_y
if self.this_state == 'start':
if time() - self.start_state < 2:
self.qr_codes = {}
return True
if self.Nochange_gaze(agaze_x, agaze_y, avg_col):
self.this_state = 'want'
self.start_state = time()
self.gaze_x = agaze_x
self.gaze_y = agaze_y
return True
if self.this_state == 'want':
if time() - self.start_state < 2:
self.qr_codes.update(QRs)
return True
if self.Nochange_gaze(agaze_x, agaze_y, avg_col) and self.qr_codes:
self.this_state = 'pick'
self.choice = self.choose(frame)
#print self.qr_codes, self.choice
else:
self.this_state = 'start'
self.start_state = time()
self.gaze_x = agaze_x
self.gaze_y = agaze_y
return True
if self.this_state == 'pick':
if time() - self.start_state < 4:
return True
if self.Nochange_gaze(agaze_x, agaze_y, avg_col):
del self.qr_codes[self.choice]
if not self.qr_codes:
self.this_state = 'start'
else:
self.choice = self.choose(frame)
#print self.choice
else:
print self.choice, " chosen"
self.this_state = 'start'
self.start_state = time()
self.gaze_x = agaze_x
self.gaze_y = agaze_y
return True
def Nochange_gaze(self, agaze_x, agaze_y, avg_col):
#print self.gaze_x, self.gaze_y, agaze_x, agaze_y
return (abs(self.gaze_x - agaze_x) < .1
and abs(self.gaze_y - agaze_y) < .1)
def choose(self, frame):
h = frame.img.shape[0]
w = frame.img.shape[1]
gaze_x = self.gaze_x
gaze_y = self.gaze_y
if (gaze_x < 0 or gaze_y < 0):
gaze_x = .5
gaz_y = .5
k = self.qr_codes.keys()
chosen = k[0]
min_dist = 1000000
for name, mc in self.qr_codes.items():
dis = (mc[0] - gaze_x * w) * (mc[0] - gaze_x * w) + (
mc[1] - gaze_y * h) * (mc[1] - gaze_y * h)
if dis < min_dist:
min_dist = dis
chosen = name
return chosen
class Recorder(Plugin):
"""Capture Recorder"""
def __init__(self,g_pool,session_name = get_auto_name(),rec_dir=None, user_info={'name':'','additional_field':'change_me'},info_menu_conf={},show_info_menu=False, record_eye = False, audio_src = 'No Audio', raw_jpeg=False):
super(Recorder, self).__init__(g_pool)
self.states = ['start', 'want', 'pick']
self.all_qr = {'termo1' : 75, 'light1' : False, 'light2' : False, 'light3': False}
self.this_state = 'start'
self.start_state = time()
self.qr_codes = {}
self.gaze_x = -1
self.gaze_y = -1
self.avg_col = 0
self.choice = ""
#update name if it was autogenerated.
if session_name.startswith('20') and len(session_name)==10:
session_name = get_auto_name()
base_dir = self.g_pool.user_dir.rsplit(os.path.sep,1)[0]
default_rec_dir = os.path.join(base_dir, 'recordings')
if rec_dir and rec_dir != default_rec_dir and self.verify_path(rec_dir):
self.rec_dir = rec_dir
else:
try:
os.makedirs(default_rec_dir)
except OSError as e:
if 'File exists' in '%s'%e:
pass
else:
logger.error("Could not create Rec dir")
raise e
else:
logger.info('Created standard Rec dir at "%s"'%default_rec_dir)
self.rec_dir = default_rec_dir
# if rec_dir and rec_dir != default_rec_dir and self.verify_path(rec_dir):
# self.rec_dir = rec_dir
# else:
# #lets make a rec dir next to the user dir
# base_dir = self.g_pool.user_dir.rsplit(os.path.sep,1)[0]
# self.rec_dir = os.path.join(base_dir,'recordings')
# if not os.path.isdir(self.rec_dir):
# os.mkdir(self.rec_dir)
self.raw_jpeg = raw_jpeg
self.order = .9
self.record_eye = record_eye
self.session_name = session_name
self.audio_devices_dict = Audio_Input_Dict()
if audio_src in self.audio_devices_dict.keys():
self.audio_src = audio_src
else:
self.audio_src = 'No Audio'
self.running = False
self.menu = None
self.button = None
self.user_info = user_info
self.show_info_menu = show_info_menu
self.info_menu = None
self.info_menu_conf = info_menu_conf
self.height, self.width = self.g_pool.capture.frame_size
def get_init_dict(self):
d = {}
d['record_eye'] = self.record_eye
d['audio_src'] = self.audio_src
d['session_name'] = self.session_name
d['user_info'] = self.user_info
d['info_menu_conf'] = self.info_menu_conf
d['show_info_menu'] = self.show_info_menu
d['rec_dir'] = self.rec_dir
d['raw_jpeg'] = self.raw_jpeg
# if self.menu:
# d['menu_conf'] = self.menu.configuration
# else:
# d['menu_conf'] = self.menu_conf
return d
def init_gui(self):
self.menu = ui.Growing_Menu('Recorder')
# self.menu.configuration = self.menu_conf
self.g_pool.sidebar.insert(3,self.menu)
self.menu.append(ui.Info_Text('Pupil recordings are saved like this: "path_to_recordings/recording_session_name/nnn" where "nnn" is an increasing number to avoid overwrites. You can use "/" in your session name to create subdirectories.'))
self.menu.append(ui.Info_Text('Recordings are saved to "~/pupil_recordings". You can change the path here but note that invalid input will be ignored.'))
self.menu.append(ui.Text_Input('rec_dir',self,setter=self.set_rec_dir,label='Path to recordings'))
self.menu.append(ui.Text_Input('session_name',self,setter=self.set_session_name,label='Recording session name'))
self.menu.append(ui.Switch('show_info_menu',self,on_val=True,off_val=False,label='Request additional user info'))
self.menu.append(ui.Selector('raw_jpeg',self,selection = [True,False], labels=["bigger file, less CPU", "smaller file, more CPU"],label='compression'))
self.menu.append(ui.Info_Text('Recording the raw eye video is optional. We use it for debugging.'))
self.menu.append(ui.Switch('record_eye',self,on_val=True,off_val=False,label='Record eye'))
self.menu.append(ui.Selector('audio_src',self, selection=self.audio_devices_dict.keys()))
self.button = ui.Thumb('running',self,setter=self.toggle,label='QR',hotkey='r')
self.button.on_color[:] = (1,.0,.0,.8)
self.g_pool.quickbar.insert(1,self.button)
def deinit_gui(self):
if self.menu:
# self.menu_conf = self.menu.configuration
self.g_pool.sidebar.remove(self.menu)
self.menu = None
if self.button:
self.g_pool.quickbar.remove(self.button)
self.button = None
def toggle(self, _=None):
if self.running:
self.stop()
else:
self.start()
def get_rec_time_str(self):
rec_time = gmtime(time()-self.start_time)
return strftime("%H:%M:%S", rec_time)
def start(self):
self.timestamps = []
# add self.data
self.data = {'pupil_positions':[], 'gaze_positions': []}
self.pupil_list = []
self.gaze_list = []
self.frame_count = 0
self.running = True
self.menu.read_only = True
self.start_time = time()
self.start_state = time()
session = os.path.join(self.rec_dir, self.session_name)
try:
os.makedirs(session)
logger.debug("Created new recordings session dir %s"%session)
except:
logger.debug("Recordings session dir %s already exists, using it." %session)
# set up self incrementing folder within session folder
counter = 0
while True:
self.rec_path = os.path.join(session, "%03d/" % counter)
try:
os.mkdir(self.rec_path)
logger.debug("Created new recording dir %s"%self.rec_path)
break
except:
logger.debug("We dont want to overwrite data, incrementing counter & trying to make new data folder")
counter += 1
self.meta_info_path = os.path.join(self.rec_path, "info.csv")
with open(self.meta_info_path, 'w') as f:
f.write("Recording Name\t"+self.session_name+ "\n")
f.write("Start Date\t"+ strftime("%d.%m.%Y", localtime(self.start_time))+ "\n")
f.write("Start Time\t"+ strftime("%H:%M:%S", localtime(self.start_time))+ "\n")
if self.audio_src != 'No Audio':
audio_path = os.path.join(self.rec_path, "world.wav")
self.audio_writer = Audio_Capture(self.audio_devices_dict[self.audio_src],audio_path)
else:
self.audio_writer = None
if self.raw_jpeg and "uvc_capture" in str(self.g_pool.capture.__class__):
self.video_path = os.path.join(self.rec_path, "world.mp4")
self.writer = JPEG_Writer(self.video_path,int(self.g_pool.capture.frame_rate))
# elif 1:
# self.writer = av_writer.AV_Writer(self.video_path)
else:
self.video_path = os.path.join(self.rec_path, "world.mkv")
self.writer = CV_Writer(self.video_path, float(self.g_pool.capture.frame_rate), self.g_pool.capture.frame_size)
# positions path to eye process
if self.record_eye:
for tx in self.g_pool.eye_tx:
tx.send((self.rec_path,self.raw_jpeg))
if self.show_info_menu:
self.open_info_menu()
# self.video_path = os.path.join(self.rec_path, "world.mkv")
# self.writer = cv2.VideoWriter(self.video_path, int(cv2.cv.CV_FOURCC(*'DIVX')), float(self.g_pool.capture.frame_rate), (1280,720))
# # positions path to eye process
# if self.record_eye:
# for tx in self.g_pool.eye_tx:
# tx.send(self.rec_path)
# if self.show_info_menu:
# self.open_info_menu()
###############
def open_info_menu(self):
self.info_menu = ui.Growing_Menu('additional Recording Info',size=(300,300),pos=(300,300))
self.info_menu.configuration = self.info_menu_conf
def populate_info_menu():
self.info_menu.elements[:-2] = []
for name in self.user_info.iterkeys():
self.info_menu.insert(0,ui.Text_Input(name,self.user_info))
def set_user_info(new_string):
self.user_info = new_string
populate_info_menu()
populate_info_menu()
self.info_menu.append(ui.Info_Text('Use the *user info* field to add/remove additional fields and their values. The format must be a valid Python dictionary. For example -- {"key":"value"}. You can add as many fields as you require. Your custom fields will be saved for your next session.'))
self.info_menu.append(ui.Text_Input('user_info',self,setter=set_user_info,label="User info"))
self.g_pool.gui.append(self.info_menu)
def close_info_menu(self):
if self.info_menu:
self.info_menu_conf = self.info_menu.configuration
self.g_pool.gui.remove(self.info_menu)
self.info_menu = None
def update(self,frame,events):
if self.running:
# update data
self.data['pupil_positions'] += events['pupil_positions']
self.data['gaze_positions'] += events['gaze_positions']
#cv2.putText(frame.img, "Frame %s"%self.frame_count,(200,200), cv2.FONT_HERSHEY_SIMPLEX,1,(255,100,100)
cv2.putText(frame.img, self.this_state,(200,100), cv2.FONT_HERSHEY_SIMPLEX,1,(255,100,100))
if self.this_state == 'pick':
cv2.putText(frame.img, self.choice,(300,100), cv2.FONT_HERSHEY_SIMPLEX,1,(100,100,255))
QRs = qr_detect(frame)
for p in events['pupil_positions']:
pupil_pos = p['timestamp'],p['confidence'],p['id'],p['norm_pos'][0],p['norm_pos'][1],p['diameter']
self.pupil_list.append(pupil_pos)
c = 0
avg_x = -1
avg_y = -1
for g in events.get('gaze_positions',[]):
gaze_pos = g['timestamp'],g['confidence'],g['norm_pos'][0],g['norm_pos'][1]
avg_x = (avg_x * c + g['norm_pos'][0]) / (c+1)
avg_y = (avg_y * c + g['norm_pos'][0]) / (c+1)
c = c + 1
self.gaze_list.append(gaze_pos)
self.timestamps.append(frame.timestamp)
avg_col = 0
self.stateT(frame, QRs, avg_x, avg_y, avg_col)
self.writer.write_video_frame(frame)
self.frame_count += 1
self.button.status_text = self.get_rec_time_str()
def stop(self):
#explicit release of VideoWriter
self.writer.release()
self.writer = None
if self.record_eye:
for tx in self.g_pool.eye_tx:
try:
tx.send(None)
except:
logger.warning("Could not stop eye-recording. Please report this bug!")
save_object(self.data, os.path.join(self.rec_path, "pupil_data"))
gaze_list_path = os.path.join(self.rec_path, "gaze_positions.npy")
np.save(gaze_list_path,np.asarray(self.gaze_list))
pupil_list_path = os.path.join(self.rec_path, "pupil_positions.npy")
np.save(pupil_list_path,np.asarray(self.pupil_list))
timestamps_path = os.path.join(self.rec_path, "world_timestamps.npy")
ts = sanitize_timestamps(np.array(self.timestamps))
np.save(timestamps_path,ts)
try:
copy2(os.path.join(self.g_pool.user_dir,"surface_definitions"),os.path.join(self.rec_path,"surface_definitions"))
except:
logger.info("No surface_definitions data found. You may want this if you do marker tracking.")
try:
copy2(os.path.join(self.g_pool.user_dir,"cal_pt_cloud.npy"),os.path.join(self.rec_path,"cal_pt_cloud.npy"))
except:
logger.warning("No calibration data found. Please calibrate first.")
try:
copy2(os.path.join(self.g_pool.user_dir,"camera_matrix.npy"),os.path.join(self.rec_path,"camera_matrix.npy"))
copy2(os.path.join(self.g_pool.user_dir,"dist_coefs.npy"),os.path.join(self.rec_path,"dist_coefs.npy"))
except:
logger.info("No camera intrinsics found.")
try:
with open(self.meta_info_path, 'a') as f:
f.write("Duration Time\t"+ self.get_rec_time_str()+ "\n")
if self.g_pool.binocular:
f.write("Eye Mode\tbinocular\n")
else:
f.write("Eye Mode\tmonocular\n")
f.write("Duration Time\t"+ self.get_rec_time_str()+ "\n")
f.write("World Camera Frames\t"+ str(self.frame_count)+ "\n")
f.write("World Camera Resolution\t"+ str(self.width)+"x"+str(self.height)+"\n")
f.write("Capture Software Version\t%s\n"%self.g_pool.version)
if platform.system() == "Windows":
username = os.environ["USERNAME"]
sysname, nodename, release, version, machine, _ = platform.uname()
else:
username = os.getlogin()
try:
sysname, nodename, release, version, machine = os.uname()
except:
sysname, nodename, release, version, machine = sys.platform,None,None,None,None
f.write("User\t"+username+"\n")
f.write("Platform\t"+sysname+"\n")
f.write("Machine\t"+nodename+"\n")
f.write("Release\t"+release+"\n")
f.write("Version\t"+version+"\n")
except Exception:
logger.exception("Could not save metadata. Please report this bug!")
try:
with open(os.path.join(self.rec_path, "user_info.csv"), 'w') as f:
for name,val in self.user_info.iteritems():
f.write("%s\t%s\n"%(name,val))
except Exception:
logger.exception("Could not save userdata. Please report this bug!")
self.close_info_menu()
if self.audio_writer:
self.audio_writer = None
self.running = False
self.menu.read_only = False
self.button.status_text = ''
def cleanup(self):
"""gets called when the plugin get terminated.
either volunatily or forced.
"""
if self.running:
self.stop()
self.deinit_gui()
def verify_path(self, val):
try:
n_path = os.path.expanduser(val)
logger.debug("Expanded user path.")
except:
n_path = val
if not n_path:
logger.waring("Please specify a path.")
return False
elif not os.path.isdir(n_path):
logger.warning("This is not a valid path.")
return False
elif not writable_dir(n_path):
logger.warning("Do not have write access to '%s'." %n_path)
return False
else:
return n_path
def set_rec_dir(self,val):
# try:
# n_path = os.path.expanduser(val)
# logger.debug("Expanded user path.")
# except:
# n_path = val
# if not n_path:
# logger.warning("Please specify a path.")
# elif not os.path.isdir(n_path):
# logger.warning("This is not a valid path.")
# # elif not os.access(n_path, os.W_OK):
# elif not writable_dir(n_path):
# logger.warning("Do not have write access to '%s'."%n_path)
# else:
# self.rec_dir = n_path
n_path = self.verify_path(val)
if n_path:
self.rec_dir = n_path
def set_session_name(self, val):
if not val:
self.session_name = get_auto_name()
else:
if '/' in val:
logger.warning('You session name with create one or more subdirectories')
self.session_name = val
def stateT(self, frame, QRs, agaze_x, agaze_y, avg_col):
if time() - self.start_time < 1:
self.gaze_x = agaze_x
self.gaze_y = agaze_y
if self.this_state == 'start':
if time() - self.start_state < 2:
self.qr_codes = {}
return True
if self.Nochange_gaze(agaze_x, agaze_y, avg_col):
self.this_state = 'want'
self.start_state = time()
self.gaze_x = agaze_x
self.gaze_y = agaze_y
return True
if self.this_state == 'want':
if time() - self.start_state < 2:
self.qr_codes.update(QRs)
return True
if self.Nochange_gaze(agaze_x, agaze_y, avg_col) and self.qr_codes:
self.this_state = 'pick'
self.choice = self.choose(frame)
#print self.qr_codes, self.choice
else:
self.this_state = 'start'
self.start_state = time()
self.gaze_x = agaze_x
self.gaze_y = agaze_y
return True
if self.this_state == 'pick':
if time() - self.start_state < 4:
return True
if self.Nochange_gaze(agaze_x, agaze_y, avg_col):
del self.qr_codes[self.choice]
if not self.qr_codes:
self.this_state = 'start'
else:
self.choice = self.choose(frame)
#print self.choice
else:
print self.choice, " chosen"
self.this_state = 'start'
self.start_state = time()
self.gaze_x = agaze_x
self.gaze_y = agaze_y
return True
def Nochange_gaze(self, agaze_x, agaze_y, avg_col):
#print self.gaze_x, self.gaze_y, agaze_x, agaze_y
return (abs(self.gaze_x - agaze_x) < .1 and abs(self.gaze_y - agaze_y) < .1)
def choose(self, frame):
h = frame.img.shape[0]
w = frame.img.shape[1]
gaze_x = self.gaze_x
gaze_y = self.gaze_y
if (gaze_x < 0 or gaze_y < 0):
gaze_x = .5
gaz_y = .5
k = self.qr_codes.keys()
chosen = k[0]
min_dist = 1000000
for name, mc in self.qr_codes.items():
dis = (mc[0] - gaze_x * w) * (mc[0] - gaze_x * w) + (mc[1] - gaze_y * h) * (mc[1] - gaze_y * h)
if dis < min_dist:
min_dist = dis
chosen = name
return chosen