def decode(self, *args, **kwargs):
# fail with EAGAIN once
if not self.__failed:
self.__failed = True
raise av.AVError(errno.EAGAIN, "EAGAIN")
return self.__real.decode(*args, **kwargs)
def check_valid(self):
try:
cont = av.open(self.path)
# Three failure scenarios:
# 1. Broken video -> AVError
# 2. decode() does not yield anything
# 3. decode() yields None
first_frame = next(cont.decode(video=0), None)
if first_frame is None:
raise av.AVError("Video does not contain any frames")
except av.AVError:
return False
else:
cont.seek(0)
self.cont = cont
return True
def capture_thread(self, audio_src, running):
try:
if platform.system() == "Darwin":
in_container = av.open('none:{}'.format(audio_src),
format="avfoundation")
elif platform.system() == "Linux":
in_container = av.open('hw:{}'.format(audio_src),
format="alsa")
else:
raise av.AVError('Platform does not support audio capture.')
except av.AVError:
running.clear()
return
in_stream = None
for stream in in_container.streams:
if stream.type == 'audio':
in_stream = stream
break
if not in_stream:
logger.warning('No audio stream found for selected device.')
running.clear()
return
stream_start_ts = self.g_pool.get_now()
for packet in in_container.demux(in_stream):
try:
# ffmpeg timestamps - in_stream.startime = packte pts relative to startime
# multiply with stream_timebase to get seconds
# add start time of this stream in pupil time unadjusted
# finally add pupil timebase offset to adjust for settable timebase.
packet.timestamp = (
packet.pts - in_stream.start_time
) * in_stream.time_base + stream_start_ts - self.g_pool.timebase.value
self.queue.put_nowait(packet)
except queue.Full:
pass # drop packet
if not running.is_set():
return
self.audio_src = 'No Audio'
running.clear() # in_stream stopped yielding packets
def capture_thread(self, audio_src, running, recording):
try:
if platform.system() == "Darwin":
in_container = av.open(
"none:{}".format(audio_src), format="avfoundation"
)
elif platform.system() == "Linux":
print("audio src = {}".format(audio_src))
dev_str = re.search("(hw:\s*\d+,\s*\d+)", audio_src)
in_container = av.open(dev_str.group(0), format="alsa")
elif platform.system() == "Windows":
in_container = av.open(
"audio={}".format(audio_src),
format="dshow",
options={"audio_buffer_size": "23"},
)
else:
raise av.AVError("Platform does not support audio capture.")
except av.AVError as err:
running.clear()
self.audio_src = "No Audio"
logger.warning("Error starting audio capture: {}".format(err))
return
in_stream = None
try:
in_stream = in_container.streams.audio[0]
except IndexError:
logger.warning("No audio stream found for selected device.")
running.clear()
self.audio_src = "No Audio"
return
out_container = None
out_stream = None
timestamps = None
in_frame_size = 0
stream_epoch = in_stream.start_time * in_stream.time_base
uvc_clock_dif = abs(stream_epoch - self.g_pool.get_now())
pyt_clock_dif = abs(stream_epoch - time())
max_clock_dif = 4.0
if uvc_clock_dif > max_clock_dif and pyt_clock_dif > max_clock_dif:
logger.error("Could not identify audio stream clock.")
running.clear()
self.audio_src = "No Audio"
return
elif uvc_clock_dif > pyt_clock_dif:
logger.info(
"Audio stream uses time.time() as clock (Δ {}s)".format(pyt_clock_dif)
)
clock_differences = self.g_pool.get_now() - time()
else:
logger.info("Audio stream uses uvc.get_time_monotonic() as clock")
clock_differences = 0
def close_recording():
# Bind nonlocal variables, https://www.python.org/dev/peps/pep-3104/
nonlocal out_container, out_stream, in_stream, in_frame_size, timestamps, out_frame_num
if out_container is not None:
timestamps.append(timestamp)
audio_frame.pts = None
out_packets = [out_stream.encode(audio_frame)]
while out_packets[-1]:
out_packets.append(out_stream.encode(None))
for out_packet in out_packets:
if out_packet is not None:
out_container.mux(out_packet)
out_container.close()
out_frame_num = out_stream.frames
in_frame_rate = in_stream.rate
# in_stream.frame_size does not return the correct value.
out_frame_size = out_stream.frame_size
out_frame_rate = out_stream.rate
new_ts_idx = (
np.arange(0, out_frame_num * out_frame_size, out_frame_size)
/ out_frame_rate
)
if in_frame_rate != out_frame_rate:
old_ts_idx = (
np.arange(0, len(timestamps) * in_frame_size, in_frame_size)
/ in_frame_rate
)
interpolate = interp1d(
old_ts_idx,
timestamps,
bounds_error=False,
fill_value="extrapolate",
)
new_ts = interpolate(new_ts_idx)
else:
new_ts = timestamps[0] + new_ts_idx
ts_loc = os.path.join(self.rec_dir, "audio_timestamps.npy")
np.save(ts_loc, new_ts)
out_container = None
out_stream = None
timestamps = None
for packet in in_container.demux(in_stream):
# ffmpeg timestamps - in_stream.startime = packte pts relative to startime
# multiply with stream_timebase to get seconds
# add start time of this stream in pupil time unadjusted
# finally add pupil timebase offset to adjust for settable timebase.
for audio_frame in packet.decode():
timestamp = (
audio_frame.pts * in_stream.time_base
+ clock_differences
- self.g_pool.timebase.value
)
if recording.is_set():
if out_container is None:
rec_file = os.path.join(self.rec_dir, "audio.mp4")
out_container = av.open(rec_file, "w")
out_stream = out_container.add_stream(
"aac", rate=in_stream.rate
)
out_frame_num = 0
in_frame_size = (
audio_frame.samples
) # set here to make sure full packet size is used
timestamps = []
timestamps.append(timestamp)
audio_frame.pts = None
out_packets = [out_stream.encode(audio_frame)]
for out_packet in out_packets:
if out_packet is not None:
out_container.mux(out_packet)
elif out_container is not None:
# recording stopped
close_recording()
if not running.is_set():
close_recording()
return
close_recording()
self.audio_src = "No Audio"
running.clear() # in_stream stopped yielding packets
def capture_thread(self, audio_src, running, recording):
try:
if platform.system() == "Darwin":
in_container = av.open('none:{}'.format(audio_src), format="avfoundation")
elif platform.system() == "Linux":
in_container = av.open('hw:{}'.format(audio_src), format="alsa")
else:
raise av.AVError('Platform does not support audio capture.')
except av.AVError as err:
running.clear()
self.audio_src = 'No Audio'
logger.warning('Error starting audio capture: {}'.format(err))
return
in_stream = None
try:
in_stream = in_container.streams.audio[0]
except IndexError:
logger.warning('No audio stream found for selected device.')
running.clear()
self.audio_src = 'No Audio'
return
out_container = None
out_stream = None
timestamps = None
out_frame_num = 0
in_frame_size = 0
stream_epoch = in_stream.start_time * in_stream.time_base
uvc_clock_dif = abs(stream_epoch - self.g_pool.get_now())
pyt_clock_dif = abs(stream_epoch - time())
max_clock_dif = 4.
if uvc_clock_dif > max_clock_dif and pyt_clock_dif > max_clock_dif:
logger.error('Could not identify audio stream clock.')
running.clear()
self.audio_src = 'No Audio'
return
elif uvc_clock_dif > pyt_clock_dif:
logger.info('Audio stream uses time.time() as clock (Δ {}s)'.format(pyt_clock_dif))
clock_differences = self.g_pool.get_now() - time()
else:
logger.info('Audio stream uses uvc.get_time_monotonic() as clock')
clock_differences = 0
def close_recording():
# Bind nonlocal variables, https://www.python.org/dev/peps/pep-3104/
nonlocal out_container, out_stream, in_stream, in_frame_size, timestamps, out_frame_num
if out_container is not None:
packet = out_stream.encode(audio_frame)
while packet is not None:
out_frame_num += 1
out_container.mux(packet)
packet = out_stream.encode(audio_frame)
out_container.close()
in_frame_rate = in_stream.rate
# in_stream.frame_size does not return the correct value.
out_frame_size = out_stream.frame_size
out_frame_rate = out_stream.rate
old_ts_idx = np.arange(0, len(timestamps) * in_frame_size, in_frame_size) / in_frame_rate
new_ts_idx = np.arange(0, out_frame_num * out_frame_size, out_frame_size) / out_frame_rate
interpolate = interp1d(old_ts_idx, timestamps, bounds_error=False, fill_value='extrapolate')
new_ts = interpolate(new_ts_idx)
ts_loc = os.path.join(self.rec_dir, 'audio_timestamps.npy')
np.save(ts_loc, new_ts)
out_container = None
out_stream = None
timestamps = None
for packet in in_container.demux(in_stream):
# ffmpeg timestamps - in_stream.startime = packte pts relative to startime
# multiply with stream_timebase to get seconds
# add start time of this stream in pupil time unadjusted
# finally add pupil timebase offset to adjust for settable timebase.
for audio_frame in packet.decode():
timestamp = audio_frame.pts * in_stream.time_base + clock_differences - self.g_pool.timebase.value
if recording.is_set():
if out_container is None:
rec_file = os.path.join(self.rec_dir, 'audio.mp4')
out_container = av.open(rec_file, 'w')
out_stream = out_container.add_stream('aac')
out_frame_num = 0
in_frame_size = audio_frame.samples # set here to make sure full packet size is used
timestamps = []
timestamps.append(timestamp)
packet = out_stream.encode(audio_frame)
if packet is not None:
out_frame_num += 1
out_container.mux(packet)
elif out_container is not None:
# recording stopped
close_recording()
if not running.is_set():
close_recording()
return
close_recording()
self.audio_src = 'No Audio'
running.clear() # in_stream stopped yielding packets