async def handle_get_devices(
self, get_devices: AudioGetDevices
) -> typing.AsyncIterable[typing.Union[AudioDevices, AudioRecordError]]:
"""Get available microphones and optionally test them."""
if get_devices.modes and (AudioDeviceMode.INPUT not in get_devices.modes):
_LOGGER.debug("Not a request for input devices")
return
devices: typing.List[AudioDevice] = []
if self.list_command:
try:
# Run list command
_LOGGER.debug(self.list_command)
output = subprocess.check_output(
self.list_command, universal_newlines=True
).splitlines()
# Parse output (assume like arecord -L)
name, description = None, ""
first_mic = True
for line in output:
line = line.rstrip()
if re.match(r"^\s", line):
description = line.strip()
if first_mic:
description += "*"
first_mic = False
else:
if name is not None:
working = None
if get_devices.test:
working = self.get_microphone_working(name)
devices.append(
AudioDevice(
mode=AudioDeviceMode.INPUT,
id=name,
name=name,
description=description,
working=working,
)
)
name = line.strip()
except Exception as e:
_LOGGER.exception("handle_get_devices")
yield AudioRecordError(
error=str(e), context=get_devices.id, site_id=get_devices.site_id
)
else:
_LOGGER.warning("No device list command. Cannot list microphones.")
yield AudioDevices(
devices=devices, id=get_devices.id, site_id=get_devices.site_id
)
async def handle_get_devices(
self, get_devices: AudioGetDevices
) -> typing.AsyncIterable[typing.Union[AudioDevices, AudioRecordError]]:
"""Get available microphones and optionally test them."""
if get_devices.modes and (AudioDeviceMode.INPUT
not in get_devices.modes):
_LOGGER.debug("Not a request for input devices")
return
devices: typing.List[AudioDevice] = []
try:
audio = pyaudio.PyAudio()
default_name = audio.get_default_input_device_info().get("name")
for device_index in range(audio.get_device_count()):
device_info = audio.get_device_info_by_index(device_index)
device_name = device_info.get("name")
if device_name == default_name:
device_name += "*"
working: typing.Optional[bool] = None
if get_devices.test:
working = self.get_microphone_working(
device_name, device_index, audio)
devices.append(
AudioDevice(
mode=AudioDeviceMode.INPUT,
id=str(device_index),
name=device_name,
description="",
working=working,
))
except Exception as e:
_LOGGER.exception("handle_get_devices")
yield AudioRecordError(error=str(e),
context=get_devices.id,
site_id=get_devices.site_id)
finally:
audio.terminate()
yield AudioDevices(devices=devices,
id=get_devices.id,
site_id=get_devices.site_id)
def publish_odas(self):
"""Publish the latest ."""
try:
while not self._exit_requested:
for _n in range(MAX_ODAS_SOURCES):
_ssl_latest = SSL_latest[_n]
if self._is_source_localized(_ssl_latest):
_ts = _ssl_latest[0]
_ssl_latest = _ssl_latest[1]
_publish = SSL_src_msg(timestamp=_ts,
channel=_n,
E=_ssl_latest.E,
x=_ssl_latest.x,
y=_ssl_latest.y,
z=_ssl_latest.z)
self.publish(
_publish,
site_id=self.output_site_id,
)
# print('Published SSL', _publish.topic(), _publish)
_sst_latest = SST_latest[_n]
if self._is_source_tracked(_sst_latest):
_ts = _sst_latest[0]
_sst_latest = _sst_latest[1]
_publish = SST_src_msg(timestamp=_ts,
channel=_n,
activity=_sst_latest.activity,
x=_sst_latest.x,
y=_sst_latest.y,
z=_sst_latest.z,
id=_sst_latest.id)
self.publish(
_publish,
site_id=self.output_site_id,
)
# print('Published SST', _publish.topic(), _publish)
sleep(1.0 / REFRESH_RATE)
except Exception as e:
_LOGGER.exception("publish_odas: " + str(e))
self.publish(
AudioRecordError(error=str(e),
context="publish_chunks",
site_id=self.site_id))
def record(self):
"""Record audio from PyAudio device."""
try:
audio = pyaudio.PyAudio()
def callback(in_data, frame_count, time_info, status):
if in_data:
self.chunk_queue.put(in_data)
return (None, pyaudio.paContinue)
# Open device
mic = audio.open(
input_device_index=self.device_index,
channels=self.channels,
format=audio.get_format_from_width(self.sample_width),
rate=self.sample_rate,
frames_per_buffer=self.frames_per_buffer,
input=True,
stream_callback=callback,
)
assert mic is not None
mic.start_stream()
_LOGGER.debug("Recording audio")
while mic.is_active():
time.sleep(0.1)
mic.stop_stream()
audio.terminate()
except Exception as e:
_LOGGER.exception("record")
self.publish(
AudioRecordError(
error=str(e),
context=f"Device index: {self.device_index}",
site_id=self.output_site_id,
))
def record(self):
"""Record audio from external program's stdout."""
try:
_LOGGER.debug(self.record_command)
record_proc = subprocess.Popen(self.record_command, stdout=subprocess.PIPE)
_LOGGER.debug("Recording audio")
while True:
chunk = record_proc.stdout.read(self.chunk_size)
if chunk:
self.chunk_queue.put(chunk)
else:
# Avoid 100% CPU usage
time.sleep(0.01)
except Exception as e:
_LOGGER.exception("record")
self.publish(
AudioRecordError(
error=str(e),
context=str(self.record_command),
site_id=self.output_site_id,
)
)
def publish_chunks(self):
"""Publish audio chunks to MQTT or UDP."""
try:
udp_dest = (self.udp_audio_host, self.udp_audio_port)
while True:
chunk = self.chunk_queue.get()
if chunk:
# MQTT output
with io.BytesIO() as wav_buffer:
wav_file: wave.Wave_write = wave.open(wav_buffer, "wb")
with wav_file:
wav_file.setframerate(self.sample_rate)
wav_file.setsampwidth(self.sample_width)
wav_file.setnchannels(self.channels)
wav_file.writeframes(chunk)
wav_bytes = wav_buffer.getvalue()
if self.udp_output:
# UDP output
self.udp_socket.sendto(wav_bytes, udp_dest)
else:
# Publish to output site_id
self.publish(
AudioFrame(wav_bytes=wav_bytes),
site_id=self.output_site_id,
)
if self.enable_summary:
self.summary_frames_left -= 1
if self.summary_frames_left > 0:
continue
self.summary_frames_left = self.summary_skip_frames
if not self.vad:
# Create voice activity detector
self.vad = webrtcvad.Vad()
self.vad.set_mode(self.vad_mode)
# webrtcvad needs 16-bit 16Khz mono
self.vad_audio_data += self.maybe_convert_wav(
wav_bytes,
sample_rate=16000,
sample_width=2,
channels=1)
is_speech = False
# Process in chunks of 30ms for webrtcvad
while len(self.vad_audio_data) >= self.vad_chunk_size:
vad_chunk = self.vad_audio_data[:self.
vad_chunk_size]
self.vad_audio_data = self.vad_audio_data[
self.vad_chunk_size:]
# Speech in any chunk counts as speech
is_speech = is_speech or self.vad.is_speech(
vad_chunk, 16000)
# Publish audio summary
self.publish(
AudioSummary(
debiased_energy=AudioSummary.
get_debiased_energy(chunk),
is_speech=is_speech,
),
site_id=self.output_site_id,
)
except Exception as e:
_LOGGER.exception("publish_chunks")
self.publish(
AudioRecordError(error=str(e),
context="publish_chunks",
site_id=self.site_id))
def publish_chunks(self):
"""Publish audio chunks to MQTT or UDP."""
try:
udp_dest = (self.udp_audio_host, self.udp_audio_port)
while not self._exit_requested:
chunk = self.chunk_queue.get()
if chunk:
# MQTT output
with io.BytesIO() as wav_buffer:
wav_file: wave.Wave_write = wave.open(wav_buffer, "wb")
with wav_file:
wav_file.setframerate(self.sample_rate)
wav_file.setsampwidth(self.sample_width)
wav_file.setnchannels(self.channels)
wav_file.writeframes(chunk)
wav_bytes = wav_buffer.getvalue()
if self.udp_output:
# UDP output
self.udp_socket.sendto(wav_bytes, udp_dest)
else:
# Publish to output site_id
self.publish(
AudioFrame(wav_bytes=wav_bytes),
site_id=self.output_site_id,
)
if self._dump_file is not None:
# print("tell is", self._dump_file.tell(), end=' ')
# write_wave( self._dump_file, wav_bytes, remove_header=True)
if USE_SOUNDFILE:
# soultion soundfile
self._dump_file.write(
np.frombuffer(
wav_bytes[44:],
np.int16)) # removing header!
else:
# Solution wave, this write always the latest buffer and doesnt happend it!
self._dump_file.writeframesraw(
wav_bytes[44:]) # removing header!
if self.enable_summary:
self.summary_frames_left -= 1
if self.summary_frames_left > 0:
continue
self.summary_frames_left = self.summary_skip_frames
if not self.vad:
# Create voice activity detector
self.vad = webrtcvad.Vad()
self.vad.set_mode(self.vad_mode)
# webrtcvad needs 16-bit 16Khz mono
# TODO: would be possible to split here if demux is not selected? this would avoid resampling,
# which is called continuously. (uncomment this code). With the switch --demux a proper channel
# is produced
# with io.BytesIO(wav_bytes) as wav_io:
# with wave.open(wav_io, "rb") as wav_file:
# if (wav_file.getframerate() != 16000) or \
# (wav_file.getsampwidth() != 2) or \
# (wav_file.getnchannels() != 1):
# print("Need Resample: sr={}, width={}, n_ch={}".format(wav_file.getframerate(),
# wav_file.getsampwidth(),
# wav_file.getnchannels()))
# else:
# print("No resample")
# webrtcvad needs 16-bit 16Khz mono
self.vad_audio_data += self.maybe_convert_wav(
wav_bytes,
sample_rate=16000,
sample_width=2,
channels=1)
is_speech = False
# Process in chunks of 30ms for webrtcvad
while len(self.vad_audio_data) >= self.vad_chunk_size:
vad_chunk = self.vad_audio_data[:self.
vad_chunk_size]
self.vad_audio_data = self.vad_audio_data[
self.vad_chunk_size:]
# Speech in any chunk counts as speech
is_speech = is_speech or self.vad.is_speech(
vad_chunk, 16000)
# Publish audio summary
self.publish(
AudioSummary(
debiased_energy=AudioSummary.
get_debiased_energy(chunk),
is_speech=is_speech,
),
site_id=self.output_site_id,
)
except Exception as e:
_LOGGER.exception("publish_chunks")
self.publish(
AudioRecordError(error=str(e),
context="publish_chunks",
site_id=self.site_id))
def record(self):
"""Record audio from ODAS receiver."""
_HIGHER_ID = _HigherID()
def _get_postion_message(source_id=None):
if source_id is None:
return {"SST": SST_latest, "SSL": SSL_latest}
else:
return {
"SST": SST_latest[source_id],
"SSL": SSL_latest[source_id]
}
def _print_localization(n, position, audio_chunk):
"""Debug only purpose"""
PRINT_AUDIO_CHUNK = True
def _print(n_id, pos, aud_cnk=None):
print("[{}]-SST_latest_{}-id_{}: {} {} - Audio Chunk len={}".
format(pos[0], n_id, pos[1].id, pos[1].tag,
pos[1].activity, len(aud_cnk)))
if PRINT_AUDIO_CHUNK:
print('{}...{}\n---------------------'.format(
aud_cnk[:30], aud_cnk[-10:]))
if isinstance(position, tuple):
p = position
elif position['SST'] is not None:
p = position['SST']
if len(p[1].tag) or p[1].activity > ACTIVITY_THRESHOLD:
_print(n, p, audio_chunk)
def _acquire_streaming_id(position): #, higher_id):
"""Determine the stream with the higher id (latest)
TODO: here could stay the speaker identification process? How?"""
if position['SST'] is not None and self._is_source_tracked(
position['SST']
): # position['SST'] is not None and (len(position['SST'][1].tag) or position['SST'][1].activity > ACTIVITY_THRESHOLD):
if position['SST'][1].id >= _HIGHER_ID.value:
_HIGHER_ID.set_value(position['SST'][1].id)
# print(position['SST'][1].id , higher_id.value)
return True
else:
return False
else:
return False
def _source_listening(source_id, higher_id):
"""Only for demux operations, used as a thread listen to one channel and the source with a certain policy
is """
_LOGGER.debug("Recording audio {}".format(source_id))
try:
while not self._exit_requested:
audio_chunk = SSS_queue[source_id].get().tobytes()
SSS_queue[source_id].task_done(
) # sign the last job as done
position = _get_postion_message(
source_id
) # metadata with position. to add a subscriber
if audio_chunk is None:
raise Exception(
"Received buffer none for source_id_{}".format(
source_id)
) # stop processing if the main thread is done. this means data are not anymore a
if _acquire_streaming_id(
position
): #, _HIGHER_ID): # source_id >= higher_id.value():
if self._is_source_tracked(position['SST']):
self.chunk_queue.put(audio_chunk)
if DEBUG_LOCALIZATION:
_print_localization(
str(source_id) + '_Acquired Priority',
position, audio_chunk)
else:
if DEBUG_LOCALIZATION:
_print_localization(
str(source_id) + '_Discarded(Higher is ' +
str(higher_id.value) + ')', position,
audio_chunk)
except Exception as e:
_LOGGER.warning(
"Got an error listening source {} -> {}".format(
source_id, e))
finally:
pass
# TODO: Profiling?
_LOGGER.debug(
"Stop recording audio {}, exit thread".format(source_id))
try:
# Start Thread
listener_threads = []
try:
# I need to retrieve ALL available channels and simply discard (this looks not very efficient though)
if self.demux:
for _n in range(
MAX_ODAS_SOURCES
): # What about range(channels)? TODO: channels < MAX_ODAS_SOURCES in init
listener_threads.append(
threading.Thread(target=_source_listening,
args=(
_n,
_HIGHER_ID,
),
daemon=True))
listener_threads[-1].start()
while not self._exit_requested:
sleep(1.0 / REFRESH_RATE) # Refresh rate
# check and in case reset the max id (e.g. a higher id source terminated)
_max_ID = -1
positions = _get_postion_message()
for _n in range(MAX_ODAS_SOURCES):
if self._is_source_tracked(positions['SST'][_n]):
_max_ID = max(positions['SST'][_n][1].id,
_max_ID)
if _HIGHER_ID.value > _max_ID:
# reset the max id with the actual higher value
if DEBUG_LOCALIZATION:
print("Reset max id from {} to {}".format(
_HIGHER_ID.value, _max_ID))
_HIGHER_ID.set_value(_max_ID)
else:
loop = 0
assert N_BITS_INCOME_STREAM == 16, \
"Unsupported format, only 16 bits are accepted (this condition should be removed in future with a local cast)"
while not self._exit_requested:
loop += 1
audio_chunk = SSS_queue.get(
) # It will wait until data are available
out_chunk = np.zeros(shape=(audio_chunk.shape[0],
self.channels),
dtype=np.int16)
positions = _get_postion_message(
) # metadata with position. to add a subscriber
for _n in range(MAX_ODAS_SOURCES):
if _n < self.channels:
# the channel is added only if the activity threshold is above a certain level
if self._is_source_tracked(
positions['SST'][_n]
): # len(positions['SST'][_n][1].tag) or positions['SST'][_n][1].activity > ACTIVITY_THRESHOLD:
out_chunk[:, _n] = audio_chunk[:, _n]
if DEBUG_LOCALIZATION:
_print_localization(
'CH_' + str(_n),
positions['SST'][_n],
out_chunk[:, _n])
else:
if DEBUG_LOCALIZATION:
print("CH_" + str(_n) +
" Discarded, activity=" +
str(positions['SST'][_n]
[1].activity))
self.chunk_queue.put(out_chunk.tobytes())
except Exception as e:
_LOGGER.warning("Got an error recording -> {}".format(e))
finally:
if self.demux:
for _n in range(MAX_ODAS_SOURCES):
_LOGGER.debug(
"Recording shutdown, wait for exit acquistion thread {}"
.format(_n))
listener_threads[_n].join()
_LOGGER.debug("Thread {} exit done".format(_n))
else:
_LOGGER.debug("Recording shutdown")
except Exception as e:
_LOGGER.exception("record")
self.publish(
AudioRecordError(
error=str(e),
context=f"Device index: {self.device_index}",
site_id=self.output_site_id,
))