async def stop_listening(
self, message: AsrStopListening
) -> typing.AsyncIterable[
typing.Union[
AsrTextCaptured,
AsrError,
typing.Tuple[AsrAudioCaptured, typing.Dict[str, typing.Any]],
]
]:
"""Stop recording audio data for a session."""
try:
session = self.sessions.pop(message.session_id, None)
if session:
# Stop session
if session.recorder:
audio_data = session.recorder.stop()
else:
assert session.audio_buffer is not None
audio_data = session.audio_buffer
wav_bytes = self.to_wav_bytes(audio_data)
_LOGGER.debug(
"Received a total of %s byte(s) for WAV data for session %s",
session.num_wav_bytes,
message.session_id,
)
if not session.transcription_sent:
# Send transcription
session.transcription_sent = True
yield (
await self.transcribe(
wav_bytes,
site_id=message.site_id,
session_id=message.session_id,
lang=session.start_listening.lang,
)
)
if session.start_listening.send_audio_captured:
# Send audio data
yield (
AsrAudioCaptured(wav_bytes=wav_bytes),
{
"site_id": message.site_id,
"session_id": message.session_id,
},
)
_LOGGER.debug("Stopping listening (session_id=%s)", message.session_id)
except Exception as e:
_LOGGER.exception("stop_listening")
yield AsrError(
error=str(e),
context="stop_listening",
site_id=message.site_id,
session_id=message.session_id,
)
async def handle_audio_frame(
self,
frame_wav_bytes: bytes,
site_id: str = "default",
session_id: typing.Optional[str] = None,
) -> typing.AsyncIterable[typing.Union[AsrTextCaptured, AsrError,
typing.Tuple[AsrAudioCaptured,
TopicArgs]]]:
"""Process single frame of WAV audio"""
# Don't process audio if no sessions
if not self.sessions:
return
audio_data = self.maybe_convert_wav(frame_wav_bytes)
if session_id is None:
# Add to every open session
target_sessions = list(self.sessions.items())
else:
# Add to single session
target_sessions = [(session_id, self.sessions[session_id])]
# Add to every open session with matching site_id
for target_id, info in target_sessions:
try:
assert info.start_listening is not None
# Match site_id
if info.start_listening.site_id != site_id:
continue
# Push to transcription thread
info.frame_queue.put(audio_data)
if info.recorder is not None:
# Check for voice command end
command = info.recorder.process_chunk(audio_data)
if command:
# Trigger publishing of transcription on silence
async for result in self.finish_session(
info, site_id=site_id, session_id=target_id):
yield result
else:
# Use session audio buffer
assert info.audio_buffer is not None
info.audio_buffer += audio_data
except Exception as e:
_LOGGER.exception("handle_audio_frame")
yield AsrError(
error=str(e),
context=repr(info.transcriber),
site_id=site_id,
session_id=target_id,
)
async def handle_asr_train(
self, train: AsrTrain, site_id: str = "default"
) -> typing.AsyncIterable[
typing.Union[typing.Tuple[AsrTrainSuccess, TopicArgs], AsrError]
]:
"""Re-trains ASR system"""
try:
# Load gzipped graph pickle
_LOGGER.debug("Loading %s", train.graph_path)
with gzip.GzipFile(train.graph_path, mode="rb") as graph_gzip:
intent_graph = nx.readwrite.gpickle.read_gpickle(graph_gzip)
# Get JSON intent graph
json_graph = rhasspynlu.graph_to_json(intent_graph)
if self.asr_train_url:
# Remote ASR server
_LOGGER.debug(self.asr_train_url)
async with self.http_session.post(
self.asr_train_url, json=json_graph, ssl=self.ssl_context
) as response:
# No data expected back
response.raise_for_status()
elif self.asr_train_command:
# Local ASR training command
_LOGGER.debug(self.asr_train_command)
proc = await asyncio.create_subprocess_exec(
*self.asr_train_command,
stdin=asyncio.subprocess.PIPE,
sterr=asyncio.subprocess.PIPE,
)
output, error = await proc.communicate(json.dumps(json_graph).encode())
if output:
_LOGGER.debug(output.decode())
if error:
_LOGGER.debug(error.decode())
else:
_LOGGER.warning("Can't train ASR system. No train URL or command.")
# Report success
yield (AsrTrainSuccess(id=train.id), {"site_id": site_id})
except Exception as e:
_LOGGER.exception("handle_asr_train")
yield AsrError(
error=str(e),
context=f"url='{self.asr_train_url}', command='{self.asr_train_command}'",
site_id=site_id,
session_id=train.id,
)
async def handle_train(
self, train: AsrTrain, site_id: str = "default"
) -> typing.AsyncIterable[
typing.Union[typing.Tuple[AsrTrainSuccess, TopicArgs], AsrError]
]:
"""Re-trains ASR system"""
try:
if (
not self.no_overwrite_train
and self.model_path
and self.scorer_path
and self.alphabet_path
):
_LOGGER.debug("no overwrite: %s, model_path: %s, scorer_path: %s, alphabet: %s", self.no_overwrite_train, self.model_path, self.scorer_path, self.alphabet_path)
_LOGGER.debug("Loading %s", train.graph_path)
with gzip.GzipFile(train.graph_path, mode="rb") as graph_gzip:
graph = nx.readwrite.gpickle.read_gpickle(graph_gzip)
# Generate language model/trie
_LOGGER.debug("Starting training")
rhasspyasr_deepspeech.train(
graph,
self.model_path,
self.scorer_path,
self.alphabet_path,
base_language_model_fst=self.base_language_model_fst,
base_language_model_weight=self.base_language_model_weight,
mixed_language_model_fst=self.mixed_language_model_fst,
)
else:
_LOGGER.debug("no overwrite: %s, model_path: %s, scorer_path: %s, alphabet: %s", self.no_overwrite_train, self.model_path, self.scorer_path, self.alphabet_path)
_LOGGER.warning("Not overwriting language model/trie")
# Model will reload
self.transcriber = None
yield (AsrTrainSuccess(id=train.id), {"site_id": site_id})
except Exception as e:
_LOGGER.exception("handle_train")
yield AsrError(
error=str(e),
context="handle_train",
site_id=site_id,
session_id=train.id,
)
async def handle_train(
self, train: AsrTrain, site_id: str = "default"
) -> typing.AsyncIterable[
typing.Union[typing.Tuple[AsrTrainSuccess, TopicArgs], AsrError]
]:
"""Re-trains ASR system"""
try:
if (
not self.no_overwrite_train
and self.language_model_path
and self.scorer_path
and self.alphabet_path
):
_LOGGER.debug("Loading %s", train.graph_path)
with gzip.GzipFile(train.graph_path, mode="rb") as graph_gzip:
graph = nx.readwrite.gpickle.read_gpickle(graph_gzip)
# Generate language model/scorer
_LOGGER.debug("Starting training")
rhasspyasr_deepspeech.train(
graph=graph,
language_model=self.language_model_path,
scorer_path=self.scorer_path,
alphabet_path=self.alphabet_path,
base_language_model_fst=self.base_language_model_fst,
base_language_model_weight=self.base_language_model_weight,
mixed_language_model_fst=self.mixed_language_model_fst,
)
else:
_LOGGER.warning("Not overwriting language model/scorer")
# Clear out existing transcribers so models can reload on next voice command
self.free_transcribers = []
for info in self.sessions.values():
info.reuse = False
yield (AsrTrainSuccess(id=train.id), {"site_id": site_id})
except Exception as e:
_LOGGER.exception("handle_train")
yield AsrError(
error=str(e),
context="handle_train",
site_id=site_id,
session_id=train.id,
)
async def handle_start_listening(
self, start_listening: AsrStartListening
) -> typing.AsyncIterable[AsrError]:
"""Start ASR session."""
_LOGGER.debug("<- %s", start_listening)
try:
session = AsrSession(
start_listening=start_listening, recorder=self.make_recorder()
)
self.asr_sessions[start_listening.session_id] = session
session.recorder.start()
except Exception as e:
_LOGGER.exception("handle_start_listening")
yield AsrError(
error=str(e),
context="",
site_id=start_listening.site_id,
session_id=start_listening.session_id,
)
async def stop_listening(
self, message: AsrStopListening
) -> typing.AsyncIterable[StopListeningType]:
"""Stop recording audio data for a session."""
info = self.sessions.pop(message.session_id, None)
if info:
try:
# Trigger publishing of transcription on end of session
async for result in self.finish_session(
info, message.site_id, message.session_id
):
yield result
if info.reuse and (info.transcriber is not None):
# Reset state
info.result = None
info.result_event.clear()
info.result_sent = False
info.result = None
info.start_listening = None
info.audio_buffer = None
while info.frame_queue.qsize() > 0:
info.frame_queue.get_nowait()
# Add to free pool
self.free_transcribers.append(info)
except Exception as e:
_LOGGER.exception("stop_listening")
yield AsrError(
error=str(e),
context=repr(info.transcriber),
site_id=message.site_id,
session_id=message.session_id,
)
_LOGGER.debug("Stopping listening (session_id=%s)", message.session_id)
async def handle_audio_frame(
self,
frame_wav_bytes: bytes,
site_id: str = "default",
session_id: typing.Optional[str] = None,
) -> typing.AsyncIterable[typing.Union[
AsrTextCaptured, AsrError, typing.Tuple[
AsrAudioCaptured, typing.Dict[str, typing.Any]], ]]:
"""Process single frame of WAV audio"""
# Don't process audio if no sessions
if not self.sessions:
return
audio_data = self.maybe_convert_wav(frame_wav_bytes)
if session_id is None:
# Add to every open session
target_sessions = list(self.sessions.items())
else:
# Add to single session
target_sessions = [(session_id, self.sessions[session_id])]
# Add audio to session(s)
for target_id, session in target_sessions:
try:
# Skip if site_id doesn't match
if session.start_listening.site_id != site_id:
continue
session.num_wav_bytes += len(frame_wav_bytes)
if session.recorder:
# Check for end of voice command
command = session.recorder.process_chunk(audio_data)
if command and (command.result
== VoiceCommandResult.SUCCESS):
assert command.audio_data is not None
_LOGGER.debug(
"Voice command recorded for session %s (%s byte(s))",
target_id,
len(command.audio_data),
)
session.transcription_sent = True
wav_bytes = self.to_wav_bytes(command.audio_data)
yield (await self.transcribe(
wav_bytes,
site_id=site_id,
session_id=target_id,
wakeword_id=session.start_listening.wakeword_id))
if session.start_listening.send_audio_captured:
# Send audio data
yield (
AsrAudioCaptured(wav_bytes=wav_bytes),
{
"site_id": site_id,
"session_id": target_id
},
)
# Reset session (but keep open)
session.recorder.stop()
session.recorder.start()
else:
# Add to buffer
assert session.audio_buffer is not None
session.audio_buffer += audio_data
except Exception as e:
_LOGGER.exception("handle_audio_frame")
yield AsrError(
error=str(e),
context=repr(self.transcriber),
site_id=site_id,
session_id=target_id,
)
async def handle_train(
self,
train: AsrTrain,
site_id: str = "default"
) -> typing.AsyncIterable[typing.Union[typing.Tuple[AsrTrainSuccess,
TopicArgs], AsrError]]:
"""Re-trains ASR system."""
try:
assert (self.model_dir and self.graph_dir
), "Model and graph dirs are required to train"
# Load base dictionaries
pronunciations: PronunciationsType = defaultdict(list)
for base_dict in self.base_dictionaries:
if not os.path.exists(base_dict.path):
_LOGGER.warning("Base dictionary does not exist: %s",
base_dict.path)
continue
# Re-load dictionary if modification time has changed
dict_mtime_ns = os.stat(base_dict.path).st_mtime_ns
if (base_dict.mtime_ns is None) or (base_dict.mtime_ns !=
dict_mtime_ns):
base_dict.mtime_ns = dict_mtime_ns
_LOGGER.debug("Loading base dictionary from %s",
base_dict.path)
with open(base_dict.path, "r") as base_dict_file:
rhasspynlu.g2p.read_pronunciations(
base_dict_file, word_dict=base_dict.pronunciations)
for word in base_dict.pronunciations:
pronunciations[word].extend(base_dict.pronunciations[word])
if not self.no_overwrite_train:
_LOGGER.debug("Loading %s", train.graph_path)
with gzip.GzipFile(train.graph_path, mode="rb") as graph_gzip:
graph = nx.readwrite.gpickle.read_gpickle(graph_gzip)
# Re-generate HCLG.fst
_LOGGER.debug("Starting training")
rhasspyasr_kaldi.train(
graph,
pronunciations,
self.model_dir,
self.graph_dir,
dictionary=self.dictionary_path,
language_model=self.language_model_path,
dictionary_word_transform=self.dictionary_word_transform,
g2p_model=self.g2p_model,
g2p_word_transform=self.g2p_word_transform,
missing_words_path=self.unknown_words,
base_language_model_fst=self.base_language_model_fst,
base_language_model_weight=self.base_language_model_weight,
mixed_language_model_fst=self.mixed_language_model_fst,
)
else:
_LOGGER.warning("Not overwriting HCLG.fst")
kaldi_dir = rhasspyasr_kaldi.get_kaldi_dir()
rhasspyasr_kaldi.train_prepare_online_decoding(
self.model_dir, self.graph_dir, kaldi_dir)
yield (AsrTrainSuccess(id=train.id), {"site_id": site_id})
except Exception as e:
_LOGGER.exception("train")
yield AsrError(error=str(e), site_id=site_id, session_id=train.id)
async def start_listening(
self, message: AsrStartListening
) -> typing.AsyncIterable[typing.Union[StopListeningType, AsrError]]:
"""Start recording audio data for a session."""
try:
if message.session_id in self.sessions:
# Stop existing session
async for stop_message in self.stop_listening(
AsrStopListening(site_id=message.site_id,
session_id=message.session_id)):
yield stop_message
if self.free_transcribers:
# Re-use existing transcriber
info = self.free_transcribers.pop()
_LOGGER.debug("Re-using existing transcriber (session_id=%s)",
message.session_id)
else:
# Create new transcriber
info = TranscriberInfo(reuse=self.reuse_transcribers)
_LOGGER.debug("Creating new transcriber session %s",
message.session_id)
def transcribe_proc(info, transcriber_factory, sample_rate,
sample_width, channels):
def audio_stream(frame_queue) -> typing.Iterable[bytes]:
# Pull frames from the queue
frames = frame_queue.get()
while frames:
yield frames
frames = frame_queue.get()
try:
info.transcriber = transcriber_factory(
port_num=self.kaldi_port)
assert (info.transcriber
is not None), "Failed to create transcriber"
while True:
# Wait for session to start
info.ready_event.wait()
info.ready_event.clear()
# Get result of transcription
result = info.transcriber.transcribe_stream(
audio_stream(info.frame_queue),
sample_rate,
sample_width,
channels,
)
_LOGGER.debug("Transcription result: %s", result)
assert (result is not None
and result.text), "Null transcription"
# Signal completion
info.result = result
info.result_event.set()
if not self.reuse_transcribers:
try:
info.transcriber.stop()
except Exception:
_LOGGER.exception("Transcriber stop")
break
except Exception:
_LOGGER.exception("session proc")
# Mark as not reusable
info.reuse = False
# Stop transcriber
if info.transcriber is not None:
try:
info.transcriber.stop()
except Exception:
_LOGGER.exception("Transcriber stop")
# Signal failure
info.transcriber = None
info.result = Transcription(text="",
likelihood=0,
transcribe_seconds=0,
wav_seconds=0)
info.result_event.set()
# Run in separate thread
info.thread = threading.Thread(
target=transcribe_proc,
args=(
info,
self.transcriber_factory,
self.sample_rate,
self.sample_width,
self.channels,
),
daemon=True,
)
info.thread.start()
# ---------------------------------------------------------------------
# Settings for session
info.start_listening = message
# Signal session thread to start
info.ready_event.set()
if message.stop_on_silence:
# Begin silence detection
if info.recorder is None:
info.recorder = self.recorder_factory()
info.recorder.start()
else:
# Use internal buffer (no silence detection)
info.audio_buffer = bytes()
self.sessions[message.session_id] = info
_LOGGER.debug("Starting listening (session_id=%s)",
message.session_id)
self.first_audio = True
except Exception as e:
_LOGGER.exception("start_listening")
yield AsrError(
error=str(e),
context=repr(message),
site_id=message.site_id,
session_id=message.session_id,
)
async def handle_train(
self, train: AsrTrain, site_id: str = "default"
) -> typing.AsyncIterable[
typing.Union[typing.Tuple[AsrTrainSuccess, TopicArgs], AsrError]
]:
"""Re-trains ASR system"""
try:
if not self.base_dictionaries:
_LOGGER.warning(
"No base dictionaries provided. Training will likely fail."
)
# Load base dictionaries
pronunciations: PronunciationsType = defaultdict(list)
for base_dict in self.base_dictionaries:
if not os.path.exists(base_dict.path):
_LOGGER.warning(
"Base dictionary does not exist: %s", base_dict.path
)
continue
# Re-load dictionary if modification time has changed
dict_mtime_ns = os.stat(base_dict.path).st_mtime_ns
if (base_dict.mtime_ns is None) or (
base_dict.mtime_ns != dict_mtime_ns
):
base_dict.mtime_ns = dict_mtime_ns
_LOGGER.debug("Loading base dictionary from %s", base_dict.path)
with open(base_dict.path, "r") as base_dict_file:
rhasspynlu.g2p.read_pronunciations(
base_dict_file, word_dict=base_dict.pronunciations
)
for word in base_dict.pronunciations:
pronunciations[word].extend(base_dict.pronunciations[word])
# Load intent graph
_LOGGER.debug("Loading %s", train.graph_path)
with gzip.GzipFile(train.graph_path, mode="rb") as graph_gzip:
self.intent_graph = nx.readwrite.gpickle.read_gpickle(graph_gzip)
# Clean LM cache completely
for lm_path in self.lm_cache_paths.values():
try:
lm_path.unlink()
except Exception:
pass
self.lm_cache_paths = {}
self.lm_cache_transcribers = {}
# Generate dictionary/language model
if not self.no_overwrite_train:
_LOGGER.debug("Starting training")
rhasspyasr_pocketsphinx.train(
self.intent_graph,
self.dictionary,
self.language_model,
pronunciations,
dictionary_word_transform=self.dictionary_word_transform,
g2p_model=self.g2p_model,
g2p_word_transform=self.g2p_word_transform,
missing_words_path=self.unknown_words,
base_language_model_fst=self.base_language_model_fst,
base_language_model_weight=self.base_language_model_weight,
mixed_language_model_fst=self.mixed_language_model_fst,
)
else:
_LOGGER.warning("Not overwriting dictionary/language model")
_LOGGER.debug("Re-loading transcriber")
self.transcriber = self.make_transcriber(self.language_model)
yield (AsrTrainSuccess(id=train.id), {"site_id": site_id})
except Exception as e:
_LOGGER.exception("handle_train")
yield AsrError(
error=str(e),
context=repr(self.transcriber),
site_id=site_id,
session_id=train.id,
)
async def handle_stop_listening(
self, stop_listening: AsrStopListening
) -> typing.AsyncIterable[
typing.Union[
AsrTextCaptured, typing.Tuple[AsrAudioCaptured, TopicArgs], AsrError
]
]:
"""Stop ASR session."""
_LOGGER.debug("<- %s", stop_listening)
try:
session = self.asr_sessions.pop(stop_listening.session_id, None)
if session is None:
_LOGGER.warning("Session not found for %s", stop_listening.session_id)
return
assert session.sample_rate is not None, "No sample rate"
assert session.sample_width is not None, "No sample width"
assert session.channels is not None, "No channels"
if session.start_listening.stop_on_silence:
# Use recorded voice command
audio_data = session.recorder.stop()
else:
# Use entire audio
audio_data = session.audio_data
# Process entire WAV file
wav_bytes = self.to_wav_bytes(
audio_data, session.sample_rate, session.sample_width, session.channels
)
_LOGGER.debug("Received %s byte(s) of WAV data", len(wav_bytes))
if self.asr_url:
_LOGGER.debug(self.asr_url)
# Remote ASR server
async with self.http_session.post(
self.asr_url,
data=wav_bytes,
headers={"Content-Type": "audio/wav", "Accept": "application/json"},
ssl=self.ssl_context,
) as response:
response.raise_for_status()
transcription_dict = await response.json()
elif self.asr_command:
# Local ASR command
_LOGGER.debug(self.asr_command)
start_time = time.perf_counter()
proc = await asyncio.create_subprocess_exec(
*self.asr_command,
stdin=asyncio.subprocess.PIPE,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
output, error = await proc.communicate(wav_bytes)
if error:
_LOGGER.debug(error.decode())
text = output.decode()
end_time = time.perf_counter()
transcription_dict = {
"text": text,
"transcribe_seconds": (end_time - start_time),
}
else:
# Empty transcription
_LOGGER.warning(
"No ASR URL or command. Only empty transcriptions will be returned."
)
transcription_dict = {}
# Publish transcription
yield AsrTextCaptured(
text=transcription_dict.get("text", ""),
likelihood=float(transcription_dict.get("likelihood", 0)),
seconds=float(transcription_dict.get("transcribe_seconds", 0)),
site_id=stop_listening.site_id,
session_id=stop_listening.session_id,
lang=session.start_listening.lang,
)
if session.start_listening.send_audio_captured:
# Send audio data
yield (
AsrAudioCaptured(wav_bytes=wav_bytes),
{
"site_id": stop_listening.site_id,
"session_id": stop_listening.session_id,
},
)
except Exception as e:
_LOGGER.exception("handle_stop_listening")
yield AsrError(
error=str(e),
context=f"url='{self.asr_url}', command='{self.asr_command}'",
site_id=stop_listening.site_id,
session_id=stop_listening.session_id,
)
