async def end_session( self, reason: DialogueSessionTerminationReason, site_id: str ) -> typing.AsyncIterable[typing.Union[EndSessionType, StartSessionType, SayType]]: """End current session and start queued session.""" assert self.session is not None, "No session" session = self.session if session.start_session.init.type != DialogueActionType.NOTIFICATION: # Stop listening yield AsrStopListening(site_id=session.site_id, session_id=session.session_id) yield DialogueSessionEnded( site_id=site_id, session_id=session.session_id, custom_data=session.custom_data, termination=DialogueSessionTermination(reason=reason), ) self.session = None # Check session queue if self.session_queue: _LOGGER.debug("Handling queued session") async for start_result in self.start_session( self.session_queue.popleft()): yield start_result else: # Enable hotword if no queued sessions yield HotwordToggleOn(site_id=session.site_id, reason=HotwordToggleReason.DIALOGUE_SESSION)
def messages(): yield AsrStartListening(siteId=self.siteId, sessionId=sessionId) # Break WAV into chunks with io.BytesIO(wav_bytes) as wav_buffer: with wave.open(wav_buffer, "rb") as wav_file: frames_left = wav_file.getnframes() while frames_left > 0: with io.BytesIO() as chunk_buffer: chunk_file: wave.Wave_write = wave.open( chunk_buffer, "wb") with chunk_file: chunk_file.setframerate( wav_file.getframerate()) chunk_file.setsampwidth( wav_file.getsampwidth()) chunk_file.setnchannels( wav_file.getnchannels()) chunk_file.writeframes( wav_file.readframes(frames_per_chunk)) yield ( AudioFrame(wav_data=chunk_buffer.getvalue()), { "siteId": self.siteId }, ) frames_left -= frames_per_chunk yield AsrStopListening(siteId=self.siteId, sessionId=sessionId)
async def async_test_transcriber_error(self): """Check start/stop session with error in transcriber.""" def fake_transcribe(stream, *args): """Raise an exception.""" raise FakeException() self.transcriber.transcribe_stream = fake_transcribe # Start session start_listening = AsrStartListening( site_id=self.site_id, session_id=self.session_id, stop_on_silence=False ) result = None async for response in self.hermes.on_message_blocking(start_listening): result = response # No response expected self.assertIsNone(result) # Send in "audio" fake_wav_bytes = self.hermes.to_wav_bytes(secrets.token_bytes(100)) fake_frame = AudioFrame(wav_bytes=fake_wav_bytes) async for response in self.hermes.on_message_blocking( fake_frame, site_id=self.site_id ): result = response # No response expected self.assertIsNone(result) # Stop session stop_listening = AsrStopListening( site_id=self.site_id, session_id=self.session_id ) results = [] async for response in self.hermes.on_message_blocking(stop_listening): results.append(response) # Check results for empty transcription self.assertEqual( results, [ AsrRecordingFinished(site_id=self.site_id, session_id=self.session_id), AsrTextCaptured( text="", likelihood=0, seconds=0, site_id=self.site_id, session_id=self.session_id, ), ], )
def on_message(self, client, userdata, msg): """Received message from MQTT broker.""" try: if not msg.topic.endswith("/audioFrame"): _LOGGER.debug("Received %s byte(s) on %s", len(msg.payload), msg.topic) # Check enable/disable messages if msg.topic == AsrToggleOn.topic(): json_payload = json.loads(msg.payload or "{}") if self._check_siteId(json_payload): self.enabled = True _LOGGER.debug("Enabled") elif msg.topic == AsrToggleOn.topic(): json_payload = json.loads(msg.payload or "{}") if self._check_siteId(json_payload): self.enabled = False _LOGGER.debug("Disabled") if not self.enabled: # Disabled return if AudioFrame.is_topic(msg.topic): # Check siteId if (not self.audioframe_topics) or ( msg.topic in self.audioframe_topics ): # Add to all active sessions if self.first_audio: _LOGGER.debug("Receiving audio") self.first_audio = False siteId = AudioFrame.get_siteId(msg.topic) for result in self.handle_audio_frame(msg.payload, siteId=siteId): self.publish(result) elif msg.topic == AsrStartListening.topic(): # hermes/asr/startListening json_payload = json.loads(msg.payload) if self._check_siteId(json_payload): for result in self.start_listening( AsrStartListening(**json_payload) ): self.publish(result) elif msg.topic == AsrStopListening.topic(): # hermes/asr/stopListening json_payload = json.loads(msg.payload) if self._check_siteId(json_payload): for result in self.stop_listening(AsrStopListening(**json_payload)): self.publish(result) except Exception: _LOGGER.exception("on_message")
async def handle_text_captured( self, text_captured: AsrTextCaptured ) -> typing.AsyncIterable[typing.Union[AsrStopListening, HotwordToggleOn, NluQuery]]: """Handle ASR text captured for session.""" try: if not text_captured.session_id: _LOGGER.warning("Missing session id on text captured message.") return site_session = self.all_sessions.get(text_captured.session_id) if site_session is None: _LOGGER.warning( "No session for id %s. Dropping captured text from ASR.", text_captured.session_id, ) return _LOGGER.debug("Received text: %s", text_captured.text) # Record result site_session.text_captured = text_captured # Stop listening yield AsrStopListening(site_id=text_captured.site_id, session_id=site_session.session_id) # Enable hotword yield HotwordToggleOn( site_id=text_captured.site_id, reason=HotwordToggleReason.DIALOGUE_SESSION, ) # Perform query yield NluQuery( input=text_captured.text, intent_filter=site_session.intent_filter or self.default_intent_filter, session_id=site_session.session_id, site_id=site_session.site_id, wakeword_id=text_captured.wakeword_id or site_session.wakeword_id, lang=text_captured.lang or site_session.lang, ) except Exception: _LOGGER.exception("handle_text_captured")
def handle_text_captured( self, text_captured: AsrTextCaptured ) -> typing.Iterable[typing.Union[AsrStopListening, NluQuery]]: """Handle ASR text captured for session.""" try: assert self.session, "No session" _LOGGER.debug("Received text: %s", text_captured.text) # Stop listening yield AsrStopListening(siteId=self.siteId, sessionId=self.session.sessionId) # Perform query yield NluQuery( input=text_captured.text, intentFilter=self.session.intentFilter, sessionId=self.session.sessionId, ) except Exception: _LOGGER.exception("handle_text_captured")
async def end_session( self, reason: DialogueSessionTerminationReason, site_id: str, session_id: str, start_next_session: bool, ) -> typing.AsyncIterable[typing.Union[EndSessionType, StartSessionType, SayType]]: """End current session and start queued session.""" site_session = self.all_sessions.pop(session_id, None) if site_session: # Remove session for site self.session_by_site.pop(site_session.site_id, None) # End the existing session if site_session.start_session.init.type != DialogueActionType.NOTIFICATION: # Stop listening yield AsrStopListening( site_id=site_session.site_id, session_id=site_session.session_id ) yield DialogueSessionEnded( site_id=site_id, session_id=site_session.session_id, custom_data=site_session.custom_data, termination=DialogueSessionTermination(reason=reason), ) else: _LOGGER.warning("No session for id %s", session_id) # Check session queue session_queue = self.session_queue_by_site[site_id] if session_queue: if start_next_session: _LOGGER.debug("Handling queued session") async for start_result in self.start_session(session_queue.popleft()): yield start_result else: # Enable hotword if no queued sessions yield HotwordToggleOn( site_id=site_id, reason=HotwordToggleReason.DIALOGUE_SESSION )
def messages(): yield AsrStartListening( site_id=self.site_id, session_id=session_id, stop_on_silence=stop_on_silence, send_audio_captured=send_audio_captured, intent_filter=intent_filter, ) # Break WAV into chunks num_bytes_sent: int = 0 with io.BytesIO(wav_bytes) as wav_buffer: for wav_chunk in AudioFrame.iter_wav_chunked( wav_buffer, frames_per_chunk ): num_bytes_sent += len(wav_chunk) yield ( AudioSessionFrame(wav_bytes=wav_chunk), {"site_id": self.site_id, "session_id": session_id}, ) _LOGGER.debug("Sent %s byte(s) of WAV data", num_bytes_sent) yield AsrStopListening(site_id=self.site_id, session_id=session_id)
async def handle_text_captured( self, text_captured: AsrTextCaptured ) -> typing.AsyncIterable[ typing.Union[ AsrStopListening, HotwordToggleOn, NluQuery, NluIntentNotRecognized ] ]: """Handle ASR text captured for session.""" try: if not text_captured.session_id: _LOGGER.warning("Missing session id on text captured message.") return site_session = self.all_sessions.get(text_captured.session_id) if site_session is None: _LOGGER.warning( "No session for id %s. Dropping captured text from ASR.", text_captured.session_id, ) return _LOGGER.debug("Received text: %s", text_captured.text) # Record result site_session.text_captured = text_captured # Stop listening yield AsrStopListening( site_id=text_captured.site_id, session_id=site_session.session_id ) # Enable hotword yield HotwordToggleOn( site_id=text_captured.site_id, reason=HotwordToggleReason.DIALOGUE_SESSION, ) if (self.min_asr_confidence is not None) and ( text_captured.likelihood < self.min_asr_confidence ): # Transcription is below thresold. # Don't actually do an NLU query, just reject as "not recognized". _LOGGER.debug( "Transcription is below confidence threshold (%s < %s): %s", text_captured.likelihood, self.min_asr_confidence, text_captured.text, ) yield NluIntentNotRecognized( input=text_captured.text, site_id=site_session.site_id, session_id=site_session.session_id, ) else: # Perform query custom_entities: typing.Optional[typing.Dict[str, typing.Any]] = None # Copy custom entities from hotword detected if site_session.detected: custom_entities = site_session.detected.custom_entities yield NluQuery( input=text_captured.text, intent_filter=site_session.intent_filter or self.default_intent_filter, session_id=site_session.session_id, site_id=site_session.site_id, wakeword_id=text_captured.wakeword_id or site_session.wakeword_id, lang=text_captured.lang or site_session.lang, custom_data=site_session.custom_data, asr_confidence=text_captured.likelihood, custom_entities=custom_entities, ) except Exception: _LOGGER.exception("handle_text_captured")
async def handle_continue( self, continue_session: DialogueContinueSession ) -> typing.AsyncIterable[ typing.Union[AsrStartListening, AsrStopListening, SayType, DialogueError] ]: """Continue the existing session.""" site_session = self.all_sessions.get(continue_session.session_id) if site_session is None: _LOGGER.warning( "No session for id %s. Cannot continue.", continue_session.session_id ) return try: if continue_session.custom_data is not None: # Overwrite custom data site_session.custom_data = continue_session.custom_data if continue_session.lang is not None: # Overwrite language site_session.lang = continue_session.lang site_session.intent_filter = continue_session.intent_filter site_session.send_intent_not_recognized = ( continue_session.send_intent_not_recognized ) site_session.step += 1 _LOGGER.debug( "Continuing session %s (step=%s)", site_session.session_id, site_session.step, ) # Stop listening yield AsrStopListening( site_id=site_session.site_id, session_id=site_session.session_id ) # Ensure hotword is disabled for session yield HotwordToggleOff( site_id=site_session.site_id, reason=HotwordToggleReason.DIALOGUE_SESSION, ) if continue_session.text: # Forward to TTS async for tts_result in self.say( continue_session.text, site_id=site_session.site_id, session_id=continue_session.session_id, ): yield tts_result # Start ASR listening _LOGGER.debug("Listening for session %s", site_session.session_id) yield AsrStartListening( site_id=site_session.site_id, session_id=site_session.session_id, send_audio_captured=site_session.send_audio_captured, lang=site_session.lang, ) # Set up timeout asyncio.create_task( self.handle_session_timeout( site_session.site_id, site_session.session_id, site_session.step ) ) except Exception as e: _LOGGER.exception("handle_continue") yield DialogueError( error=str(e), context=str(continue_session), site_id=site_session.site_id, session_id=continue_session.session_id, )
async def start_listening( self, message: AsrStartListening ) -> typing.AsyncIterable[typing.Union[StopListeningType, AsrError]]: """Start recording audio data for a session.""" try: if message.session_id in self.sessions: # Stop existing session async for stop_message in self.stop_listening( AsrStopListening(site_id=message.site_id, session_id=message.session_id)): yield stop_message if self.free_transcribers: # Re-use existing transcriber info = self.free_transcribers.pop() _LOGGER.debug("Re-using existing transcriber (session_id=%s)", message.session_id) else: # Create new transcriber info = TranscriberInfo(reuse=self.reuse_transcribers) _LOGGER.debug("Creating new transcriber session %s", message.session_id) def transcribe_proc(info, transcriber_factory, sample_rate, sample_width, channels): def audio_stream(frame_queue) -> typing.Iterable[bytes]: # Pull frames from the queue frames = frame_queue.get() while frames: yield frames frames = frame_queue.get() try: info.transcriber = transcriber_factory( port_num=self.kaldi_port) assert (info.transcriber is not None), "Failed to create transcriber" while True: # Wait for session to start info.ready_event.wait() info.ready_event.clear() # Get result of transcription result = info.transcriber.transcribe_stream( audio_stream(info.frame_queue), sample_rate, sample_width, channels, ) _LOGGER.debug("Transcription result: %s", result) assert (result is not None and result.text), "Null transcription" # Signal completion info.result = result info.result_event.set() if not self.reuse_transcribers: try: info.transcriber.stop() except Exception: _LOGGER.exception("Transcriber stop") break except Exception: _LOGGER.exception("session proc") # Mark as not reusable info.reuse = False # Stop transcriber if info.transcriber is not None: try: info.transcriber.stop() except Exception: _LOGGER.exception("Transcriber stop") # Signal failure info.transcriber = None info.result = Transcription(text="", likelihood=0, transcribe_seconds=0, wav_seconds=0) info.result_event.set() # Run in separate thread info.thread = threading.Thread( target=transcribe_proc, args=( info, self.transcriber_factory, self.sample_rate, self.sample_width, self.channels, ), daemon=True, ) info.thread.start() # --------------------------------------------------------------------- # Settings for session info.start_listening = message # Signal session thread to start info.ready_event.set() if message.stop_on_silence: # Begin silence detection if info.recorder is None: info.recorder = self.recorder_factory() info.recorder.start() else: # Use internal buffer (no silence detection) info.audio_buffer = bytes() self.sessions[message.session_id] = info _LOGGER.debug("Starting listening (session_id=%s)", message.session_id) self.first_audio = True except Exception as e: _LOGGER.exception("start_listening") yield AsrError( error=str(e), context=repr(message), site_id=message.site_id, session_id=message.session_id, )
async def async_test_session(self): """Check good start/stop session.""" fake_transcription = Transcription( text="this is a test", likelihood=1, transcribe_seconds=0, wav_seconds=0 ) def fake_transcribe(stream, *args): """Return test trancription.""" for chunk in stream: if not chunk: break return fake_transcription self.transcriber.transcribe_stream = fake_transcribe # Start session start_listening = AsrStartListening( site_id=self.site_id, session_id=self.session_id, stop_on_silence=False, send_audio_captured=True, ) result = None async for response in self.hermes.on_message_blocking(start_listening): result = response # No response expected self.assertIsNone(result) # Send in "audio" fake_wav_bytes = self.hermes.to_wav_bytes(secrets.token_bytes(100)) fake_frame = AudioFrame(wav_bytes=fake_wav_bytes) async for response in self.hermes.on_message_blocking( fake_frame, site_id=self.site_id ): result = response # No response expected self.assertIsNone(result) # Stop session stop_listening = AsrStopListening( site_id=self.site_id, session_id=self.session_id ) results = [] async for response in self.hermes.on_message_blocking(stop_listening): results.append(response) # Check results self.assertEqual( results, [ AsrRecordingFinished(site_id=self.site_id, session_id=self.session_id), AsrTextCaptured( text=fake_transcription.text, likelihood=fake_transcription.likelihood, seconds=fake_transcription.transcribe_seconds, site_id=self.site_id, session_id=self.session_id, ), ( AsrAudioCaptured(wav_bytes=fake_wav_bytes), {"site_id": self.site_id, "session_id": self.session_id}, ), ], )
def start_listening( self, message: AsrStartListening ) -> typing.Iterable[typing.Union[AsrTextCaptured, AsrError]]: """Start recording audio data for a session.""" try: if message.sessionId in self.sessions: # Stop existing session for result in self.stop_listening( AsrStopListening(sessionId=message.sessionId) ): yield result if self.free_transcribers: # Re-use existing transcriber info = self.free_transcribers.pop() _LOGGER.debug( "Re-using existing transcriber (sessionId=%s)", message.sessionId ) else: # Create new transcriber info = TranscriberInfo(recorder=self.recorder_factory()) # type: ignore _LOGGER.debug("Creating new transcriber session %s", message.sessionId) def transcribe_proc( info, transcriber_factory, sample_rate, sample_width, channels ): def audio_stream(frame_queue): # Pull frames from the queue frames = frame_queue.get() while frames: yield frames frames = frame_queue.get() try: # Create transcriber in this thread info.transcriber = transcriber_factory() while True: # Wait for session to start info.ready_event.wait() info.ready_event.clear() # Get result of transcription result = info.transcriber.transcribe_stream( audio_stream(info.frame_queue), sample_rate, sample_width, channels, ) _LOGGER.debug(result) # Signal completion info.result = result info.result_event.set() except Exception: _LOGGER.exception("session proc") # Run in separate thread info.thread = threading.Thread( target=transcribe_proc, args=( info, self.transcriber_factory, self.sample_rate, self.sample_width, self.channels, ), daemon=True, ) info.thread.start() # --------------------------------------------------------------------- # Signal session thread to start info.ready_event.set() # Begin silence detection assert info.recorder is not None info.recorder.start() self.sessions[message.sessionId] = info _LOGGER.debug("Starting listening (sessionId=%s)", message.sessionId) self.first_audio = True except Exception as e: _LOGGER.exception("start_listening") yield AsrError( error=str(e), context=repr(message), siteId=message.siteId, sessionId=message.sessionId, )
async def handle_audio_frame( self, wav_bytes: bytes, site_id: str = "default", session_id: typing.Optional[str] = None, ) -> typing.AsyncIterable[ typing.Union[ typing.Tuple[HotwordDetected, TopicArgs], AsrTextCaptured, typing.Tuple[AsrAudioCaptured, TopicArgs], AsrError, ] ]: """Add audio frame to open sessions.""" try: if self.asr_enabled: if session_id is None: # Add to every open session target_sessions = list(self.asr_sessions.items()) else: # Add to single session target_sessions = [(session_id, self.asr_sessions[session_id])] with io.BytesIO(wav_bytes) as in_io: with wave.open(in_io) as in_wav: # Get WAV details from first frame sample_rate = in_wav.getframerate() sample_width = in_wav.getsampwidth() channels = in_wav.getnchannels() audio_data = in_wav.readframes(in_wav.getnframes()) # Add to target ASR sessions for target_id, session in target_sessions: # Skip non-matching site_id if session.start_listening.site_id != site_id: continue session.sample_rate = sample_rate session.sample_width = sample_width session.channels = channels session.audio_data += audio_data if session.start_listening.stop_on_silence: # Detect silence (end of command) audio_data = self.maybe_convert_wav( wav_bytes, self.recorder_sample_rate, self.recorder_sample_width, self.recorder_channels, ) command = session.recorder.process_chunk(audio_data) if command and (command.result == VoiceCommandResult.SUCCESS): # Complete session stop_listening = AsrStopListening( site_id=site_id, session_id=target_id ) async for message in self.handle_stop_listening( stop_listening ): yield message if self.wake_enabled and (session_id is None) and self.wake_proc: # Convert and send to wake command audio_bytes = self.maybe_convert_wav( wav_bytes, self.wake_sample_rate, self.wake_sample_width, self.wake_channels, ) assert self.wake_proc.stdin self.wake_proc.stdin.write(audio_bytes) if self.wake_proc.poll(): stdout, stderr = self.wake_proc.communicate() if stderr: _LOGGER.debug(stderr.decode()) wakeword_id = stdout.decode().strip() _LOGGER.debug("Detected wake word %s", wakeword_id) yield ( HotwordDetected( model_id=wakeword_id, model_version="", model_type="personal", current_sensitivity=1.0, site_id=site_id, ), {"wakeword_id": wakeword_id}, ) # Restart wake process self.start_wake_command() except Exception: _LOGGER.exception("handle_audio_frame")