def test_recognized(self):
"""Verify valid input leads to a recognition."""
query_id = str(uuid.uuid4())
text = "set the bedroom light to red"
self.hermes.publish = MagicMock()
self.hermes.handle_query(
NluQuery(input=text,
id=query_id,
siteId=self.siteId,
sessionId=self.sessionId))
self.hermes.publish.assert_called_with(
NluIntent(
input=text,
id=query_id,
intent=Intent(intentName="SetLightColor", confidenceScore=1),
slots=[
Slot(
entity="name",
slotName="name",
value="bedroom",
raw_value="bedroom",
confidence=1,
range=SlotRange(8, 15),
),
Slot(
entity="color",
slotName="color",
value="red",
raw_value="red",
confidence=1,
range=SlotRange(25, 28),
),
],
siteId=self.siteId,
sessionId=self.sessionId,
),
intent_name="SetLightColor",
)
async def handle_text_captured(
self, text_captured: AsrTextCaptured
) -> typing.AsyncIterable[typing.Union[AsrStopListening, HotwordToggleOn,
NluQuery]]:
"""Handle ASR text captured for session."""
try:
if self.session is None:
return
_LOGGER.debug("Received text: %s", text_captured.text)
# Record result
self.session.text_captured = text_captured
# Stop listening
yield AsrStopListening(site_id=text_captured.site_id,
session_id=self.session.session_id)
# Enable hotword
yield HotwordToggleOn(
site_id=text_captured.site_id,
reason=HotwordToggleReason.DIALOGUE_SESSION,
)
# Perform query
yield NluQuery(
input=text_captured.text,
intent_filter=self.session.intent_filter
or self.default_intent_filter,
session_id=self.session.session_id,
site_id=self.session.site_id,
wakeword_id=text_captured.wakeword_id
or self.session.wakeword_id,
lang=text_captured.lang or self.session.lang,
)
except Exception:
_LOGGER.exception("handle_text_captured")
async def handle_text_captured(
self, text_captured: AsrTextCaptured
) -> typing.AsyncIterable[
typing.Union[
AsrStopListening, HotwordToggleOn, NluQuery, NluIntentNotRecognized
]
]:
"""Handle ASR text captured for session."""
try:
if not text_captured.session_id:
_LOGGER.warning("Missing session id on text captured message.")
return
site_session = self.all_sessions.get(text_captured.session_id)
if site_session is None:
_LOGGER.warning(
"No session for id %s. Dropping captured text from ASR.",
text_captured.session_id,
)
return
_LOGGER.debug("Received text: %s", text_captured.text)
# Record result
site_session.text_captured = text_captured
# Stop listening
yield AsrStopListening(
site_id=text_captured.site_id, session_id=site_session.session_id
)
# Enable hotword
yield HotwordToggleOn(
site_id=text_captured.site_id,
reason=HotwordToggleReason.DIALOGUE_SESSION,
)
if (self.min_asr_confidence is not None) and (
text_captured.likelihood < self.min_asr_confidence
):
# Transcription is below thresold.
# Don't actually do an NLU query, just reject as "not recognized".
_LOGGER.debug(
"Transcription is below confidence threshold (%s < %s): %s",
text_captured.likelihood,
self.min_asr_confidence,
text_captured.text,
)
yield NluIntentNotRecognized(
input=text_captured.text,
site_id=site_session.site_id,
session_id=site_session.session_id,
)
else:
# Perform query
custom_entities: typing.Optional[typing.Dict[str, typing.Any]] = None
# Copy custom entities from hotword detected
if site_session.detected:
custom_entities = site_session.detected.custom_entities
yield NluQuery(
input=text_captured.text,
intent_filter=site_session.intent_filter
or self.default_intent_filter,
session_id=site_session.session_id,
site_id=site_session.site_id,
wakeword_id=text_captured.wakeword_id or site_session.wakeword_id,
lang=text_captured.lang or site_session.lang,
custom_data=site_session.custom_data,
asr_confidence=text_captured.likelihood,
custom_entities=custom_entities,
)
except Exception:
_LOGGER.exception("handle_text_captured")
async def handle_query(
self, query: NluQuery
) -> typing.AsyncIterable[typing.Union[NluIntentParsed, typing.Tuple[
NluIntent, TopicArgs], NluIntentNotRecognized, NluError, ]]:
"""Do intent recognition."""
original_input = query.input
try:
if not self.intent_graph and self.graph_path and self.graph_path.is_file(
):
# Load graph from file
_LOGGER.debug("Loading %s", self.graph_path)
with open(self.graph_path, mode="rb") as graph_file:
self.intent_graph = rhasspynlu.gzip_pickle_to_graph(
graph_file)
if self.intent_graph:
def intent_filter(intent_name: str) -> bool:
"""Filter out intents."""
if query.intent_filter:
return intent_name in query.intent_filter
return True
# Replace digits with words
if self.replace_numbers:
# Have to assume whitespace tokenization
words = rhasspynlu.replace_numbers(query.input.split(),
self.language)
query.input = " ".join(words)
input_text = query.input
# Fix casing for output event
if self.word_transform:
input_text = self.word_transform(input_text)
if self.failure_token and (self.failure_token
in query.input.split()):
# Failure token was found in input
recognitions = []
else:
# Pass in raw query input so raw values will be correct
recognitions = recognize(
query.input,
self.intent_graph,
intent_filter=intent_filter,
word_transform=self.word_transform,
fuzzy=self.fuzzy,
extra_converters=self.extra_converters,
)
else:
_LOGGER.error("No intent graph loaded")
recognitions = []
if NluHermesMqtt.is_success(recognitions):
# Use first recognition only.
recognition = recognitions[0]
assert recognition is not None
assert recognition.intent is not None
intent = Intent(
intent_name=recognition.intent.name,
confidence_score=recognition.intent.confidence,
)
slots = [
Slot(
entity=(e.source or e.entity),
slot_name=e.entity,
confidence=1.0,
value=e.value_dict,
raw_value=e.raw_value,
range=SlotRange(
start=e.start,
end=e.end,
raw_start=e.raw_start,
raw_end=e.raw_end,
),
) for e in recognition.entities
]
if query.custom_entities:
# Copy user-defined entities
for entity_name, entity_value in query.custom_entities.items(
):
slots.append(
Slot(
entity=entity_name,
confidence=1.0,
value={"value": entity_value},
))
# intentParsed
yield NluIntentParsed(
input=recognition.text,
id=query.id,
site_id=query.site_id,
session_id=query.session_id,
intent=intent,
slots=slots,
)
# intent
yield (
NluIntent(
input=recognition.text,
id=query.id,
site_id=query.site_id,
session_id=query.session_id,
intent=intent,
slots=slots,
asr_tokens=[
NluIntent.make_asr_tokens(recognition.tokens)
],
asr_confidence=query.asr_confidence,
raw_input=original_input,
wakeword_id=query.wakeword_id,
lang=(query.lang or self.lang),
custom_data=query.custom_data,
),
{
"intent_name": recognition.intent.name
},
)
else:
# Not recognized
yield NluIntentNotRecognized(
input=query.input,
id=query.id,
site_id=query.site_id,
session_id=query.session_id,
custom_data=query.custom_data,
)
except Exception as e:
_LOGGER.exception("handle_query")
yield NluError(
site_id=query.site_id,
session_id=query.session_id,
error=str(e),
context=original_input,
)
async def async_test_handle_query(self):
"""Verify valid input leads to a query message."""
query_id = str(uuid.uuid4())
text = "set the bedroom light to red"
query = NluQuery(input=text,
id=query_id,
site_id=self.site_id,
session_id=self.session_id)
results = []
async for result in self.hermes.on_message(query):
results.append(result)
# Check results
intent = Intent(intent_name="SetLightColor", confidence_score=1.0)
slots = [
Slot(
entity="name",
slot_name="name",
value={
"kind": "Unknown",
"value": "bedroom"
},
raw_value="bedroom",
confidence=1.0,
range=SlotRange(start=8, end=15, raw_start=8, raw_end=15),
),
Slot(
entity="color",
slot_name="color",
value={
"kind": "Unknown",
"value": "red"
},
raw_value="red",
confidence=1.0,
range=SlotRange(start=25, end=28, raw_start=25, raw_end=28),
),
]
self.assertEqual(
results,
[
NluIntentParsed(
input=text,
id=query_id,
site_id=self.site_id,
session_id=self.session_id,
intent=intent,
slots=slots,
),
(
NluIntent(
input=text,
id=query_id,
site_id=self.site_id,
session_id=self.session_id,
intent=intent,
slots=slots,
asr_tokens=[NluIntent.make_asr_tokens(text.split())],
raw_input=text,
),
{
"intent_name": intent.intent_name
},
),
],
)
async def handle_query(
self, query: NluQuery
) -> typing.AsyncIterable[typing.Union[NluIntentParsed, typing.Tuple[
NluIntent, TopicArgs], NluIntentNotRecognized, NluError, ]]:
"""Do intent recognition."""
# Check intent graph
try:
if (not self.intent_graph and self.intent_graph_path
and self.intent_graph_path.is_file()):
_LOGGER.debug("Loading %s", self.intent_graph_path)
with open(self.intent_graph_path, mode="rb") as graph_file:
self.intent_graph = rhasspynlu.gzip_pickle_to_graph(
graph_file)
# Check examples
if (self.intent_graph and self.examples_path
and self.examples_path.is_file()):
def intent_filter(intent_name: str) -> bool:
"""Filter out intents."""
if query.intent_filter:
return intent_name in query.intent_filter
return True
original_text = query.input
# Replace digits with words
if self.replace_numbers:
# Have to assume whitespace tokenization
words = rhasspynlu.replace_numbers(query.input.split(),
self.language)
query.input = " ".join(words)
input_text = query.input
# Fix casing
if self.word_transform:
input_text = self.word_transform(input_text)
recognitions: typing.List[rhasspynlu.intent.Recognition] = []
if input_text:
recognitions = rhasspyfuzzywuzzy.recognize(
input_text,
self.intent_graph,
str(self.examples_path),
intent_filter=intent_filter,
extra_converters=self.extra_converters,
)
else:
_LOGGER.error("No intent graph or examples loaded")
recognitions = []
# Use first recognition only if above threshold
if (recognitions and recognitions[0] and recognitions[0].intent
and (recognitions[0].intent.confidence >=
self.confidence_threshold)):
recognition = recognitions[0]
assert recognition.intent
intent = Intent(
intent_name=recognition.intent.name,
confidence_score=recognition.intent.confidence,
)
slots = [
Slot(
entity=(e.source or e.entity),
slot_name=e.entity,
confidence=1.0,
value=e.value_dict,
raw_value=e.raw_value,
range=SlotRange(
start=e.start,
end=e.end,
raw_start=e.raw_start,
raw_end=e.raw_end,
),
) for e in recognition.entities
]
if query.custom_entities:
# Copy user-defined entities
for entity_name, entity_value in query.custom_entities.items(
):
slots.append(
Slot(
entity=entity_name,
confidence=1.0,
value={"value": entity_value},
))
# intentParsed
yield NluIntentParsed(
input=recognition.text,
id=query.id,
site_id=query.site_id,
session_id=query.session_id,
intent=intent,
slots=slots,
)
# intent
yield (
NluIntent(
input=recognition.text,
id=query.id,
site_id=query.site_id,
session_id=query.session_id,
intent=intent,
slots=slots,
asr_tokens=[
NluIntent.make_asr_tokens(recognition.tokens)
],
asr_confidence=query.asr_confidence,
raw_input=original_text,
wakeword_id=query.wakeword_id,
lang=(query.lang or self.lang),
custom_data=query.custom_data,
),
{
"intent_name": recognition.intent.name
},
)
else:
# Not recognized
yield NluIntentNotRecognized(
input=query.input,
id=query.id,
site_id=query.site_id,
session_id=query.session_id,
custom_data=query.custom_data,
)
except Exception as e:
_LOGGER.exception("handle_query")
yield NluError(
site_id=query.site_id,
session_id=query.session_id,
error=str(e),
context=original_text,
)
def test_nlu_query():
"""Test NluQuery."""
assert NluQuery.topic() == "hermes/nlu/query"
async def handle_query(
self, query: NluQuery
) -> typing.AsyncIterable[typing.Union[
NluIntentParsed, NluIntentNotRecognized, NluError, ]]:
"""Do intent recognition."""
try:
# Replace digits with words
if self.replace_numbers:
# Have to assume whitespace tokenization
words = rhasspynlu.replace_numbers(query.input.split(),
self.number_language)
query.input = " ".join(words)
input_text = query.input
# Fix casing for output event
if self.word_transform:
input_text = self.word_transform(input_text)
parse_url = urljoin(self.rasa_url, "model/parse")
_LOGGER.debug(parse_url)
async with self.http_session.post(
parse_url,
json={
"text": input_text,
"project": self.rasa_project
},
ssl=self.ssl_context,
) as response:
response.raise_for_status()
intent_json = await response.json()
intent = intent_json.get("intent", {})
intent_name = intent.get("name", "")
if intent_name and (query.intent_filter is None
or intent_name in query.intent_filter):
confidence_score = float(intent.get("confidence", 0.0))
slots = [
Slot(
entity=e.get("entity", ""),
slot_name=e.get("entity", ""),
confidence=float(e.get("confidence", 0.0)),
value={
"kind": "Unknown",
"value": e.get("value", ""),
"additional_info":
e.get("additional_info", {}),
"extractor": e.get("extractor", None),
},
raw_value=e.get("value", ""),
range=SlotRange(
start=int(e.get("start", 0)),
end=int(e.get("end", 1)),
raw_start=int(e.get("start", 0)),
raw_end=int(e.get("end", 1)),
),
) for e in intent_json.get("entities", [])
]
# intentParsed
yield NluIntentParsed(
input=input_text,
id=query.id,
site_id=query.site_id,
session_id=query.session_id,
intent=Intent(intent_name=intent_name,
confidence_score=confidence_score),
slots=slots,
)
else:
# Not recognized
yield NluIntentNotRecognized(
input=query.input,
id=query.id,
site_id=query.site_id,
session_id=query.session_id,
)
except Exception as e:
_LOGGER.exception("nlu query")
yield NluError(
site_id=query.site_id,
session_id=query.session_id,
error=str(e),
context=query.input,
)
async def handle_query(
self, query: NluQuery
) -> typing.AsyncIterable[typing.Union[NluIntentParsed, typing.Tuple[
NluIntent, TopicArgs], NluIntentNotRecognized, NluError, ]]:
"""Do intent recognition."""
try:
original_input = query.input
# Replace digits with words
if self.replace_numbers:
# Have to assume whitespace tokenization
words = rhasspynlu.replace_numbers(query.input.split(),
self.number_language)
query.input = " ".join(words)
input_text = query.input
# Fix casing for output event
if self.word_transform:
input_text = self.word_transform(input_text)
parse_url = urljoin(self.rasa_url, "model/parse")
_LOGGER.debug(parse_url)
async with self.http_session.post(
parse_url,
json={
"text": input_text,
"project": self.rasa_project
},
ssl=self.ssl_context,
) as response:
response.raise_for_status()
intent_json = await response.json()
intent = intent_json.get("intent", {})
intent_name = intent.get("name", "")
if intent_name and (query.intent_filter is None
or intent_name in query.intent_filter):
confidence_score = float(intent.get("confidence", 0.0))
slots = [
Slot(
entity=e.get("entity", ""),
slot_name=e.get("entity", ""),
confidence=float(e.get("confidence", 0.0)),
value={
"kind": "Unknown",
"value": e.get("value", "")
},
raw_value=e.get("value", ""),
range=SlotRange(
start=int(e.get("start", 0)),
end=int(e.get("end", 1)),
raw_start=int(e.get("start", 0)),
raw_end=int(e.get("end", 1)),
),
) for e in intent_json.get("entities", [])
]
if query.custom_entities:
# Copy user-defined entities
for entity_name, entity_value in query.custom_entities.items(
):
slots.append(
Slot(
entity=entity_name,
confidence=1.0,
value={"value": entity_value},
))
# intentParsed
yield NluIntentParsed(
input=input_text,
id=query.id,
site_id=query.site_id,
session_id=query.session_id,
intent=Intent(intent_name=intent_name,
confidence_score=confidence_score),
slots=slots,
)
# intent
yield (
NluIntent(
input=input_text,
id=query.id,
site_id=query.site_id,
session_id=query.session_id,
intent=Intent(
intent_name=intent_name,
confidence_score=confidence_score,
),
slots=slots,
asr_tokens=[
NluIntent.make_asr_tokens(input_text.split())
],
asr_confidence=query.asr_confidence,
raw_input=original_input,
lang=(query.lang or self.lang),
custom_data=query.custom_data,
),
{
"intent_name": intent_name
},
)
else:
# Not recognized
yield NluIntentNotRecognized(
input=query.input,
id=query.id,
site_id=query.site_id,
session_id=query.session_id,
custom_data=query.custom_data,
)
except Exception as e:
_LOGGER.exception("nlu query")
yield NluError(
site_id=query.site_id,
session_id=query.session_id,
error=str(e),
context=query.input,
)
