def test_rules(self):
"""Make sure local and remote rules work."""
intents = parse_ini("""
[Intent1]
rule = a test
this is <rule>
[Intent2]
rule = this is
<rule> <Intent1.rule>
""")
graph = intents_to_graph(intents)
# Lower confidence with no stop words
recognitions = zero_times(recognize("this is a test", graph))
self.assertEqual(
recognitions,
[
Recognition(
intent=Intent(name="Intent1", confidence=1.0),
text="this is a test",
raw_text="this is a test",
tokens=["this", "is", "a", "test"],
raw_tokens=["this", "is", "a", "test"],
),
Recognition(
intent=Intent(name="Intent2", confidence=1.0),
text="this is a test",
raw_text="this is a test",
tokens=["this", "is", "a", "test"],
raw_tokens=["this", "is", "a", "test"],
),
],
)
def test_multiple_sentences(self):
"""Identical sentences from two different intents."""
intents = parse_ini("""
[TestIntent1]
this is a test
[TestIntent2]
this is a test
""")
graph = intents_to_graph(intents)
# Should produce a recognition for each intent
recognitions = zero_times(recognize("this is a test", graph))
self.assertEqual(len(recognitions), 2)
self.assertIn(
Recognition(
intent=Intent(name="TestIntent1", confidence=1.0),
text="this is a test",
raw_text="this is a test",
tokens=["this", "is", "a", "test"],
raw_tokens=["this", "is", "a", "test"],
),
recognitions,
)
self.assertIn(
Recognition(
intent=Intent(name="TestIntent2", confidence=1.0),
text="this is a test",
raw_text="this is a test",
tokens=["this", "is", "a", "test"],
raw_tokens=["this", "is", "a", "test"],
),
recognitions,
)
def test_stop_words(self):
"""Check sentence with stop words."""
intents = parse_ini("""
[TestIntent]
this is a test
""")
graph = intents_to_graph(intents)
# Failure without stop words
recognitions = zero_times(
recognize("this is a abcd test", graph, fuzzy=False))
self.assertFalse(recognitions)
# Success with stop words
recognitions = zero_times(
recognize("this is a abcd test",
graph,
stop_words={"abcd"},
fuzzy=False))
self.assertEqual(
recognitions,
[
Recognition(
intent=Intent(name="TestIntent", confidence=1.0),
text="this is a test",
raw_text="this is a test",
tokens=["this", "is", "a", "test"],
raw_tokens=["this", "is", "a", "test"],
)
],
)
def test_stop_words(self):
"""Check sentence with stop words."""
intents = parse_ini("""
[TestIntent]
this is a test
""")
graph = intents_to_graph(intents)
# Lower confidence with no stop words
recognitions = zero_times(recognize("this is a abcd test", graph))
self.assertEqual(len(recognitions), 1)
self.assertEqual(recognitions[0].intent.confidence, 1 - (1 / 4))
# Higher confidence with stop words
recognitions = zero_times(
recognize("this is a abcd test", graph, stop_words={"abcd"}))
self.assertEqual(
recognitions,
[
Recognition(
intent=Intent(name="TestIntent",
confidence=float(1 - (0.1 / 4))),
text="this is a test",
raw_text="this is a test",
tokens=["this", "is", "a", "test"],
raw_tokens=["this", "is", "a", "test"],
)
],
)
def test_intent_filter(self):
"""Identical sentences from two different intents with filter."""
intents = parse_ini("""
[TestIntent1]
this is a test
[TestIntent2]
this is a test
""")
graph = intents_to_graph(intents)
def intent_filter(name):
return name == "TestIntent1"
# Should produce a recognition for first intent only
recognitions = zero_times(
recognize("this is a test", graph, intent_filter=intent_filter))
self.assertEqual(
recognitions,
[
Recognition(
intent=Intent(name="TestIntent1", confidence=1.0),
text="this is a test",
raw_text="this is a test",
tokens=["this", "is", "a", "test"],
raw_tokens=["this", "is", "a", "test"],
)
],
)
def test_converter_args(self):
"""Check converter with arguments."""
intents = parse_ini("""
[TestIntent]
this is a test ten:10!int!pow,3
""")
graph = intents_to_graph(intents)
def pow_converter(*args, converter_args=None):
exponent = int(converter_args[0]) if converter_args else 1
return [x**exponent for x in args]
# Should convert "ten" -> 10 -> 1000
recognitions = zero_times(
recognize(
"this is a test ten",
graph,
fuzzy=False,
extra_converters={"pow": pow_converter},
))
self.assertEqual(
recognitions,
[
Recognition(
intent=Intent(name="TestIntent", confidence=1.0),
text="this is a test 1000",
raw_text="this is a test ten",
tokens=["this", "is", "a", "test", 1000],
raw_tokens=["this", "is", "a", "test", "ten"],
)
],
)
def test_single_sentence(self):
"""Single intent, single sentence."""
intents = parse_ini("""
[TestIntent]
this is a test
""")
graph = intents_to_graph(intents)
# Exact
recognitions = zero_times(
recognize("this is a test", graph, fuzzy=False))
print(recognitions)
self.assertEqual(
recognitions,
[
Recognition(
intent=Intent(name="TestIntent", confidence=1.0),
text="this is a test",
raw_text="this is a test",
tokens=["this", "is", "a", "test"],
raw_tokens=["this", "is", "a", "test"],
)
],
)
# Too many tokens (lower confidence)
recognitions = zero_times(
recognize("this is a bad test", graph, fuzzy=False))
self.assertFalse(recognitions)
# Too few tokens (failure)
recognitions = zero_times(recognize("this is a", graph, fuzzy=False))
self.assertFalse(recognitions)
def test_converters(self):
"""Check sentence with converters."""
intents = parse_ini("""
[TestIntent]
this is a test!upper ten:10!int!square
""")
graph = intents_to_graph(intents)
# Should upper-case "test" and convert "ten" -> 10 -> 100
recognitions = zero_times(
recognize(
"this is a test ten",
graph,
fuzzy=False,
extra_converters={
"square": lambda *args: [x**2 for x in args]
},
))
self.assertEqual(
recognitions,
[
Recognition(
intent=Intent(name="TestIntent", confidence=1.0),
text="this is a TEST 100",
raw_text="this is a test ten",
tokens=["this", "is", "a", "TEST", 100],
raw_tokens=["this", "is", "a", "test", "ten"],
)
],
)
def test_single_sentence(self):
"""Single intent, single sentence."""
intents = parse_ini("""
[TestIntent]
this is a test?
""")
graph = intents_to_graph(intents)
examples = train(graph)
# Exact
recognitions = zero_times(recognize("this is a test", graph, examples))
self.assertEqual(
recognitions,
[
Recognition(
intent=Intent(name="TestIntent", confidence=1),
text="this is a test?",
raw_text="this is a test",
tokens=["this", "is", "a", "test?"],
raw_tokens=["this", "is", "a", "test"],
)
],
)
# Mispellings, too many tokens (lower confidence)
for sentence in ["this is a bad test", "this iz b tst"]:
recognitions = zero_times(recognize(sentence, graph, examples))
self.assertEqual(len(recognitions), 1)
intent = recognitions[0].intent
self.assertIsNotNone(intent)
self.assertLess(intent.confidence, 1.0)
def recognize(args: argparse.Namespace):
"""Do intent recognition from query text."""
try:
# Convert to Paths
args.examples = Path(args.examples)
args.intent_graph = Path(args.intent_graph)
# Load graph/examples
_LOGGER.debug("Loading intent graph from %s", str(args.intent_graph))
with open(args.intent_graph, "r") as intent_graph_file:
graph_dict = json.load(intent_graph_file)
intent_graph = rhasspynlu.json_to_graph(graph_dict)
_LOGGER.debug("Loading examples from %s", str(args.examples))
with open(args.examples, "r") as examples_file:
examples = json.load(examples_file)
_LOGGER.debug("Processing sentences")
word_transform = get_word_transform(args.word_casing)
# Process queries
if args.query:
sentences = args.query
else:
if os.isatty(sys.stdin.fileno()):
print("Reading queries from stdin...", file=sys.stderr)
sentences = sys.stdin
for sentence in sentences:
# Handle casing
sentence = sentence.strip()
sentence = word_transform(sentence)
# Do recognition
recognitions = fuzzywuzzy_recognize(sentence, intent_graph,
examples)
if recognitions:
# Intent recognized
recognition = recognitions[0]
else:
# Intent not recognized
recognition = Recognition.empty()
# Print as a line of JSON
json.dump(recognition.asdict(), sys.stdout)
print("")
sys.stdout.flush()
except KeyboardInterrupt:
pass
def test_evaluate_perfect(self):
"""Test intent evaluation with a perfect match."""
expected = {
"test1": Recognition(
intent=Intent(name="TestIntent"),
entities=[Entity(entity="testEntity", value="testValue")],
text="this is a test",
)
}
actual = expected
report = evaluate_intents(expected, actual)
self.assertEqual(1, report.num_wavs)
self.assertEqual(1, report.num_intents)
self.assertEqual(1, report.num_entities)
self.assertEqual(1, report.correct_intent_names)
self.assertEqual(1, report.correct_entities)
self.assertEqual(1, report.correct_intent_and_entities)
def test_sequence_converters(self):
"""Check sentence with sequence converters."""
intents = parse_ini("""
[TestIntent]
this (is a test)!upper
""")
graph = intents_to_graph(intents)
# Should upper-case "is a test"
recognitions = zero_times(
recognize("this is a test", graph, fuzzy=False))
self.assertEqual(
recognitions,
[
Recognition(
intent=Intent(name="TestIntent", confidence=1.0),
text="this IS A TEST",
raw_text="this is a test",
tokens=["this", "IS", "A", "TEST"],
raw_tokens=["this", "is", "a", "test"],
)
],
)
def test_drop_group(self):
"""Test dropping a group."""
intents = parse_ini("""
[TestIntent]
this is (a | another): test
""")
graph = intents_to_graph(intents)
recognitions = zero_times(
recognize("this is a test", graph, fuzzy=False))
self.assertEqual(
recognitions,
[
Recognition(
intent=Intent(name="TestIntent", confidence=1.0),
text="this is test",
raw_text="this is a test",
tokens=["this", "is", "test"],
raw_tokens=["this", "is", "a", "test"],
)
],
)
async def test_examples(args: argparse.Namespace, core: Voice2JsonCore) -> None:
"""Test speech/intent recognition against a directory of expected results."""
from rhasspynlu.evaluate import evaluate_intents
from rhasspynlu.intent import Recognition
# Make sure profile has been trained
assert core.check_trained(), "Not trained"
# Expected/actual intents
expected: typing.Dict[str, Recognition] = {}
actual: typing.Dict[str, Recognition] = {}
if args.expected:
_LOGGER.debug("Loading expected intents from %s", args.expected)
# Load expected results from jsonl file.
# Each line is an intent with a wav_name key.
with open(args.expected, "r") as expected_file:
for line in expected_file:
expected_intent = Recognition.from_dict(json.loads(line))
assert expected_intent.wav_name, f"No wav_name for {line}"
expected[expected_intent.wav_name] = expected_intent
else:
# Load expected results from examples directory
assert args.directory, "Examples directory required if no --expected"
examples_dir = Path(args.directory)
_LOGGER.debug("Loading expected intents from %s", examples_dir)
for wav_path in examples_dir.glob("*.wav"):
json_path = wav_path.with_suffix(".json")
text_path = wav_path.with_suffix(".txt")
if json_path.is_file():
# JSON file with expected intent and transcripion
with open(json_path, "r") as json_file:
expected[wav_path.name] = Recognition.from_dict(
json.load(json_file)
)
elif text_path.is_file():
# Text file with expected transcripion only
transcription = text_path.read_text().strip()
expected[wav_path.name] = Recognition(text=transcription)
else:
_LOGGER.warning("No JSON or text file for %s", wav_path)
if not expected:
_LOGGER.fatal("No expected examples provided")
return
temp_dir = None
try:
if not args.actual:
# Generate actual results from examples directory
assert args.directory, "Examples directory required if no --expected"
examples_dir = Path(args.directory)
_LOGGER.debug("Generating actual intents from %s", examples_dir)
# Use voice2json and GNU parallel
assert shutil.which("parallel"), "GNU parallel is required"
if args.results:
# Save results to user-specified directory
results_dir = Path(args.results)
results_dir.mkdir(parents=True, exist_ok=True)
_LOGGER.debug("Saving results to %s", results_dir)
else:
# Save resuls to temporary directory
temp_dir = tempfile.TemporaryDirectory()
results_dir = Path(temp_dir.name)
_LOGGER.debug(
"Saving results to temporary directory (use --results to specify)"
)
# Transcribe WAV files
actual_wavs_path = results_dir / "actual_wavs.txt"
actual_transcriptions_path = results_dir / "actual_transcriptions.jsonl"
# Write list of WAV files to a text file
with open(actual_wavs_path, "w") as actual_wavs_file:
for wav_path in examples_dir.glob("*.wav"):
print(str(wav_path.absolute()), file=actual_wavs_file)
# Transcribe in parallel
transcribe_args = ["--stdin-files"]
if args.open:
transcribe_args.append("--open")
transcribe_command = (
[
"parallel",
"-k",
"--pipe",
"-n",
str(args.threads),
"voice2json",
"-p",
shlex.quote(str(core.profile_file)),
"transcribe-wav",
]
+ transcribe_args
+ ["<", str(actual_wavs_path), ">", str(actual_transcriptions_path)]
)
_LOGGER.debug(transcribe_command)
transcribe_process = await asyncio.create_subprocess_shell(
" ".join(transcribe_command)
)
await transcribe_process.wait()
assert transcribe_process.returncode == 0, "Transcription failed"
# Recognize intents from transcriptions in parallel
actual_intents_path = results_dir / "actual_intents.jsonl"
recognize_command = [
"parallel",
"-k",
"--pipe",
"-n",
str(args.threads),
"voice2json",
"-p",
shlex.quote(str(core.profile_file)),
"recognize-intent",
"<",
str(actual_transcriptions_path),
">",
str(actual_intents_path),
]
_LOGGER.debug(recognize_command)
recognize_process = await asyncio.create_subprocess_shell(
" ".join(recognize_command)
)
await recognize_process.wait()
assert recognize_process.returncode == 0, "Recognition failed"
# Load actual intents
args.actual = actual_intents_path
assert args.actual, "No actual intents to load"
_LOGGER.debug("Loading actual intents from %s", args.actual)
# Load actual results from jsonl file
with open(args.actual, "r") as actual_file:
for line in actual_file:
actual_intent = Recognition.from_dict(json.loads(line))
assert actual_intent.wav_name, f"No wav_name for {line}"
actual[actual_intent.wav_name] = actual_intent
if not actual:
_LOGGER.fatal("No actual examples provided")
return
report = evaluate_intents(expected, actual)
print_json(dataclasses.asdict(report))
finally:
# Delete temporary directory
if temp_dir:
temp_dir.cleanup()
def recognize(
text: str,
engine: SnipsNLUEngine,
slots_dict: typing.Optional[typing.Dict[str, typing.List[str]]] = None,
slot_graphs: typing.Optional[typing.Dict[str, nx.DiGraph]] = None,
**parse_args,
) -> typing.List[Recognition]:
"""Recognize intent using Snips NLU."""
result = engine.parse(text, **parse_args)
intent_name = result.get("intent", {}).get("intentName")
if not intent_name:
# Recognition failure
return []
slots_dict = slots_dict or {}
slot_graphs = slot_graphs or {}
recognition = Recognition(text=text,
raw_text=text,
intent=Intent(name=intent_name, confidence=1.0))
# Replace Snips slot values with Rhasspy slot values (substituted)
for slot in result.get("slots", []):
slot_name = slot.get("slotName")
slot_value_dict = slot.get("value", {})
slot_value = slot_value_dict.get("value")
entity = Entity(
entity=slot_name,
source=slot.get("entity", ""),
value=slot_value,
raw_value=slot.get("rawValue", slot_value),
start=slot["range"]["start"],
end=slot["range"]["end"],
)
recognition.entities.append(entity)
if (not slot_name) or (not slot_value):
continue
slot_graph = slot_graphs.get(slot_name)
if not slot_graph and (slot_name in slots_dict):
# Convert slot values to graph
slot_graph = rhasspynlu.sentences_to_graph({
slot_name: [
rhasspynlu.jsgf.Sentence.parse(slot_line)
for slot_line in slots_dict[slot_name]
if slot_line.strip()
]
})
slot_graphs[slot_name] = slot_graph
entity.tokens = slot_value.split()
entity.raw_tokens = list(entity.tokens)
if slot_graph:
# Pass Snips value through graph
slot_recognitions = rhasspynlu.recognize(entity.tokens, slot_graph)
if slot_recognitions:
# Pull out substituted value and replace in Rhasspy entitiy
new_slot_value = slot_recognitions[0].text
entity.value = new_slot_value
entity.tokens = new_slot_value.split()
return [recognition]