def test_intent_evaluation_report(tmpdir_factory):
path = tmpdir_factory.mktemp("evaluation").strpath
report_folder = os.path.join(path, "reports")
report_filename = os.path.join(report_folder, "intent_report.json")
utils.create_dir(report_folder)
intent_results = [
IntentEvaluationResult("", "restaurant_search",
"I am hungry", 0.12345),
IntentEvaluationResult("greet", "greet",
"hello", 0.98765)]
result = evaluate_intents(intent_results,
report_folder,
successes_filename=None,
errors_filename=None,
confmat_filename=None,
intent_hist_filename=None)
report = json.loads(utils.read_file(report_filename))
greet_results = {"precision": 1.0,
"recall": 1.0,
"f1-score": 1.0,
"support": 1}
prediction = {'text': 'hello',
'intent': 'greet',
'predicted': 'greet',
'confidence': 0.98765}
assert len(report.keys()) == 4
assert report["greet"] == greet_results
assert result["predictions"][0] == prediction
def test_empty_intent_removal():
intent_results = [
IntentEvaluationResult("", "restaurant_search", "I am hungry", 0.12345),
IntentEvaluationResult("greet", "greet", "hello", 0.98765),
]
intent_results = remove_empty_intent_examples(intent_results)
assert len(intent_results) == 1
assert intent_results[0].intent_target == "greet"
assert intent_results[0].intent_prediction == "greet"
assert intent_results[0].confidence == 0.98765
assert intent_results[0].message == "hello"
def test_intent_evaluation_report(tmp_path: Path):
path = tmp_path / "evaluation"
path.mkdir()
report_folder = str(path / "reports")
report_filename = os.path.join(report_folder, "intent_report.json")
rasa.shared.utils.io.create_directory(report_folder)
intent_results = [
IntentEvaluationResult("", "restaurant_search", "I am hungry",
0.12345),
IntentEvaluationResult("greet", "greet", "hello", 0.98765),
]
result = evaluate_intents(
intent_results,
report_folder,
successes=True,
errors=True,
disable_plotting=False,
)
report = json.loads(rasa.shared.utils.io.read_file(report_filename))
greet_results = {
"precision": 1.0,
"recall": 1.0,
"f1-score": 1.0,
"support": 1,
"confused_with": {},
}
prediction = {
"text": "hello",
"intent": "greet",
"predicted": "greet",
"confidence": 0.98765,
}
assert len(report.keys()) == 4
assert report["greet"] == greet_results
assert result["predictions"][0] == prediction
assert os.path.exists(
os.path.join(report_folder, "intent_confusion_matrix.png"))
assert os.path.exists(os.path.join(report_folder, "intent_histogram.png"))
assert not os.path.exists(os.path.join(report_folder,
"intent_errors.json"))
assert os.path.exists(os.path.join(report_folder, "intent_successes.json"))
def test_intent_evaluation_report(tmpdir_factory):
path = tmpdir_factory.mktemp("evaluation").strpath
report_folder = os.path.join(path, "reports")
report_filename = os.path.join(report_folder, "intent_report.json")
rasa.utils.io.create_directory(report_folder)
intent_results = [
IntentEvaluationResult("", "restaurant_search", "I am hungry",
0.12345),
IntentEvaluationResult("greet", "greet", "hello", 0.98765),
]
result = evaluate_intents(
intent_results,
report_folder,
successes=False,
errors=False,
confmat_filename=None,
intent_hist_filename=None,
disable_plotting=False,
)
report = json.loads(rasa.utils.io.read_file(report_filename))
greet_results = {
"precision": 1.0,
"recall": 1.0,
"f1-score": 1.0,
"support": 1,
"confused_with": {},
}
prediction = {
"text": "hello",
"intent": "greet",
"predicted": "greet",
"confidence": 0.98765,
}
assert len(report.keys()) == 4
assert report["greet"] == greet_results
assert result["predictions"][0] == prediction
def get_intent_predictions(targets, interpreter,
test_data): # pragma: no cover
"""Runs the model for the test set and extracts intent predictions.
Returns intent predictions, the original messages
and the confidences of the predictions"""
intent_results = []
for e, target in zip(test_data.training_examples, targets):
res = interpreter.parse(e.text, only_output_properties=False)
intent_results.append(
IntentEvaluationResult(target,
res.get('intent', {}).get('name'),
res.get('text', {}),
res.get('intent', {}).get('confidence')))
return intent_results
def incorrect(label: Text, _label: Text) -> IntentEvaluationResult:
return IntentEvaluationResult(label, _label, "", 1.0)
