def test_empty_response_removal():
response_results = [
ResponseSelectionEvaluationResult(None, None, "What's the weather",
0.65432),
ResponseSelectionEvaluationResult("chitchat/ask_name",
"chitchat/ask_name",
"What's your name?", 0.98765),
# This happens if response selection test data is present but no response
# selector is part of the model
ResponseSelectionEvaluationResult("chitchat/ask_name", None,
"What's your name?", None),
]
response_results = remove_empty_response_examples(response_results)
assert len(response_results) == 2
assert response_results[
0].intent_response_key_target == "chitchat/ask_name"
assert response_results[
0].intent_response_key_prediction == "chitchat/ask_name"
assert response_results[0].confidence == 0.98765
assert response_results[0].message == "What's your name?"
assert response_results[
1].intent_response_key_target == "chitchat/ask_name"
assert response_results[1].intent_response_key_prediction == ""
assert response_results[1].confidence == 0.0
assert response_results[1].message == "What's your name?"
def test_response_evaluation_report(tmp_path: Path):
path = tmp_path / "evaluation"
path.mkdir()
report_folder = str(path / "reports")
report_filename = os.path.join(report_folder,
"response_selection_report.json")
rasa.shared.utils.io.create_directory(report_folder)
response_results = [
ResponseSelectionEvaluationResult(
"chitchat/ask_weather",
"chitchat/ask_weather",
"What's the weather",
0.65432,
),
ResponseSelectionEvaluationResult("chitchat/ask_name",
"chitchat/ask_name",
"What's your name?", 0.98765),
]
result = evaluate_response_selections(
response_results,
report_folder,
successes=True,
errors=True,
disable_plotting=False,
)
report = json.loads(rasa.shared.utils.io.read_file(report_filename))
name_query_results = {
"precision": 1.0,
"recall": 1.0,
"f1-score": 1.0,
"support": 1,
"confused_with": {},
}
prediction = {
"text": "What's your name?",
"intent_response_key_target": "chitchat/ask_name",
"intent_response_key_prediction": "chitchat/ask_name",
"confidence": 0.98765,
}
assert len(report.keys()) == 5
assert report["chitchat/ask_name"] == name_query_results
assert result["predictions"][1] == prediction
assert os.path.exists(
os.path.join(report_folder, "response_selection_confusion_matrix.png"))
assert os.path.exists(
os.path.join(report_folder, "response_selection_histogram.png"))
assert not os.path.exists(
os.path.join(report_folder, "response_selection_errors.json"))
assert os.path.exists(
os.path.join(report_folder, "response_selection_successes.json"))
def test_response_evaluation_report(tmpdir_factory):
path = tmpdir_factory.mktemp("evaluation").strpath
report_folder = os.path.join(path, "reports")
report_filename = os.path.join(report_folder,
"response_selection_report.json")
rasa.utils.io.create_directory(report_folder)
response_results = [
ResponseSelectionEvaluationResult(
"chitchat",
"It's sunny in Berlin",
"It's sunny in Berlin",
"What's the weather",
0.65432,
),
ResponseSelectionEvaluationResult(
"chitchat",
"My name is Mr.bot",
"My name is Mr.bot",
"What's your name?",
0.98765,
),
]
result = evaluate_response_selections(
response_results,
report_folder,
successes=False,
errors=False,
disable_plotting=True,
)
report = json.loads(rasa.utils.io.read_file(report_filename))
name_query_results = {
"precision": 1.0,
"recall": 1.0,
"f1-score": 1.0,
"support": 1,
"confused_with": {},
}
prediction = {
"text": "What's your name?",
"intent_target": "chitchat",
"response_target": "My name is Mr.bot",
"response_predicted": "My name is Mr.bot",
"confidence": 0.98765,
}
assert len(report.keys()) == 5
assert report["My name is Mr.bot"] == name_query_results
assert result["predictions"][1] == prediction
def test_empty_response_removal():
response_results = [
ResponseSelectionEvaluationResult(None, None, "What's the weather", 0.65432),
ResponseSelectionEvaluationResult(
"chitchat/ask_name", "chitchat/ask_name", "What's your name?", 0.98765
),
]
response_results = remove_empty_response_examples(response_results)
assert len(response_results) == 1
assert response_results[0].intent_response_key_target == "chitchat/ask_name"
assert response_results[0].intent_response_key_prediction == "chitchat/ask_name"
assert response_results[0].confidence == 0.98765
assert response_results[0].message == "What's your name?"
def test_empty_response_removal():
response_results = [
ResponseSelectionEvaluationResult("chitchat", None,
"It's sunny in Berlin",
"What's the weather", 0.65432),
ResponseSelectionEvaluationResult(
"chitchat",
"My name is Mr.bot",
"My name is Mr.bot",
"What's your name?",
0.98765,
),
]
response_results = remove_empty_response_examples(response_results)
assert len(response_results) == 1
assert response_results[0].intent_target == "chitchat"
assert response_results[0].response_target == "My name is Mr.bot"
assert response_results[0].response_prediction == "My name is Mr.bot"
assert response_results[0].confidence == 0.98765
assert response_results[0].message == "What's your name?"
