def test_empty_response_removal(): response_results = [ ResponseSelectionEvaluationResult(None, None, "What's the weather", 0.65432), ResponseSelectionEvaluationResult("chitchat/ask_name", "chitchat/ask_name", "What's your name?", 0.98765), # This happens if response selection test data is present but no response # selector is part of the model ResponseSelectionEvaluationResult("chitchat/ask_name", None, "What's your name?", None), ] response_results = remove_empty_response_examples(response_results) assert len(response_results) == 2 assert response_results[ 0].intent_response_key_target == "chitchat/ask_name" assert response_results[ 0].intent_response_key_prediction == "chitchat/ask_name" assert response_results[0].confidence == 0.98765 assert response_results[0].message == "What's your name?" assert response_results[ 1].intent_response_key_target == "chitchat/ask_name" assert response_results[1].intent_response_key_prediction == "" assert response_results[1].confidence == 0.0 assert response_results[1].message == "What's your name?"
def test_response_evaluation_report(tmp_path: Path): path = tmp_path / "evaluation" path.mkdir() report_folder = str(path / "reports") report_filename = os.path.join(report_folder, "response_selection_report.json") rasa.shared.utils.io.create_directory(report_folder) response_results = [ ResponseSelectionEvaluationResult( "chitchat/ask_weather", "chitchat/ask_weather", "What's the weather", 0.65432, ), ResponseSelectionEvaluationResult("chitchat/ask_name", "chitchat/ask_name", "What's your name?", 0.98765), ] result = evaluate_response_selections( response_results, report_folder, successes=True, errors=True, disable_plotting=False, ) report = json.loads(rasa.shared.utils.io.read_file(report_filename)) name_query_results = { "precision": 1.0, "recall": 1.0, "f1-score": 1.0, "support": 1, "confused_with": {}, } prediction = { "text": "What's your name?", "intent_response_key_target": "chitchat/ask_name", "intent_response_key_prediction": "chitchat/ask_name", "confidence": 0.98765, } assert len(report.keys()) == 5 assert report["chitchat/ask_name"] == name_query_results assert result["predictions"][1] == prediction assert os.path.exists( os.path.join(report_folder, "response_selection_confusion_matrix.png")) assert os.path.exists( os.path.join(report_folder, "response_selection_histogram.png")) assert not os.path.exists( os.path.join(report_folder, "response_selection_errors.json")) assert os.path.exists( os.path.join(report_folder, "response_selection_successes.json"))
def test_response_evaluation_report(tmpdir_factory): path = tmpdir_factory.mktemp("evaluation").strpath report_folder = os.path.join(path, "reports") report_filename = os.path.join(report_folder, "response_selection_report.json") rasa.utils.io.create_directory(report_folder) response_results = [ ResponseSelectionEvaluationResult( "chitchat", "It's sunny in Berlin", "It's sunny in Berlin", "What's the weather", 0.65432, ), ResponseSelectionEvaluationResult( "chitchat", "My name is Mr.bot", "My name is Mr.bot", "What's your name?", 0.98765, ), ] result = evaluate_response_selections( response_results, report_folder, successes=False, errors=False, disable_plotting=True, ) report = json.loads(rasa.utils.io.read_file(report_filename)) name_query_results = { "precision": 1.0, "recall": 1.0, "f1-score": 1.0, "support": 1, "confused_with": {}, } prediction = { "text": "What's your name?", "intent_target": "chitchat", "response_target": "My name is Mr.bot", "response_predicted": "My name is Mr.bot", "confidence": 0.98765, } assert len(report.keys()) == 5 assert report["My name is Mr.bot"] == name_query_results assert result["predictions"][1] == prediction
def test_empty_response_removal(): response_results = [ ResponseSelectionEvaluationResult(None, None, "What's the weather", 0.65432), ResponseSelectionEvaluationResult( "chitchat/ask_name", "chitchat/ask_name", "What's your name?", 0.98765 ), ] response_results = remove_empty_response_examples(response_results) assert len(response_results) == 1 assert response_results[0].intent_response_key_target == "chitchat/ask_name" assert response_results[0].intent_response_key_prediction == "chitchat/ask_name" assert response_results[0].confidence == 0.98765 assert response_results[0].message == "What's your name?"
def test_empty_response_removal(): response_results = [ ResponseSelectionEvaluationResult("chitchat", None, "It's sunny in Berlin", "What's the weather", 0.65432), ResponseSelectionEvaluationResult( "chitchat", "My name is Mr.bot", "My name is Mr.bot", "What's your name?", 0.98765, ), ] response_results = remove_empty_response_examples(response_results) assert len(response_results) == 1 assert response_results[0].intent_target == "chitchat" assert response_results[0].response_target == "My name is Mr.bot" assert response_results[0].response_prediction == "My name is Mr.bot" assert response_results[0].confidence == 0.98765 assert response_results[0].message == "What's your name?"