def test_intent_evaluation_report(tmpdir_factory): path = tmpdir_factory.mktemp("evaluation").strpath report_folder = os.path.join(path, "reports") report_filename = os.path.join(report_folder, "intent_report.json") utils.create_dir(report_folder) intent_results = [ IntentEvaluationResult("", "restaurant_search", "I am hungry", 0.12345), IntentEvaluationResult("greet", "greet", "hello", 0.98765)] result = evaluate_intents(intent_results, report_folder, successes_filename=None, errors_filename=None, confmat_filename=None, intent_hist_filename=None) report = json.loads(utils.read_file(report_filename)) greet_results = {"precision": 1.0, "recall": 1.0, "f1-score": 1.0, "support": 1} prediction = {'text': 'hello', 'intent': 'greet', 'predicted': 'greet', 'confidence': 0.98765} assert len(report.keys()) == 4 assert report["greet"] == greet_results assert result["predictions"][0] == prediction
def test_empty_intent_removal(): intent_results = [ IntentEvaluationResult("", "restaurant_search", "I am hungry", 0.12345), IntentEvaluationResult("greet", "greet", "hello", 0.98765), ] intent_results = remove_empty_intent_examples(intent_results) assert len(intent_results) == 1 assert intent_results[0].intent_target == "greet" assert intent_results[0].intent_prediction == "greet" assert intent_results[0].confidence == 0.98765 assert intent_results[0].message == "hello"
def test_intent_evaluation_report(tmp_path: Path): path = tmp_path / "evaluation" path.mkdir() report_folder = str(path / "reports") report_filename = os.path.join(report_folder, "intent_report.json") rasa.shared.utils.io.create_directory(report_folder) intent_results = [ IntentEvaluationResult("", "restaurant_search", "I am hungry", 0.12345), IntentEvaluationResult("greet", "greet", "hello", 0.98765), ] result = evaluate_intents( intent_results, report_folder, successes=True, errors=True, disable_plotting=False, ) report = json.loads(rasa.shared.utils.io.read_file(report_filename)) greet_results = { "precision": 1.0, "recall": 1.0, "f1-score": 1.0, "support": 1, "confused_with": {}, } prediction = { "text": "hello", "intent": "greet", "predicted": "greet", "confidence": 0.98765, } assert len(report.keys()) == 4 assert report["greet"] == greet_results assert result["predictions"][0] == prediction assert os.path.exists( os.path.join(report_folder, "intent_confusion_matrix.png")) assert os.path.exists(os.path.join(report_folder, "intent_histogram.png")) assert not os.path.exists(os.path.join(report_folder, "intent_errors.json")) assert os.path.exists(os.path.join(report_folder, "intent_successes.json"))
def test_intent_evaluation_report(tmpdir_factory): path = tmpdir_factory.mktemp("evaluation").strpath report_folder = os.path.join(path, "reports") report_filename = os.path.join(report_folder, "intent_report.json") rasa.utils.io.create_directory(report_folder) intent_results = [ IntentEvaluationResult("", "restaurant_search", "I am hungry", 0.12345), IntentEvaluationResult("greet", "greet", "hello", 0.98765), ] result = evaluate_intents( intent_results, report_folder, successes=False, errors=False, confmat_filename=None, intent_hist_filename=None, disable_plotting=False, ) report = json.loads(rasa.utils.io.read_file(report_filename)) greet_results = { "precision": 1.0, "recall": 1.0, "f1-score": 1.0, "support": 1, "confused_with": {}, } prediction = { "text": "hello", "intent": "greet", "predicted": "greet", "confidence": 0.98765, } assert len(report.keys()) == 4 assert report["greet"] == greet_results assert result["predictions"][0] == prediction
def get_intent_predictions(targets, interpreter, test_data): # pragma: no cover """Runs the model for the test set and extracts intent predictions. Returns intent predictions, the original messages and the confidences of the predictions""" intent_results = [] for e, target in zip(test_data.training_examples, targets): res = interpreter.parse(e.text, only_output_properties=False) intent_results.append( IntentEvaluationResult(target, res.get('intent', {}).get('name'), res.get('text', {}), res.get('intent', {}).get('confidence'))) return intent_results
def incorrect(label: Text, _label: Text) -> IntentEvaluationResult: return IntentEvaluationResult(label, _label, "", 1.0)