def test_end_to_end_train_test_metrics(self): # Given dataset_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "resources", "beverage_dataset.json") with io.open(dataset_path, encoding="utf8") as f: dataset = json.load(f) # When/Then try: engine_class = build_nlu_engine_class(MockTrainingEngine, MockInferenceEngine) compute_train_test_metrics(train_dataset=dataset, test_dataset=dataset, engine_class=engine_class) except Exception as e: self.fail(e.args[0])
def test_compute_train_test_metrics_without_slots_metrics( logger, beverage_dataset): # When/Then try: res = compute_train_test_metrics( train_dataset=beverage_dataset, test_dataset=beverage_dataset, engine_class=MockEngine, include_slot_metrics=False, ) except Exception as e: raise AssertionError(e.args[0]) expected_metrics = { "MakeCoffee": { "intent": { "true_positive": 0, "false_positive": 0, "false_negative": 7, "precision": 0.0, "recall": 0.0, "f1": 0.0, }, "intent_utterances": 7, "exact_parsings": 0, }, "null": { "intent": { "true_positive": 0, "false_positive": 11, "false_negative": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, }, "intent_utterances": 0, "exact_parsings": 0, }, "MakeTea": { "intent": { "true_positive": 0, "false_positive": 0, "false_negative": 4, "precision": 0.0, "recall": 0.0, "f1": 0.0, }, "intent_utterances": 4, "exact_parsings": 0, }, } assert expected_metrics, res["metrics"]
def test_compute_train_test_metrics_without_slots_metrics(self): # Given dataset_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "resources", "beverage_dataset.json") with io.open(dataset_path, encoding="utf8") as f: dataset = json.load(f) # When/Then try: res = compute_train_test_metrics(train_dataset=dataset, test_dataset=dataset, engine_class=MockEngine, include_slot_metrics=False) except Exception as e: self.fail(e.args[0]) expected_metrics = { "MakeCoffee": { "intent": { "true_positive": 0, "false_positive": 0, "false_negative": 7, "precision": 0.0, "recall": 0.0, "f1": 0.0 }, "intent_utterances": 7 }, "null": { "intent": { "true_positive": 0, "false_positive": 11, "false_negative": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0 }, "intent_utterances": 0 }, "MakeTea": { "intent": { "true_positive": 0, "false_positive": 0, "false_negative": 4, "precision": 0.0, "recall": 0.0, "f1": 0.0 }, "intent_utterances": 4 } } self.assertDictEqual(expected_metrics, res["metrics"])