def test_end_to_end_train_test_metrics(self):
# Given
dataset_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
"resources", "beverage_dataset.json")
with io.open(dataset_path, encoding="utf8") as f:
dataset = json.load(f)
# When/Then
try:
engine_class = build_nlu_engine_class(MockTrainingEngine,
MockInferenceEngine)
compute_train_test_metrics(train_dataset=dataset,
test_dataset=dataset,
engine_class=engine_class)
except Exception as e:
self.fail(e.args[0])
def test_compute_train_test_metrics_without_slots_metrics(
logger, beverage_dataset):
# When/Then
try:
res = compute_train_test_metrics(
train_dataset=beverage_dataset,
test_dataset=beverage_dataset,
engine_class=MockEngine,
include_slot_metrics=False,
)
except Exception as e:
raise AssertionError(e.args[0])
expected_metrics = {
"MakeCoffee": {
"intent": {
"true_positive": 0,
"false_positive": 0,
"false_negative": 7,
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
},
"intent_utterances": 7,
"exact_parsings": 0,
},
"null": {
"intent": {
"true_positive": 0,
"false_positive": 11,
"false_negative": 0,
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
},
"intent_utterances": 0,
"exact_parsings": 0,
},
"MakeTea": {
"intent": {
"true_positive": 0,
"false_positive": 0,
"false_negative": 4,
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
},
"intent_utterances": 4,
"exact_parsings": 0,
},
}
assert expected_metrics, res["metrics"]
def test_compute_train_test_metrics_without_slots_metrics(self):
# Given
dataset_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
"resources", "beverage_dataset.json")
with io.open(dataset_path, encoding="utf8") as f:
dataset = json.load(f)
# When/Then
try:
res = compute_train_test_metrics(train_dataset=dataset,
test_dataset=dataset,
engine_class=MockEngine,
include_slot_metrics=False)
except Exception as e:
self.fail(e.args[0])
expected_metrics = {
"MakeCoffee": {
"intent": {
"true_positive": 0,
"false_positive": 0,
"false_negative": 7,
"precision": 0.0,
"recall": 0.0,
"f1": 0.0
},
"intent_utterances": 7
},
"null": {
"intent": {
"true_positive": 0,
"false_positive": 11,
"false_negative": 0,
"precision": 0.0,
"recall": 0.0,
"f1": 0.0
},
"intent_utterances": 0
},
"MakeTea": {
"intent": {
"true_positive": 0,
"false_positive": 0,
"false_negative": 4,
"precision": 0.0,
"recall": 0.0,
"f1": 0.0
},
"intent_utterances": 4
}
}
self.assertDictEqual(expected_metrics, res["metrics"])