def test_should_use_slot_matching_lambda_to_compute_metrics(self):
# Given
text = "this is intent1 with slot1_value and slot2_value"
actual_intent = "intent1"
actual_slots = [{
"text": "slot1_value2",
"entity": "entity1",
"slot_name": "slot1"
}, {
"text": "slot2_value",
"entity": "entity2",
"slot_name": "slot2"
}]
predicted_intent = actual_intent
predicted_slots = [{
"rawValue": "slot1_value",
"value": {
"kind": "Custom",
"value": "slot1_value"
},
"range": {
"start": 21,
"end": 32
},
"entity": "entity1",
"slotName": "slot1"
}]
def slot_matching_lambda(l, r):
return l[TEXT].split("_")[0] == r["rawValue"].split("_")[0]
# When
metrics = compute_utterance_metrics(predicted_intent, predicted_slots,
actual_intent, actual_slots, True,
slot_matching_lambda)
# Then
expected_metrics = {
"intent1": {
"intent": {
"false_negative": 0,
"false_positive": 0,
"true_positive": 1
},
"slots": {
"slot1": {
"false_negative": 0,
"false_positive": 0,
"true_positive": 1
},
"slot2": {
"false_negative": 1,
"false_positive": 0,
"true_positive": 0
}
}
}
}
self.assertDictEqual(expected_metrics, metrics)
def test_should_compute_utterance_metrics_when_correct_intent(self):
# Given
text = "this is intent1 with slot1_value and slot2_value"
actual_intent = "intent1"
actual_slots = [{
"text": "slot1_value",
"entity": "entity1",
"slot_name": "slot1"
}, {
"text": "slot2_value",
"entity": "entity2",
"slot_name": "slot2"
}]
predicted_intent = actual_intent
predicted_slots = [{
"rawValue": "slot1_value",
"value": {
"kind": "Custom",
"value": "slot1_value"
},
"range": {
"start": 21,
"end": 32
},
"entity": "entity1",
"slotName": "slot1"
}]
# When
metrics = compute_utterance_metrics(predicted_intent, predicted_slots,
actual_intent, actual_slots, True,
exact_match)
# Then
expected_metrics = {
"intent1": {
"intent": {
"false_negative": 0,
"false_positive": 0,
"true_positive": 1
},
"slots": {
"slot1": {
"false_negative": 0,
"false_positive": 0,
"true_positive": 1
},
"slot2": {
"false_negative": 1,
"false_positive": 0,
"true_positive": 0
}
}
}
}
self.assertDictEqual(expected_metrics, metrics)
def test_should_compute_utterance_metrics_when_wrong_intent(self):
# Given
text = "utterance of intent1"
parsing = {
"text":
text,
"intent": {
"intentName": "intent2",
"probability": 0.32
},
"slots": [{
"rawValue": "utterance",
"value": {
"kind": "Custom",
"value": "utterance"
},
"range": {
"start": 0,
"end": 9
},
"entity": "erroneous_entity",
"slotName": "erroneous_slot"
}]
}
utterance = {"data": [{"text": text}]}
intent_name = "intent1"
# When
metrics = compute_utterance_metrics(parsing, utterance, intent_name,
exact_match)
# Then
expected_metrics = {
"intent1": {
"intent": {
"false_negative": 1,
"false_positive": 0,
"true_positive": 0
},
"slots": {}
},
"intent2": {
"intent": {
"false_negative": 0,
"false_positive": 1,
"true_positive": 0
},
"slots": {
"erroneous_slot": {
"false_negative": 0,
"false_positive": 0,
"true_positive": 0
}
}
}
}
self.assertDictEqual(expected_metrics, metrics)
def test_should_compute_utterance_metrics_when_wrong_intent():
# Given
actual_intent = "intent1"
actual_slots = []
predicted_intent = "intent2"
predicted_slots = [{
"rawValue": "utterance",
"value": {
"kind": "Custom",
"value": "utterance"
},
"range": {
"start": 0,
"end": 9
},
"entity": "erroneous_entity",
"slotName": "erroneous_slot",
}]
# When
metrics = compute_utterance_metrics(
predicted_intent,
predicted_slots,
actual_intent,
actual_slots,
True,
exact_match,
)
# Then
expected_metrics = {
"intent1": {
"intent": {
"false_negative": 1,
"false_positive": 0,
"true_positive": 0
},
"slots": {},
},
"intent2": {
"intent": {
"false_negative": 0,
"false_positive": 1,
"true_positive": 0
},
"slots": {
"erroneous_slot": {
"false_negative": 0,
"false_positive": 0,
"true_positive": 0,
}
},
},
}
assert expected_metrics == metrics
def test_should_exclude_slot_metrics_when_specified():
# Given
actual_intent = "intent1"
actual_slots = [
{
"text": "slot1_value",
"entity": "entity1",
"slot_name": "slot1"
},
{
"text": "slot2_value",
"entity": "entity2",
"slot_name": "slot2"
},
]
predicted_intent = actual_intent
predicted_slots = [{
"rawValue": "slot1_value",
"value": {
"kind": "Custom",
"value": "slot1_value"
},
"range": {
"start": 21,
"end": 32
},
"entity": "entity1",
"slotName": "slot1",
}]
# When
include_slot_metrics = False
metrics = compute_utterance_metrics(
predicted_intent,
predicted_slots,
actual_intent,
actual_slots,
include_slot_metrics,
exact_match,
)
# Then
expected_metrics = {
"intent1": {
"intent": {
"false_negative": 0,
"false_positive": 0,
"true_positive": 1
}
}
}
assert expected_metrics == metrics
def test_should_use_slot_matching_lambda_to_compute_metrics(self):
# Given
text = "this is intent1 with slot1_value and slot2_value"
intent_name = "intent1"
parsing = {
"text":
text,
"intent": {
"intentName": intent_name,
"probability": 0.32
},
"slots": [{
"rawValue": "slot1_value",
"value": {
"kind": "Custom",
"value": "slot1_value"
},
"range": {
"start": 21,
"end": 32
},
"entity": "entity1",
"slotName": "slot1"
}]
}
utterance = {
"data": [{
"text": "this is intent1 with "
}, {
"text": "slot1_value2",
"entity": "entity1",
"slot_name": "slot1"
}, {
"text": " and "
}, {
"text": "slot2_value",
"entity": "entity2",
"slot_name": "slot2"
}]
}
def slot_matching_lambda(l, r):
return l[TEXT].split("_")[0] == r["rawValue"].split("_")[0]
# When
metrics = compute_utterance_metrics(parsing, utterance, intent_name,
slot_matching_lambda)
# Then
expected_metrics = {
"intent1": {
"intent": {
"false_negative": 0,
"false_positive": 0,
"true_positive": 1
},
"slots": {
"slot1": {
"false_negative": 0,
"false_positive": 0,
"true_positive": 1
},
"slot2": {
"false_negative": 1,
"false_positive": 0,
"true_positive": 0
}
}
}
}
self.assertDictEqual(expected_metrics, metrics)
def test_should_compute_utterance_metrics_when_correct_intent(self):
# Given
text = "this is intent1 with slot1_value and slot2_value"
intent_name = "intent1"
parsing = {
"text":
text,
"intent": {
"intentName": intent_name,
"probability": 0.32
},
"slots": [{
"rawValue": "slot1_value",
"value": {
"kind": "Custom",
"value": "slot1_value"
},
"range": {
"start": 21,
"end": 32
},
"entity": "entity1",
"slotName": "slot1"
}]
}
utterance = {
"data": [{
"text": "this is intent1 with "
}, {
"text": "slot1_value",
"entity": "entity1",
"slot_name": "slot1"
}, {
"text": " and "
}, {
"text": "slot2_value",
"entity": "entity2",
"slot_name": "slot2"
}]
}
# When
metrics = compute_utterance_metrics(parsing, utterance, intent_name,
exact_match)
# Then
expected_metrics = {
"intent1": {
"intent": {
"false_negative": 0,
"false_positive": 0,
"true_positive": 1
},
"slots": {
"slot1": {
"false_negative": 0,
"false_positive": 0,
"true_positive": 1
},
"slot2": {
"false_negative": 1,
"false_positive": 0,
"true_positive": 0
}
}
}
}
self.assertDictEqual(expected_metrics, metrics)