def _collect_user_uttered_predictions(event,
partial_tracker,
fail_on_prediction_errors):
user_uttered_eval_store = EvaluationStore()
intent_gold = event.parse_data.get("true_intent")
predicted_intent = event.parse_data.get("intent").get("name")
if predicted_intent is None:
predicted_intent = "None"
user_uttered_eval_store.add_to_store(intent_predictions=predicted_intent,
intent_targets=intent_gold)
entity_gold = event.parse_data.get("true_entities")
predicted_entities = event.parse_data.get("entities")
if entity_gold or predicted_entities:
if len(entity_gold) > len(predicted_entities):
predicted_entities = pad_list_to_size(predicted_entities,
len(entity_gold),
"None")
elif len(predicted_entities) > len(entity_gold):
entity_gold = pad_list_to_size(entity_gold,
len(predicted_entities),
"None")
user_uttered_eval_store.add_to_store(
entity_targets=_clean_entity_results(entity_gold),
entity_predictions=_clean_entity_results(predicted_entities)
)
if user_uttered_eval_store.has_prediction_target_mismatch():
partial_tracker.update(
WronglyClassifiedUserUtterance(
event.text, intent_gold,
user_uttered_eval_store.entity_predictions,
event.parse_data,
event.timestamp,
event.input_channel,
predicted_intent,
user_uttered_eval_store.entity_targets)
)
if fail_on_prediction_errors:
raise ValueError(
"NLU model predicted a wrong intent. Failed Story:"
" \n\n{}".format(partial_tracker.export_stories()))
else:
end_to_end_user_utterance = EndToEndUserUtterance(
event.text, event.intent, event.entities)
partial_tracker.update(end_to_end_user_utterance)
return user_uttered_eval_store
def test_pad_list_to_size():
assert (utils.pad_list_to_size(["e1", "e2"], 4,
"other") == ["e1", "e2", "other", "other"])
