def _collect_user_uttered_predictions(
event, partial_tracker, fail_on_prediction_errors
):
from rasa.core.utils import pad_list_to_size
user_uttered_eval_store = EvaluationStore()
intent_gold = event.parse_data.get("true_intent")
predicted_intent = event.parse_data.get("intent").get("name")
if predicted_intent is None:
predicted_intent = "None"
user_uttered_eval_store.add_to_store(
intent_predictions=predicted_intent, intent_targets=intent_gold
)
entity_gold = event.parse_data.get("true_entities")
predicted_entities = event.parse_data.get("entities")
if entity_gold or predicted_entities:
if len(entity_gold) > len(predicted_entities):
predicted_entities = pad_list_to_size(
predicted_entities, len(entity_gold), "None"
)
elif len(predicted_entities) > len(entity_gold):
entity_gold = pad_list_to_size(entity_gold, len(predicted_entities), "None")
user_uttered_eval_store.add_to_store(
entity_targets=_clean_entity_results(entity_gold),
entity_predictions=_clean_entity_results(predicted_entities),
)
if user_uttered_eval_store.has_prediction_target_mismatch():
partial_tracker.update(
WronglyClassifiedUserUtterance(
event.text,
intent_gold,
user_uttered_eval_store.entity_predictions,
event.parse_data,
event.timestamp,
event.input_channel,
predicted_intent,
user_uttered_eval_store.entity_targets,
)
)
if fail_on_prediction_errors:
raise ValueError(
"NLU model predicted a wrong intent. Failed Story:"
" \n\n{}".format(partial_tracker.export_stories())
)
else:
end_to_end_user_utterance = EndToEndUserUtterance(
event.text, event.intent, event.entities
)
partial_tracker.update(end_to_end_user_utterance)
return user_uttered_eval_store
def test_pad_list_to_size():
assert utils.pad_list_to_size(["e1", "e2"], 4, "other") == [
"e1",
"e2",
"other",
"other",
]
