def _collect_user_uttered_predictions( event, partial_tracker, fail_on_prediction_errors ): from rasa.core.utils import pad_list_to_size user_uttered_eval_store = EvaluationStore() intent_gold = event.parse_data.get("true_intent") predicted_intent = event.parse_data.get("intent").get("name") if predicted_intent is None: predicted_intent = "None" user_uttered_eval_store.add_to_store( intent_predictions=predicted_intent, intent_targets=intent_gold ) entity_gold = event.parse_data.get("true_entities") predicted_entities = event.parse_data.get("entities") if entity_gold or predicted_entities: if len(entity_gold) > len(predicted_entities): predicted_entities = pad_list_to_size( predicted_entities, len(entity_gold), "None" ) elif len(predicted_entities) > len(entity_gold): entity_gold = pad_list_to_size(entity_gold, len(predicted_entities), "None") user_uttered_eval_store.add_to_store( entity_targets=_clean_entity_results(entity_gold), entity_predictions=_clean_entity_results(predicted_entities), ) if user_uttered_eval_store.has_prediction_target_mismatch(): partial_tracker.update( WronglyClassifiedUserUtterance( event.text, intent_gold, user_uttered_eval_store.entity_predictions, event.parse_data, event.timestamp, event.input_channel, predicted_intent, user_uttered_eval_store.entity_targets, ) ) if fail_on_prediction_errors: raise ValueError( "NLU model predicted a wrong intent. Failed Story:" " \n\n{}".format(partial_tracker.export_stories()) ) else: end_to_end_user_utterance = EndToEndUserUtterance( event.text, event.intent, event.entities ) partial_tracker.update(end_to_end_user_utterance) return user_uttered_eval_store
def test_pad_list_to_size(): assert utils.pad_list_to_size(["e1", "e2"], 4, "other") == [ "e1", "e2", "other", "other", ]