def _collect_action_executed_predictions(
processor: "MessageProcessor",
partial_tracker: DialogueStateTracker,
event: ActionExecuted,
fail_on_prediction_errors: bool,
circuit_breaker_tripped: bool,
) -> Tuple[EvaluationStore, Optional[Text], Optional[float]]:
from rasa.core.policies.form_policy import FormPolicy
action_executed_eval_store = EvaluationStore()
gold = event.action_name
if circuit_breaker_tripped:
predicted = "circuit breaker tripped"
policy = None
confidence = None
else:
action, policy, confidence = processor.predict_next_action(
partial_tracker)
predicted = action.name()
if (policy and predicted != gold and _form_might_have_been_rejected(
processor.domain, partial_tracker, predicted)):
# Wrong action was predicted,
# but it might be Ok if form action is rejected.
_emulate_form_rejection(partial_tracker)
# try again
action, policy, confidence = processor.predict_next_action(
partial_tracker)
# Even if the prediction is also wrong, we don't have to undo the emulation
# of the action rejection as we know that the user explicitly specified
# that something else than the form was supposed to run.
predicted = action.name()
action_executed_eval_store.add_to_store(action_predictions=[predicted],
action_targets=[gold])
if action_executed_eval_store.has_prediction_target_mismatch():
partial_tracker.update(
WronglyPredictedAction(gold, predicted, event.policy,
event.confidence, event.timestamp))
if fail_on_prediction_errors:
error_msg = ("Model predicted a wrong action. Failed Story: "
"\n\n{}".format(YAMLStoryWriter().dumps(
partial_tracker.as_story().story_steps)))
if FormPolicy.__name__ in policy:
error_msg += ("FormAction is not run during "
"evaluation therefore it is impossible to know "
"if validation failed or this story is wrong. "
"If the story is correct, add it to the "
"training stories and retrain.")
raise ValueError(error_msg)
else:
partial_tracker.update(event)
return action_executed_eval_store, policy, confidence
def _collect_user_uttered_predictions(
event: UserUttered,
predicted: Dict[Text, Any],
partial_tracker: DialogueStateTracker,
fail_on_prediction_errors: bool,
) -> EvaluationStore:
user_uttered_eval_store = EvaluationStore()
intent_gold = event.intent.get("name")
predicted_intent = predicted.get(INTENT, {}).get("name")
user_uttered_eval_store.add_to_store(
intent_predictions=[predicted_intent], intent_targets=[intent_gold]
)
entity_gold = event.entities
predicted_entities = predicted.get(ENTITIES)
if entity_gold or predicted_entities:
user_uttered_eval_store.add_to_store(
entity_targets=_clean_entity_results(event.text, entity_gold),
entity_predictions=_clean_entity_results(event.text, predicted_entities),
)
if user_uttered_eval_store.has_prediction_target_mismatch():
partial_tracker.update(
WronglyClassifiedUserUtterance(event, user_uttered_eval_store)
)
if fail_on_prediction_errors:
raise ValueError(
"NLU model predicted a wrong intent. Failed Story:"
" \n\n{}".format(
YAMLStoryWriter().dumps(partial_tracker.as_story().story_steps)
)
)
else:
end_to_end_user_utterance = EndToEndUserUtterance(
event.text, event.intent, event.entities
)
partial_tracker.update(end_to_end_user_utterance)
return user_uttered_eval_store
