def _collect_action_executed_predictions(
processor: "MessageProcessor",
partial_tracker: DialogueStateTracker,
event: ActionExecuted,
fail_on_prediction_errors: bool,
circuit_breaker_tripped: bool,
) -> Tuple[EvaluationStore, Optional[Text], Optional[float]]:
from rasa.core.policies.form_policy import FormPolicy
action_executed_eval_store = EvaluationStore()
gold = event.action_name or event.action_text
if circuit_breaker_tripped:
predicted = "circuit breaker tripped"
policy = None
confidence = None
else:
action, policy, confidence = processor.predict_next_action(
partial_tracker)
predicted = action.name()
if (policy and predicted != gold and _form_might_have_been_rejected(
processor.domain, partial_tracker, predicted)):
# Wrong action was predicted,
# but it might be Ok if form action is rejected.
emulate_loop_rejection(partial_tracker)
# try again
action, policy, confidence = processor.predict_next_action(
partial_tracker)
# Even if the prediction is also wrong, we don't have to undo the emulation
# of the action rejection as we know that the user explicitly specified
# that something else than the form was supposed to run.
predicted = action.name()
action_executed_eval_store.add_to_store(action_predictions=[predicted],
action_targets=[gold])
if action_executed_eval_store.has_prediction_target_mismatch():
partial_tracker.update(
WronglyPredictedAction(gold, predicted, policy, confidence,
event.timestamp))
if fail_on_prediction_errors:
story_dump = YAMLStoryWriter().dumps(
partial_tracker.as_story().story_steps)
error_msg = (f"Model predicted a wrong action. Failed Story: "
f"\n\n{story_dump}")
if FormPolicy.__name__ in policy:
error_msg += ("FormAction is not run during "
"evaluation therefore it is impossible to know "
"if validation failed or this story is wrong. "
"If the story is correct, add it to the "
"training stories and retrain.")
raise WrongPredictionException(error_msg)
else:
partial_tracker.update(
ActionExecuted(predicted, policy, confidence, event.timestamp))
return action_executed_eval_store, policy, confidence
def _collect_user_uttered_predictions(
event: UserUttered,
predicted: Dict[Text, Any],
partial_tracker: DialogueStateTracker,
fail_on_prediction_errors: bool,
) -> EvaluationStore:
user_uttered_eval_store = EvaluationStore()
# intent from the test story, may either be base intent or full retrieval intent
base_intent = event.intent.get(INTENT_NAME_KEY)
full_retrieval_intent = event.intent.get(FULL_RETRIEVAL_INTENT_NAME_KEY)
intent_gold = full_retrieval_intent if full_retrieval_intent else base_intent
# predicted intent: note that this is only the base intent at this point
predicted_base_intent = predicted.get(INTENT, {}).get(INTENT_NAME_KEY)
# if the test story only provides the base intent AND the prediction was correct,
# we are not interested in full retrieval intents and skip this section.
# In any other case we are interested in the full retrieval intent (e.g. for report)
if intent_gold != predicted_base_intent:
predicted_base_intent = _get_full_retrieval_intent(predicted)
user_uttered_eval_store.add_to_store(
intent_targets=[intent_gold],
intent_predictions=[predicted_base_intent])
entity_gold = event.entities
predicted_entities = predicted.get(ENTITIES)
if entity_gold or predicted_entities:
user_uttered_eval_store.add_to_store(
entity_targets=_clean_entity_results(event.text, entity_gold),
entity_predictions=_clean_entity_results(event.text,
predicted_entities),
)
if user_uttered_eval_store.check_prediction_target_mismatch():
partial_tracker.update(
WronglyClassifiedUserUtterance(event, user_uttered_eval_store))
if fail_on_prediction_errors:
story_dump = YAMLStoryWriter().dumps(
partial_tracker.as_story().story_steps)
raise WrongPredictionException(
f"NLU model predicted a wrong intent or entities. Failed Story:"
f" \n\n{story_dump}")
else:
response_selector_info = ({
RESPONSE_SELECTOR_PROPERTY_NAME:
predicted[RESPONSE_SELECTOR_PROPERTY_NAME]
} if RESPONSE_SELECTOR_PROPERTY_NAME in predicted else None)
end_to_end_user_utterance = EndToEndUserUtterance(
text=event.text,
intent=event.intent,
entities=event.entities,
parse_data=response_selector_info,
)
partial_tracker.update(end_to_end_user_utterance)
return user_uttered_eval_store
def _collect_user_uttered_predictions(
event: UserUttered,
predicted: Dict[Text, Any],
partial_tracker: DialogueStateTracker,
fail_on_prediction_errors: bool,
) -> EvaluationStore:
user_uttered_eval_store = EvaluationStore()
intent_gold = event.intent.get("name")
predicted_intent = predicted.get(INTENT, {}).get("name")
user_uttered_eval_store.add_to_store(
intent_predictions=[predicted_intent], intent_targets=[intent_gold]
)
entity_gold = event.entities
predicted_entities = predicted.get(ENTITIES)
if entity_gold or predicted_entities:
user_uttered_eval_store.add_to_store(
entity_targets=_clean_entity_results(event.text, entity_gold),
entity_predictions=_clean_entity_results(event.text, predicted_entities),
)
if user_uttered_eval_store.has_prediction_target_mismatch():
partial_tracker.update(
WronglyClassifiedUserUtterance(event, user_uttered_eval_store)
)
if fail_on_prediction_errors:
raise ValueError(
"NLU model predicted a wrong intent. Failed Story:"
" \n\n{}".format(
YAMLStoryWriter().dumps(partial_tracker.as_story().story_steps)
)
)
else:
end_to_end_user_utterance = EndToEndUserUtterance(
event.text, event.intent, event.entities
)
partial_tracker.update(end_to_end_user_utterance)
return user_uttered_eval_store
def _collect_action_executed_predictions(
processor: "MessageProcessor",
partial_tracker: DialogueStateTracker,
event: ActionExecuted,
fail_on_prediction_errors: bool,
) -> Tuple[EvaluationStore, PolicyPrediction,
Optional[EntityEvaluationResult]]:
action_executed_eval_store = EvaluationStore()
expected_action_name = event.action_name
expected_action_text = event.action_text
expected_action = expected_action_name or expected_action_text
policy_entity_result = None
prev_action_unlikely_intent = False
try:
predicted_action, prediction, policy_entity_result = _run_action_prediction(
processor, partial_tracker, expected_action)
except ActionLimitReached:
prediction = PolicyPrediction([], policy_name=None)
predicted_action = "circuit breaker tripped"
predicted_action_unlikely_intent = predicted_action == ACTION_UNLIKELY_INTENT_NAME
if predicted_action_unlikely_intent and predicted_action != expected_action:
partial_tracker.update(
WronglyPredictedAction(
predicted_action,
expected_action_text,
predicted_action,
prediction.policy_name,
prediction.max_confidence,
event.timestamp,
metadata=prediction.action_metadata,
))
prev_action_unlikely_intent = True
try:
predicted_action, prediction, policy_entity_result = _run_action_prediction(
processor, partial_tracker, expected_action)
except ActionLimitReached:
prediction = PolicyPrediction([], policy_name=None)
predicted_action = "circuit breaker tripped"
action_executed_eval_store.add_to_store(
action_predictions=[predicted_action],
action_targets=[expected_action])
if action_executed_eval_store.has_prediction_target_mismatch():
partial_tracker.update(
WronglyPredictedAction(
expected_action_name,
expected_action_text,
predicted_action,
prediction.policy_name,
prediction.max_confidence,
event.timestamp,
metadata=prediction.action_metadata,
predicted_action_unlikely_intent=prev_action_unlikely_intent,
))
if (fail_on_prediction_errors
and predicted_action != ACTION_UNLIKELY_INTENT_NAME
and predicted_action != expected_action):
story_dump = YAMLStoryWriter().dumps(
partial_tracker.as_story().story_steps)
error_msg = (f"Model predicted a wrong action. Failed Story: "
f"\n\n{story_dump}")
raise WrongPredictionException(error_msg)
elif prev_action_unlikely_intent:
partial_tracker.update(
WarningPredictedAction(
ACTION_UNLIKELY_INTENT_NAME,
predicted_action,
prediction.policy_name,
prediction.max_confidence,
event.timestamp,
prediction.action_metadata,
))
else:
partial_tracker.update(
ActionExecuted(
predicted_action,
prediction.policy_name,
prediction.max_confidence,
event.timestamp,
metadata=prediction.action_metadata,
))
return action_executed_eval_store, prediction, policy_entity_result
