def _collect_action_executed_predictions( processor: "MessageProcessor", partial_tracker: DialogueStateTracker, event: ActionExecuted, fail_on_prediction_errors: bool, circuit_breaker_tripped: bool, ) -> Tuple[EvaluationStore, Optional[Text], Optional[float]]: from rasa.core.policies.form_policy import FormPolicy action_executed_eval_store = EvaluationStore() gold = event.action_name or event.action_text if circuit_breaker_tripped: predicted = "circuit breaker tripped" policy = None confidence = None else: action, policy, confidence = processor.predict_next_action( partial_tracker) predicted = action.name() if (policy and predicted != gold and _form_might_have_been_rejected( processor.domain, partial_tracker, predicted)): # Wrong action was predicted, # but it might be Ok if form action is rejected. emulate_loop_rejection(partial_tracker) # try again action, policy, confidence = processor.predict_next_action( partial_tracker) # Even if the prediction is also wrong, we don't have to undo the emulation # of the action rejection as we know that the user explicitly specified # that something else than the form was supposed to run. predicted = action.name() action_executed_eval_store.add_to_store(action_predictions=[predicted], action_targets=[gold]) if action_executed_eval_store.has_prediction_target_mismatch(): partial_tracker.update( WronglyPredictedAction(gold, predicted, policy, confidence, event.timestamp)) if fail_on_prediction_errors: story_dump = YAMLStoryWriter().dumps( partial_tracker.as_story().story_steps) error_msg = (f"Model predicted a wrong action. Failed Story: " f"\n\n{story_dump}") if FormPolicy.__name__ in policy: error_msg += ("FormAction is not run during " "evaluation therefore it is impossible to know " "if validation failed or this story is wrong. " "If the story is correct, add it to the " "training stories and retrain.") raise WrongPredictionException(error_msg) else: partial_tracker.update( ActionExecuted(predicted, policy, confidence, event.timestamp)) return action_executed_eval_store, policy, confidence
def _collect_user_uttered_predictions( event: UserUttered, predicted: Dict[Text, Any], partial_tracker: DialogueStateTracker, fail_on_prediction_errors: bool, ) -> EvaluationStore: user_uttered_eval_store = EvaluationStore() # intent from the test story, may either be base intent or full retrieval intent base_intent = event.intent.get(INTENT_NAME_KEY) full_retrieval_intent = event.intent.get(FULL_RETRIEVAL_INTENT_NAME_KEY) intent_gold = full_retrieval_intent if full_retrieval_intent else base_intent # predicted intent: note that this is only the base intent at this point predicted_base_intent = predicted.get(INTENT, {}).get(INTENT_NAME_KEY) # if the test story only provides the base intent AND the prediction was correct, # we are not interested in full retrieval intents and skip this section. # In any other case we are interested in the full retrieval intent (e.g. for report) if intent_gold != predicted_base_intent: predicted_base_intent = _get_full_retrieval_intent(predicted) user_uttered_eval_store.add_to_store( intent_targets=[intent_gold], intent_predictions=[predicted_base_intent]) entity_gold = event.entities predicted_entities = predicted.get(ENTITIES) if entity_gold or predicted_entities: user_uttered_eval_store.add_to_store( entity_targets=_clean_entity_results(event.text, entity_gold), entity_predictions=_clean_entity_results(event.text, predicted_entities), ) if user_uttered_eval_store.check_prediction_target_mismatch(): partial_tracker.update( WronglyClassifiedUserUtterance(event, user_uttered_eval_store)) if fail_on_prediction_errors: story_dump = YAMLStoryWriter().dumps( partial_tracker.as_story().story_steps) raise WrongPredictionException( f"NLU model predicted a wrong intent or entities. Failed Story:" f" \n\n{story_dump}") else: response_selector_info = ({ RESPONSE_SELECTOR_PROPERTY_NAME: predicted[RESPONSE_SELECTOR_PROPERTY_NAME] } if RESPONSE_SELECTOR_PROPERTY_NAME in predicted else None) end_to_end_user_utterance = EndToEndUserUtterance( text=event.text, intent=event.intent, entities=event.entities, parse_data=response_selector_info, ) partial_tracker.update(end_to_end_user_utterance) return user_uttered_eval_store
def _collect_user_uttered_predictions( event: UserUttered, predicted: Dict[Text, Any], partial_tracker: DialogueStateTracker, fail_on_prediction_errors: bool, ) -> EvaluationStore: user_uttered_eval_store = EvaluationStore() intent_gold = event.intent.get("name") predicted_intent = predicted.get(INTENT, {}).get("name") user_uttered_eval_store.add_to_store( intent_predictions=[predicted_intent], intent_targets=[intent_gold] ) entity_gold = event.entities predicted_entities = predicted.get(ENTITIES) if entity_gold or predicted_entities: user_uttered_eval_store.add_to_store( entity_targets=_clean_entity_results(event.text, entity_gold), entity_predictions=_clean_entity_results(event.text, predicted_entities), ) if user_uttered_eval_store.has_prediction_target_mismatch(): partial_tracker.update( WronglyClassifiedUserUtterance(event, user_uttered_eval_store) ) if fail_on_prediction_errors: raise ValueError( "NLU model predicted a wrong intent. Failed Story:" " \n\n{}".format( YAMLStoryWriter().dumps(partial_tracker.as_story().story_steps) ) ) else: end_to_end_user_utterance = EndToEndUserUtterance( event.text, event.intent, event.entities ) partial_tracker.update(end_to_end_user_utterance) return user_uttered_eval_store
def _collect_action_executed_predictions( processor: "MessageProcessor", partial_tracker: DialogueStateTracker, event: ActionExecuted, fail_on_prediction_errors: bool, ) -> Tuple[EvaluationStore, PolicyPrediction, Optional[EntityEvaluationResult]]: action_executed_eval_store = EvaluationStore() expected_action_name = event.action_name expected_action_text = event.action_text expected_action = expected_action_name or expected_action_text policy_entity_result = None prev_action_unlikely_intent = False try: predicted_action, prediction, policy_entity_result = _run_action_prediction( processor, partial_tracker, expected_action) except ActionLimitReached: prediction = PolicyPrediction([], policy_name=None) predicted_action = "circuit breaker tripped" predicted_action_unlikely_intent = predicted_action == ACTION_UNLIKELY_INTENT_NAME if predicted_action_unlikely_intent and predicted_action != expected_action: partial_tracker.update( WronglyPredictedAction( predicted_action, expected_action_text, predicted_action, prediction.policy_name, prediction.max_confidence, event.timestamp, metadata=prediction.action_metadata, )) prev_action_unlikely_intent = True try: predicted_action, prediction, policy_entity_result = _run_action_prediction( processor, partial_tracker, expected_action) except ActionLimitReached: prediction = PolicyPrediction([], policy_name=None) predicted_action = "circuit breaker tripped" action_executed_eval_store.add_to_store( action_predictions=[predicted_action], action_targets=[expected_action]) if action_executed_eval_store.has_prediction_target_mismatch(): partial_tracker.update( WronglyPredictedAction( expected_action_name, expected_action_text, predicted_action, prediction.policy_name, prediction.max_confidence, event.timestamp, metadata=prediction.action_metadata, predicted_action_unlikely_intent=prev_action_unlikely_intent, )) if (fail_on_prediction_errors and predicted_action != ACTION_UNLIKELY_INTENT_NAME and predicted_action != expected_action): story_dump = YAMLStoryWriter().dumps( partial_tracker.as_story().story_steps) error_msg = (f"Model predicted a wrong action. Failed Story: " f"\n\n{story_dump}") raise WrongPredictionException(error_msg) elif prev_action_unlikely_intent: partial_tracker.update( WarningPredictedAction( ACTION_UNLIKELY_INTENT_NAME, predicted_action, prediction.policy_name, prediction.max_confidence, event.timestamp, prediction.action_metadata, )) else: partial_tracker.update( ActionExecuted( predicted_action, prediction.policy_name, prediction.max_confidence, event.timestamp, metadata=prediction.action_metadata, )) return action_executed_eval_store, prediction, policy_entity_result