def _collect_action_executed_predictions( processor: "MessageProcessor", partial_tracker: DialogueStateTracker, event: ActionExecuted, fail_on_prediction_errors: bool, circuit_breaker_tripped: bool, ) -> Tuple[EvaluationStore, Optional[Text], Optional[float]]: from rasa.core.policies.form_policy import FormPolicy action_executed_eval_store = EvaluationStore() gold = event.action_name if circuit_breaker_tripped: predicted = "circuit breaker tripped" policy = None confidence = None else: action, policy, confidence = processor.predict_next_action( partial_tracker) predicted = action.name() if policy and predicted != gold and FormPolicy.__name__ in policy: # FormPolicy predicted wrong action # but it might be Ok if form action is rejected _emulate_form_rejection(processor, partial_tracker) # try again action, policy, confidence = processor.predict_next_action( partial_tracker) predicted = action.name() action_executed_eval_store.add_to_store(action_predictions=predicted, action_targets=gold) if action_executed_eval_store.has_prediction_target_mismatch(): partial_tracker.update( WronglyPredictedAction(gold, predicted, event.policy, event.confidence, event.timestamp)) if fail_on_prediction_errors: error_msg = ("Model predicted a wrong action. Failed Story: " "\n\n{}".format(partial_tracker.export_stories())) if FormPolicy.__name__ in policy: error_msg += ("FormAction is not run during " "evaluation therefore it is impossible to know " "if validation failed or this story is wrong. " "If the story is correct, add it to the " "training stories and retrain.") raise ValueError(error_msg) else: partial_tracker.update(event) return action_executed_eval_store, policy, confidence
def _collect_user_uttered_predictions( event: UserUttered, partial_tracker: DialogueStateTracker, fail_on_prediction_errors: bool, ) -> EvaluationStore: user_uttered_eval_store = EvaluationStore() intent_gold = event.parse_data.get("true_intent") predicted_intent = event.parse_data.get("intent", {}).get("name") if not predicted_intent: predicted_intent = [None] user_uttered_eval_store.add_to_store( intent_predictions=predicted_intent, intent_targets=intent_gold ) entity_gold = event.parse_data.get("true_entities") predicted_entities = event.parse_data.get("entities") if entity_gold or predicted_entities: user_uttered_eval_store.add_to_store( entity_targets=_clean_entity_results(event.text, entity_gold), entity_predictions=_clean_entity_results(event.text, predicted_entities), ) if user_uttered_eval_store.has_prediction_target_mismatch(): partial_tracker.update( WronglyClassifiedUserUtterance(event, user_uttered_eval_store) ) if fail_on_prediction_errors: raise ValueError( "NLU model predicted a wrong intent. Failed Story:" " \n\n{}".format(partial_tracker.export_stories()) ) else: end_to_end_user_utterance = EndToEndUserUtterance( event.text, event.intent, event.entities ) partial_tracker.update(end_to_end_user_utterance) return user_uttered_eval_store
async def test_persist_form_story(tmpdir): domain = Domain.load("data/test_domains/form.yml") tracker = DialogueStateTracker("", domain.slots) story = ("* greet\n" " - utter_greet\n" "* start_form\n" " - some_form\n" ' - form{"name": "some_form"}\n' "* default\n" " - utter_default\n" " - some_form\n" "* stop\n" " - utter_ask_continue\n" "* affirm\n" " - some_form\n" "* stop\n" " - utter_ask_continue\n" " - action_listen\n" "* form: inform\n" " - some_form\n" ' - form{"name": null}\n' "* goodbye\n" " - utter_goodbye\n") # simulate talking to the form events = [ UserUttered(intent={"name": "greet"}), ActionExecuted("utter_greet"), ActionExecuted("action_listen"), # start the form UserUttered(intent={"name": "start_form"}), ActionExecuted("some_form"), Form("some_form"), ActionExecuted("action_listen"), # out of form input UserUttered(intent={"name": "default"}), ActionExecutionRejected("some_form"), ActionExecuted("utter_default"), ActionExecuted("some_form"), ActionExecuted("action_listen"), # out of form input UserUttered(intent={"name": "stop"}), ActionExecutionRejected("some_form"), ActionExecuted("utter_ask_continue"), ActionExecuted("action_listen"), # out of form input but continue with the form UserUttered(intent={"name": "affirm"}), FormValidation(False), ActionExecuted("some_form"), ActionExecuted("action_listen"), # out of form input UserUttered(intent={"name": "stop"}), ActionExecutionRejected("some_form"), ActionExecuted("utter_ask_continue"), ActionExecuted("action_listen"), # form input UserUttered(intent={"name": "inform"}), FormValidation(True), ActionExecuted("some_form"), ActionExecuted("action_listen"), Form(None), UserUttered(intent={"name": "goodbye"}), ActionExecuted("utter_goodbye"), ActionExecuted("action_listen"), ] [tracker.update(e) for e in events] assert story in tracker.export_stories()