def _collect_action_executed_predictions(
processor: "MessageProcessor",
partial_tracker: DialogueStateTracker,
event: ActionExecuted,
fail_on_prediction_errors: bool,
circuit_breaker_tripped: bool,
) -> Tuple[EvaluationStore, Optional[Text], Optional[float]]:
from rasa.core.policies.form_policy import FormPolicy
action_executed_eval_store = EvaluationStore()
gold = event.action_name
if circuit_breaker_tripped:
predicted = "circuit breaker tripped"
policy = None
confidence = None
else:
action, policy, confidence = processor.predict_next_action(
partial_tracker)
predicted = action.name()
if policy and predicted != gold and FormPolicy.__name__ in policy:
# FormPolicy predicted wrong action
# but it might be Ok if form action is rejected
_emulate_form_rejection(processor, partial_tracker)
# try again
action, policy, confidence = processor.predict_next_action(
partial_tracker)
predicted = action.name()
action_executed_eval_store.add_to_store(action_predictions=predicted,
action_targets=gold)
if action_executed_eval_store.has_prediction_target_mismatch():
partial_tracker.update(
WronglyPredictedAction(gold, predicted, event.policy,
event.confidence, event.timestamp))
if fail_on_prediction_errors:
error_msg = ("Model predicted a wrong action. Failed Story: "
"\n\n{}".format(partial_tracker.export_stories()))
if FormPolicy.__name__ in policy:
error_msg += ("FormAction is not run during "
"evaluation therefore it is impossible to know "
"if validation failed or this story is wrong. "
"If the story is correct, add it to the "
"training stories and retrain.")
raise ValueError(error_msg)
else:
partial_tracker.update(event)
return action_executed_eval_store, policy, confidence
def _collect_user_uttered_predictions(
event: UserUttered,
partial_tracker: DialogueStateTracker,
fail_on_prediction_errors: bool,
) -> EvaluationStore:
user_uttered_eval_store = EvaluationStore()
intent_gold = event.parse_data.get("true_intent")
predicted_intent = event.parse_data.get("intent", {}).get("name")
if not predicted_intent:
predicted_intent = [None]
user_uttered_eval_store.add_to_store(
intent_predictions=predicted_intent, intent_targets=intent_gold
)
entity_gold = event.parse_data.get("true_entities")
predicted_entities = event.parse_data.get("entities")
if entity_gold or predicted_entities:
user_uttered_eval_store.add_to_store(
entity_targets=_clean_entity_results(event.text, entity_gold),
entity_predictions=_clean_entity_results(event.text, predicted_entities),
)
if user_uttered_eval_store.has_prediction_target_mismatch():
partial_tracker.update(
WronglyClassifiedUserUtterance(event, user_uttered_eval_store)
)
if fail_on_prediction_errors:
raise ValueError(
"NLU model predicted a wrong intent. Failed Story:"
" \n\n{}".format(partial_tracker.export_stories())
)
else:
end_to_end_user_utterance = EndToEndUserUtterance(
event.text, event.intent, event.entities
)
partial_tracker.update(end_to_end_user_utterance)
return user_uttered_eval_store
async def test_persist_form_story(tmpdir):
domain = Domain.load("data/test_domains/form.yml")
tracker = DialogueStateTracker("", domain.slots)
story = ("* greet\n"
" - utter_greet\n"
"* start_form\n"
" - some_form\n"
' - form{"name": "some_form"}\n'
"* default\n"
" - utter_default\n"
" - some_form\n"
"* stop\n"
" - utter_ask_continue\n"
"* affirm\n"
" - some_form\n"
"* stop\n"
" - utter_ask_continue\n"
" - action_listen\n"
"* form: inform\n"
" - some_form\n"
' - form{"name": null}\n'
"* goodbye\n"
" - utter_goodbye\n")
# simulate talking to the form
events = [
UserUttered(intent={"name": "greet"}),
ActionExecuted("utter_greet"),
ActionExecuted("action_listen"),
# start the form
UserUttered(intent={"name": "start_form"}),
ActionExecuted("some_form"),
Form("some_form"),
ActionExecuted("action_listen"),
# out of form input
UserUttered(intent={"name": "default"}),
ActionExecutionRejected("some_form"),
ActionExecuted("utter_default"),
ActionExecuted("some_form"),
ActionExecuted("action_listen"),
# out of form input
UserUttered(intent={"name": "stop"}),
ActionExecutionRejected("some_form"),
ActionExecuted("utter_ask_continue"),
ActionExecuted("action_listen"),
# out of form input but continue with the form
UserUttered(intent={"name": "affirm"}),
FormValidation(False),
ActionExecuted("some_form"),
ActionExecuted("action_listen"),
# out of form input
UserUttered(intent={"name": "stop"}),
ActionExecutionRejected("some_form"),
ActionExecuted("utter_ask_continue"),
ActionExecuted("action_listen"),
# form input
UserUttered(intent={"name": "inform"}),
FormValidation(True),
ActionExecuted("some_form"),
ActionExecuted("action_listen"),
Form(None),
UserUttered(intent={"name": "goodbye"}),
ActionExecuted("utter_goodbye"),
ActionExecuted("action_listen"),
]
[tracker.update(e) for e in events]
assert story in tracker.export_stories()
