async def _run_action( self, action, tracker, output_channel, nlg, policy=None, confidence=None ): # events and return values are used to update # the tracker state after an action has been taken try: events = await action.run(output_channel, nlg, tracker, self.domain) except ActionExecutionRejection: events = [ActionExecutionRejected(action.name(), policy, confidence)] tracker.update(events[0]) return self.should_predict_another_action(action.name(), events) except Exception as e: logger.error( "Encountered an exception while running action '{}'. " "Bot will continue, but the actions events are lost. " "Please check the logs of your action server for " "more information.".format(action.name()) ) logger.debug(e, exc_info=True) events = [] self._log_action_on_tracker(tracker, action.name(), events, policy, confidence) if action.name() != ACTION_LISTEN_NAME and not action.name().startswith( UTTER_PREFIX ): self._log_slots(tracker) await self._send_bot_messages(events, tracker, output_channel) await self._schedule_reminders(events, tracker, output_channel, nlg) await self._cancel_reminders(events, tracker) return self.should_predict_another_action(action.name(), events)
async def _run_action( self, action, tracker, dispatcher, policy=None, confidence=None ): # events and return values are used to update # the tracker state after an action has been taken try: events = await action.run(dispatcher, tracker, self.domain) except ActionExecutionRejection: events = [ActionExecutionRejected(action.name(), policy, confidence)] tracker.update(events[0]) return self.should_predict_another_action(action.name(), events) except Exception as e: logger.error( "Encountered an exception while running action '{}'. " "Bot will continue, but the actions events are lost. " "Make sure to fix the exception in your custom " "code.".format(action.name()) ) logger.debug(e, exc_info=True) events = [] self._log_action_on_tracker(tracker, action.name(), events, policy, confidence) if action.name() != "action_listen" and "utter_" not in action.name(): self._log_slots(tracker) self.log_bot_utterances_on_tracker(tracker, dispatcher) await self._schedule_reminders(events, tracker, dispatcher) await self._cancel_reminders(events, tracker) return self.should_predict_another_action(action.name(), events)
async def test_form_unhappy_path_from_general_rule(): form_name = "some_form" domain = Domain.from_yaml(f""" intents: - {GREET_INTENT_NAME} actions: - {UTTER_GREET_ACTION} - some-action slots: {REQUESTED_SLOT}: type: unfeaturized forms: - {form_name} """) policy = RulePolicy() # RulePolicy should memorize that unhappy_rule overrides GREET_RULE policy.train([GREET_RULE], domain, RegexInterpreter()) # Check that RulePolicy predicts action to handle unhappy path conversation_events = [ ActionExecuted(form_name), ActiveLoop(form_name), SlotSet(REQUESTED_SLOT, "some value"), ActionExecuted(ACTION_LISTEN_NAME), UserUttered("haha", {"name": GREET_INTENT_NAME}), ActionExecutionRejected(form_name), ] action_probabilities = policy.predict_action_probabilities( DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots), domain, RegexInterpreter(), ) # check that general rule action is predicted assert_predicted_action(action_probabilities, domain, UTTER_GREET_ACTION) # Check that RulePolicy triggers form again after handling unhappy path conversation_events.append(ActionExecuted(UTTER_GREET_ACTION)) action_probabilities = policy.predict_action_probabilities( DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots), domain, RegexInterpreter(), ) # check that action_listen from general rule is overwritten by form action assert_predicted_action(action_probabilities, domain, form_name)
def _emulate_form_rejection(processor, partial_tracker): from rasa.core.policies import FormPolicy if partial_tracker.active_form.get("name"): for p in processor.policy_ensemble.policies: if isinstance(p, FormPolicy): # emulate form rejection partial_tracker.update( ActionExecutionRejected( partial_tracker.active_form["name"])) # check if unhappy path is covered by the train stories if not p.state_is_unhappy(partial_tracker, processor.domain): # this state is not covered by the stories del partial_tracker.events[-1] partial_tracker.active_form['rejected'] = False
async def _run_action( self, action, tracker, output_channel, nlg, policy=None, confidence=None, metadata: Optional[Dict[Text, Any]] = None, ) -> bool: # events and return values are used to update # the tracker state after an action has been taken try: # Here we set optional metadata to the ActionSessionStart, which will then # be passed to the SessionStart event. Otherwise the metadata will be lost. if action.name() == ACTION_SESSION_START_NAME: action.metadata = metadata events = await action.run(output_channel, nlg, tracker, self.domain) except ActionExecutionRejection: events = [ ActionExecutionRejected(action.name(), policy, confidence) ] tracker.update(events[0]) return self.should_predict_another_action(action.name()) except Exception as e: logger.error( f"Encountered an exception while running action '{action.name()}'. " "Bot will continue, but the actions events are lost. " "Please check the logs of your action server for " "more information.") logger.debug(e, exc_info=True) events = [] self._log_action_on_tracker(tracker, action.name(), events, policy, confidence) if action.name() != ACTION_LISTEN_NAME and not action.name( ).startswith(UTTER_PREFIX): self._log_slots(tracker) await self._send_bot_messages(events, tracker, output_channel) await self._schedule_reminders(events, tracker, output_channel, nlg) await self._cancel_reminders(events, tracker) return self.should_predict_another_action(action.name())
async def test_form_unhappy_path(): form_name = "some_form" domain = Domain.from_yaml( f""" intents: - {GREET_INTENT_NAME} actions: - {UTTER_GREET_ACTION} - some-action slots: {REQUESTED_SLOT}: type: unfeaturized forms: - {form_name} """ ) policy = RulePolicy() policy.train([GREET_RULE], domain, RegexInterpreter()) unhappy_form_conversation = DialogueStateTracker.from_events( "in a form", evts=[ # We are in an active form ActionExecuted(form_name), Form(form_name), SlotSet(REQUESTED_SLOT, "some value"), # User responds to slot request ActionExecuted(ACTION_LISTEN_NAME), UserUttered("haha", {"name": GREET_INTENT_NAME}), # Form isn't happy with the answer and rejects execution ActionExecutionRejected(form_name), ], slots=domain.slots, ) # RulePolicy doesn't trigger form but FAQ action_probabilities = policy.predict_action_probabilities( unhappy_form_conversation, domain ) assert_predicted_action(action_probabilities, domain, UTTER_GREET_ACTION)
async def test_form_unhappy_path_without_rule(): form_name = "some_form" other_intent = "bye" domain = Domain.from_yaml( f""" intents: - {GREET_INTENT_NAME} - {other_intent} actions: - {UTTER_GREET_ACTION} - some-action slots: {REQUESTED_SLOT}: type: unfeaturized forms: - {form_name} """ ) policy = RulePolicy() policy.train([GREET_RULE], domain, RegexInterpreter()) conversation_events = [ ActionExecuted(form_name), Form(form_name), SlotSet(REQUESTED_SLOT, "some value"), ActionExecuted(ACTION_LISTEN_NAME), UserUttered("haha", {"name": other_intent}), Form(form_name), ActionExecutionRejected(form_name), ] # Unhappy path is not handled. No rule matches. Let's hope ML fixes our problems 🤞 action_probabilities = policy.predict_action_probabilities( DialogueStateTracker.from_events( "casd", evts=conversation_events, slots=domain.slots ), domain, ) assert max(action_probabilities) == policy._core_fallback_threshold
async def test_loop_without_deactivate(): expected_activation_events = [ ActionExecutionRejected("tada"), ActionExecuted("test"), ] expected_do_events = [ActionExecuted("do")] form_name = "my form" class MyLoop(LoopAction): def name(self) -> Text: return form_name async def activate(self, *args: Any) -> List[Event]: return expected_activation_events async def do(self, *args: Any) -> List[Event]: return expected_do_events async def deactivate(self, *args) -> List[Event]: raise ValueError("this shouldn't be called") async def is_done(self, *args) -> bool: return False tracker = DialogueStateTracker.from_events("some sender", []) domain = Domain.empty() action = MyLoop() actual = await action.run( CollectingOutputChannel(), TemplatedNaturalLanguageGenerator(domain.templates), tracker, domain, ) assert actual == [ ActiveLoop(form_name), *expected_activation_events, *expected_do_events, ]
async def test_form_unhappy_path_no_validation_from_story(): form_name = "some_form" handle_rejection_action_name = "utter_handle_rejection" domain = Domain.from_yaml(f""" intents: - {GREET_INTENT_NAME} actions: - {UTTER_GREET_ACTION} - {handle_rejection_action_name} - some-action slots: {REQUESTED_SLOT}: type: unfeaturized forms: - {form_name} """) unhappy_story = TrackerWithCachedStates.from_events( "bla", domain=domain, slots=domain.slots, evts=[ # We are in an active form ActionExecuted(form_name), ActiveLoop(form_name), # When a user says "hi", and the form is unhappy, # we want to run a specific action UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(handle_rejection_action_name), ActionExecuted(ACTION_LISTEN_NAME), # Next user utterance is an answer to the previous question # and shouldn't be validated by the form UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(form_name), ActionExecuted(ACTION_LISTEN_NAME), ], ) policy = RulePolicy() policy.train([unhappy_story], domain, RegexInterpreter()) # Check that RulePolicy predicts no validation to handle unhappy path conversation_events = [ ActionExecuted(form_name), ActiveLoop(form_name), SlotSet(REQUESTED_SLOT, "some value"), ActionExecuted(ACTION_LISTEN_NAME), UserUttered("haha", {"name": GREET_INTENT_NAME}), ActionExecutionRejected(form_name), ActionExecuted(handle_rejection_action_name), ActionExecuted(ACTION_LISTEN_NAME), UserUttered("haha", {"name": GREET_INTENT_NAME}), ] tracker = DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots) action_probabilities = policy.predict_action_probabilities( tracker, domain, RegexInterpreter()) # there is no rule for next action assert max(action_probabilities) == policy._core_fallback_threshold # check that RulePolicy added FormValidation False event based on the training story assert tracker.events[-1] == FormValidation(False)
async def test_form_unhappy_path_no_validation_from_rule(): form_name = "some_form" handle_rejection_action_name = "utter_handle_rejection" domain = Domain.from_yaml(f""" intents: - {GREET_INTENT_NAME} actions: - {UTTER_GREET_ACTION} - {handle_rejection_action_name} - some-action slots: {REQUESTED_SLOT}: type: unfeaturized forms: - {form_name} """) unhappy_rule = TrackerWithCachedStates.from_events( "bla", domain=domain, slots=domain.slots, evts=[ # We are in an active form ActiveLoop(form_name), SlotSet(REQUESTED_SLOT, "bla"), ActionExecuted(RULE_SNIPPET_ACTION_NAME), ActionExecuted(ACTION_LISTEN_NAME), # When a user says "hi", and the form is unhappy, # we want to run a specific action UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(handle_rejection_action_name), # Next user utterance is an answer to the previous question # and shouldn't be validated by the form ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(form_name), ActionExecuted(ACTION_LISTEN_NAME), ], is_rule_tracker=True, ) policy = RulePolicy() # RulePolicy should memorize that unhappy_rule overrides GREET_RULE policy.train([GREET_RULE, unhappy_rule], domain, RegexInterpreter()) # Check that RulePolicy predicts action to handle unhappy path conversation_events = [ ActionExecuted(form_name), ActiveLoop(form_name), SlotSet(REQUESTED_SLOT, "some value"), ActionExecuted(ACTION_LISTEN_NAME), UserUttered("haha", {"name": GREET_INTENT_NAME}), ActionExecutionRejected(form_name), ] action_probabilities = policy.predict_action_probabilities( DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots), domain, RegexInterpreter(), ) assert_predicted_action(action_probabilities, domain, handle_rejection_action_name) # Check that RulePolicy predicts action_listen conversation_events.append(ActionExecuted(handle_rejection_action_name)) action_probabilities = policy.predict_action_probabilities( DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots), domain, RegexInterpreter(), ) assert_predicted_action(action_probabilities, domain, ACTION_LISTEN_NAME) # Check that RulePolicy triggers form again after handling unhappy path conversation_events.append(ActionExecuted(ACTION_LISTEN_NAME)) tracker = DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots) action_probabilities = policy.predict_action_probabilities( tracker, domain, RegexInterpreter()) assert_predicted_action(action_probabilities, domain, form_name) # check that RulePolicy added FormValidation False event based on the training rule assert tracker.events[-1] == FormValidation(False)
async def test_form_unhappy_path_from_story(): form_name = "some_form" handle_rejection_action_name = "utter_handle_rejection" domain = Domain.from_yaml(f""" intents: - {GREET_INTENT_NAME} actions: - {UTTER_GREET_ACTION} - {handle_rejection_action_name} - some-action slots: {REQUESTED_SLOT}: type: unfeaturized forms: - {form_name} """) unhappy_story = TrackerWithCachedStates.from_events( "bla", domain=domain, slots=domain.slots, evts=[ # We are in an active form ActionExecuted(form_name), ActiveLoop(form_name), UserUttered("haha", {"name": GREET_INTENT_NAME}), ActionExecuted(UTTER_GREET_ACTION), # After our bot says "hi", we want to run a specific action ActionExecuted(handle_rejection_action_name), ActionExecuted(form_name), ActionExecuted(ACTION_LISTEN_NAME), ], ) policy = RulePolicy() policy.train([GREET_RULE, unhappy_story], domain, RegexInterpreter()) # Check that RulePolicy predicts action to handle unhappy path conversation_events = [ ActionExecuted(form_name), ActiveLoop(form_name), SlotSet(REQUESTED_SLOT, "some value"), ActionExecuted(ACTION_LISTEN_NAME), UserUttered("haha", {"name": GREET_INTENT_NAME}), ActionExecutionRejected(form_name), ] action_probabilities = policy.predict_action_probabilities( DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots), domain, RegexInterpreter(), ) assert_predicted_action(action_probabilities, domain, UTTER_GREET_ACTION) # Check that RulePolicy doesn't trigger form or action_listen # after handling unhappy path conversation_events.append(ActionExecuted(handle_rejection_action_name)) action_probabilities = policy.predict_action_probabilities( DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots), domain, RegexInterpreter(), ) assert max(action_probabilities) == policy._core_fallback_threshold
@pytest.mark.parametrize( "events, expected_applied_events", [ ( [ # Form is triggered and requests slot. ActionExecuted(ACTION_LISTEN_NAME), user_uttered("greet"), ActionExecuted("loop"), ActiveLoop("loop"), SlotSet(REQUESTED_SLOT, "bla"), # User sends chitchat instead of answering form. ActionExecuted(ACTION_LISTEN_NAME), user_uttered("chitchat"), # Form rejected execution. ActionExecutionRejected("loop"), # Action which deals with unhappy path. ActionExecuted("handling chitchat"), # We immediately return to form after executing an action to handle it. ActionExecuted("loop"), # Form happy path continues until all slots are filled. SlotSet(REQUESTED_SLOT, "bla"), ActionExecuted(ACTION_LISTEN_NAME), user_uttered("fill slots"), ActionExecuted("loop"), SlotSet("slot", "value"), SlotSet(REQUESTED_SLOT, None), ActiveLoop(None), ], [ ActionExecuted(ACTION_LISTEN_NAME),
def _emulate_form_rejection(partial_tracker: DialogueStateTracker) -> None: from rasa.core.events import ActionExecutionRejected rejected_action_name: Text = partial_tracker.active_loop["name"] partial_tracker.update(ActionExecutionRejected(rejected_action_name))
async def test_persist_form_story(tmpdir): domain = Domain.load("data/test_domains/form.yml") tracker = DialogueStateTracker("", domain.slots) story = ("* greet\n" " - utter_greet\n" "* start_form\n" " - some_form\n" ' - form{"name": "some_form"}\n' "* default\n" " - utter_default\n" " - some_form\n" "* stop\n" " - utter_ask_continue\n" "* affirm\n" " - some_form\n" "* stop\n" " - utter_ask_continue\n" " - action_listen\n" "* form: inform\n" " - some_form\n" ' - form{"name": null}\n' "* goodbye\n" " - utter_goodbye\n") # simulate talking to the form events = [ UserUttered(intent={"name": "greet"}), ActionExecuted("utter_greet"), ActionExecuted("action_listen"), # start the form UserUttered(intent={"name": "start_form"}), ActionExecuted("some_form"), Form("some_form"), ActionExecuted("action_listen"), # out of form input UserUttered(intent={"name": "default"}), ActionExecutionRejected("some_form"), ActionExecuted("utter_default"), ActionExecuted("some_form"), ActionExecuted("action_listen"), # out of form input UserUttered(intent={"name": "stop"}), ActionExecutionRejected("some_form"), ActionExecuted("utter_ask_continue"), ActionExecuted("action_listen"), # out of form input but continue with the form UserUttered(intent={"name": "affirm"}), FormValidation(False), ActionExecuted("some_form"), ActionExecuted("action_listen"), # out of form input UserUttered(intent={"name": "stop"}), ActionExecutionRejected("some_form"), ActionExecuted("utter_ask_continue"), ActionExecuted("action_listen"), # form input UserUttered(intent={"name": "inform"}), FormValidation(True), ActionExecuted("some_form"), ActionExecuted("action_listen"), Form(None), UserUttered(intent={"name": "goodbye"}), ActionExecuted("utter_goodbye"), ActionExecuted("action_listen"), ] [tracker.update(e) for e in events] assert story in tracker.export_stories()
async def test_whole_loop(): expected_activation_events = [ ActionExecutionRejected("tada"), ActionExecuted("test"), ] expected_do_events = [ActionExecuted("do")] expected_deactivation_events = [SlotSet("deactivated")] form_name = "my form" class MyLoop(LoopAction): def name(self) -> Text: return form_name async def activate(self, *args: Any) -> List[Event]: return expected_activation_events async def do(self, *args: Any) -> List[Event]: events_so_far = args[-1] assert events_so_far == [ ActiveLoop(form_name), *expected_activation_events ] return expected_do_events async def deactivate(self, *args) -> List[Event]: events_so_far = args[-1] assert events_so_far == [ ActiveLoop(form_name), *expected_activation_events, *expected_do_events, ActiveLoop(None), ] return expected_deactivation_events async def is_done(self, *args) -> bool: events_so_far = args[-1] return events_so_far == [ ActiveLoop(form_name), *expected_activation_events, *expected_do_events, ] tracker = DialogueStateTracker.from_events("some sender", []) domain = Domain.empty() action = MyLoop() actual = await action.run( CollectingOutputChannel(), TemplatedNaturalLanguageGenerator(domain.templates), tracker, domain, ) assert actual == [ ActiveLoop(form_name), *expected_activation_events, *expected_do_events, ActiveLoop(None), *expected_deactivation_events, ]
async def test_form_unhappy_path_triggering_form_again(): form_name = "some_form" handle_rejection_action_name = "utter_handle_rejection" domain = Domain.from_yaml(f""" intents: - {GREET_INTENT_NAME} actions: - {UTTER_GREET_ACTION} - {handle_rejection_action_name} - some-action slots: {REQUESTED_SLOT}: type: unfeaturized forms: - {form_name} """) unhappy_rule = TrackerWithCachedStates.from_events( "bla", domain=domain, slots=domain.slots, evts=[ # We are in an active form Form(form_name), SlotSet(REQUESTED_SLOT, "bla"), ActionExecuted(RULE_SNIPPET_ACTION_NAME), ActionExecuted(ACTION_LISTEN_NAME), # When a user says "hi", and the form is unhappy, we want to run a specific # action UserUttered("haha", {"name": GREET_INTENT_NAME}), ActionExecuted(handle_rejection_action_name), ActionExecuted(form_name), ActionExecuted(ACTION_LISTEN_NAME), ], is_rule_tracker=True, ) policy = RulePolicy() policy.train([unhappy_rule], domain, RegexInterpreter()) # Check that RulePolicy predicts action to handle unhappy path conversation_events = [ ActionExecuted(form_name), Form(form_name), SlotSet(REQUESTED_SLOT, "some value"), ActionExecuted(ACTION_LISTEN_NAME), UserUttered("haha", {"name": GREET_INTENT_NAME}), ActionExecutionRejected(form_name), ] action_probabilities = policy.predict_action_probabilities( DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots), domain, ) assert_predicted_action(action_probabilities, domain, handle_rejection_action_name) # Check that RulePolicy triggers form again after handling unhappy path conversation_events.append(ActionExecuted(handle_rejection_action_name)) action_probabilities = policy.predict_action_probabilities( DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots), domain, ) assert_predicted_action(action_probabilities, domain, form_name)