async def test_rule_policy_slot_filling_from_text( trained_rule_policy: RulePolicy, trained_rule_policy_domain: Domain): form_conversation = DialogueStateTracker.from_events( "in a form", evts=[ ActionExecuted(ACTION_LISTEN_NAME), # User responds and fills requested slot UserUttered("/activate_q_form", {"name": "activate_q_form"}), ActionExecuted("loop_q_form"), ActiveLoop("loop_q_form"), SlotSet(REQUESTED_SLOT, "some_slot"), ActionExecuted(ACTION_LISTEN_NAME), UserUttered("/bla", {"name": GREET_INTENT_NAME}), ActionExecuted("loop_q_form"), SlotSet("some_slot", "/bla"), ActiveLoop(None), SlotSet(REQUESTED_SLOT, None), ], slots=trained_rule_policy_domain.slots, ) # RulePolicy predicts action which handles submit action_probabilities = trained_rule_policy.predict_action_probabilities( form_conversation, trained_rule_policy_domain, RegexInterpreter()) assert_predicted_action(action_probabilities, trained_rule_policy_domain, "utter_stop")
async def test_set_slot_and_deactivate(): form_name = "my form" slot_name = "num_people" slot_value = "dasdasdfasdf" events = [ ActiveLoop(form_name), SlotSet(REQUESTED_SLOT, slot_name), ActionExecuted(ACTION_LISTEN_NAME), UserUttered(slot_value), ] tracker = DialogueStateTracker.from_events(sender_id="bla", evts=events) domain = f""" forms: - {form_name}: {slot_name}: - type: from_text slots: {slot_name}: type: unfeaturized """ domain = Domain.from_yaml(domain) action = FormAction(form_name, None) events = await action.run( CollectingOutputChannel(), TemplatedNaturalLanguageGenerator(domain.templates), tracker, domain, ) assert events == [ SlotSet(slot_name, slot_value), SlotSet(REQUESTED_SLOT, None), ActiveLoop(None), ]
async def test_2nd_affirmation_failed(intent_which_lets_action_give_up: Text): tracker = DialogueStateTracker.from_events( "some-sender", evts=[ # User sends message with low NLU confidence *_message_requiring_fallback(), ActiveLoop(ACTION_TWO_STAGE_FALLBACK_NAME), # Action asks user to affirm *_two_stage_clarification_request(), ActionExecuted(ACTION_LISTEN_NAME), # User denies suggested intents UserUttered("hi", {"name": USER_INTENT_OUT_OF_SCOPE}), # Action asks user to rephrase *_two_stage_clarification_request(), # User rephrased with low confidence *_message_requiring_fallback(), # Actions asks user to affirm for the last time *_two_stage_clarification_request(), ActionExecuted(ACTION_LISTEN_NAME), # User denies suggested intents for the second time UserUttered("hi", {"name": intent_which_lets_action_give_up}), ], ) domain = Domain.empty() action = TwoStageFallbackAction() events = await action.run( CollectingOutputChannel(), TemplatedNaturalLanguageGenerator(domain.templates), tracker, domain, ) assert events == [ActiveLoop(None), UserUtteranceReverted()]
async def test_validate_slots_on_activation_with_other_action_after_user_utterance( ): form_name = "my form" slot_name = "num_people" slot_value = "hi" events = [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(slot_value, entities=[{ "entity": "num_tables", "value": 5 }]), ActionExecuted("action_in_between"), ] tracker = DialogueStateTracker.from_events(sender_id="bla", evts=events) domain = f""" slots: {slot_name}: type: unfeaturized forms: - {form_name}: {slot_name}: - type: from_text actions: - validate_{form_name} """ domain = Domain.from_yaml(domain) action_server_url = "http:/my-action-server:5055/webhook" expected_slot_value = "✅" with aioresponses() as mocked: mocked.post( action_server_url, payload={ "events": [{ "event": "slot", "name": slot_name, "value": expected_slot_value }] }, ) action_server = EndpointConfig(action_server_url) action = FormAction(form_name, action_server) events = await action.run( CollectingOutputChannel(), TemplatedNaturalLanguageGenerator(domain.templates), tracker, domain, ) assert events == [ ActiveLoop(form_name), SlotSet(slot_name, expected_slot_value), SlotSet(REQUESTED_SLOT, None), ActiveLoop(None), ]
async def deactivate(self, *args) -> List[Event]: events_so_far = args[-1] assert events_so_far == [ ActiveLoop(form_name), *expected_activation_events, *expected_do_events, ActiveLoop(None), ] return expected_deactivation_events
def test_reading_of_trackers_with_legacy_form_events(): loop_name1 = "my loop" loop_name2 = "my form" tracker = DialogueStateTracker.from_dict( "sender", events_as_dict=[ { "event": ActiveLoop.type_name, "name": loop_name1 }, { "event": LegacyForm.type_name, "name": None }, { "event": LegacyForm.type_name, "name": loop_name2 }, ], ) expected_events = [ ActiveLoop(loop_name1), LegacyForm(None), LegacyForm(loop_name2) ] assert list(tracker.events) == expected_events assert tracker.active_loop["name"] == loop_name2
async def test_activate_with_prefilled_slot(): slot_name = "num_people" slot_value = 5 tracker = DialogueStateTracker.from_events( sender_id="bla", evts=[SlotSet(slot_name, slot_value)]) form_name = "my form" action = FormAction(form_name, None) next_slot_to_request = "next slot to request" domain = f""" forms: - {form_name}: {slot_name}: - type: from_entity entity: {slot_name} {next_slot_to_request}: - type: from_text slots: {slot_name}: type: unfeaturized """ domain = Domain.from_yaml(domain) events = await action.run( CollectingOutputChannel(), TemplatedNaturalLanguageGenerator(domain.templates), tracker, domain, ) assert events == [ ActiveLoop(form_name), SlotSet(slot_name, slot_value), SlotSet(REQUESTED_SLOT, next_slot_to_request), ]
async def test_activate(): tracker = DialogueStateTracker.from_events(sender_id="bla", evts=[]) form_name = "my form" action = FormAction(form_name, None) slot_name = "num_people" domain = f""" forms: - {form_name}: {slot_name}: - type: from_entity entity: number responses: utter_ask_num_people: - text: "How many people?" """ domain = Domain.from_yaml(domain) events = await action.run( CollectingOutputChannel(), TemplatedNaturalLanguageGenerator(domain.templates), tracker, domain, ) assert events[:-1] == [ ActiveLoop(form_name), SlotSet(REQUESTED_SLOT, slot_name) ] assert isinstance(events[-1], BotUttered)
async def test_action_rejection(): form_name = "my form" slot_to_fill = "some slot" tracker = DialogueStateTracker.from_events( sender_id="bla", evts=[ ActiveLoop(form_name), SlotSet(REQUESTED_SLOT, slot_to_fill), ActionExecuted(ACTION_LISTEN_NAME), UserUttered("haha", {"name": "greet"}), ], ) form_name = "my form" action = FormAction(form_name, None) domain = f""" forms: - {form_name}: {slot_to_fill}: - type: from_entity entity: some_entity slots: {slot_to_fill}: type: unfeaturized """ domain = Domain.from_yaml(domain) with pytest.raises(ActionExecutionRejection): await action.run( CollectingOutputChannel(), TemplatedNaturalLanguageGenerator(domain.templates), tracker, domain, )
async def is_done(self, *args) -> bool: events_so_far = args[-1] return events_so_far == [ ActiveLoop(form_name), *expected_activation_events, *expected_do_events, ]
async def do(self, *args: Any) -> List[Event]: events_so_far = args[-1] assert events_so_far == [ ActiveLoop(form_name), *expected_activation_events ] return expected_do_events
async def test_1st_affirmation_is_successful(): tracker = DialogueStateTracker.from_events( "some-sender", evts=[ # User sends message with low NLU confidence *_message_requiring_fallback(), ActiveLoop(ACTION_TWO_STAGE_FALLBACK_NAME), # Action asks user to affirm *_two_stage_clarification_request(), ActionExecuted(ACTION_LISTEN_NAME), # User affirms UserUttered("hi", {"name": "greet", "confidence": 1.0}), ], ) domain = Domain.empty() action = TwoStageFallbackAction() events = await action.run( CollectingOutputChannel(), TemplatedNaturalLanguageGenerator(domain.templates), tracker, domain, ) for events in events: tracker.update(events, domain) applied_events = tracker.applied_events() assert applied_events == [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered("hi", {"name": "greet", "confidence": 1.0}), ]
async def test_ask_affirm_after_rephrasing(): tracker = DialogueStateTracker.from_events( "some-sender", evts=[ # User sends message with low NLU confidence *_message_requiring_fallback(), ActiveLoop(ACTION_TWO_STAGE_FALLBACK_NAME), # Action asks user to affirm *_two_stage_clarification_request(), ActionExecuted(ACTION_LISTEN_NAME), # User denies suggested intents UserUttered("hi", {"name": USER_INTENT_OUT_OF_SCOPE}), # Action asks user to rephrase ActionExecuted(ACTION_TWO_STAGE_FALLBACK_NAME), BotUttered("please rephrase"), # User rephrased with low confidence *_message_requiring_fallback(), ], ) domain = Domain.empty() action = TwoStageFallbackAction() events = await action.run( CollectingOutputChannel(), TemplatedNaturalLanguageGenerator(domain.templates), tracker, domain, ) assert len(events) == 1 assert isinstance(events[0], BotUttered)
def test_form_submit_rule(): form_name = "some_form" submit_action_name = "utter_submit" domain = Domain.from_yaml(f""" intents: - {GREET_INTENT_NAME} actions: - {UTTER_GREET_ACTION} - some-action - {submit_action_name} slots: {REQUESTED_SLOT}: type: unfeaturized forms: - {form_name} """) form_submit_rule = _form_submit_rule(domain, submit_action_name, form_name) policy = RulePolicy() policy.train([GREET_RULE, form_submit_rule], domain, RegexInterpreter()) form_conversation = DialogueStateTracker.from_events( "in a form", evts=[ # Form was activated ActionExecuted(ACTION_LISTEN_NAME), UserUttered("haha", {"name": GREET_INTENT_NAME}), ActionExecuted(form_name), ActiveLoop(form_name), SlotSet(REQUESTED_SLOT, "some value"), ActionExecuted(ACTION_LISTEN_NAME), # User responds and fills requested slot UserUttered("haha", {"name": GREET_INTENT_NAME}), ActionExecuted(form_name), # Form get's deactivated ActiveLoop(None), SlotSet(REQUESTED_SLOT, None), ], slots=domain.slots, ) # RulePolicy predicts action which handles submit action_probabilities = policy.predict_action_probabilities( form_conversation, domain, RegexInterpreter()) assert_predicted_action(action_probabilities, domain, submit_action_name)
async def run( self, output_channel: "OutputChannel", nlg: "NaturalLanguageGenerator", tracker: "DialogueStateTracker", domain: "Domain", ) -> List[Event]: return [ActiveLoop(None), SlotSet(REQUESTED_SLOT, None)]
def test_immediate_submit(): form_name = "some_form" submit_action_name = "utter_submit" entity = "some_entity" slot = "some_slot" domain = Domain.from_yaml(f""" intents: - {GREET_INTENT_NAME} actions: - {UTTER_GREET_ACTION} - some-action - {submit_action_name} slots: {REQUESTED_SLOT}: type: unfeaturized {slot}: type: unfeaturized forms: - {form_name} entities: - {entity} """) form_activation_rule = _form_activation_rule(domain, form_name, GREET_INTENT_NAME) form_submit_rule = _form_submit_rule(domain, submit_action_name, form_name) policy = RulePolicy() policy.train([GREET_RULE, form_activation_rule, form_submit_rule], domain, RegexInterpreter()) form_conversation = DialogueStateTracker.from_events( "in a form", evts=[ # Form was activated ActionExecuted(ACTION_LISTEN_NAME), # The same intent which activates the form also deactivates it UserUttered( "haha", {"name": GREET_INTENT_NAME}, entities=[{ "entity": entity, "value": "Bruce Wayne" }], ), SlotSet(slot, "Bruce"), ActionExecuted(form_name), SlotSet("bla", "bla"), ActiveLoop(None), SlotSet(REQUESTED_SLOT, None), ], slots=domain.slots, ) # RulePolicy predicts action which handles submit action_probabilities = policy.predict_action_probabilities( form_conversation, domain, RegexInterpreter()) assert_predicted_action(action_probabilities, domain, submit_action_name)
async def test_dont_predict_form_if_already_finished(): form_name = "some_form" domain = Domain.from_yaml(f""" intents: - {GREET_INTENT_NAME} actions: - {UTTER_GREET_ACTION} - some-action slots: {REQUESTED_SLOT}: type: unfeaturized forms: - {form_name} """) policy = RulePolicy() policy.train([GREET_RULE], domain, RegexInterpreter()) form_conversation = DialogueStateTracker.from_events( "in a form", evts=[ # We are in an activate form ActionExecuted(form_name), ActiveLoop(form_name), SlotSet(REQUESTED_SLOT, "some value"), ActionExecuted(ACTION_LISTEN_NAME), # User sends message as response to a requested slot UserUttered("haha", {"name": GREET_INTENT_NAME}), # Form is happy and deactivates itself ActionExecuted(form_name), ActiveLoop(None), SlotSet(REQUESTED_SLOT, None), # User sends another message. Form is already done. Shouldn't get triggered # again ActionExecuted(ACTION_LISTEN_NAME), UserUttered("haha", {"name": GREET_INTENT_NAME}), ], slots=domain.slots, ) # RulePolicy triggers form again action_probabilities = policy.predict_action_probabilities( form_conversation, domain, RegexInterpreter()) assert_predicted_action(action_probabilities, domain, UTTER_GREET_ACTION)
async def test_activate_and_immediate_deactivate(): slot_name = "num_people" slot_value = 5 tracker = DialogueStateTracker.from_events( sender_id="bla", evts=[ ActionExecuted(ACTION_LISTEN_NAME), UserUttered( "haha", {"name": "greet"}, entities=[{ "entity": slot_name, "value": slot_value }], ), ], ) form_name = "my form" action = FormAction(form_name, None) domain = f""" forms: - {form_name}: {slot_name}: - type: from_entity entity: {slot_name} slots: {slot_name}: type: unfeaturized """ domain = Domain.from_yaml(domain) events = await action.run( CollectingOutputChannel(), TemplatedNaturalLanguageGenerator(domain.templates), tracker, domain, ) assert events == [ ActiveLoop(form_name), SlotSet(slot_name, slot_value), SlotSet(REQUESTED_SLOT, None), ActiveLoop(None), ]
def test_writing_trackers_with_legacy_form_events(): loop_name = "my loop" tracker = DialogueStateTracker.from_events( "sender", evts=[ActiveLoop(loop_name), LegacyForm(None), LegacyForm("some")] ) events_as_dict = [event.as_dict() for event in tracker.events] for event in events_as_dict: assert event["event"] == ActiveLoop.type_name
def test_rule_without_condition(rule_steps_without_stories: List[StoryStep]): rule = rule_steps_without_stories[1] assert rule.block_name == "Rule without condition" assert rule.events == [ ActionExecuted(RULE_SNIPPET_ACTION_NAME), UserUttered("explain", {"name": "explain", "confidence": 1.0}, []), ActionExecuted("utter_explain_some_slot"), ActionExecuted("loop_q_form"), ActiveLoop("loop_q_form"), ]
def _form_submit_rule(domain: Domain, submit_action_name: Text, form_name: Text) -> DialogueStateTracker: return TrackerWithCachedStates.from_events( "bla", domain=domain, slots=domain.slots, evts=[ ActiveLoop(form_name), # Any events in between ActionExecuted(RULE_SNIPPET_ACTION_NAME), # Form runs and deactivates itself ActionExecuted(form_name), ActiveLoop(None), SlotSet(REQUESTED_SLOT, None), ActionExecuted(submit_action_name), ActionExecuted(ACTION_LISTEN_NAME), ], is_rule_tracker=True, )
async def test_form_unhappy_path_without_rule(): form_name = "some_form" other_intent = "bye" domain = Domain.from_yaml(f""" intents: - {GREET_INTENT_NAME} - {other_intent} actions: - {UTTER_GREET_ACTION} - some-action slots: {REQUESTED_SLOT}: type: unfeaturized forms: - {form_name} """) policy = RulePolicy() policy.train([GREET_RULE], domain, RegexInterpreter()) conversation_events = [ ActionExecuted(form_name), ActiveLoop(form_name), SlotSet(REQUESTED_SLOT, "some value"), ActionExecuted(ACTION_LISTEN_NAME), UserUttered("haha", {"name": other_intent}), ActiveLoop(form_name), ActionExecutionRejected(form_name), ] # Unhappy path is not handled. No rule matches. Let's hope ML fixes our problems 🤞 action_probabilities = policy.predict_action_probabilities( DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots), domain, RegexInterpreter(), ) assert max(action_probabilities) == policy._core_fallback_threshold
def test_rule_with_condition(rule_steps_without_stories: List[StoryStep]): rule = rule_steps_without_stories[0] assert rule.block_name == "Rule with condition" assert rule.events == [ ActiveLoop("loop_q_form"), SlotSet("requested_slot", "some_slot"), ActionExecuted(RULE_SNIPPET_ACTION_NAME), UserUttered( intent={"name": "inform", "confidence": 1.0}, entities=[{"entity": "some_slot", "value": "bla"}], ), ActionExecuted("loop_q_form"), ]
async def test_form_unhappy_path_from_general_rule(): form_name = "some_form" domain = Domain.from_yaml(f""" intents: - {GREET_INTENT_NAME} actions: - {UTTER_GREET_ACTION} - some-action slots: {REQUESTED_SLOT}: type: unfeaturized forms: - {form_name} """) policy = RulePolicy() # RulePolicy should memorize that unhappy_rule overrides GREET_RULE policy.train([GREET_RULE], domain, RegexInterpreter()) # Check that RulePolicy predicts action to handle unhappy path conversation_events = [ ActionExecuted(form_name), ActiveLoop(form_name), SlotSet(REQUESTED_SLOT, "some value"), ActionExecuted(ACTION_LISTEN_NAME), UserUttered("haha", {"name": GREET_INTENT_NAME}), ActionExecutionRejected(form_name), ] action_probabilities = policy.predict_action_probabilities( DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots), domain, RegexInterpreter(), ) # check that general rule action is predicted assert_predicted_action(action_probabilities, domain, UTTER_GREET_ACTION) # Check that RulePolicy triggers form again after handling unhappy path conversation_events.append(ActionExecuted(UTTER_GREET_ACTION)) action_probabilities = policy.predict_action_probabilities( DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots), domain, RegexInterpreter(), ) # check that action_listen from general rule is overwritten by form action assert_predicted_action(action_probabilities, domain, form_name)
async def test_validate_slots(validate_return_events: List[Dict], expected_events: List[Event]): form_name = "my form" slot_name = "num_people" slot_value = "hi" events = [ ActiveLoop(form_name), SlotSet(REQUESTED_SLOT, slot_name), ActionExecuted(ACTION_LISTEN_NAME), UserUttered(slot_value, entities=[{ "entity": "num_tables", "value": 5 }]), ] tracker = DialogueStateTracker.from_events(sender_id="bla", evts=events) domain = f""" slots: {slot_name}: type: unfeaturized num_tables: type: unfeaturized forms: - {form_name}: {slot_name}: - type: from_text num_tables: - type: from_entity entity: num_tables actions: - validate_{form_name} """ domain = Domain.from_yaml(domain) action_server_url = "http:/my-action-server:5055/webhook" with aioresponses() as mocked: mocked.post(action_server_url, payload={"events": validate_return_events}) action_server = EndpointConfig(action_server_url) action = FormAction(form_name, action_server) events = await action.run( CollectingOutputChannel(), TemplatedNaturalLanguageGenerator(domain.templates), tracker, domain, ) assert events == expected_events
async def test_give_it_up_after_low_confidence_after_affirm_request(): tracker = DialogueStateTracker.from_events( "some-sender", evts=[ # User sends message with low NLU confidence *_message_requiring_fallback(), ActiveLoop(ACTION_TWO_STAGE_FALLBACK_NAME), # Action asks user to affirm *_two_stage_clarification_request(), # User's affirms with low NLU confidence again *_message_requiring_fallback(), ], ) domain = Domain.empty() action = TwoStageFallbackAction() events = await action.run( CollectingOutputChannel(), TemplatedNaturalLanguageGenerator(domain.templates), tracker, domain, ) assert events == [ActiveLoop(None), UserUtteranceReverted()]
async def test_remote_action_utterances_with_none_values( default_channel, default_tracker, default_domain): endpoint = EndpointConfig("https://example.com/webhooks/actions") remote_action = action.RemoteAction("my_action", endpoint) response = { "events": [ { "event": "form", "name": "restaurant_form", "timestamp": None }, { "event": "slot", "timestamp": None, "name": "requested_slot", "value": "cuisine", }, ], "responses": [{ "text": None, "buttons": None, "elements": [], "custom": None, "template": "utter_ask_cuisine", "image": None, "attachment": None, }], } nlg = TemplatedNaturalLanguageGenerator( {"utter_ask_cuisine": [{ "text": "what dou want to eat?" }]}) with aioresponses() as mocked: mocked.post("https://example.com/webhooks/actions", payload=response) events = await remote_action.run(default_channel, nlg, default_tracker, default_domain) assert events == [ BotUttered("what dou want to eat?", metadata={"template_name": "utter_ask_cuisine"}), ActiveLoop("restaurant_form"), SlotSet("requested_slot", "cuisine"), ]
async def test_ask_affirmation(): tracker = DialogueStateTracker.from_events( "some-sender", evts=_message_requiring_fallback() ) domain = Domain.empty() action = TwoStageFallbackAction() events = await action.run( CollectingOutputChannel(), TemplatedNaturalLanguageGenerator(domain.templates), tracker, domain, ) assert len(events) == 2 assert events[0] == ActiveLoop(ACTION_TWO_STAGE_FALLBACK_NAME) assert isinstance(events[1], BotUttered)
def _form_activation_rule( domain: Domain, form_name: Text, activation_intent_name: Text) -> DialogueStateTracker: return TrackerWithCachedStates.from_events( "bla", domain=domain, slots=domain.slots, evts=[ ActionExecuted(RULE_SNIPPET_ACTION_NAME), # The intent `other_intent` activates the form ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": activation_intent_name}), ActionExecuted(form_name), ActiveLoop(form_name), ActionExecuted(ACTION_LISTEN_NAME), ], is_rule_tracker=True, )
async def test_predict_form_action_if_multiple_turns(): form_name = "some_form" other_intent = "bye" domain = Domain.from_yaml(f""" intents: - {GREET_INTENT_NAME} - {other_intent} actions: - {UTTER_GREET_ACTION} - some-action slots: {REQUESTED_SLOT}: type: unfeaturized forms: - {form_name} """) policy = RulePolicy() policy.train([GREET_RULE], domain, RegexInterpreter()) form_conversation = DialogueStateTracker.from_events( "in a form", evts=[ # We are in an active form ActionExecuted(form_name), ActiveLoop(form_name), SlotSet(REQUESTED_SLOT, "some value"), # User responds to slot request ActionExecuted(ACTION_LISTEN_NAME), UserUttered("haha", {"name": GREET_INTENT_NAME}), # Form validates input and requests another slot ActionExecuted(form_name), SlotSet(REQUESTED_SLOT, "some other"), # User responds to 2nd slot request ActionExecuted(ACTION_LISTEN_NAME), UserUttered("haha", {"name": other_intent}), ], slots=domain.slots, ) # RulePolicy triggers form again action_probabilities = policy.predict_action_probabilities( form_conversation, domain, RegexInterpreter()) assert_predicted_action(action_probabilities, domain, form_name)