def get_tracker(events: List[Event]) -> DialogueStateTracker: return DialogueStateTracker.from_events("sender", events, [], 20)
def test_tracker_init_copy(domain: Domain): sender_id = "some-id" tracker = DialogueStateTracker(sender_id, domain.slots) tracker_copy = tracker.init_copy() assert tracker.sender_id == tracker_copy.sender_id
def test_session_start_is_not_serialised(default_domain: Domain): tracker = DialogueStateTracker("default", default_domain.slots) # the retrieved tracker should be empty assert len(tracker.events) == 0 # add SlotSet event tracker.update(SlotSet("slot", "value")) # add the two SessionStarted events and a user event tracker.update(ActionExecuted(ACTION_SESSION_START_NAME)) tracker.update(SessionStarted()) tracker.update( UserUttered("say something", intent={INTENT_NAME_KEY: "some_intent"}) ) tracker.update(DefinePrevUserUtteredFeaturization(False)) YAMLStoryWriter().dumps( Story.from_events(tracker.events, "some-story01").story_steps ) expected = """version: "2.0" stories: - story: some-story01 steps: - slot_was_set: - slot: value - intent: some_intent """ actual = YAMLStoryWriter().dumps( Story.from_events(tracker.events, "some-story01").story_steps ) assert actual == expected
def test_restart_event(domain: Domain): tracker = DialogueStateTracker("default", domain.slots) # the retrieved tracker should be empty assert len(tracker.events) == 0 intent = {"name": "greet", PREDICTED_CONFIDENCE_KEY: 1.0} tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) tracker.update(UserUttered("/greet", intent, [])) tracker.update(ActionExecuted("my_action")) tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) assert len(tracker.events) == 4 assert tracker.latest_message.text == "/greet" assert len(list(tracker.generate_all_prior_trackers())) == 4 tracker.update(Restarted()) assert len(tracker.events) == 5 assert tracker.followup_action == ACTION_SESSION_START_NAME tracker.update(SessionStarted()) assert tracker.followup_action == ACTION_LISTEN_NAME assert tracker.latest_message.text is None assert len(list(tracker.generate_all_prior_trackers())) == 1 dialogue = tracker.as_dialogue() recovered = DialogueStateTracker("default", domain.slots) recovered.recreate_from_dialogue(dialogue) assert recovered.current_state() == tracker.current_state() assert len(recovered.events) == 6 assert recovered.latest_message.text is None assert len(list(recovered.generate_all_prior_trackers())) == 1
def test_revert_user_utterance_event(domain: Domain): tracker = DialogueStateTracker("default", domain.slots) # the retrieved tracker should be empty assert len(tracker.events) == 0 intent1 = {"name": "greet", PREDICTED_CONFIDENCE_KEY: 1.0} tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) tracker.update(UserUttered("/greet", intent1, [])) tracker.update(ActionExecuted("my_action_1")) tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) intent2 = {"name": "goodbye", PREDICTED_CONFIDENCE_KEY: 1.0} tracker.update(UserUttered("/goodbye", intent2, [])) tracker.update(ActionExecuted("my_action_2")) tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) # Expecting count of 6: # +5 executed actions # +1 final state assert tracker.latest_action.get(ACTION_NAME) == ACTION_LISTEN_NAME assert len(list(tracker.generate_all_prior_trackers())) == 6 tracker.update(UserUtteranceReverted()) # Expecting count of 3: # +5 executed actions # +1 final state # -2 rewound actions associated with the /goodbye # -1 rewound action from the listen right before /goodbye assert tracker.latest_action.get(ACTION_NAME) == "my_action_1" assert len(list(tracker.generate_all_prior_trackers())) == 3 dialogue = tracker.as_dialogue() recovered = DialogueStateTracker("default", domain.slots) recovered.recreate_from_dialogue(dialogue) assert recovered.current_state() == tracker.current_state() assert tracker.latest_action.get(ACTION_NAME) == "my_action_1" assert len(list(tracker.generate_all_prior_trackers())) == 3
def default_tracker(domain: Domain) -> DialogueStateTracker: return DialogueStateTracker("my-sender", domain.slots)
async def _predict_tracker_actions( tracker: DialogueStateTracker, agent: "Agent", fail_on_prediction_errors: bool = False, use_e2e: bool = False, ) -> Tuple[EvaluationStore, DialogueStateTracker, List[Dict[Text, Any]]]: processor = agent.create_processor() tracker_eval_store = EvaluationStore() events = list(tracker.events) partial_tracker = DialogueStateTracker.from_events( tracker.sender_id, events[:1], agent.domain.slots, sender_source=tracker.sender_source, ) tracker_actions = [] should_predict_another_action = True num_predicted_actions = 0 for event in events[1:]: if isinstance(event, ActionExecuted): circuit_breaker_tripped = processor.is_action_limit_reached( num_predicted_actions, should_predict_another_action) (action_executed_result, prediction) = _collect_action_executed_predictions( processor, partial_tracker, event, fail_on_prediction_errors, circuit_breaker_tripped, ) tracker_eval_store.merge_store(action_executed_result) tracker_actions.append({ "action": action_executed_result.action_targets[0], "predicted": action_executed_result.action_predictions[0], "policy": prediction.policy_name, "confidence": prediction.max_confidence, }) should_predict_another_action = processor.should_predict_another_action( action_executed_result.action_predictions[0]) num_predicted_actions += 1 elif use_e2e and isinstance(event, UserUttered): # This means that user utterance didn't have a user message, only intent, # so we can skip the NLU part and take the parse data directly. # Indirectly that means that the test story was in YAML format. if not event.text: predicted = event.parse_data # Indirectly that means that the test story was either: # in YAML format containing a user message, or in Markdown format. # Leaving that as it is because Markdown is in legacy mode. else: predicted = await processor.parse_message( UserMessage(event.text)) user_uttered_result = _collect_user_uttered_predictions( event, predicted, partial_tracker, fail_on_prediction_errors) tracker_eval_store.merge_store(user_uttered_result) else: partial_tracker.update(event) if isinstance(event, UserUttered): num_predicted_actions = 0 return tracker_eval_store, partial_tracker, tracker_actions
async def test_form_unhappy_path_no_validation_from_rule(): form_name = "some_form" handle_rejection_action_name = "utter_handle_rejection" domain = Domain.from_yaml(f""" intents: - {GREET_INTENT_NAME} actions: - {UTTER_GREET_ACTION} - {handle_rejection_action_name} - some-action slots: {REQUESTED_SLOT}: type: unfeaturized forms: - {form_name} """) unhappy_rule = TrackerWithCachedStates.from_events( "bla", domain=domain, slots=domain.slots, evts=[ # We are in an active form ActiveLoop(form_name), SlotSet(REQUESTED_SLOT, "bla"), ActionExecuted(RULE_SNIPPET_ACTION_NAME), ActionExecuted(ACTION_LISTEN_NAME), # When a user says "hi", and the form is unhappy, # we want to run a specific action UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(handle_rejection_action_name), # Next user utterance is an answer to the previous question # and shouldn't be validated by the form ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(form_name), ActionExecuted(ACTION_LISTEN_NAME), ], is_rule_tracker=True, ) policy = RulePolicy() # RulePolicy should memorize that unhappy_rule overrides GREET_RULE policy.train([GREET_RULE, unhappy_rule], domain, RegexInterpreter()) # Check that RulePolicy predicts action to handle unhappy path conversation_events = [ ActionExecuted(form_name), ActiveLoop(form_name), SlotSet(REQUESTED_SLOT, "some value"), ActionExecuted(ACTION_LISTEN_NAME), UserUttered("haha", {"name": GREET_INTENT_NAME}), ActionExecutionRejected(form_name), ] action_probabilities = policy.predict_action_probabilities( DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots), domain, RegexInterpreter(), ) assert_predicted_action(action_probabilities, domain, handle_rejection_action_name) # Check that RulePolicy predicts action_listen conversation_events.append(ActionExecuted(handle_rejection_action_name)) action_probabilities = policy.predict_action_probabilities( DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots), domain, RegexInterpreter(), ) assert_predicted_action(action_probabilities, domain, ACTION_LISTEN_NAME) # Check that RulePolicy triggers form again after handling unhappy path conversation_events.append(ActionExecuted(ACTION_LISTEN_NAME)) tracker = DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots) action_probabilities = policy.predict_action_probabilities( tracker, domain, RegexInterpreter()) assert_predicted_action(action_probabilities, domain, form_name) # check that RulePolicy added FormValidation False event based on the training rule assert tracker.events[-1] == FormValidation(False)
async def test_form_unhappy_path_no_validation_from_story(): form_name = "some_form" handle_rejection_action_name = "utter_handle_rejection" domain = Domain.from_yaml(f""" intents: - {GREET_INTENT_NAME} actions: - {UTTER_GREET_ACTION} - {handle_rejection_action_name} - some-action slots: {REQUESTED_SLOT}: type: unfeaturized forms: - {form_name} """) unhappy_story = TrackerWithCachedStates.from_events( "bla", domain=domain, slots=domain.slots, evts=[ # We are in an active form ActionExecuted(form_name), ActiveLoop(form_name), # When a user says "hi", and the form is unhappy, # we want to run a specific action UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(handle_rejection_action_name), ActionExecuted(ACTION_LISTEN_NAME), # Next user utterance is an answer to the previous question # and shouldn't be validated by the form UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(form_name), ActionExecuted(ACTION_LISTEN_NAME), ], ) policy = RulePolicy() policy.train([unhappy_story], domain, RegexInterpreter()) # Check that RulePolicy predicts no validation to handle unhappy path conversation_events = [ ActionExecuted(form_name), ActiveLoop(form_name), SlotSet(REQUESTED_SLOT, "some value"), ActionExecuted(ACTION_LISTEN_NAME), UserUttered("haha", {"name": GREET_INTENT_NAME}), ActionExecutionRejected(form_name), ActionExecuted(handle_rejection_action_name), ActionExecuted(ACTION_LISTEN_NAME), UserUttered("haha", {"name": GREET_INTENT_NAME}), ] tracker = DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots) action_probabilities = policy.predict_action_probabilities( tracker, domain, RegexInterpreter()) # there is no rule for next action assert max(action_probabilities) == policy._core_fallback_threshold # check that RulePolicy added FormValidation False event based on the training story assert tracker.events[-1] == FormValidation(False)
FormValidation, ) from rasa.shared.nlu.interpreter import RegexInterpreter from rasa.core.nlg import TemplatedNaturalLanguageGenerator from rasa.core.policies.rule_policy import RulePolicy from rasa.shared.core.trackers import DialogueStateTracker from rasa.shared.core.generator import TrackerWithCachedStates UTTER_GREET_ACTION = "utter_greet" GREET_INTENT_NAME = "greet" GREET_RULE = DialogueStateTracker.from_events( "bla", evts=[ ActionExecuted(RULE_SNIPPET_ACTION_NAME), ActionExecuted(ACTION_LISTEN_NAME), # Greet is a FAQ here and gets triggered in any context UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(UTTER_GREET_ACTION), ActionExecuted(ACTION_LISTEN_NAME), ], ) GREET_RULE.is_rule_tracker = True def _form_submit_rule(domain: Domain, submit_action_name: Text, form_name: Text) -> DialogueStateTracker: return TrackerWithCachedStates.from_events( "bla", domain=domain, slots=domain.slots, evts=[
async def test_form_unhappy_path_from_story(): form_name = "some_form" handle_rejection_action_name = "utter_handle_rejection" domain = Domain.from_yaml(f""" intents: - {GREET_INTENT_NAME} actions: - {UTTER_GREET_ACTION} - {handle_rejection_action_name} - some-action slots: {REQUESTED_SLOT}: type: unfeaturized forms: - {form_name} """) unhappy_story = TrackerWithCachedStates.from_events( "bla", domain=domain, slots=domain.slots, evts=[ # We are in an active form ActionExecuted(form_name), ActiveLoop(form_name), UserUttered("haha", {"name": GREET_INTENT_NAME}), ActionExecuted(UTTER_GREET_ACTION), # After our bot says "hi", we want to run a specific action ActionExecuted(handle_rejection_action_name), ActionExecuted(form_name), ActionExecuted(ACTION_LISTEN_NAME), ], ) policy = RulePolicy() policy.train([GREET_RULE, unhappy_story], domain, RegexInterpreter()) # Check that RulePolicy predicts action to handle unhappy path conversation_events = [ ActionExecuted(form_name), ActiveLoop(form_name), SlotSet(REQUESTED_SLOT, "some value"), ActionExecuted(ACTION_LISTEN_NAME), UserUttered("haha", {"name": GREET_INTENT_NAME}), ActionExecutionRejected(form_name), ] action_probabilities = policy.predict_action_probabilities( DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots), domain, RegexInterpreter(), ) assert_predicted_action(action_probabilities, domain, UTTER_GREET_ACTION) # Check that RulePolicy doesn't trigger form or action_listen # after handling unhappy path conversation_events.append(ActionExecuted(handle_rejection_action_name)) action_probabilities = policy.predict_action_probabilities( DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots), domain, RegexInterpreter(), ) assert max(action_probabilities) == policy._core_fallback_threshold
def template_sender_tracker(domain_path: Text): domain = Domain.load(domain_path) return DialogueStateTracker("template-sender", domain.slots)
def tracker(self, domain_with_mapping: Domain) -> DialogueStateTracker: return DialogueStateTracker(DEFAULT_SENDER_ID, domain_with_mapping.slots)
def tracker(self, default_domain: Domain) -> DialogueStateTracker: return DialogueStateTracker(DEFAULT_SENDER_ID, default_domain.slots)
def template_sender_tracker(default_domain): return DialogueStateTracker("template-sender", default_domain.slots)
async def test_one_stage_fallback_rule(): domain = Domain.from_yaml(f""" intents: - {GREET_INTENT_NAME} - {DEFAULT_NLU_FALLBACK_INTENT_NAME} actions: - {UTTER_GREET_ACTION} """) fallback_recover_rule = TrackerWithCachedStates.from_events( "bla", domain=domain, slots=domain.slots, evts=[ ActionExecuted(RULE_SNIPPET_ACTION_NAME), ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": DEFAULT_NLU_FALLBACK_INTENT_NAME}), ActionExecuted(ACTION_DEFAULT_FALLBACK_NAME), ActionExecuted(ACTION_LISTEN_NAME), ], is_rule_tracker=True, ) greet_rule_which_only_applies_at_start = TrackerWithCachedStates.from_events( "bla", domain=domain, evts=[ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(UTTER_GREET_ACTION), ActionExecuted(ACTION_LISTEN_NAME), ], is_rule_tracker=True, ) policy = RulePolicy() policy.train( [greet_rule_which_only_applies_at_start, fallback_recover_rule], domain, RegexInterpreter(), ) # RulePolicy predicts fallback action conversation_events = [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered("dasdakl;fkasd", {"name": DEFAULT_NLU_FALLBACK_INTENT_NAME}), ] tracker = DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots) action_probabilities = policy.predict_action_probabilities( tracker, domain, RegexInterpreter()) assert_predicted_action(action_probabilities, domain, ACTION_DEFAULT_FALLBACK_NAME) # Fallback action reverts fallback events, next action is `ACTION_LISTEN` conversation_events += await ActionDefaultFallback().run( CollectingOutputChannel(), TemplatedNaturalLanguageGenerator(domain.templates), tracker, domain, ) # Rasa is back on track when user rephrased intent conversation_events += [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered("haha", {"name": GREET_INTENT_NAME}), ] tracker = DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots) action_probabilities = policy.predict_action_probabilities( tracker, domain, RegexInterpreter()) assert_predicted_action(action_probabilities, domain, UTTER_GREET_ACTION)
async def _get_tracker(conversation_id: Text) -> DialogueStateTracker: return DialogueStateTracker.from_events( conversation_id, events_for_conversation_id[conversation_id])
def predict_action_probabilities( self, tracker: DialogueStateTracker, domain: Domain, interpreter: NaturalLanguageInterpreter, **kwargs: Any, ) -> PolicyPrediction: """Predicts the next action the bot should take after seeing the tracker. Args: tracker: Tracker containing past conversation events. domain: Domain of the assistant. interpreter: Interpreter which may be used by the policies to create additional features. Returns: The policy's prediction (e.g. the probabilities for the actions). """ if self.model is None: return self._prediction(self._default_predictions(domain)) # Prediction through the policy is skipped if: # 1. If the tracker does not contain any event of type `UserUttered` # till now. # 2. There is at least one event of type `ActionExecuted` # after the last `UserUttered` event. if self._should_skip_prediction(tracker): logger.debug( f"Skipping predictions for {self.__class__.__name__} " f"as either there is no event of type `UserUttered` or " f"there is an event of type `ActionExecuted` after " f"the last `UserUttered`.") return self._prediction(self._default_predictions(domain)) # create model data from tracker tracker_state_features = self._featurize_for_prediction( tracker, domain, interpreter) model_data = self._create_model_data(tracker_state_features) output = self.model.run_inference(model_data) # take the last prediction in the sequence all_similarities: np.ndarray = output["similarities"] sequence_similarities = all_similarities[:, -1, :] # Check for unlikely intent query_intent = tracker.get_last_event_for(UserUttered).intent_name is_unlikely_intent = self._check_unlikely_intent( domain, sequence_similarities, query_intent) confidences = list(np.zeros(domain.num_actions)) if is_unlikely_intent: confidences[domain.index_for_action( ACTION_UNLIKELY_INTENT_NAME)] = 1.0 return self._prediction( confidences, action_metadata=self._collect_action_metadata( domain, sequence_similarities, query_intent), )
"path, expected_format", [ ("bla.json", RASA), ("other.md", MARKDOWN), ("other.yml", RASA_YAML), ("unknown", UNK), ], ) def test_get_nlu_target_format(path: Text, expected_format: Text): assert interactive._get_nlu_target_format(path) == expected_format @pytest.mark.parametrize( "trackers, expected_trackers", [ ([DialogueStateTracker.from_events("one", []) ], [deque([]), DEFAULT_SENDER_ID]), ( [ str(i) for i in range(interactive. MAX_NUMBER_OF_TRAINING_STORIES_FOR_VISUALIZATION + 1) ], [DEFAULT_SENDER_ID], ), ], ) async def test_initial_plotting_call( mock_endpoint: EndpointConfig, monkeypatch: MonkeyPatch, trackers: List[Text],
async def generate( self, template_name: Text, tracker: DialogueStateTracker, output_channel: Text, **kwargs: Any, ) -> List[Dict[Text, Any]]: fallback_language_slot = tracker.slots.get("fallback_language") fallback_language = (fallback_language_slot.initial_value if fallback_language_slot else None) language = tracker.latest_message.metadata.get( "language") or fallback_language body = nlg_request_format( template_name, tracker, output_channel, **kwargs, language=language, projectId=os.environ.get("BF_PROJECT_ID"), ) logger.debug("Requesting NLG for {} from {}." "".format(template_name, self.nlg_endpoint.url)) try: if "graphql" in self.nlg_endpoint.url: from sgqlc.endpoint.http import HTTPEndpoint logging.getLogger("sgqlc.endpoint.http").setLevel( logging.WARNING) api_key = os.environ.get("API_KEY") headers = [{"Authorization": api_key}] if api_key else [] response = HTTPEndpoint(self.nlg_endpoint.url, *headers)(NLG_QUERY, body) if response.get("errors"): raise urllib.error.URLError(", ".join( [e.get("message") for e in response.get("errors")])) response = response.get("data", {}).get("getResponse", {}) rewrite_url(response, self.url_substitution_patterns) if "customText" in response: response["text"] = response.pop("customText") if "customImage" in response: response["image"] = response.pop("customImage") if "customQuickReplies" in response: response["quick_replies"] = response.pop( "customQuickReplies") if "customButtons" in response: response["buttons"] = response.pop("customButtons") if "customElements" in response: response["elements"] = response.pop("customElements") if "customAttachment" in response: response["attachment"] = response.pop("customAttachment") metadata = response.pop("metadata", {}) or {} for key in metadata: response[key] = metadata[key] keys_to_interpolate = [ "text", "image", "custom", "buttons", "attachment", "quick_replies", ] for key in keys_to_interpolate: if key in response: response[key] = interpolate( response[key], tracker.current_slot_values()) else: response = await self.nlg_endpoint.request( method="post", json=body, timeout=DEFAULT_REQUEST_TIMEOUT) response = response[ 0] # legacy route, use first message in seq except urllib.error.URLError as e: message = e.reason logger.error( f"NLG web endpoint at {self.nlg_endpoint.url} returned errors: {message}" ) return {"text": template_name} if self.validate_response(response): return response else: logger.error( f"NLG web endpoint at {self.nlg_endpoint.url} returned an invalid response." ) return {"text": template_name}
def test_autofill_slots_for_policy_entities(): policy_entity, policy_entity_value = "policy_entity", "end-to-end" nlu_entity, nlu_entity_value = "nlu_entity", "nlu rocks" domain = Domain.from_yaml( textwrap.dedent(f""" entities: - {nlu_entity} - {policy_entity} slots: {nlu_entity}: type: text {policy_entity}: type: text """)) tracker = DialogueStateTracker.from_events( "some sender", evts=[ ActionExecuted(ACTION_LISTEN_NAME), UserUttered( "hi", intent={"name": "greet"}, entities=[{ "entity": nlu_entity, "value": nlu_entity_value }], ), DefinePrevUserUtteredFeaturization(True), EntitiesAdded(entities=[ { "entity": policy_entity, "value": policy_entity_value }, { "entity": nlu_entity, "value": nlu_entity_value }, ]), ], domain=domain, slots=domain.slots, ) # Slots are correctly set assert tracker.slots[nlu_entity].value == nlu_entity_value assert tracker.slots[policy_entity].value == policy_entity_value expected_events = [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered( "hi", intent={"name": "greet"}, entities=[ { "entity": nlu_entity, "value": nlu_entity_value }, # Added by `DefinePrevUserUtteredEntities` { "entity": policy_entity, "value": policy_entity_value }, ], ), # SlotSet event added for entity predicted by NLU SlotSet(nlu_entity, nlu_entity_value), DefinePrevUserUtteredFeaturization(True), EntitiesAdded(entities=[ { "entity": policy_entity, "value": policy_entity_value }, { "entity": nlu_entity, "value": nlu_entity_value }, ]), # SlotSet event added for entity predicted by policies # This event is somewhat duplicate. We don't deduplicate as this is a true # reflection of the given events and it doesn't change the actual state. SlotSet(nlu_entity, nlu_entity_value), SlotSet(policy_entity, policy_entity_value), ] for actual, expected in zip(tracker.events, expected_events): assert actual == expected
async def test_request_correct_slots_after_unhappy_path_with_custom_required_slots( ): form_name = "some_form" slot_name_1 = "slot_1" slot_name_2 = "slot_2" domain = f""" slots: {slot_name_1}: type: any {slot_name_2}: type: any forms: {form_name}: {slot_name_1}: - type: from_intent intent: some_intent value: some_value {slot_name_2}: - type: from_intent intent: some_intent value: some_value actions: - validate_{form_name} """ domain = Domain.from_yaml(domain) tracker = DialogueStateTracker.from_events( "default", [ ActiveLoop(form_name), SlotSet(REQUESTED_SLOT, "slot_2"), ActionExecuted(ACTION_LISTEN_NAME), UserUttered( "hello", intent={ "name": "greet", "confidence": 1.0 }, ), ActionExecutionRejected(form_name), ActionExecuted("utter_greet"), ], ) action_server_url = "http://my-action-server:5055/webhook" # Custom form validation action changes the order of the requested slots validate_return_events = [ { "event": "slot", "name": REQUESTED_SLOT, "value": slot_name_2 }, ] # The form should ask the same slot again when coming back after unhappy path expected_events = [SlotSet(REQUESTED_SLOT, slot_name_2)] with aioresponses() as mocked: mocked.post(action_server_url, payload={"events": validate_return_events}) action_server = EndpointConfig(action_server_url) action = FormAction(form_name, action_server) events = await action.run( CollectingOutputChannel(), TemplatedNaturalLanguageGenerator(domain.templates), tracker, domain, ) assert events == expected_events
def test_revert_action_event(domain: Domain): tracker = DialogueStateTracker("default", domain.slots) # the retrieved tracker should be empty assert len(tracker.events) == 0 intent = {"name": "greet", PREDICTED_CONFIDENCE_KEY: 1.0} tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) tracker.update(UserUttered("/greet", intent, [])) tracker.update(ActionExecuted("my_action")) tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) # Expecting count of 4: # +3 executed actions # +1 final state assert tracker.latest_action.get(ACTION_NAME) == ACTION_LISTEN_NAME assert len(list(tracker.generate_all_prior_trackers())) == 4 tracker.update(ActionReverted()) # Expecting count of 3: # +3 executed actions # +1 final state # -1 reverted action assert tracker.latest_action.get(ACTION_NAME) == "my_action" assert len(list(tracker.generate_all_prior_trackers())) == 3 dialogue = tracker.as_dialogue() recovered = DialogueStateTracker("default", domain.slots) recovered.recreate_from_dialogue(dialogue) assert recovered.current_state() == tracker.current_state() assert tracker.latest_action.get(ACTION_NAME) == "my_action" assert len(list(tracker.generate_all_prior_trackers())) == 3
async def test_switch_forms_with_same_slot(default_agent: Agent): """Tests switching of forms, where the first slot is the same in both forms. Tests the fix for issue 7710""" # Define two forms in the domain, with same first slot slot_a = "my_slot_a" form_1 = "my_form_1" utter_ask_form_1 = f"Please provide the value for {slot_a} of form 1" form_2 = "my_form_2" utter_ask_form_2 = f"Please provide the value for {slot_a} of form 2" domain = f""" version: "2.0" nlu: - intent: order_status examples: | - check status of my order - when are my shoes coming in - intent: return examples: | - start a return - I don't want my shoes anymore forms: {form_1}: {slot_a}: - type: from_entity entity: number {form_2}: {slot_a}: - type: from_entity entity: number responses: utter_ask_{form_1}_{slot_a}: - text: {utter_ask_form_1} utter_ask_{form_2}_{slot_a}: - text: {utter_ask_form_2} """ domain = Domain.from_yaml(domain) # Driving it like rasa/core/processor processor = MessageProcessor( default_agent.interpreter, default_agent.policy_ensemble, domain, InMemoryTrackerStore(domain), TemplatedNaturalLanguageGenerator(domain.templates), ) # activate the first form tracker = DialogueStateTracker.from_events( "some-sender", evts=[ ActionExecuted(ACTION_LISTEN_NAME), UserUttered("order status", { "name": "form_1", "confidence": 1.0 }), DefinePrevUserUtteredFeaturization(False), ], ) # rasa/core/processor.predict_next_action prediction = PolicyPrediction([], "some_policy") action_1 = FormAction(form_1, None) await processor._run_action( action_1, tracker, CollectingOutputChannel(), TemplatedNaturalLanguageGenerator(domain.templates), prediction, ) events_expected = [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered("order status", { "name": "form_1", "confidence": 1.0 }), DefinePrevUserUtteredFeaturization(False), ActionExecuted(form_1), ActiveLoop(form_1), SlotSet(REQUESTED_SLOT, slot_a), BotUttered( text=utter_ask_form_1, metadata={"template_name": f"utter_ask_{form_1}_{slot_a}"}, ), ] assert tracker.applied_events() == events_expected next_events = [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered("return my shoes", { "name": "form_2", "confidence": 1.0 }), DefinePrevUserUtteredFeaturization(False), ] tracker.update_with_events( next_events, domain, ) events_expected.extend(next_events) # form_1 is still active, and bot will first validate if the user utterance # provides valid data for the requested slot, which is rejected await processor._run_action( action_1, tracker, CollectingOutputChannel(), TemplatedNaturalLanguageGenerator(domain.templates), prediction, ) events_expected.extend([ActionExecutionRejected(action_name=form_1)]) assert tracker.applied_events() == events_expected # Next, bot predicts form_2 action_2 = FormAction(form_2, None) await processor._run_action( action_2, tracker, CollectingOutputChannel(), TemplatedNaturalLanguageGenerator(domain.templates), prediction, ) events_expected.extend([ ActionExecuted(form_2), ActiveLoop(form_2), SlotSet(REQUESTED_SLOT, slot_a), BotUttered( text=utter_ask_form_2, metadata={"template_name": f"utter_ask_{form_2}_{slot_a}"}, ), ]) assert tracker.applied_events() == events_expected
def test_traveling_back_in_time(domain: Domain): tracker = DialogueStateTracker("default", domain.slots) # the retrieved tracker should be empty assert len(tracker.events) == 0 intent = {"name": "greet", PREDICTED_CONFIDENCE_KEY: 1.0} tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) tracker.update(UserUttered("/greet", intent, [])) time.sleep(1) time_for_timemachine = time.time() time.sleep(1) tracker.update(ActionExecuted("my_action")) tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) # Expecting count of 4: # +3 executed actions # +1 final state assert tracker.latest_action.get(ACTION_NAME) == ACTION_LISTEN_NAME assert len(tracker.events) == 4 assert len(list(tracker.generate_all_prior_trackers())) == 4 tracker = tracker.travel_back_in_time(time_for_timemachine) # Expecting count of 2: # +1 executed actions # +1 final state assert tracker.latest_action.get(ACTION_NAME) == ACTION_LISTEN_NAME assert len(tracker.events) == 2 assert len(list(tracker.generate_all_prior_trackers())) == 2
def _should_handle_message(tracker: DialogueStateTracker): return ( not tracker.is_paused() or tracker.latest_message.intent.get(INTENT_NAME_KEY) == USER_INTENT_RESTART )
def test_applied_events_with_loop_happy_path( events: List[Event], expected_applied_events: List[Event]): tracker = DialogueStateTracker.from_events("👋", events) applied = tracker.applied_events() assert applied == expected_applied_events
def _find_action_from_rules(self, tracker: DialogueStateTracker, domain: Domain) -> Optional[Text]: tracker_as_states = self.featurizer.prediction_states([tracker], domain) states = tracker_as_states[0] logger.debug(f"Current tracker state: {states}") rule_keys = self._get_possible_keys(self.lookup[RULES], states) predicted_action_name = None best_rule_key = "" if rule_keys: # if there are several rules, # it should mean that some rule is a subset of another rule # therefore we pick a rule of maximum length best_rule_key = max(rule_keys, key=len) predicted_action_name = self.lookup[RULES].get(best_rule_key) active_loop_name = tracker.active_loop_name if active_loop_name: # find rules for unhappy path of the loop loop_unhappy_keys = self._get_possible_keys( self.lookup[RULES_FOR_LOOP_UNHAPPY_PATH], states) # there could be several unhappy path conditions unhappy_path_conditions = [ self.lookup[RULES_FOR_LOOP_UNHAPPY_PATH].get(key) for key in loop_unhappy_keys ] # Check if a rule that predicted action_listen # was applied inside the loop. # Rules might not explicitly switch back to the loop. # Hence, we have to take care of that. predicted_listen_from_general_rule = ( predicted_action_name == ACTION_LISTEN_NAME and not get_active_loop_name( self._rule_key_to_state(best_rule_key)[-1])) if predicted_listen_from_general_rule: if DO_NOT_PREDICT_LOOP_ACTION not in unhappy_path_conditions: # negative rules don't contain a key that corresponds to # the fact that active_loop shouldn't be predicted logger.debug( f"Predicted loop '{active_loop_name}' by overwriting " f"'{ACTION_LISTEN_NAME}' predicted by general rule.") return active_loop_name # do not predict anything predicted_action_name = None if DO_NOT_VALIDATE_LOOP in unhappy_path_conditions: logger.debug("Added `FormValidation(False)` event.") tracker.update(FormValidation(False)) if predicted_action_name is not None: logger.debug( f"There is a rule for the next action '{predicted_action_name}'." ) else: logger.debug("There is no applicable rule.") return predicted_action_name
def _collect_action_executed_predictions( processor: "MessageProcessor", partial_tracker: DialogueStateTracker, event: ActionExecuted, fail_on_prediction_errors: bool, circuit_breaker_tripped: bool, ) -> Tuple[EvaluationStore, PolicyPrediction]: from rasa.core.policies.form_policy import FormPolicy action_executed_eval_store = EvaluationStore() gold_action_name = event.action_name gold_action_text = event.action_text gold = gold_action_name or gold_action_text if circuit_breaker_tripped: prediction = PolicyPrediction([], policy_name=None) predicted = "circuit breaker tripped" else: action, prediction = processor.predict_next_action(partial_tracker) predicted = action.name() if (prediction.policy_name and predicted != gold and _form_might_have_been_rejected( processor.domain, partial_tracker, predicted)): # Wrong action was predicted, # but it might be Ok if form action is rejected. emulate_loop_rejection(partial_tracker) # try again action, prediction = processor.predict_next_action(partial_tracker) # Even if the prediction is also wrong, we don't have to undo the emulation # of the action rejection as we know that the user explicitly specified # that something else than the form was supposed to run. predicted = action.name() action_executed_eval_store.add_to_store(action_predictions=[predicted], action_targets=[gold]) if action_executed_eval_store.has_prediction_target_mismatch(): partial_tracker.update( WronglyPredictedAction( gold_action_name, gold_action_text, predicted, prediction.policy_name, prediction.max_confidence, event.timestamp, )) if fail_on_prediction_errors: story_dump = YAMLStoryWriter().dumps( partial_tracker.as_story().story_steps) error_msg = (f"Model predicted a wrong action. Failed Story: " f"\n\n{story_dump}") if FormPolicy.__name__ in prediction.policy_name: error_msg += ("FormAction is not run during " "evaluation therefore it is impossible to know " "if validation failed or this story is wrong. " "If the story is correct, add it to the " "training stories and retrain.") raise WrongPredictionException(error_msg) else: partial_tracker.update( ActionExecuted( predicted, prediction.policy_name, prediction.max_confidence, event.timestamp, )) return action_executed_eval_store, prediction
def tracker_from_dialogue(dialogue: "Dialogue", domain: Domain) -> DialogueStateTracker: tracker = DialogueStateTracker(dialogue.name, domain.slots) tracker.recreate_from_dialogue(dialogue) return tracker