def test_no_incomplete_rules_due_to_slots_after_listen(): some_action = "some_action" some_slot = "some_slot" domain = Domain.from_yaml(f""" intents: - {GREET_INTENT_NAME} actions: - {some_action} entities: - {some_slot} slots: {some_slot}: type: text """) policy = RulePolicy() complete_rule = TrackerWithCachedStates.from_events( "complete_rule", domain=domain, slots=domain.slots, evts=[ ActionExecuted(RULE_SNIPPET_ACTION_NAME), ActionExecuted(ACTION_LISTEN_NAME), UserUttered( intent={"name": GREET_INTENT_NAME}, entities=[{ "entity": some_slot, "value": "bla" }], ), SlotSet(some_slot, "bla"), ActionExecuted(some_action), ActionExecuted(ACTION_LISTEN_NAME), ], is_rule_tracker=True, ) potentially_incomplete_rule = TrackerWithCachedStates.from_events( "potentially_incomplete_rule", domain=domain, slots=domain.slots, evts=[ ActionExecuted(RULE_SNIPPET_ACTION_NAME), ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(some_action), ActionExecuted(ACTION_LISTEN_NAME), ], is_rule_tracker=True, ) policy.train([complete_rule, potentially_incomplete_rule], domain, RegexInterpreter())
def test_train_with_e2e_data( default_model_storage: ModelStorage, default_execution_context: ExecutionContext, tracker_events: List[List[Event]], skip_training: bool, domain: Domain, ): policy = UnexpecTEDIntentPolicy( UnexpecTEDIntentPolicy.get_default_config(), default_model_storage, Resource("UnexpecTEDIntentPolicy"), default_execution_context, featurizer=IntentMaxHistoryTrackerFeaturizer( IntentTokenizerSingleStateFeaturizer() ), ) trackers_for_training = [ TrackerWithCachedStates.from_events( sender_id=f"{tracker_index}", evts=events, domain=domain ) for tracker_index, events in enumerate(tracker_events) ] if skip_training: with pytest.warns(UserWarning): policy.train(trackers_for_training, domain, precomputations=None) else: policy.train(trackers_for_training, domain, precomputations=None)
def test_contradicting_rules_and_stories(): utter_anti_greet_action = "utter_anti_greet" domain = Domain.from_yaml(f""" intents: - {GREET_INTENT_NAME} actions: - {UTTER_GREET_ACTION} - {utter_anti_greet_action} """) policy = RulePolicy() anti_greet_story = TrackerWithCachedStates.from_events( "anti greet story", domain=domain, slots=domain.slots, evts=[ ActionExecuted(RULE_SNIPPET_ACTION_NAME), ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(utter_anti_greet_action), ActionExecuted(ACTION_LISTEN_NAME), ], ) with pytest.raises(InvalidRule) as execinfo: policy.train([GREET_RULE, anti_greet_story], domain, RegexInterpreter()) assert all( name in execinfo.value.message for name in {utter_anti_greet_action, anti_greet_story.sender_id})
async def test_finetune_after_load(self, trained_policy: MemoizationPolicy, default_domain: Domain, tmp_path: Path): trained_policy.persist(tmp_path) loaded_policy = MemoizationPolicy.load(tmp_path, should_finetune=True) assert loaded_policy.finetune_mode new_story = TrackerWithCachedStates.from_events( "channel", domain=default_domain, slots=default_domain.slots, evts=[ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": "why"}), ActionExecuted("utter_channel"), ActionExecuted(ACTION_LISTEN_NAME), ], ) original_train_data = await train_trackers(default_domain, augmentation_factor=20) loaded_policy.train(original_train_data + [new_story], default_domain, RegexInterpreter()) # Get the hash of the tracker state of new story new_story_states, _ = loaded_policy.featurizer.training_states_and_actions( [new_story], default_domain) # Feature keys for each new state should be present in the lookup for states in new_story_states: state_key = loaded_policy._create_feature_key(states) assert state_key in loaded_policy.lookup
def test_finetune_after_load( self, trained_policy: MemoizationPolicy, resource: Resource, model_storage: ModelStorage, execution_context: ExecutionContext, default_domain: Domain, stories_path: Text, ): execution_context = dataclasses.replace(execution_context, is_finetuning=True) loaded_policy = MemoizationPolicy.load(trained_policy.config, model_storage, resource, execution_context) assert loaded_policy.finetune_mode new_story = TrackerWithCachedStates.from_events( "channel", domain=default_domain, slots=default_domain.slots, evts=[ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": "why"}), ActionExecuted("utter_channel"), ActionExecuted(ACTION_LISTEN_NAME), ], ) original_train_data = train_trackers(default_domain, stories_path, augmentation_factor=20) loaded_policy.train( original_train_data + [new_story], default_domain, ) # Get the hash of the tracker state of new story new_story_states, _ = loaded_policy.featurizer.training_states_and_labels( [new_story], default_domain) # Feature keys for each new state should be present in the lookup for states in new_story_states: state_key = loaded_policy._create_feature_key(states) assert state_key in loaded_policy.lookup
def _form_activation_rule( domain: Domain, form_name: Text, activation_intent_name: Text) -> TrackerWithCachedStates: return TrackerWithCachedStates.from_events( "form activation rule", domain=domain, slots=domain.slots, evts=[ ActionExecuted(RULE_SNIPPET_ACTION_NAME), # The intent `other_intent` activates the form ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": activation_intent_name}), ActionExecuted(form_name), ActiveLoop(form_name), ActionExecuted(ACTION_LISTEN_NAME), ], is_rule_tracker=True, )
def _form_submit_rule(domain: Domain, submit_action_name: Text, form_name: Text) -> TrackerWithCachedStates: return TrackerWithCachedStates.from_events( "form submit rule", domain=domain, slots=domain.slots, evts=[ ActiveLoop(form_name), # Any events in between ActionExecuted(RULE_SNIPPET_ACTION_NAME), # Form runs and deactivates itself ActionExecuted(form_name), ActiveLoop(None), SlotSet(REQUESTED_SLOT, None), ActionExecuted(submit_action_name), ActionExecuted(ACTION_LISTEN_NAME), ], is_rule_tracker=True, )
def test_filter_training_trackers( self, tracker_events_for_training: List[List[Event]], expected_trackers_with_events: List[List[Event]], domain: Domain, ): trackers_for_training = [ TrackerWithCachedStates.from_events(sender_id=f"{tracker_index}", evts=events, domain=domain) for tracker_index, events in enumerate(tracker_events_for_training) ] filtered_trackers = UnexpecTEDIntentPolicy._get_trackers_for_training( trackers_for_training) assert len(filtered_trackers) == len(expected_trackers_with_events) for collected_tracker, expected_tracker_events in zip( filtered_trackers, expected_trackers_with_events): collected_tracker_events = list(collected_tracker.events) assert collected_tracker_events == expected_tracker_events
async def test_form_unhappy_path_no_validation_from_story(): form_name = "some_form" handle_rejection_action_name = "utter_handle_rejection" domain = Domain.from_yaml(f""" intents: - {GREET_INTENT_NAME} actions: - {UTTER_GREET_ACTION} - {handle_rejection_action_name} - some-action slots: {REQUESTED_SLOT}: type: unfeaturized forms: - {form_name} """) unhappy_story = TrackerWithCachedStates.from_events( "bla", domain=domain, slots=domain.slots, evts=[ # We are in an active form ActionExecuted(form_name), ActiveLoop(form_name), ActionExecuted(ACTION_LISTEN_NAME), # When a user says "hi", and the form is unhappy, # we want to run a specific action UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(handle_rejection_action_name), ActionExecuted(ACTION_LISTEN_NAME), # Next user utterance is an answer to the previous question # and shouldn't be validated by the form UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(form_name), ActionExecuted(ACTION_LISTEN_NAME), ], ) policy = RulePolicy() policy.train([unhappy_story], domain, RegexInterpreter()) # Check that RulePolicy predicts no validation to handle unhappy path conversation_events = [ ActionExecuted(form_name), ActiveLoop(form_name), SlotSet(REQUESTED_SLOT, "some value"), ActionExecuted(ACTION_LISTEN_NAME), UserUttered("haha", {"name": GREET_INTENT_NAME}), ActionExecutionRejected(form_name), ActionExecuted(handle_rejection_action_name), ActionExecuted(ACTION_LISTEN_NAME), UserUttered("haha", {"name": GREET_INTENT_NAME}), ] tracker = DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots) action_probabilities = policy.predict_action_probabilities( tracker, domain, RegexInterpreter()) # there is no rule for next action assert max(action_probabilities) == policy._core_fallback_threshold # check that RulePolicy entered unhappy path based on the training story assert tracker.events[-1] == LoopInterrupted(True)
async def test_form_unhappy_path_from_story(): form_name = "some_form" handle_rejection_action_name = "utter_handle_rejection" domain = Domain.from_yaml(f""" intents: - {GREET_INTENT_NAME} actions: - {UTTER_GREET_ACTION} - {handle_rejection_action_name} - some-action slots: {REQUESTED_SLOT}: type: unfeaturized forms: - {form_name} """) unhappy_story = TrackerWithCachedStates.from_events( "bla", domain=domain, slots=domain.slots, evts=[ # We are in an active form ActionExecuted(form_name), ActiveLoop(form_name), ActionExecuted(ACTION_LISTEN_NAME), # in training stories there is either intent or text, never both UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(UTTER_GREET_ACTION), # After our bot says "hi", we want to run a specific action ActionExecuted(handle_rejection_action_name), ActionExecuted(form_name), ActionExecuted(ACTION_LISTEN_NAME), ], ) policy = RulePolicy() policy.train([GREET_RULE, unhappy_story], domain, RegexInterpreter()) # Check that RulePolicy predicts action to handle unhappy path conversation_events = [ ActionExecuted(form_name), ActiveLoop(form_name), SlotSet(REQUESTED_SLOT, "some value"), ActionExecuted(ACTION_LISTEN_NAME), UserUttered("haha", {"name": GREET_INTENT_NAME}), ActionExecutionRejected(form_name), ] action_probabilities = policy.predict_action_probabilities( DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots), domain, RegexInterpreter(), ) assert_predicted_action(action_probabilities, domain, UTTER_GREET_ACTION) # Check that RulePolicy doesn't trigger form or action_listen # after handling unhappy path conversation_events.append(ActionExecuted(handle_rejection_action_name)) action_probabilities = policy.predict_action_probabilities( DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots), domain, RegexInterpreter(), ) assert max(action_probabilities) == policy._core_fallback_threshold
async def test_form_unhappy_path_no_validation_from_rule(): form_name = "some_form" handle_rejection_action_name = "utter_handle_rejection" domain = Domain.from_yaml(f""" intents: - {GREET_INTENT_NAME} actions: - {UTTER_GREET_ACTION} - {handle_rejection_action_name} - some-action slots: {REQUESTED_SLOT}: type: unfeaturized forms: - {form_name} """) unhappy_rule = TrackerWithCachedStates.from_events( "bla", domain=domain, slots=domain.slots, evts=[ # We are in an active form ActiveLoop(form_name), SlotSet(REQUESTED_SLOT, "bla"), ActionExecuted(RULE_SNIPPET_ACTION_NAME), ActionExecuted(ACTION_LISTEN_NAME), # When a user says "hi", and the form is unhappy, # we want to run a specific action UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(handle_rejection_action_name), # Next user utterance is an answer to the previous question # and shouldn't be validated by the form ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(form_name), ActionExecuted(ACTION_LISTEN_NAME), ], is_rule_tracker=True, ) # unhappy rule is multi user turn rule, therefore remove restriction for policy policy = RulePolicy(restrict_rules=False) # RulePolicy should memorize that unhappy_rule overrides GREET_RULE policy.train([GREET_RULE, unhappy_rule], domain, RegexInterpreter()) # Check that RulePolicy predicts action to handle unhappy path conversation_events = [ ActionExecuted(form_name), ActiveLoop(form_name), SlotSet(REQUESTED_SLOT, "some value"), ActionExecuted(ACTION_LISTEN_NAME), UserUttered("haha", {"name": GREET_INTENT_NAME}), ActionExecutionRejected(form_name), ] action_probabilities = policy.predict_action_probabilities( DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots), domain, RegexInterpreter(), ) assert_predicted_action(action_probabilities, domain, handle_rejection_action_name) # Check that RulePolicy predicts action_listen conversation_events.append(ActionExecuted(handle_rejection_action_name)) action_probabilities = policy.predict_action_probabilities( DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots), domain, RegexInterpreter(), ) assert_predicted_action(action_probabilities, domain, ACTION_LISTEN_NAME) # Check that RulePolicy triggers form again after handling unhappy path conversation_events.append(ActionExecuted(ACTION_LISTEN_NAME)) tracker = DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots) action_probabilities = policy.predict_action_probabilities( tracker, domain, RegexInterpreter()) assert_predicted_action(action_probabilities, domain, form_name) # check that RulePolicy entered unhappy path based on the training story assert tracker.events[-1] == LoopInterrupted(True)
async def test_form_unhappy_path_from_in_form_rule(): form_name = "some_form" handle_rejection_action_name = "utter_handle_rejection" domain = Domain.from_yaml(f""" intents: - {GREET_INTENT_NAME} actions: - {UTTER_GREET_ACTION} - {handle_rejection_action_name} - some-action slots: {REQUESTED_SLOT}: type: unfeaturized forms: - {form_name} """) unhappy_rule = TrackerWithCachedStates.from_events( "bla", domain=domain, slots=domain.slots, evts=[ # We are in an active form ActiveLoop(form_name), SlotSet(REQUESTED_SLOT, "bla"), ActionExecuted(RULE_SNIPPET_ACTION_NAME), ActionExecuted(ACTION_LISTEN_NAME), # When a user says "hi", and the form is unhappy, # we want to run a specific action UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(handle_rejection_action_name), ActionExecuted(form_name), ActionExecuted(ACTION_LISTEN_NAME), ], is_rule_tracker=True, ) policy = RulePolicy() # RulePolicy should memorize that unhappy_rule overrides GREET_RULE policy.train([GREET_RULE, unhappy_rule], domain, RegexInterpreter()) # Check that RulePolicy predicts action to handle unhappy path conversation_events = [ ActionExecuted(form_name), ActiveLoop(form_name), SlotSet(REQUESTED_SLOT, "some value"), ActionExecuted(ACTION_LISTEN_NAME), UserUttered("haha", {"name": GREET_INTENT_NAME}), ActionExecutionRejected(form_name), ] action_probabilities = policy.predict_action_probabilities( DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots), domain, RegexInterpreter(), ) assert_predicted_action(action_probabilities, domain, handle_rejection_action_name) # Check that RulePolicy triggers form again after handling unhappy path conversation_events.append(ActionExecuted(handle_rejection_action_name)) action_probabilities = policy.predict_action_probabilities( DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots), domain, RegexInterpreter(), ) assert_predicted_action(action_probabilities, domain, form_name)
def test_prediction( self, max_history: Optional[int], model_storage: ModelStorage, resource: Resource, execution_context: ExecutionContext, ): policy = self.create_policy( featurizer=MaxHistoryTrackerFeaturizer(max_history=max_history), model_storage=model_storage, resource=resource, execution_context=execution_context, config={POLICY_MAX_HISTORY: max_history}, ) GREET_INTENT_NAME = "greet" UTTER_GREET_ACTION = "utter_greet" UTTER_BYE_ACTION = "utter_goodbye" domain = Domain.from_yaml(f""" intents: - {GREET_INTENT_NAME} actions: - {UTTER_GREET_ACTION} - {UTTER_BYE_ACTION} slots: slot_1: type: bool mappings: - type: from_text slot_2: type: bool mappings: - type: from_text slot_3: type: bool mappings: - type: from_text slot_4: type: bool mappings: - type: from_text """) events = [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(UTTER_GREET_ACTION), SlotSet("slot_1", True), ActionExecuted(UTTER_GREET_ACTION), SlotSet("slot_2", True), SlotSet("slot_3", True), ActionExecuted(UTTER_GREET_ACTION), ActionExecuted(UTTER_GREET_ACTION), ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(UTTER_GREET_ACTION), SlotSet("slot_4", True), ActionExecuted(UTTER_BYE_ACTION), ActionExecuted(ACTION_LISTEN_NAME), ] training_story = TrackerWithCachedStates.from_events( "training story", evts=events, domain=domain, slots=domain.slots) test_story = TrackerWithCachedStates.from_events("training story", events[:-2], domain=domain, slots=domain.slots) policy.train([training_story], domain) prediction = policy.predict_action_probabilities(test_story, domain) assert (domain.action_names_or_texts[prediction.probabilities.index( max(prediction.probabilities))] == UTTER_BYE_ACTION)
def test_augmented_prediction_across_max_history_actions( self, max_history: Optional[int], model_storage: ModelStorage, resource: Resource, execution_context: ExecutionContext, ): """Tests that the last user utterance is preserved in action states even when the utterance occurs prior to `max_history` actions in the past. """ policy = self.create_policy( featurizer=MaxHistoryTrackerFeaturizer(max_history=max_history), model_storage=model_storage, resource=resource, execution_context=execution_context, config={POLICY_MAX_HISTORY: max_history}, ) GREET_INTENT_NAME = "greet" UTTER_GREET_ACTION = "utter_greet" UTTER_ACTION_1 = "utter_1" UTTER_ACTION_2 = "utter_2" UTTER_ACTION_3 = "utter_3" UTTER_ACTION_4 = "utter_4" UTTER_ACTION_5 = "utter_5" UTTER_BYE_ACTION = "utter_goodbye" domain = Domain.from_yaml(f""" intents: - {GREET_INTENT_NAME} actions: - {UTTER_GREET_ACTION} - {UTTER_ACTION_1} - {UTTER_ACTION_2} - {UTTER_ACTION_3} - {UTTER_ACTION_4} - {UTTER_ACTION_5} - {UTTER_BYE_ACTION} """) training_story = TrackerWithCachedStates.from_events( "training story", [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(UTTER_ACTION_1), ActionExecuted(UTTER_ACTION_2), ActionExecuted(UTTER_ACTION_3), ActionExecuted(UTTER_ACTION_4), ActionExecuted(UTTER_ACTION_5), ActionExecuted(UTTER_BYE_ACTION), ActionExecuted(ACTION_LISTEN_NAME), ], domain=domain, slots=domain.slots, ) test_story = TrackerWithCachedStates.from_events( "test story", [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(UTTER_ACTION_1), ActionExecuted(UTTER_ACTION_2), ActionExecuted(UTTER_ACTION_3), ActionExecuted(UTTER_ACTION_4), ActionExecuted(UTTER_ACTION_5), # ActionExecuted(UTTER_BYE_ACTION), ], domain=domain, slots=domain.slots, ) policy.train([training_story], domain) prediction = policy.predict_action_probabilities(test_story, domain) assert (domain.action_names_or_texts[prediction.probabilities.index( max(prediction.probabilities))] == UTTER_BYE_ACTION)
def test_aug_pred_sensitive_to_intent_across_max_history_actions( self, max_history: Optional[int], model_storage: ModelStorage, resource: Resource, execution_context: ExecutionContext, ): """Tests that only the most recent user utterance propagates to state creation of following actions. """ policy = self.create_policy( featurizer=MaxHistoryTrackerFeaturizer(max_history=max_history), model_storage=model_storage, resource=resource, execution_context=execution_context, config={POLICY_MAX_HISTORY: max_history}, ) GREET_INTENT_NAME = "greet" GOODBYE_INTENT_NAME = "goodbye" UTTER_GREET_ACTION = "utter_greet" UTTER_ACTION_1 = "utter_1" UTTER_ACTION_2 = "utter_2" UTTER_ACTION_3 = "utter_3" UTTER_ACTION_4 = "utter_4" UTTER_ACTION_5 = "utter_5" UTTER_BYE_ACTION = "utter_goodbye" domain = Domain.from_yaml(f""" intents: - {GREET_INTENT_NAME} - {GOODBYE_INTENT_NAME} actions: - {UTTER_GREET_ACTION} - {UTTER_ACTION_1} - {UTTER_ACTION_2} - {UTTER_ACTION_3} - {UTTER_ACTION_4} - {UTTER_ACTION_5} - {UTTER_BYE_ACTION} """) training_story = TrackerWithCachedStates.from_events( "training story", [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(UTTER_ACTION_1), ActionExecuted(UTTER_ACTION_2), ActionExecuted(UTTER_ACTION_3), ActionExecuted(UTTER_ACTION_4), ActionExecuted(UTTER_ACTION_5), ActionExecuted(UTTER_BYE_ACTION), ActionExecuted(ACTION_LISTEN_NAME), ], domain=domain, slots=domain.slots, ) test_story1 = TrackerWithCachedStates.from_events( "test story", [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": GOODBYE_INTENT_NAME}), ActionExecuted(UTTER_BYE_ACTION), ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(UTTER_ACTION_1), ActionExecuted(UTTER_ACTION_2), ActionExecuted(UTTER_ACTION_3), ActionExecuted(UTTER_ACTION_4), ActionExecuted(UTTER_ACTION_5), # ActionExecuted(UTTER_BYE_ACTION), ], domain=domain, slots=domain.slots, ) policy.train([training_story], domain) prediction1 = policy.predict_action_probabilities(test_story1, domain) assert (domain.action_names_or_texts[prediction1.probabilities.index( max(prediction1.probabilities))] == UTTER_BYE_ACTION) test_story2_no_match_expected = TrackerWithCachedStates.from_events( "test story", [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(UTTER_BYE_ACTION), ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": GOODBYE_INTENT_NAME}), ActionExecuted(UTTER_ACTION_1), ActionExecuted(UTTER_ACTION_2), ActionExecuted(UTTER_ACTION_3), ActionExecuted(UTTER_ACTION_4), ActionExecuted(UTTER_ACTION_5), # No prediction should be made here. ], domain=domain, slots=domain.slots, ) prediction2 = policy.predict_action_probabilities( test_story2_no_match_expected, domain, ) assert all([prob == 0.0 for prob in prediction2.probabilities])
def test_aug_pred_without_intent( self, max_history: Optional[int], model_storage: ModelStorage, resource: Resource, execution_context: ExecutionContext, ): """Tests memoization works for a memoized state sequence that does not have a user utterance. """ policy = self.create_policy( featurizer=MaxHistoryTrackerFeaturizer(max_history=max_history), model_storage=model_storage, resource=resource, execution_context=execution_context, config={POLICY_MAX_HISTORY: max_history}, ) GREET_INTENT_NAME = "greet" GOODBYE_INTENT_NAME = "goodbye" UTTER_GREET_ACTION = "utter_greet" UTTER_ACTION_1 = "utter_1" UTTER_ACTION_2 = "utter_2" UTTER_ACTION_3 = "utter_3" UTTER_ACTION_4 = "utter_4" domain = Domain.from_yaml(f""" intents: - {GREET_INTENT_NAME} - {GOODBYE_INTENT_NAME} actions: - {UTTER_GREET_ACTION} - {UTTER_ACTION_1} - {UTTER_ACTION_2} - {UTTER_ACTION_3} - {UTTER_ACTION_4} """) training_story = TrackerWithCachedStates.from_events( "training story", [ ActionExecuted(UTTER_ACTION_3), ActionExecuted(UTTER_ACTION_4), ActionExecuted(ACTION_LISTEN_NAME), ], domain=domain, slots=domain.slots, ) policy.train([training_story], domain) test_story = TrackerWithCachedStates.from_events( "test story", [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(UTTER_ACTION_1), ActionExecuted(UTTER_ACTION_2), ActionExecuted(UTTER_ACTION_3), # ActionExecuted(UTTER_ACTION_4), ], domain=domain, slots=domain.slots, ) prediction = policy.predict_action_probabilities(test_story, domain) assert (domain.action_names_or_texts[prediction.probabilities.index( max(prediction.probabilities))] == UTTER_ACTION_4)
def test_augmented_prediction( self, max_history: Optional[int], model_storage: ModelStorage, resource: Resource, execution_context: ExecutionContext, ): policy = self.create_policy( featurizer=MaxHistoryTrackerFeaturizer(max_history=max_history), model_storage=model_storage, resource=resource, execution_context=execution_context, ) GREET_INTENT_NAME = "greet" UTTER_GREET_ACTION = "utter_greet" UTTER_BYE_ACTION = "utter_goodbye" domain = Domain.from_yaml(f""" intents: - {GREET_INTENT_NAME} actions: - {UTTER_GREET_ACTION} - {UTTER_BYE_ACTION} slots: slot_1: type: bool initial_value: true slot_2: type: bool slot_3: type: bool """) training_story = TrackerWithCachedStates.from_events( "training story", [ ActionExecuted(UTTER_GREET_ACTION), UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(UTTER_GREET_ACTION), SlotSet("slot_3", True), ActionExecuted(UTTER_BYE_ACTION), ], domain=domain, slots=domain.slots, ) test_story = TrackerWithCachedStates.from_events( "test story", [ UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(UTTER_GREET_ACTION), SlotSet("slot_1", False), ActionExecuted(UTTER_GREET_ACTION), ActionExecuted(UTTER_GREET_ACTION), UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(UTTER_GREET_ACTION), SlotSet("slot_2", True), ActionExecuted(UTTER_GREET_ACTION), UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(UTTER_GREET_ACTION), SlotSet("slot_3", True), # ActionExecuted(UTTER_BYE_ACTION), ], domain=domain, slots=domain.slots, ) policy.train([training_story], domain) prediction = policy.predict_action_probabilities(test_story, domain) assert (domain.action_names_or_texts[prediction.probabilities.index( max(prediction.probabilities))] == UTTER_BYE_ACTION)
async def test_one_stage_fallback_rule(): domain = Domain.from_yaml(f""" intents: - {GREET_INTENT_NAME} - {DEFAULT_NLU_FALLBACK_INTENT_NAME} actions: - {UTTER_GREET_ACTION} """) fallback_recover_rule = TrackerWithCachedStates.from_events( "bla", domain=domain, slots=domain.slots, evts=[ ActionExecuted(RULE_SNIPPET_ACTION_NAME), ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": DEFAULT_NLU_FALLBACK_INTENT_NAME}), ActionExecuted(ACTION_DEFAULT_FALLBACK_NAME), ActionExecuted(ACTION_LISTEN_NAME), ], is_rule_tracker=True, ) greet_rule_which_only_applies_at_start = TrackerWithCachedStates.from_events( "bla", domain=domain, evts=[ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(UTTER_GREET_ACTION), ActionExecuted(ACTION_LISTEN_NAME), ], is_rule_tracker=True, ) policy = RulePolicy() policy.train( [greet_rule_which_only_applies_at_start, fallback_recover_rule], domain, RegexInterpreter(), ) # RulePolicy predicts fallback action conversation_events = [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered("dasdakl;fkasd", {"name": DEFAULT_NLU_FALLBACK_INTENT_NAME}), ] tracker = DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots) action_probabilities = policy.predict_action_probabilities( tracker, domain, RegexInterpreter()) assert_predicted_action(action_probabilities, domain, ACTION_DEFAULT_FALLBACK_NAME) # Fallback action reverts fallback events, next action is `ACTION_LISTEN` conversation_events += await ActionDefaultFallback().run( CollectingOutputChannel(), TemplatedNaturalLanguageGenerator(domain.templates), tracker, domain, ) # Rasa is back on track when user rephrased intent conversation_events += [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered("haha", {"name": GREET_INTENT_NAME}), ] tracker = DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots) action_probabilities = policy.predict_action_probabilities( tracker, domain, RegexInterpreter()) assert_predicted_action(action_probabilities, domain, UTTER_GREET_ACTION)
def test_predict_next_action_with_hidden_rules(): rule_intent = "rule_intent" rule_action = "rule_action" story_intent = "story_intent" story_action = "story_action" rule_slot = "rule_slot" story_slot = "story_slot" domain = Domain.from_yaml(f""" version: "2.0" intents: - {rule_intent} - {story_intent} actions: - {rule_action} - {story_action} slots: {rule_slot}: type: text {story_slot}: type: text """) rule = TrackerWithCachedStates.from_events( "rule", domain=domain, slots=domain.slots, evts=[ ActionExecuted(RULE_SNIPPET_ACTION_NAME), ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": rule_intent}), ActionExecuted(rule_action), SlotSet(rule_slot, rule_slot), ActionExecuted(ACTION_LISTEN_NAME), ], is_rule_tracker=True, ) story = TrackerWithCachedStates.from_events( "story", domain=domain, slots=domain.slots, evts=[ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": story_intent}), ActionExecuted(story_action), SlotSet(story_slot, story_slot), ActionExecuted(ACTION_LISTEN_NAME), ], ) interpreter = RegexInterpreter() ensemble = SimplePolicyEnsemble( policies=[RulePolicy(), MemoizationPolicy()]) ensemble.train([rule, story], domain, interpreter) tracker_store = InMemoryTrackerStore(domain) lock_store = InMemoryLockStore() processor = MessageProcessor( interpreter, ensemble, domain, tracker_store, lock_store, TemplatedNaturalLanguageGenerator(domain.responses), ) tracker = DialogueStateTracker.from_events( "casd", evts=[ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": rule_intent}), ], slots=domain.slots, ) action, prediction = processor.predict_next_action(tracker) assert action._name == rule_action assert prediction.hide_rule_turn processor._log_action_on_tracker(tracker, action, [SlotSet(rule_slot, rule_slot)], prediction) action, prediction = processor.predict_next_action(tracker) assert isinstance(action, ActionListen) assert prediction.hide_rule_turn processor._log_action_on_tracker(tracker, action, None, prediction) tracker.events.append(UserUttered(intent={"name": story_intent})) # rules are hidden correctly if memo policy predicts next actions correctly action, prediction = processor.predict_next_action(tracker) assert action._name == story_action assert not prediction.hide_rule_turn processor._log_action_on_tracker(tracker, action, [SlotSet(story_slot, story_slot)], prediction) action, prediction = processor.predict_next_action(tracker) assert isinstance(action, ActionListen) assert not prediction.hide_rule_turn
def test_incomplete_rules_due_to_loops(): some_form = "some_form" domain = Domain.from_yaml(f""" intents: - {GREET_INTENT_NAME} forms: - {some_form} """) policy = RulePolicy() complete_rule = TrackerWithCachedStates.from_events( "complete_rule", domain=domain, slots=domain.slots, evts=[ ActionExecuted(RULE_SNIPPET_ACTION_NAME), ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(some_form), ActiveLoop(some_form), ActionExecuted(ACTION_LISTEN_NAME), ], is_rule_tracker=True, ) incomplete_rule = TrackerWithCachedStates.from_events( "incomplete_rule", domain=domain, slots=domain.slots, evts=[ ActionExecuted(RULE_SNIPPET_ACTION_NAME), ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(some_form), ActionExecuted(ACTION_LISTEN_NAME), ], is_rule_tracker=True, ) with pytest.raises(InvalidRule) as execinfo: policy.train([complete_rule, incomplete_rule], domain, RegexInterpreter()) assert all(name in execinfo.value.message for name in { some_form, incomplete_rule.sender_id, }) fixed_incomplete_rule = TrackerWithCachedStates.from_events( "fixed_incomplete_rule", domain=domain, slots=domain.slots, evts=[ ActionExecuted(RULE_SNIPPET_ACTION_NAME), ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(some_form), ActionExecuted(RULE_SNIPPET_ACTION_NAME), ActionExecuted(ACTION_LISTEN_NAME), ], is_rule_tracker=True, ) policy.train([complete_rule, fixed_incomplete_rule], domain, RegexInterpreter())