def test_prediction_applies_optional_policy_events(default_domain: Domain): optional_events = [ActionExecuted("my action")] must_have_events = [SlotSet("some slot", "some value")] ensemble = SimplePolicyEnsemble([ ConstantPolicy( priority=10, predict_index=1, events=must_have_events, optional_events=optional_events, ), ConstantPolicy(priority=1, predict_index=2), ]) tracker = DialogueStateTracker.from_events("test", evts=[]) prediction = ensemble.probabilities_using_best_policy( tracker, default_domain, RegexInterpreter()) # Policy 0 won due to higher prio assert prediction.policy_name == f"policy_0_{ConstantPolicy.__name__}" # Events of losing policy were applied nevertheless assert len( prediction.events) == len(optional_events) + len(must_have_events) assert all(event in prediction.events for event in optional_events) assert all(event in prediction.events for event in must_have_events)
def test_no_user_prediction_supersedes_others(default_domain: Domain): expected_action_index = 2 expected_confidence = 0.5 ensemble = SimplePolicyEnsemble([ ConstantPolicy(priority=100, predict_index=0), ConstantPolicy(priority=1, predict_index=1, is_end_to_end_prediction=True), ConstantPolicy( priority=1, predict_index=expected_action_index, confidence=expected_confidence, is_no_user_prediction=True, ), ]) tracker = DialogueStateTracker.from_events("test", evts=[]) prediction = ensemble.probabilities_using_best_policy( tracker, default_domain, RegexInterpreter()) assert prediction.max_confidence == expected_confidence assert prediction.max_confidence_index == expected_action_index assert prediction.policy_name == f"policy_2_{ConstantPolicy.__name__}" assert prediction.is_no_user_prediction assert not prediction.is_end_to_end_prediction
def test_mapping_wins_over_form(events: List[Event]): domain = """ forms: - test-form """ domain = Domain.from_yaml(domain) tracker = DialogueStateTracker.from_events("test", events, []) ensemble = SimplePolicyEnsemble( [ MappingPolicy(), ConstantPolicy(priority=1, predict_index=0), FormPolicy(), FallbackPolicy(), ] ) result, best_policy = ensemble.probabilities_using_best_policy( tracker, domain, RegexInterpreter() ) max_confidence_index = result.index(max(result)) next_action = domain.action_for_index(max_confidence_index, None) index_of_mapping_policy = 0 assert best_policy == f"policy_{index_of_mapping_policy}_{MappingPolicy.__name__}" assert next_action.name() == ACTION_RESTART_NAME
def test_fallback_mapping_restart(): domain = Domain.load("data/test_domains/default.yml") events = [ ActionExecuted(ACTION_DEFAULT_FALLBACK_NAME, timestamp=1), utilities.user_uttered(USER_INTENT_RESTART, 1, timestamp=2), ] tracker = DialogueStateTracker.from_events("test", events, []) two_stage_fallback_policy = TwoStageFallbackPolicy( priority=2, deny_suggestion_intent_name="deny" ) mapping_policy = MappingPolicy(priority=1) mapping_fallback_ensemble = SimplePolicyEnsemble( [two_stage_fallback_policy, mapping_policy] ) result, best_policy = mapping_fallback_ensemble.probabilities_using_best_policy( tracker, domain, RegexInterpreter() ) max_confidence_index = result.index(max(result)) index_of_mapping_policy = 1 next_action = domain.action_for_index(max_confidence_index, None) assert best_policy == f"policy_{index_of_mapping_policy}_{MappingPolicy.__name__}" assert next_action.name() == ACTION_RESTART_NAME
def test_with_float_returning_policy(default_domain: Domain): expected_index = 3 class OldPolicy(Policy): def predict_action_probabilities( self, tracker: DialogueStateTracker, domain: Domain, interpreter: NaturalLanguageInterpreter, **kwargs: Any, ) -> List[float]: prediction = [0.0] * default_domain.num_actions prediction[expected_index] = 3 return prediction ensemble = SimplePolicyEnsemble( [ConstantPolicy(priority=1, predict_index=1), OldPolicy(priority=1)]) tracker = DialogueStateTracker.from_events("test", evts=[]) with pytest.warns(FutureWarning): prediction = ensemble.probabilities_using_best_policy( tracker, default_domain, RegexInterpreter()) assert prediction.policy_name == f"policy_1_{OldPolicy.__name__}" assert prediction.max_confidence_index == expected_index
def test_form_wins_over_everything_else(ensemble: SimplePolicyEnsemble): form_name = "test-form" domain = f""" forms: - {form_name} """ domain = Domain.from_yaml(domain) events = [ ActiveLoop("test-form"), ActionExecuted(ACTION_LISTEN_NAME), utilities.user_uttered("test", 1), ] tracker = DialogueStateTracker.from_events("test", events, []) result, best_policy = ensemble.probabilities_using_best_policy( tracker, domain, RegexInterpreter() ) max_confidence_index = result.index(max(result)) next_action = rasa.core.actions.action.action_for_index( max_confidence_index, domain, None ) index_of_form_policy = 0 assert best_policy == f"policy_{index_of_form_policy}_{FormPolicy.__name__}" assert next_action.name() == form_name
def test_intent_prediction_does_not_apply_define_featurization_events( default_domain: Domain, ): ensemble = SimplePolicyEnsemble([ ConstantPolicy(priority=100, predict_index=0), ConstantPolicy(priority=1, predict_index=1, is_end_to_end_prediction=False), ]) # no events should be added if latest action is not action listen tracker = DialogueStateTracker.from_events("test", evts=[]) prediction = ensemble.probabilities_using_best_policy( tracker, default_domain, RegexInterpreter()) assert prediction.events == [] # DefinePrevUserUtteredFeaturization should be added after action listen tracker = DialogueStateTracker.from_events( "test", evts=[ActionExecuted(ACTION_LISTEN_NAME)]) prediction = ensemble.probabilities_using_best_policy( tracker, default_domain, RegexInterpreter()) assert prediction.events == [DefinePrevUserUtteredFeaturization(False)]
def test_policy_priority(): domain = Domain.load("data/test_domains/default.yml") tracker = DialogueStateTracker.from_events("test", [UserUttered("hi")], []) priority_1 = ConstantPolicy(priority=1, predict_index=0) priority_2 = ConstantPolicy(priority=2, predict_index=1) policy_ensemble_0 = SimplePolicyEnsemble([priority_1, priority_2]) policy_ensemble_1 = SimplePolicyEnsemble([priority_2, priority_1]) priority_2_result = priority_2.predict_action_probabilities( tracker, domain) i = 1 # index of priority_2 in ensemble_0 result, best_policy = policy_ensemble_0.probabilities_using_best_policy( tracker, domain) assert best_policy == "policy_{}_{}".format(i, type(priority_2).__name__) assert result.tolist() == priority_2_result i = 0 # index of priority_2 in ensemble_1 result, best_policy = policy_ensemble_1.probabilities_using_best_policy( tracker, domain) assert best_policy == "policy_{}_{}".format(i, type(priority_2).__name__) assert result.tolist() == priority_2_result
def test_prediction_applies_must_have_policy_events(default_domain: Domain): must_have_events = [ActionExecuted("my action")] ensemble = SimplePolicyEnsemble([ ConstantPolicy(priority=10, predict_index=1), ConstantPolicy(priority=1, predict_index=2, events=must_have_events), ]) tracker = DialogueStateTracker.from_events("test", evts=[]) prediction = ensemble.probabilities_using_best_policy( tracker, default_domain, RegexInterpreter()) # Policy 0 won due to higher prio assert prediction.policy_name == f"policy_0_{ConstantPolicy.__name__}" # Events of losing policy were applied nevertheless assert prediction.events == must_have_events
def test_fallback_wins_over_mapping(): domain = Domain.load("data/test_domains/default.yml") events = [ ActionExecuted(ACTION_LISTEN_NAME), # Low confidence should trigger fallback utilities.user_uttered(USER_INTENT_RESTART, 0.0001), ] tracker = DialogueStateTracker.from_events("test", events, []) ensemble = SimplePolicyEnsemble([FallbackPolicy(), MappingPolicy()]) result, best_policy = ensemble.probabilities_using_best_policy( tracker, domain) max_confidence_index = result.index(max(result)) index_of_fallback_policy = 0 next_action = domain.action_for_index(max_confidence_index, None) assert best_policy == f"policy_{index_of_fallback_policy}_{FallbackPolicy.__name__}" assert next_action.name() == ACTION_DEFAULT_FALLBACK_NAME