def test_prediction_applies_optional_policy_events(default_domain: Domain):
optional_events = [ActionExecuted("my action")]
must_have_events = [SlotSet("some slot", "some value")]
ensemble = SimplePolicyEnsemble([
ConstantPolicy(
priority=10,
predict_index=1,
events=must_have_events,
optional_events=optional_events,
),
ConstantPolicy(priority=1, predict_index=2),
])
tracker = DialogueStateTracker.from_events("test", evts=[])
prediction = ensemble.probabilities_using_best_policy(
tracker, default_domain, RegexInterpreter())
# Policy 0 won due to higher prio
assert prediction.policy_name == f"policy_0_{ConstantPolicy.__name__}"
# Events of losing policy were applied nevertheless
assert len(
prediction.events) == len(optional_events) + len(must_have_events)
assert all(event in prediction.events for event in optional_events)
assert all(event in prediction.events for event in must_have_events)
def test_no_user_prediction_supersedes_others(default_domain: Domain):
expected_action_index = 2
expected_confidence = 0.5
ensemble = SimplePolicyEnsemble([
ConstantPolicy(priority=100, predict_index=0),
ConstantPolicy(priority=1,
predict_index=1,
is_end_to_end_prediction=True),
ConstantPolicy(
priority=1,
predict_index=expected_action_index,
confidence=expected_confidence,
is_no_user_prediction=True,
),
])
tracker = DialogueStateTracker.from_events("test", evts=[])
prediction = ensemble.probabilities_using_best_policy(
tracker, default_domain, RegexInterpreter())
assert prediction.max_confidence == expected_confidence
assert prediction.max_confidence_index == expected_action_index
assert prediction.policy_name == f"policy_2_{ConstantPolicy.__name__}"
assert prediction.is_no_user_prediction
assert not prediction.is_end_to_end_prediction
def test_mapping_wins_over_form(events: List[Event]):
domain = """
forms:
- test-form
"""
domain = Domain.from_yaml(domain)
tracker = DialogueStateTracker.from_events("test", events, [])
ensemble = SimplePolicyEnsemble(
[
MappingPolicy(),
ConstantPolicy(priority=1, predict_index=0),
FormPolicy(),
FallbackPolicy(),
]
)
result, best_policy = ensemble.probabilities_using_best_policy(
tracker, domain, RegexInterpreter()
)
max_confidence_index = result.index(max(result))
next_action = domain.action_for_index(max_confidence_index, None)
index_of_mapping_policy = 0
assert best_policy == f"policy_{index_of_mapping_policy}_{MappingPolicy.__name__}"
assert next_action.name() == ACTION_RESTART_NAME
def test_fallback_mapping_restart():
domain = Domain.load("data/test_domains/default.yml")
events = [
ActionExecuted(ACTION_DEFAULT_FALLBACK_NAME, timestamp=1),
utilities.user_uttered(USER_INTENT_RESTART, 1, timestamp=2),
]
tracker = DialogueStateTracker.from_events("test", events, [])
two_stage_fallback_policy = TwoStageFallbackPolicy(
priority=2, deny_suggestion_intent_name="deny"
)
mapping_policy = MappingPolicy(priority=1)
mapping_fallback_ensemble = SimplePolicyEnsemble(
[two_stage_fallback_policy, mapping_policy]
)
result, best_policy = mapping_fallback_ensemble.probabilities_using_best_policy(
tracker, domain, RegexInterpreter()
)
max_confidence_index = result.index(max(result))
index_of_mapping_policy = 1
next_action = domain.action_for_index(max_confidence_index, None)
assert best_policy == f"policy_{index_of_mapping_policy}_{MappingPolicy.__name__}"
assert next_action.name() == ACTION_RESTART_NAME
def test_with_float_returning_policy(default_domain: Domain):
expected_index = 3
class OldPolicy(Policy):
def predict_action_probabilities(
self,
tracker: DialogueStateTracker,
domain: Domain,
interpreter: NaturalLanguageInterpreter,
**kwargs: Any,
) -> List[float]:
prediction = [0.0] * default_domain.num_actions
prediction[expected_index] = 3
return prediction
ensemble = SimplePolicyEnsemble(
[ConstantPolicy(priority=1, predict_index=1),
OldPolicy(priority=1)])
tracker = DialogueStateTracker.from_events("test", evts=[])
with pytest.warns(FutureWarning):
prediction = ensemble.probabilities_using_best_policy(
tracker, default_domain, RegexInterpreter())
assert prediction.policy_name == f"policy_1_{OldPolicy.__name__}"
assert prediction.max_confidence_index == expected_index
def test_form_wins_over_everything_else(ensemble: SimplePolicyEnsemble):
form_name = "test-form"
domain = f"""
forms:
- {form_name}
"""
domain = Domain.from_yaml(domain)
events = [
ActiveLoop("test-form"),
ActionExecuted(ACTION_LISTEN_NAME),
utilities.user_uttered("test", 1),
]
tracker = DialogueStateTracker.from_events("test", events, [])
result, best_policy = ensemble.probabilities_using_best_policy(
tracker, domain, RegexInterpreter()
)
max_confidence_index = result.index(max(result))
next_action = rasa.core.actions.action.action_for_index(
max_confidence_index, domain, None
)
index_of_form_policy = 0
assert best_policy == f"policy_{index_of_form_policy}_{FormPolicy.__name__}"
assert next_action.name() == form_name
def test_intent_prediction_does_not_apply_define_featurization_events(
default_domain: Domain, ):
ensemble = SimplePolicyEnsemble([
ConstantPolicy(priority=100, predict_index=0),
ConstantPolicy(priority=1,
predict_index=1,
is_end_to_end_prediction=False),
])
# no events should be added if latest action is not action listen
tracker = DialogueStateTracker.from_events("test", evts=[])
prediction = ensemble.probabilities_using_best_policy(
tracker, default_domain, RegexInterpreter())
assert prediction.events == []
# DefinePrevUserUtteredFeaturization should be added after action listen
tracker = DialogueStateTracker.from_events(
"test", evts=[ActionExecuted(ACTION_LISTEN_NAME)])
prediction = ensemble.probabilities_using_best_policy(
tracker, default_domain, RegexInterpreter())
assert prediction.events == [DefinePrevUserUtteredFeaturization(False)]
def test_policy_priority():
domain = Domain.load("data/test_domains/default.yml")
tracker = DialogueStateTracker.from_events("test", [UserUttered("hi")], [])
priority_1 = ConstantPolicy(priority=1, predict_index=0)
priority_2 = ConstantPolicy(priority=2, predict_index=1)
policy_ensemble_0 = SimplePolicyEnsemble([priority_1, priority_2])
policy_ensemble_1 = SimplePolicyEnsemble([priority_2, priority_1])
priority_2_result = priority_2.predict_action_probabilities(
tracker, domain)
i = 1 # index of priority_2 in ensemble_0
result, best_policy = policy_ensemble_0.probabilities_using_best_policy(
tracker, domain)
assert best_policy == "policy_{}_{}".format(i, type(priority_2).__name__)
assert result.tolist() == priority_2_result
i = 0 # index of priority_2 in ensemble_1
result, best_policy = policy_ensemble_1.probabilities_using_best_policy(
tracker, domain)
assert best_policy == "policy_{}_{}".format(i, type(priority_2).__name__)
assert result.tolist() == priority_2_result
def test_prediction_applies_must_have_policy_events(default_domain: Domain):
must_have_events = [ActionExecuted("my action")]
ensemble = SimplePolicyEnsemble([
ConstantPolicy(priority=10, predict_index=1),
ConstantPolicy(priority=1, predict_index=2, events=must_have_events),
])
tracker = DialogueStateTracker.from_events("test", evts=[])
prediction = ensemble.probabilities_using_best_policy(
tracker, default_domain, RegexInterpreter())
# Policy 0 won due to higher prio
assert prediction.policy_name == f"policy_0_{ConstantPolicy.__name__}"
# Events of losing policy were applied nevertheless
assert prediction.events == must_have_events
def test_fallback_wins_over_mapping():
domain = Domain.load("data/test_domains/default.yml")
events = [
ActionExecuted(ACTION_LISTEN_NAME),
# Low confidence should trigger fallback
utilities.user_uttered(USER_INTENT_RESTART, 0.0001),
]
tracker = DialogueStateTracker.from_events("test", events, [])
ensemble = SimplePolicyEnsemble([FallbackPolicy(), MappingPolicy()])
result, best_policy = ensemble.probabilities_using_best_policy(
tracker, domain)
max_confidence_index = result.index(max(result))
index_of_fallback_policy = 0
next_action = domain.action_for_index(max_confidence_index, None)
assert best_policy == f"policy_{index_of_fallback_policy}_{FallbackPolicy.__name__}"
assert next_action.name() == ACTION_DEFAULT_FALLBACK_NAME