def test_policy_priority():
domain = Domain.load("data/test_domains/default.yml")
tracker = DialogueStateTracker.from_events("test", [UserUttered("hi")], [])
priority_1 = ConstantPolicy(priority=1, predict_index=0)
priority_2 = ConstantPolicy(priority=2, predict_index=1)
policy_ensemble_0 = SimplePolicyEnsemble([priority_1, priority_2])
policy_ensemble_1 = SimplePolicyEnsemble([priority_2, priority_1])
priority_2_result = priority_2.predict_action_probabilities(tracker, domain)
i = 1 # index of priority_2 in ensemble_0
result, best_policy = policy_ensemble_0.probabilities_using_best_policy(
tracker, domain, RegexInterpreter()
)
assert best_policy == "policy_{}_{}".format(i, type(priority_2).__name__)
assert result == priority_2_result
i = 0 # index of priority_2 in ensemble_1
result, best_policy = policy_ensemble_1.probabilities_using_best_policy(
tracker, domain, RegexInterpreter()
)
assert best_policy == "policy_{}_{}".format(i, type(priority_2).__name__)
assert result == priority_2_result
def test_mapping_wins_over_form(events: List[Event]):
domain = """
forms:
- test-form
"""
domain = Domain.from_yaml(domain)
tracker = DialogueStateTracker.from_events("test", events, [])
ensemble = SimplePolicyEnsemble(
[
MappingPolicy(),
ConstantPolicy(priority=1, predict_index=0),
FormPolicy(),
FallbackPolicy(),
]
)
result, best_policy = ensemble.probabilities_using_best_policy(
tracker, domain, RegexInterpreter()
)
max_confidence_index = result.index(max(result))
next_action = domain.action_for_index(max_confidence_index, None)
index_of_mapping_policy = 0
assert best_policy == f"policy_{index_of_mapping_policy}_{MappingPolicy.__name__}"
assert next_action.name() == ACTION_RESTART_NAME
def test_fallback_mapping_restart():
domain = Domain.load("data/test_domains/default.yml")
events = [
ActionExecuted(ACTION_DEFAULT_FALLBACK_NAME),
utilities.user_uttered(USER_INTENT_RESTART, 1),
]
tracker = DialogueStateTracker.from_events("test", events, [])
two_stage_fallback_policy = TwoStageFallbackPolicy(
priority=2, deny_suggestion_intent_name="deny"
)
mapping_policy = MappingPolicy(priority=1)
mapping_fallback_ensemble = SimplePolicyEnsemble(
[two_stage_fallback_policy, mapping_policy]
)
result, best_policy = mapping_fallback_ensemble.probabilities_using_best_policy(
tracker, domain, RegexInterpreter()
)
max_confidence_index = result.index(max(result))
index_of_mapping_policy = 1
next_action = domain.action_for_index(max_confidence_index, None)
assert best_policy == f"policy_{index_of_mapping_policy}_{MappingPolicy.__name__}"
assert next_action.name() == ACTION_RESTART_NAME
def test_with_float_returning_policy(default_domain: Domain):
expected_index = 3
class OldPolicy(Policy):
def predict_action_probabilities(
self,
tracker: DialogueStateTracker,
domain: Domain,
interpreter: NaturalLanguageInterpreter,
**kwargs: Any,
) -> List[float]:
prediction = [0.0] * default_domain.num_actions
prediction[expected_index] = 3
return prediction
ensemble = SimplePolicyEnsemble(
[ConstantPolicy(priority=1, predict_index=1), OldPolicy(priority=1)]
)
tracker = DialogueStateTracker.from_events("test", evts=[])
with pytest.warns(FutureWarning):
prediction = ensemble.probabilities_using_best_policy(
tracker, default_domain, RegexInterpreter()
)
assert prediction.policy_name == f"policy_1_{OldPolicy.__name__}"
assert prediction.max_confidence_index == expected_index
def test_prediction_applies_optional_policy_events(default_domain: Domain):
optional_events = [ActionExecuted("my action")]
must_have_events = [SlotSet("some slot", "some value")]
ensemble = SimplePolicyEnsemble(
[
ConstantPolicy(
priority=10,
predict_index=1,
events=must_have_events,
optional_events=optional_events,
),
ConstantPolicy(priority=1, predict_index=2),
]
)
tracker = DialogueStateTracker.from_events("test", evts=[])
prediction = ensemble.probabilities_using_best_policy(
tracker, default_domain, RegexInterpreter()
)
# Policy 0 won due to higher prio
assert prediction.policy_name == f"policy_0_{ConstantPolicy.__name__}"
# Events of losing policy were applied nevertheless
assert len(prediction.events) == len(optional_events) + len(must_have_events)
assert all(event in prediction.events for event in optional_events)
assert all(event in prediction.events for event in must_have_events)
def test_intent_prediction_does_not_apply_define_featurization_events(
default_domain: Domain,
):
ensemble = SimplePolicyEnsemble(
[
ConstantPolicy(priority=100, predict_index=0),
ConstantPolicy(priority=1, predict_index=1, is_end_to_end_prediction=False),
]
)
# no events should be added if latest action is not action listen
tracker = DialogueStateTracker.from_events("test", evts=[])
prediction = ensemble.probabilities_using_best_policy(
tracker, default_domain, RegexInterpreter()
)
assert prediction.events == []
# DefinePrevUserUtteredFeaturization should be added after action listen
tracker = DialogueStateTracker.from_events(
"test", evts=[ActionExecuted(ACTION_LISTEN_NAME)]
)
prediction = ensemble.probabilities_using_best_policy(
tracker, default_domain, RegexInterpreter()
)
assert prediction.events == [DefinePrevUserUtteredFeaturization(False)]
def test_no_user_prediction_supersedes_others(default_domain: Domain):
expected_action_index = 2
expected_confidence = 0.5
ensemble = SimplePolicyEnsemble([
ConstantPolicy(priority=100, predict_index=0),
ConstantPolicy(priority=1,
predict_index=1,
is_end_to_end_prediction=True),
ConstantPolicy(
priority=1,
predict_index=expected_action_index,
confidence=expected_confidence,
is_no_user_prediction=True,
),
])
tracker = DialogueStateTracker.from_events("test", evts=[])
prediction = ensemble.probabilities_using_best_policy(
tracker, default_domain, RegexInterpreter())
assert prediction.max_confidence == expected_confidence
assert prediction.max_confidence_index == expected_action_index
assert prediction.policy_name == f"policy_2_{ConstantPolicy.__name__}"
assert prediction.is_no_user_prediction
assert not prediction.is_end_to_end_prediction
def _train_rule_based_agent(
moodbot_domain: Domain,
train_file_name: Path,
monkeypatch: MonkeyPatch,
ignore_action_unlikely_intent: bool,
) -> Agent:
# We need `RulePolicy` to predict the correct actions
# in a particular conversation context as seen during training.
# Since it can get affected by `action_unlikely_intent` being triggered in
# some cases. We monkey-patch the method which creates
# prediction states to ignore `action_unlikely_intent`s if needed.
monkeypatch.setattr(
RulePolicy,
"_prediction_states",
_custom_prediction_states_for_rules(ignore_action_unlikely_intent),
)
deterministic_policy = RulePolicy(restrict_rules=False)
agent = Agent(moodbot_domain, SimplePolicyEnsemble([deterministic_policy]))
training_data = agent.load_data(str(train_file_name))
# Make the trackers compatible with rules
# so that they are picked up by the policy.
for tracker in training_data:
tracker.is_rule_tracker = True
agent.train(training_data)
return agent
async def default_processor(default_domain, default_nlg):
agent = Agent(default_domain,
SimplePolicyEnsemble([AugmentedMemoizationPolicy()]),
interpreter=RegexInterpreter())
training_data = await agent.load_data(DEFAULT_STORIES_FILE)
agent.train(training_data)
tracker_store = InMemoryTrackerStore(default_domain)
return MessageProcessor(agent.interpreter, agent.policy_ensemble,
default_domain, tracker_store, default_nlg)
def test_rule_based_data_warnings_no_rule_policy():
trackers = [DialogueStateTracker("some-id", slots=[], is_rule_tracker=True)]
policies = [FallbackPolicy()]
ensemble = SimplePolicyEnsemble(policies)
with pytest.warns(UserWarning) as record:
ensemble.train(trackers, Domain.empty(), RegexInterpreter())
assert (
"Found rule-based training data but no policy supporting rule-based data."
) in record[0].message.args[0]
def test_rule_based_data_warnings_no_rule_trackers():
trackers = [DialogueStateTracker("some-id", slots=[], is_rule_tracker=False)]
policies = [RulePolicy()]
ensemble = SimplePolicyEnsemble(policies)
with pytest.warns(UserWarning) as record:
ensemble.train(trackers, Domain.empty(), RegexInterpreter())
assert (
"Found a rule-based policy in your pipeline but no rule-based training data."
) in record[0].message.args[0]
def _create_ensemble(
policies: Union[List[Policy], PolicyEnsemble, None]
) -> Optional[PolicyEnsemble]:
if policies is None:
return None
if isinstance(policies, list):
return SimplePolicyEnsemble(policies)
elif isinstance(policies, PolicyEnsemble):
return policies
else:
passed_type = type(policies).__name__
raise InvalidParameterException(
f"Invalid param `policies`. Passed object is "
f"of type '{passed_type}', but should be policy, an array of "
f"policies, or a policy ensemble.")
def _create_ensemble(
policies: Union[List[Policy], PolicyEnsemble, None]
) -> Optional[PolicyEnsemble]:
if policies is None:
return None
if isinstance(policies, list):
return SimplePolicyEnsemble(policies)
elif isinstance(policies, PolicyEnsemble):
return policies
else:
passed_type = type(policies).__name__
raise ValueError(
"Invalid param `policies`. Passed object is "
"of type '{}', but should be policy, an array of "
"policies, or a policy ensemble.".format(passed_type))
def test_prediction_applies_must_have_policy_events(default_domain: Domain):
must_have_events = [ActionExecuted("my action")]
ensemble = SimplePolicyEnsemble([
ConstantPolicy(priority=10, predict_index=1),
ConstantPolicy(priority=1, predict_index=2, events=must_have_events),
])
tracker = DialogueStateTracker.from_events("test", evts=[])
prediction = ensemble.probabilities_using_best_policy(
tracker, default_domain, RegexInterpreter())
# Policy 0 won due to higher prio
assert prediction.policy_name == f"policy_0_{ConstantPolicy.__name__}"
# Events of losing policy were applied nevertheless
assert prediction.events == must_have_events
def test_fallback_wins_over_mapping():
domain = Domain.load("data/test_domains/default.yml")
events = [
ActionExecuted(ACTION_LISTEN_NAME),
# Low confidence should trigger fallback
utilities.user_uttered(USER_INTENT_RESTART, 0.0001),
]
tracker = DialogueStateTracker.from_events("test", events, [])
ensemble = SimplePolicyEnsemble([FallbackPolicy(), MappingPolicy()])
result, best_policy = ensemble.probabilities_using_best_policy(
tracker, domain)
max_confidence_index = result.index(max(result))
index_of_fallback_policy = 0
next_action = domain.action_for_index(max_confidence_index, None)
assert best_policy == f"policy_{index_of_fallback_policy}_{FallbackPolicy.__name__}"
assert next_action.name() == ACTION_DEFAULT_FALLBACK_NAME
max_confidence_index = result.index(max(result))
next_action = domain.action_for_index(max_confidence_index, None)
index_of_mapping_policy = 0
assert best_policy == f"policy_{index_of_mapping_policy}_{MappingPolicy.__name__}"
assert next_action.name() == ACTION_RESTART_NAME
@pytest.mark.parametrize(
"ensemble",
[
SimplePolicyEnsemble(
[
FormPolicy(),
ConstantPolicy(FORM_POLICY_PRIORITY - 1, 0),
FallbackPolicy(),
]
),
SimplePolicyEnsemble([FormPolicy(), MappingPolicy()]),
],
)
def test_form_wins_over_everything_else(ensemble: SimplePolicyEnsemble):
form_name = "test-form"
domain = f"""
forms:
- {form_name}
"""
domain = Domain.from_yaml(domain)
events = [