def test_normalization(
self,
trained_policy: TEDPolicy,
tracker: DialogueStateTracker,
default_domain: Domain,
monkeypatch: MonkeyPatch,
):
# first check the output is what we expect
prediction = trained_policy.predict_action_probabilities(
tracker, default_domain, RegexInterpreter())
assert not prediction.is_end_to_end_prediction
# count number of non-zero confidences
assert (sum([
confidence > 0 for confidence in prediction.probabilities
]) == trained_policy.config[RANKING_LENGTH])
# check that the norm is still 1
assert sum(prediction.probabilities) == pytest.approx(1)
# also check our function is called
mock = Mock()
monkeypatch.setattr(train_utils, "normalize", mock.normalize)
trained_policy.predict_action_probabilities(tracker, default_domain,
RegexInterpreter())
mock.normalize.assert_called_once()
def test_ignore_action_unlikely_intent(
self,
trained_policy: TEDPolicy,
default_domain: Domain,
tracker_events_with_action: List[Event],
tracker_events_without_action: List[Event],
):
precomputations = None
tracker_with_action = DialogueStateTracker.from_events(
"test 1", evts=tracker_events_with_action)
tracker_without_action = DialogueStateTracker.from_events(
"test 2", evts=tracker_events_without_action)
prediction_with_action = trained_policy.predict_action_probabilities(
tracker_with_action,
default_domain,
precomputations,
)
prediction_without_action = trained_policy.predict_action_probabilities(
tracker_without_action,
default_domain,
precomputations,
)
# If the weights didn't change then both trackers
# should result in same prediction.
assert (prediction_with_action.probabilities ==
prediction_without_action.probabilities)
def test_get_next_action_probabilities_passes_interpreter_to_policies(
monkeypatch: MonkeyPatch, ):
policy = TEDPolicy()
test_interpreter = Mock()
def predict_action_probabilities(
tracker: DialogueStateTracker,
domain: Domain,
interpreter: NaturalLanguageInterpreter,
**kwargs,
) -> List[float]:
assert interpreter == test_interpreter
return [1, 0]
policy.predict_action_probabilities = predict_action_probabilities
ensemble = SimplePolicyEnsemble(policies=[policy])
domain = Domain.empty()
processor = MessageProcessor(test_interpreter, ensemble, domain,
InMemoryTrackerStore(domain), Mock())
# This should not raise
processor._get_next_action_probabilities(
DialogueStateTracker.from_events("lala",
[ActionExecuted(ACTION_LISTEN_NAME)]))
def test_get_next_action_probabilities_pass_policy_predictions_without_interpreter_arg(
predict_function: Callable,
):
policy = TEDPolicy()
policy.predict_action_probabilities = predict_function
ensemble = SimplePolicyEnsemble(policies=[policy])
interpreter = Mock()
domain = Domain.empty()
processor = MessageProcessor(
interpreter,
ensemble,
domain,
InMemoryTrackerStore(domain),
InMemoryLockStore(),
Mock(),
)
with pytest.warns(DeprecationWarning):
processor._get_next_action_probabilities(
DialogueStateTracker.from_events(
"lala", [ActionExecuted(ACTION_LISTEN_NAME)]
)
)
def test_diagnostics(default_model_storage: ModelStorage,
default_execution_context: ExecutionContext):
domain = Domain.from_yaml(DOMAIN_YAML)
policy = TEDPolicy(
TEDPolicy.get_default_config(),
default_model_storage,
Resource("TEDPolicy"),
default_execution_context,
)
GREET_RULE = DialogueStateTracker.from_events(
"greet rule",
evts=[
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(UTTER_GREET_ACTION),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(ACTION_LISTEN_NAME),
],
)
precomputations = None
policy.train([GREET_RULE], domain, precomputations)
prediction = policy.predict_action_probabilities(
GREET_RULE,
domain,
precomputations,
)
assert prediction.diagnostic_data
assert "attention_weights" in prediction.diagnostic_data
assert isinstance(prediction.diagnostic_data.get("attention_weights"),
np.ndarray)
def test_ranking_length_and_renormalization(
self,
trained_policy: TEDPolicy,
tracker: DialogueStateTracker,
default_domain: Domain,
monkeypatch: MonkeyPatch,
):
precomputations = None
prediction = trained_policy.predict_action_probabilities(
tracker,
default_domain,
precomputations,
)
# first check the output is what we expect
assert not prediction.is_end_to_end_prediction
# check that ranking length is applied - without normalization
if trained_policy.config[RANKING_LENGTH] == 0:
assert sum([confidence for confidence in prediction.probabilities
]) == pytest.approx(1)
assert all(confidence > 0
for confidence in prediction.probabilities)
else:
assert (sum([
confidence > 0 for confidence in prediction.probabilities
]) == trained_policy.config[RANKING_LENGTH])
assert sum([confidence for confidence in prediction.probabilities
]) != pytest.approx(1)
def test_diagnostics():
domain = Domain.from_yaml(DOMAIN_YAML)
policy = TEDPolicy()
GREET_RULE = DialogueStateTracker.from_events(
"greet rule",
evts=[
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(UTTER_GREET_ACTION),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(ACTION_LISTEN_NAME),
],
)
policy.train([GREET_RULE], domain, RegexInterpreter())
prediction = policy.predict_action_probabilities(GREET_RULE, domain,
RegexInterpreter())
assert prediction.diagnostic_data
assert "attention_weights" in prediction.diagnostic_data
assert isinstance(prediction.diagnostic_data.get("attention_weights"),
np.ndarray)
