def test_get_next_action_probabilities_pass_policy_predictions_without_interpreter_arg(
predict_function: Callable,
):
policy = TEDPolicy()
policy.predict_action_probabilities = predict_function
ensemble = SimplePolicyEnsemble(policies=[policy])
interpreter = Mock()
domain = Domain.empty()
processor = MessageProcessor(
interpreter,
ensemble,
domain,
InMemoryTrackerStore(domain),
InMemoryLockStore(),
Mock(),
)
with pytest.warns(DeprecationWarning):
processor._get_next_action_probabilities(
DialogueStateTracker.from_events(
"lala", [ActionExecuted(ACTION_LISTEN_NAME)]
)
)
def test_ignore_action_unlikely_intent(
self,
trained_policy: TEDPolicy,
default_domain: Domain,
tracker_events_with_action: List[Event],
tracker_events_without_action: List[Event],
):
precomputations = None
tracker_with_action = DialogueStateTracker.from_events(
"test 1", evts=tracker_events_with_action)
tracker_without_action = DialogueStateTracker.from_events(
"test 2", evts=tracker_events_without_action)
prediction_with_action = trained_policy.predict_action_probabilities(
tracker_with_action,
default_domain,
precomputations,
)
prediction_without_action = trained_policy.predict_action_probabilities(
tracker_without_action,
default_domain,
precomputations,
)
# If the weights didn't change then both trackers
# should result in same prediction.
assert (prediction_with_action.probabilities ==
prediction_without_action.probabilities)
def test_normalization(
self,
trained_policy: TEDPolicy,
tracker: DialogueStateTracker,
default_domain: Domain,
monkeypatch: MonkeyPatch,
):
# first check the output is what we expect
prediction = trained_policy.predict_action_probabilities(
tracker, default_domain, RegexInterpreter())
assert not prediction.is_end_to_end_prediction
# count number of non-zero confidences
assert (sum([
confidence > 0 for confidence in prediction.probabilities
]) == trained_policy.config[RANKING_LENGTH])
# check that the norm is still 1
assert sum(prediction.probabilities) == pytest.approx(1)
# also check our function is called
mock = Mock()
monkeypatch.setattr(train_utils, "normalize", mock.normalize)
trained_policy.predict_action_probabilities(tracker, default_domain,
RegexInterpreter())
mock.normalize.assert_called_once()
def test_get_next_action_probabilities_passes_interpreter_to_policies(
monkeypatch: MonkeyPatch, ):
policy = TEDPolicy()
test_interpreter = Mock()
def predict_action_probabilities(
tracker: DialogueStateTracker,
domain: Domain,
interpreter: NaturalLanguageInterpreter,
**kwargs,
) -> List[float]:
assert interpreter == test_interpreter
return [1, 0]
policy.predict_action_probabilities = predict_action_probabilities
ensemble = SimplePolicyEnsemble(policies=[policy])
domain = Domain.empty()
processor = MessageProcessor(test_interpreter, ensemble, domain,
InMemoryTrackerStore(domain), Mock())
# This should not raise
processor._get_next_action_probabilities(
DialogueStateTracker.from_events("lala",
[ActionExecuted(ACTION_LISTEN_NAME)]))
def test_fingerprint_stays_same():
key1 = fingerprinting.calculate_fingerprint_key(
TEDPolicy, TEDPolicy.get_default_config(),
{"input": FingerprintableText("Hi")})
key2 = fingerprinting.calculate_fingerprint_key(
TEDPolicy, TEDPolicy.get_default_config(),
{"input": FingerprintableText("Hi")})
assert key1 == key2
def test_fingerprint_changes_due_to_class():
key1 = fingerprinting.calculate_fingerprint_key(
DIETClassifier,
TEDPolicy.get_default_config(),
{"input": FingerprintableText("Hi")},
)
key2 = fingerprinting.calculate_fingerprint_key(
ResponseSelector,
TEDPolicy.get_default_config(),
{"input": FingerprintableText("Hi")},
)
assert key1 != key2
def test_label_data_assembly(self, trained_policy: TEDPolicy,
default_domain: Domain):
state_featurizer = trained_policy.featurizer.state_featurizer
encoded_all_labels = state_featurizer.encode_all_labels(
default_domain, precomputations=None)
attribute_data, _ = model_data_utils.convert_to_data_format(
encoded_all_labels)
assembled_label_data = trained_policy._assemble_label_data(
attribute_data, default_domain)
assembled_label_data_signature = assembled_label_data.get_signature()
assert list(assembled_label_data_signature.keys()) == [
f"{LABEL}_{ACTION_NAME}",
f"{LABEL}",
]
assert assembled_label_data.num_examples == default_domain.num_actions
assert list(assembled_label_data_signature[f"{LABEL}_{ACTION_NAME}"].
keys()) == [
MASK,
SENTENCE,
]
assert list(assembled_label_data_signature[LABEL].keys()) == [IDS]
assert (assembled_label_data_signature[f"{LABEL}_{ACTION_NAME}"]
[SENTENCE][0].units == default_domain.num_actions)
def test_ranking_length_and_renormalization(
self,
trained_policy: TEDPolicy,
tracker: DialogueStateTracker,
default_domain: Domain,
monkeypatch: MonkeyPatch,
):
precomputations = None
prediction = trained_policy.predict_action_probabilities(
tracker,
default_domain,
precomputations,
)
# first check the output is what we expect
assert not prediction.is_end_to_end_prediction
# check that ranking length is applied - without normalization
if trained_policy.config[RANKING_LENGTH] == 0:
assert sum([confidence for confidence in prediction.probabilities
]) == pytest.approx(1)
assert all(confidence > 0
for confidence in prediction.probabilities)
else:
assert (sum([
confidence > 0 for confidence in prediction.probabilities
]) == trained_policy.config[RANKING_LENGTH])
assert sum([confidence for confidence in prediction.probabilities
]) != pytest.approx(1)
def test_diagnostics(default_model_storage: ModelStorage,
default_execution_context: ExecutionContext):
domain = Domain.from_yaml(DOMAIN_YAML)
policy = TEDPolicy(
TEDPolicy.get_default_config(),
default_model_storage,
Resource("TEDPolicy"),
default_execution_context,
)
GREET_RULE = DialogueStateTracker.from_events(
"greet rule",
evts=[
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(UTTER_GREET_ACTION),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(ACTION_LISTEN_NAME),
],
)
precomputations = None
policy.train([GREET_RULE], domain, precomputations)
prediction = policy.predict_action_probabilities(
GREET_RULE,
domain,
precomputations,
)
assert prediction.diagnostic_data
assert "attention_weights" in prediction.diagnostic_data
assert isinstance(prediction.diagnostic_data.get("attention_weights"),
np.ndarray)
def trained_ted(tmp_path_factory: TempPathFactory,
moodbot_domain_path: Path) -> TEDPolicy:
training_files = "data/test_moodbot/data/stories.yml"
domain = Domain.load(moodbot_domain_path)
trackers = training.load_data(str(training_files), domain)
policy = TEDPolicy.create(
{
**TEDPolicy.get_default_config(), EPOCHS: 1
},
LocalModelStorage.create(tmp_path_factory.mktemp("storage")),
Resource("ted"),
ExecutionContext(GraphSchema({})),
)
policy.train(trackers, domain)
return policy
def create_policy(self, featurizer, priority):
p = TEDPolicy(
featurizer=featurizer,
priority=priority,
**{SCALE_LOSS: False, EVAL_NUM_EXAMPLES: 4},
)
return p
def create_policy(
self, featurizer: Optional[TrackerFeaturizer], priority: int
) -> Policy:
# use standard featurizer from TEDPolicy,
# since it is using MaxHistoryTrackerFeaturizer
# if max_history is specified
return TEDPolicy(priority=priority, max_history=self.max_history)
def create_policy(
self, featurizer: Optional[TrackerFeaturizer], priority: int
) -> Policy:
return TEDPolicy(
featurizer=featurizer,
priority=priority,
**{SCALE_LOSS: False, EVAL_NUM_EXAMPLES: 4},
)
def _config(
self,
config_override: Optional[Dict[Text,
Any]] = None) -> Dict[Text, Any]:
config_override = config_override or {}
return {
**TEDPolicy.get_default_config(),
**config_override,
}
def _config(
self,
config_override: Optional[Dict[Text,
Any]] = None) -> Dict[Text, Any]:
config_override = config_override or {}
return {
**TEDPolicy.get_default_config(),
POLICY_MAX_HISTORY: self.max_history,
**config_override,
}
def create_policy(self, featurizer, priority):
p = TEDPolicy(
featurizer=featurizer,
priority=priority,
**{
KEY_RELATIVE_ATTENTION: True,
VALUE_RELATIVE_ATTENTION: True,
MAX_RELATIVE_POSITION: 5,
},
)
return p
def _config(
self,
config_override: Optional[Dict[Text,
Any]] = None) -> Dict[Text, Any]:
config_override = config_override or {}
return {
**TEDPolicy.get_default_config(),
SCALE_LOSS: False,
EVAL_NUM_EXAMPLES: 4,
**config_override,
}
def create_policy(self, featurizer: Optional[TrackerFeaturizer],
priority: int) -> Policy:
return TEDPolicy(
featurizer=featurizer,
priority=priority,
**{
KEY_RELATIVE_ATTENTION: True,
VALUE_RELATIVE_ATTENTION: True,
MAX_RELATIVE_POSITION: 5,
},
)
def _config(
self,
config_override: Optional[Dict[Text,
Any]] = None) -> Dict[Text, Any]:
config_override = config_override or {}
return {
**TEDPolicy.get_default_config(),
RANKING_LENGTH: 4,
RENORMALIZE_CONFIDENCES: True,
**config_override,
}
def test_diagnostics():
domain = Domain.from_yaml(DOMAIN_YAML)
policy = TEDPolicy()
GREET_RULE = DialogueStateTracker.from_events(
"greet rule",
evts=[
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(UTTER_GREET_ACTION),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(ACTION_LISTEN_NAME),
],
)
policy.train([GREET_RULE], domain, RegexInterpreter())
prediction = policy.predict_action_probabilities(GREET_RULE, domain,
RegexInterpreter())
assert prediction.diagnostic_data
assert "attention_weights" in prediction.diagnostic_data
assert isinstance(prediction.diagnostic_data.get("attention_weights"),
np.ndarray)
def _config(
self,
config_override: Optional[Dict[Text,
Any]] = None) -> Dict[Text, Any]:
config_override = config_override or {}
return {
**TEDPolicy.get_default_config(),
KEY_RELATIVE_ATTENTION: True,
VALUE_RELATIVE_ATTENTION: True,
MAX_RELATIVE_POSITION: 5,
**config_override,
}
async def test_infer():
default_domain = Domain.load(DEFAULT_DOMAIN_PATH_WITH_SLOTS)
print("action names: {}".format(default_domain.action_names))
trackers = await training.load_data(DEFAULT_STORIES_FILE,
default_domain,
augmentation_factor=0,
debug_plots=True)
policy = TEDPolicy.load("{}/models/ted".format(prj_dir))
for tracker in trackers:
y_pred = policy.predict_action_probabilities(tracker, default_domain)
index = y_pred.index(max(y_pred))
print(default_domain.action_names[index])
async def test_train_keras_policy():
default_domain = Domain.load(DEFAULT_DOMAIN_PATH_WITH_SLOTS)
trackers = await training.load_data(DEFAULT_STORIES_FILE,
default_domain,
augmentation_factor=0,
debug_plots=False)
config = {"epochs": 100}
policy = TEDPolicy(featurizer=featurizer(), priority=1, **config)
policy.train(trackers, default_domain)
policy.persist("{}/models/ted".format(prj_dir))
def create_policy(
self,
featurizer: Optional[TrackerFeaturizer],
model_storage: ModelStorage,
resource: Resource,
execution_context: ExecutionContext,
config: Optional[Dict[Text, Any]] = None,
) -> Policy:
# use standard featurizer from TEDPolicy,
# since it is using MaxHistoryTrackerFeaturizer
# if max_history is specified
return TEDPolicy(
config=self._config(config),
model_storage=model_storage,
resource=resource,
execution_context=execution_context,
)
def create_policy(self, featurizer, priority):
# use standard featurizer from TEDPolicy,
# since it is using MaxHistoryTrackerFeaturizer
# if max_history is specified
p = TEDPolicy(priority=priority, max_history=self.max_history)
return p
def create_policy(self, featurizer, priority):
# use standard featurizer from TEDPolicy,
# since it is using FullDialogueTrackerFeaturizer
# if max_history is not specified
p = TEDPolicy(priority=priority)
return p
def create_policy(self, featurizer, priority):
p = TEDPolicy(featurizer=featurizer,
priority=priority,
**{RANKING_LENGTH: 11})
return p
def create_policy(self, featurizer, priority):
p = TEDPolicy(featurizer=featurizer,
priority=priority,
**{LOSS_TYPE: "margin"})
return p
def create_policy(self, featurizer, priority):
p = TEDPolicy(featurizer=featurizer, priority=priority)
return p
)
def test_supported_data(policy: Type[Policy], supported_data: SupportedData):
assert policy.supported_data() == supported_data
class OnlyRulePolicy(Policy):
"""Test policy that supports both rule-based and ML-based training data."""
@staticmethod
def supported_data() -> SupportedData:
return SupportedData.RULE_DATA
@pytest.mark.parametrize(
"policy,n_rule_trackers,n_ml_trackers",
[
(TEDPolicy(), 0, 3),
(RulePolicy(), 2, 3),
(OnlyRulePolicy, 2, 0), # policy can be passed as a `type` as well
],
)
def test_get_training_trackers_for_policy(policy: Policy, n_rule_trackers: int,
n_ml_trackers):
# create five trackers (two rule-based and three ML trackers)
trackers = [
DialogueStateTracker("id1", slots=[], is_rule_tracker=True),
DialogueStateTracker("id2", slots=[], is_rule_tracker=False),
DialogueStateTracker("id3", slots=[], is_rule_tracker=False),
DialogueStateTracker("id4", slots=[], is_rule_tracker=True),
DialogueStateTracker("id5", slots=[], is_rule_tracker=False),
]
