async def test_create_train_data_with_history(default_domain):
featurizer = MaxHistoryTrackerFeaturizer(max_history=4)
training_trackers = await training.load_data(DEFAULT_STORIES_FILE,
default_domain,
augmentation_factor=0)
assert len(training_trackers) == 3
(decoded,
_) = featurizer.training_states_and_actions(training_trackers,
default_domain)
# decoded needs to be sorted
hashed = []
for states in decoded:
hashed.append(json.dumps(states, sort_keys=True))
hashed = sorted(hashed)
assert hashed == [
'[{"prev_action": {"action_name": "action_listen"}, "slots": {"name": [1.0]}, "user": {"entities": ["name"], "intent": "greet"}}, {"prev_action": {"action_name": "utter_greet"}, "slots": {"name": [1.0]}, "user": {"entities": ["name"], "intent": "greet"}}, {"prev_action": {"action_name": "action_listen"}, "slots": {"name": [1.0]}, "user": {"intent": "default"}}, {"prev_action": {"action_name": "utter_default"}, "slots": {"name": [1.0]}, "user": {"intent": "default"}}]',
'[{"prev_action": {"action_name": "action_listen"}, "user": {"intent": "default"}}, {"prev_action": {"action_name": "utter_default"}, "user": {"intent": "default"}}, {"prev_action": {"action_name": "action_listen"}, "user": {"intent": "goodbye"}}, {"prev_action": {"action_name": "utter_goodbye"}, "user": {"intent": "goodbye"}}]',
'[{"prev_action": {"action_name": "action_listen"}, "user": {"intent": "greet"}}, {"prev_action": {"action_name": "utter_greet"}, "user": {"intent": "greet"}}, {"prev_action": {"action_name": "action_listen"}, "user": {"intent": "default"}}, {"prev_action": {"action_name": "utter_default"}, "user": {"intent": "default"}}]',
'[{"prev_action": {"action_name": "utter_greet"}, "user": {"intent": "greet"}}, {"prev_action": {"action_name": "action_listen"}, "user": {"intent": "default"}}, {"prev_action": {"action_name": "utter_default"}, "user": {"intent": "default"}}, {"prev_action": {"action_name": "action_listen"}, "user": {"intent": "goodbye"}}]',
'[{}, {"prev_action": {"action_name": "action_listen"}, "slots": {"name": [1.0]}, "user": {"entities": ["name"], "intent": "greet"}}, {"prev_action": {"action_name": "utter_greet"}, "slots": {"name": [1.0]}, "user": {"entities": ["name"], "intent": "greet"}}, {"prev_action": {"action_name": "action_listen"}, "slots": {"name": [1.0]}, "user": {"intent": "default"}}]',
'[{}, {"prev_action": {"action_name": "action_listen"}, "slots": {"name": [1.0]}, "user": {"entities": ["name"], "intent": "greet"}}, {"prev_action": {"action_name": "utter_greet"}, "slots": {"name": [1.0]}, "user": {"entities": ["name"], "intent": "greet"}}]',
'[{}, {"prev_action": {"action_name": "action_listen"}, "slots": {"name": [1.0]}, "user": {"entities": ["name"], "intent": "greet"}}]',
'[{}, {"prev_action": {"action_name": "action_listen"}, "user": {"intent": "greet"}}, {"prev_action": {"action_name": "utter_greet"}, "user": {"intent": "greet"}}, {"prev_action": {"action_name": "action_listen"}, "user": {"intent": "default"}}]',
'[{}, {"prev_action": {"action_name": "action_listen"}, "user": {"intent": "greet"}}, {"prev_action": {"action_name": "utter_greet"}, "user": {"intent": "greet"}}]',
'[{}, {"prev_action": {"action_name": "action_listen"}, "user": {"intent": "greet"}}]',
"[{}]",
]
async def test_create_train_data_no_history(domain: Domain,
stories_path: Text):
featurizer = MaxHistoryTrackerFeaturizer(max_history=1)
training_trackers = await training.load_data(stories_path,
domain,
augmentation_factor=0)
assert len(training_trackers) == 4
(decoded,
_) = featurizer.training_states_and_actions(training_trackers, domain)
# decoded needs to be sorted
hashed = []
for states in decoded:
hashed.append(json.dumps(states, sort_keys=True))
hashed = sorted(hashed, reverse=True)
assert hashed == [
"[{}]",
'[{"prev_action": {"action_name": "utter_greet"}, "user": {"intent": "greet"}}]',
'[{"prev_action": {"action_name": "utter_greet"}, "slots": {"name": [1.0]}, "user": {"entities": ["name"], "intent": "greet"}}]',
'[{"prev_action": {"action_name": "utter_goodbye"}, "user": {"intent": "goodbye"}}]',
'[{"prev_action": {"action_name": "utter_default"}, "user": {"intent": "default"}}]',
'[{"prev_action": {"action_name": "utter_default"}, "slots": {"name": [1.0]}, "user": {"intent": "default"}}]',
'[{"prev_action": {"action_name": "action_listen"}, "user": {"intent": "greet"}}]',
'[{"prev_action": {"action_name": "action_listen"}, "user": {"intent": "goodbye"}}]',
'[{"prev_action": {"action_name": "action_listen"}, "user": {"intent": "default"}}]',
'[{"prev_action": {"action_name": "action_listen"}, "slots": {"name": [1.0]}, "user": {"intent": "default"}}]',
'[{"prev_action": {"action_name": "action_listen"}, "slots": {"name": [1.0]}, "user": {"entities": ["name"], "intent": "greet"}}]',
]
def test_persist_and_load_tracker_featurizer(tmp_path: Text,
moodbot_domain: Domain):
state_featurizer = SingleStateFeaturizer()
state_featurizer.prepare_for_training(moodbot_domain, RegexInterpreter())
tracker_featurizer = MaxHistoryTrackerFeaturizer(state_featurizer)
tracker_featurizer.persist(tmp_path)
loaded_tracker_featurizer = TrackerFeaturizer.load(tmp_path)
assert loaded_tracker_featurizer is not None
assert loaded_tracker_featurizer.state_featurizer is not None
def test_slots_states_before_user_utterance(default_domain):
featurizer = MaxHistoryTrackerFeaturizer()
tracker = DialogueStateTracker.from_events(
"bla",
evts=[
SlotSet(default_domain.slots[0].name, "some_value"),
ActionExecuted("utter_default"),
],
slots=default_domain.slots,
)
trackers_as_states, _ = featurizer.training_states_and_actions(
[tracker], default_domain)
expected_states = [[{"slots": {"name": (1.0, )}}]]
assert trackers_as_states == expected_states
def test_load_multi_file_training_data(domain: Domain):
featurizer = MaxHistoryTrackerFeaturizer(SingleStateFeaturizer(), max_history=2)
trackers = training.load_data(
"data/test_yaml_stories/stories.yml", domain, augmentation_factor=0
)
trackers = sorted(trackers, key=lambda t: t.sender_id)
(tr_as_sts, tr_as_acts) = featurizer.training_states_and_labels(trackers, domain)
hashed = []
for sts, acts in zip(tr_as_sts, tr_as_acts):
hashed.append(json.dumps(sts + acts, sort_keys=True))
hashed = sorted(hashed, reverse=True)
data, label_ids, _ = featurizer.featurize_trackers(
trackers, domain, precomputations=None
)
featurizer_mul = MaxHistoryTrackerFeaturizer(SingleStateFeaturizer(), max_history=2)
trackers_mul = training.load_data(
"data/test_multifile_yaml_stories", domain, augmentation_factor=0
)
trackers_mul = sorted(trackers_mul, key=lambda t: t.sender_id)
(tr_as_sts_mul, tr_as_acts_mul) = featurizer.training_states_and_labels(
trackers_mul, domain
)
hashed_mul = []
for sts_mul, acts_mul in zip(tr_as_sts_mul, tr_as_acts_mul):
hashed_mul.append(json.dumps(sts_mul + acts_mul, sort_keys=True))
hashed_mul = sorted(hashed_mul, reverse=True)
data_mul, label_ids_mul, _ = featurizer_mul.featurize_trackers(
trackers_mul, domain, precomputations=None
)
assert hashed == hashed_mul
# we check for intents, action names and entities -- the features which
# are included in the story files
data = _surface_attributes(data)
data_mul = _surface_attributes(data_mul)
for attribute in [INTENT, ACTION_NAME, ENTITIES]:
if attribute not in data or attribute not in data_mul:
continue
assert len(data.get(attribute)) == len(data_mul.get(attribute))
for idx_tracker in range(len(data.get(attribute))):
for idx_dialogue in range(len(data.get(attribute)[idx_tracker])):
f1 = data.get(attribute)[idx_tracker][idx_dialogue]
f2 = data_mul.get(attribute)[idx_tracker][idx_dialogue]
if f1 is None or f2 is None:
assert f1 == f2
continue
for idx_turn in range(len(f1)):
f1 = data.get(attribute)[idx_tracker][idx_dialogue][idx_turn]
f2 = data_mul.get(attribute)[idx_tracker][idx_dialogue][idx_turn]
assert np.all((f1 == f2).data)
assert np.all(label_ids == label_ids_mul)
def test_featurize_trackers_with_max_history_tracker_featurizer(
moodbot_domain: Domain):
state_featurizer = SingleStateFeaturizer()
tracker_featurizer = MaxHistoryTrackerFeaturizer(state_featurizer)
tracker = tracker_from_dialogue_file("data/test_dialogues/moodbot.json",
moodbot_domain)
state_features, labels, entity_tags = tracker_featurizer.featurize_trackers(
[tracker], moodbot_domain, RegexInterpreter())
assert state_features is not None
assert len(state_features) == 7
assert labels is not None
assert len(labels) == 7
# moodbot doesn't contain e2e entities
assert not any([any(turn_tags) for turn_tags in entity_tags])
def _standard_featurizer(
max_history: int = DEFAULT_MAX_HISTORY,
) -> MaxHistoryTrackerFeaturizer:
# Sklearn policy always uses MaxHistoryTrackerFeaturizer
return MaxHistoryTrackerFeaturizer(
state_featurizer=SingleStateFeaturizer(), max_history=5
)
def _sliced_states_iterator(
trackers: List[TrackerWithCachedStates],
domain: Domain,
max_history: Optional[int],
tokenizer: Optional[Tokenizer],
) -> Generator[TrackerEventStateTuple, None, None]:
"""Creates an iterator over sliced states.
Iterate over all given trackers and all sliced states within each tracker,
where the slicing is based on `max_history`.
Args:
trackers: List of trackers.
domain: Domain (used for tracker.past_states).
max_history: Assumed `max_history` value for slicing.
tokenizer: A tokenizer to tokenize the user messages.
Yields:
A (tracker, event, sliced_states) triplet.
"""
for tracker in trackers:
states = tracker.past_states(domain)
idx = 0
for event in tracker.events:
if isinstance(event, ActionExecuted):
sliced_states = MaxHistoryTrackerFeaturizer.slice_state_history(
states[:idx + 1], max_history)
if tokenizer:
_apply_tokenizer_to_states(tokenizer, sliced_states)
# TODO: deal with oov (different tokens can lead to identical features
# if some of those tokens are out of vocabulary for all featurizers)
yield TrackerEventStateTuple(tracker, event, sliced_states)
idx += 1
def _sliced_states_iterator(
trackers: List[TrackerWithCachedStates], domain: Domain, max_history: int
) -> Generator[TrackerEventStateTuple, None, None]:
"""Creates an iterator over sliced states.
Iterate over all given trackers and all sliced states within each tracker,
where the slicing is based on `max_history`.
Args:
trackers: List of trackers.
domain: Domain (used for tracker.past_states).
max_history: Assumed `max_history` value for slicing.
Yields:
A (tracker, event, sliced_states) triplet.
"""
for tracker in trackers:
states = tracker.past_states(domain)
idx = 0
for event in tracker.events:
if isinstance(event, ActionExecuted):
sliced_states = MaxHistoryTrackerFeaturizer.slice_state_history(
states[: idx + 1], max_history
)
yield TrackerEventStateTuple(tracker, event, sliced_states)
idx += 1
def _standard_featurizer(
max_history: Optional[int] = None,
) -> MaxHistoryTrackerFeaturizer:
# Memoization policy always uses MaxHistoryTrackerFeaturizer
# without state_featurizer
return MaxHistoryTrackerFeaturizer(
state_featurizer=None, max_history=max_history
)
async def test_create_train_data_unfeaturized_entities():
import copy
domain_file = "data/test_domains/default_unfeaturized_entities.yml"
stories_file = "data/test_stories/stories_unfeaturized_entities.md"
domain = Domain.load(domain_file)
featurizer = MaxHistoryTrackerFeaturizer(max_history=1)
training_trackers = await training.load_data(stories_file,
domain,
augmentation_factor=0)
assert len(training_trackers) == 2
(decoded,
_) = featurizer.training_states_and_actions(training_trackers, domain)
# decoded needs to be sorted
hashed = []
for states in decoded:
new_states = [
check_for_too_many_entities_and_remove_them(state)
for state in states
]
hashed.append(json.dumps(new_states, sort_keys=True))
hashed = sorted(hashed, reverse=True)
assert hashed == [
"[{}]",
'[{"prev_action": {"action_name": "utter_greet"}, "user": {"intent": "greet"}}]',
'[{"prev_action": {"action_name": "utter_greet"}, "user": {"entities": ["name"], "intent": "greet"}}]',
'[{"prev_action": {"action_name": "utter_goodbye"}, "user": {"intent": "goodbye"}}]',
'[{"prev_action": {"action_name": "utter_default"}, "user": {"intent": "why"}}]',
'[{"prev_action": {"action_name": "utter_default"}, "user": {"intent": "thank"}}]',
'[{"prev_action": {"action_name": "utter_default"}, "user": {"entities": [], "intent": "default"}}]',
'[{"prev_action": {"action_name": "utter_default"}, "user": {"entities": [], "intent": "ask"}}]',
'[{"prev_action": {"action_name": "action_listen"}, "user": {"intent": "why"}}]',
'[{"prev_action": {"action_name": "action_listen"}, "user": {"intent": "thank"}}]',
'[{"prev_action": {"action_name": "action_listen"}, "user": {"intent": "greet"}}]',
'[{"prev_action": {"action_name": "action_listen"}, "user": {"intent": "goodbye"}}]',
'[{"prev_action": {"action_name": "action_listen"}, "user": {"entities": [], "intent": "default"}}]',
'[{"prev_action": {"action_name": "action_listen"}, "user": {"entities": [], "intent": "ask"}}]',
'[{"prev_action": {"action_name": "action_listen"}, "user": {"entities": ["name"], "intent": "greet"}}]',
]
def test_generate_training_data_with_cycles(domain: Domain):
featurizer = MaxHistoryTrackerFeaturizer(SingleStateFeaturizer(), max_history=4)
training_trackers = training.load_data(
"data/test_yaml_stories/stories_with_cycle.yml", domain, augmentation_factor=0,
)
_, label_ids, _ = featurizer.featurize_trackers(
training_trackers, domain, precomputations=None
)
# how many there are depends on the graph which is not created in a
# deterministic way but should always be 3 or 4
assert len(training_trackers) == 3 or len(training_trackers) == 4
# if we have 4 trackers, there is going to be one example more for label 10
num_tens = len(training_trackers) - 1
# if new default actions are added the keys of the actions will be changed
all_label_ids = [id for ids in label_ids for id in ids]
assert Counter(all_label_ids) == {0: 6, 15: 3, 14: num_tens, 1: 2, 16: 1}
async def test_generate_training_data_with_cycles(stories_file: Text,
default_domain: Domain):
featurizer = MaxHistoryTrackerFeaturizer(SingleStateFeaturizer(),
max_history=4)
training_trackers = await training.load_data(stories_file,
default_domain,
augmentation_factor=0)
training_data, label_ids = featurizer.featurize_trackers(
training_trackers, default_domain, interpreter=RegexInterpreter())
# how many there are depends on the graph which is not created in a
# deterministic way but should always be 3 or 4
assert len(training_trackers) == 3 or len(training_trackers) == 4
# if we have 4 trackers, there is going to be one example more for label 10
num_tens = len(training_trackers) - 1
# if new default actions are added the keys of the actions will be changed
all_label_ids = [id for ids in label_ids for id in ids]
assert Counter(all_label_ids) == {0: 6, 12: num_tens, 14: 1, 1: 2, 13: 3}
def test_prediction(
self,
max_history: Optional[int],
model_storage: ModelStorage,
resource: Resource,
execution_context: ExecutionContext,
):
policy = self.create_policy(
featurizer=MaxHistoryTrackerFeaturizer(max_history=max_history),
model_storage=model_storage,
resource=resource,
execution_context=execution_context,
config={POLICY_MAX_HISTORY: max_history},
)
GREET_INTENT_NAME = "greet"
UTTER_GREET_ACTION = "utter_greet"
UTTER_BYE_ACTION = "utter_goodbye"
domain = Domain.from_yaml(f"""
intents:
- {GREET_INTENT_NAME}
actions:
- {UTTER_GREET_ACTION}
- {UTTER_BYE_ACTION}
slots:
slot_1:
type: bool
mappings:
- type: from_text
slot_2:
type: bool
mappings:
- type: from_text
slot_3:
type: bool
mappings:
- type: from_text
slot_4:
type: bool
mappings:
- type: from_text
""")
events = [
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(UTTER_GREET_ACTION),
SlotSet("slot_1", True),
ActionExecuted(UTTER_GREET_ACTION),
SlotSet("slot_2", True),
SlotSet("slot_3", True),
ActionExecuted(UTTER_GREET_ACTION),
ActionExecuted(UTTER_GREET_ACTION),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(UTTER_GREET_ACTION),
SlotSet("slot_4", True),
ActionExecuted(UTTER_BYE_ACTION),
ActionExecuted(ACTION_LISTEN_NAME),
]
training_story = TrackerWithCachedStates.from_events(
"training story", evts=events, domain=domain, slots=domain.slots)
test_story = TrackerWithCachedStates.from_events("training story",
events[:-2],
domain=domain,
slots=domain.slots)
policy.train([training_story], domain)
prediction = policy.predict_action_probabilities(test_story, domain)
assert (domain.action_names_or_texts[prediction.probabilities.index(
max(prediction.probabilities))] == UTTER_BYE_ACTION)
class TestMemoizationPolicy(PolicyTestCollection):
@staticmethod
def _policy_class_to_test() -> Type[PolicyGraphComponent]:
return MemoizationPolicy
@pytest.fixture(scope="class")
def featurizer(self) -> TrackerFeaturizer:
featurizer = MaxHistoryTrackerFeaturizer(None,
max_history=self.max_history)
return featurizer
def test_featurizer(
self,
trained_policy: PolicyGraphComponent,
resource: Resource,
model_storage: ModelStorage,
tmp_path: Path,
execution_context: ExecutionContext,
) -> None:
assert isinstance(trained_policy.featurizer,
MaxHistoryTrackerFeaturizer)
assert trained_policy.featurizer.state_featurizer is None
loaded = trained_policy.__class__.load(
self._config(trained_policy.config),
model_storage,
resource,
execution_context,
)
assert isinstance(loaded.featurizer, MaxHistoryTrackerFeaturizer)
assert loaded.featurizer.state_featurizer is None
def test_memorise(
self,
trained_policy: MemoizationPolicy,
default_domain: Domain,
stories_path: Text,
):
trackers = train_trackers(default_domain,
stories_path,
augmentation_factor=20)
trained_policy.train(trackers, default_domain)
lookup_with_augmentation = trained_policy.lookup
trackers = [
t for t in trackers
if not hasattr(t, "is_augmented") or not t.is_augmented
]
(
all_states,
all_actions,
) = trained_policy.featurizer.training_states_and_labels(
trackers, default_domain)
for tracker, states, actions in zip(trackers, all_states, all_actions):
recalled = trained_policy.recall(states, tracker, default_domain,
None)
assert recalled == actions[0]
nums = np.random.randn(default_domain.num_states)
random_states = [{
f: num
for f, num in zip(default_domain.input_states, nums)
}]
assert trained_policy._recall_states(random_states) is None
# compare augmentation for augmentation_factor of 0 and 20:
trackers_no_augmentation = train_trackers(default_domain,
stories_path,
augmentation_factor=0)
trained_policy.train(trackers_no_augmentation, default_domain)
lookup_no_augmentation = trained_policy.lookup
assert lookup_no_augmentation == lookup_with_augmentation
def test_memorise_with_nlu(self, trained_policy: MemoizationPolicy,
default_domain: Domain):
tracker = tracker_from_dialogue(TEST_DEFAULT_DIALOGUE, default_domain)
states = trained_policy._prediction_states(tracker, default_domain)
recalled = trained_policy.recall(states, tracker, default_domain, None)
assert recalled is not None
def test_finetune_after_load(
self,
trained_policy: MemoizationPolicy,
resource: Resource,
model_storage: ModelStorage,
execution_context: ExecutionContext,
default_domain: Domain,
stories_path: Text,
):
execution_context = dataclasses.replace(execution_context,
is_finetuning=True)
loaded_policy = MemoizationPolicy.load(trained_policy.config,
model_storage, resource,
execution_context)
assert loaded_policy.finetune_mode
new_story = TrackerWithCachedStates.from_events(
"channel",
domain=default_domain,
slots=default_domain.slots,
evts=[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": "why"}),
ActionExecuted("utter_channel"),
ActionExecuted(ACTION_LISTEN_NAME),
],
)
original_train_data = train_trackers(default_domain,
stories_path,
augmentation_factor=20)
loaded_policy.train(
original_train_data + [new_story],
default_domain,
)
# Get the hash of the tracker state of new story
new_story_states, _ = loaded_policy.featurizer.training_states_and_labels(
[new_story], default_domain)
# Feature keys for each new state should be present in the lookup
for states in new_story_states:
state_key = loaded_policy._create_feature_key(states)
assert state_key in loaded_policy.lookup
@pytest.mark.parametrize(
"tracker_events_with_action, tracker_events_without_action",
[
(
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="hello", intent={"name": "greet"}),
ActionExecuted(ACTION_UNLIKELY_INTENT_NAME),
],
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="hello", intent={"name": "greet"}),
],
),
(
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="hello", intent={"name": "greet"}),
EntitiesAdded(entities=[{
"entity": "name",
"value": "Peter"
}]),
SlotSet("name", "Peter"),
ActionExecuted(ACTION_UNLIKELY_INTENT_NAME),
],
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="hello", intent={"name": "greet"}),
SlotSet("name", "Peter"),
EntitiesAdded(entities=[{
"entity": "name",
"value": "Peter"
}]),
],
),
],
)
def test_ignore_action_unlikely_intent(
self,
trained_policy: MemoizationPolicy,
default_domain: Domain,
tracker_events_with_action: List[Event],
tracker_events_without_action: List[Event],
):
tracker_with_action = DialogueStateTracker.from_events(
"test 1",
evts=tracker_events_with_action,
slots=default_domain.slots)
tracker_without_action = DialogueStateTracker.from_events(
"test 2",
evts=tracker_events_without_action,
slots=default_domain.slots)
prediction_with_action = trained_policy.predict_action_probabilities(
tracker_with_action,
default_domain,
)
prediction_without_action = trained_policy.predict_action_probabilities(
tracker_without_action,
default_domain,
)
# Memoization shouldn't be affected with the
# presence of action_unlikely_intent.
assert (prediction_with_action.probabilities ==
prediction_without_action.probabilities)
@pytest.mark.parametrize(
"featurizer_config, tracker_featurizer, state_featurizer",
[
(None, MaxHistoryTrackerFeaturizer(), type(None)),
([], MaxHistoryTrackerFeaturizer(), type(None)),
],
)
def test_empty_featurizer_configs(
self,
featurizer_config: Optional[Dict[Text, Any]],
model_storage: ModelStorage,
resource: Resource,
execution_context: ExecutionContext,
tracker_featurizer: MaxHistoryTrackerFeaturizer,
state_featurizer: Type[SingleStateFeaturizer],
):
featurizer_config_override = ({
"featurizer": featurizer_config
} if featurizer_config else {})
policy = self.create_policy(
None,
model_storage=model_storage,
resource=resource,
execution_context=execution_context,
config=self._config(featurizer_config_override),
)
featurizer = policy.featurizer
assert isinstance(featurizer, tracker_featurizer.__class__)
if featurizer_config:
expected_max_history = featurizer_config[0].get(POLICY_MAX_HISTORY)
else:
expected_max_history = self._config().get(POLICY_MAX_HISTORY)
assert featurizer.max_history == expected_max_history
assert isinstance(featurizer.state_featurizer, state_featurizer)
@pytest.mark.parametrize("max_history", [1, 2, 3, 4, None])
def test_prediction(
self,
max_history: Optional[int],
model_storage: ModelStorage,
resource: Resource,
execution_context: ExecutionContext,
):
policy = self.create_policy(
featurizer=MaxHistoryTrackerFeaturizer(max_history=max_history),
model_storage=model_storage,
resource=resource,
execution_context=execution_context,
)
GREET_INTENT_NAME = "greet"
UTTER_GREET_ACTION = "utter_greet"
UTTER_BYE_ACTION = "utter_goodbye"
domain = Domain.from_yaml(f"""
intents:
- {GREET_INTENT_NAME}
actions:
- {UTTER_GREET_ACTION}
- {UTTER_BYE_ACTION}
slots:
slot_1:
type: bool
slot_2:
type: bool
slot_3:
type: bool
slot_4:
type: bool
""")
events = [
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(UTTER_GREET_ACTION),
SlotSet("slot_1", True),
ActionExecuted(UTTER_GREET_ACTION),
SlotSet("slot_2", True),
SlotSet("slot_3", True),
ActionExecuted(UTTER_GREET_ACTION),
ActionExecuted(UTTER_GREET_ACTION),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(UTTER_GREET_ACTION),
SlotSet("slot_4", True),
ActionExecuted(UTTER_BYE_ACTION),
]
training_story = TrackerWithCachedStates.from_events(
"training story",
evts=events,
domain=domain,
slots=domain.slots,
)
test_story = TrackerWithCachedStates.from_events(
"training story",
events[:-1],
domain=domain,
slots=domain.slots,
)
policy.train([training_story], domain)
prediction = policy.predict_action_probabilities(test_story, domain)
assert (domain.action_names_or_texts[prediction.probabilities.index(
max(prediction.probabilities))] == UTTER_BYE_ACTION)
def _standard_featurizer(max_history: Optional[int] = None) -> TrackerFeaturizer:
return MaxHistoryTrackerFeaturizer(
SingleStateFeaturizer(), max_history=max_history
)
def featurizer(self) -> TrackerFeaturizer:
featurizer = MaxHistoryTrackerFeaturizer(None,
max_history=self.max_history)
return featurizer
def test_aug_pred_without_intent(
self,
max_history: Optional[int],
model_storage: ModelStorage,
resource: Resource,
execution_context: ExecutionContext,
):
"""Tests memoization works for a memoized state sequence that does
not have a user utterance.
"""
policy = self.create_policy(
featurizer=MaxHistoryTrackerFeaturizer(max_history=max_history),
model_storage=model_storage,
resource=resource,
execution_context=execution_context,
config={POLICY_MAX_HISTORY: max_history},
)
GREET_INTENT_NAME = "greet"
GOODBYE_INTENT_NAME = "goodbye"
UTTER_GREET_ACTION = "utter_greet"
UTTER_ACTION_1 = "utter_1"
UTTER_ACTION_2 = "utter_2"
UTTER_ACTION_3 = "utter_3"
UTTER_ACTION_4 = "utter_4"
domain = Domain.from_yaml(f"""
intents:
- {GREET_INTENT_NAME}
- {GOODBYE_INTENT_NAME}
actions:
- {UTTER_GREET_ACTION}
- {UTTER_ACTION_1}
- {UTTER_ACTION_2}
- {UTTER_ACTION_3}
- {UTTER_ACTION_4}
""")
training_story = TrackerWithCachedStates.from_events(
"training story",
[
ActionExecuted(UTTER_ACTION_3),
ActionExecuted(UTTER_ACTION_4),
ActionExecuted(ACTION_LISTEN_NAME),
],
domain=domain,
slots=domain.slots,
)
policy.train([training_story], domain)
test_story = TrackerWithCachedStates.from_events(
"test story",
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(UTTER_ACTION_1),
ActionExecuted(UTTER_ACTION_2),
ActionExecuted(UTTER_ACTION_3),
# ActionExecuted(UTTER_ACTION_4),
],
domain=domain,
slots=domain.slots,
)
prediction = policy.predict_action_probabilities(test_story, domain)
assert (domain.action_names_or_texts[prediction.probabilities.index(
max(prediction.probabilities))] == UTTER_ACTION_4)
async def test_load_multi_file_training_data(
stories_resources: List, default_domain: Domain
):
# the stories file in `data/test_multifile_stories` is the same as in
# `data/test_stories/stories.md`, but split across multiple files
featurizer = MaxHistoryTrackerFeaturizer(SingleStateFeaturizer(), max_history=2)
trackers = await training.load_data(
stories_resources[0], default_domain, augmentation_factor=0
)
(tr_as_sts, tr_as_acts) = featurizer.training_states_and_actions(
trackers, default_domain
)
hashed = []
for sts, acts in zip(tr_as_sts, tr_as_acts):
hashed.append(json.dumps(sts + acts, sort_keys=True))
hashed = sorted(hashed, reverse=True)
data, label_ids = featurizer.featurize_trackers(
trackers, default_domain, interpreter=RegexInterpreter()
)
featurizer_mul = MaxHistoryTrackerFeaturizer(SingleStateFeaturizer(), max_history=2)
trackers_mul = await training.load_data(
stories_resources[1], default_domain, augmentation_factor=0
)
(tr_as_sts_mul, tr_as_acts_mul) = featurizer.training_states_and_actions(
trackers_mul, default_domain
)
hashed_mul = []
for sts_mul, acts_mul in zip(tr_as_sts_mul, tr_as_acts_mul):
hashed_mul.append(json.dumps(sts_mul + acts_mul, sort_keys=True))
hashed_mul = sorted(hashed_mul, reverse=True)
data_mul, label_ids_mul = featurizer_mul.featurize_trackers(
trackers_mul, default_domain, interpreter=RegexInterpreter()
)
assert hashed == hashed_mul
# we check for intents, action names and entities -- the features which
# are included in the story files
data = surface_attributes(data)
data_mul = surface_attributes(data_mul)
for attribute in [INTENT, ACTION_NAME, ENTITIES]:
if attribute not in data or attribute not in data_mul:
continue
assert len(data.get(attribute)) == len(data_mul.get(attribute))
for idx_tracker in range(len(data.get(attribute))):
for idx_dialogue in range(len(data.get(attribute)[idx_tracker])):
f1 = data.get(attribute)[idx_tracker][idx_dialogue]
f2 = data_mul.get(attribute)[idx_tracker][idx_dialogue]
if f1 is None or f2 is None:
assert f1 == f2
continue
for idx_turn in range(len(f1)):
f1 = data.get(attribute)[idx_tracker][idx_dialogue][idx_turn]
f2 = data_mul.get(attribute)[idx_tracker][idx_dialogue][idx_turn]
assert np.all((f1 == f2).data)
assert np.all(label_ids == label_ids_mul)
def test_augmented_prediction(
self,
max_history: Optional[int],
model_storage: ModelStorage,
resource: Resource,
execution_context: ExecutionContext,
):
policy = self.create_policy(
featurizer=MaxHistoryTrackerFeaturizer(max_history=max_history),
model_storage=model_storage,
resource=resource,
execution_context=execution_context,
)
GREET_INTENT_NAME = "greet"
UTTER_GREET_ACTION = "utter_greet"
UTTER_BYE_ACTION = "utter_goodbye"
domain = Domain.from_yaml(f"""
intents:
- {GREET_INTENT_NAME}
actions:
- {UTTER_GREET_ACTION}
- {UTTER_BYE_ACTION}
slots:
slot_1:
type: bool
initial_value: true
slot_2:
type: bool
slot_3:
type: bool
""")
training_story = TrackerWithCachedStates.from_events(
"training story",
[
ActionExecuted(UTTER_GREET_ACTION),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(UTTER_GREET_ACTION),
SlotSet("slot_3", True),
ActionExecuted(UTTER_BYE_ACTION),
],
domain=domain,
slots=domain.slots,
)
test_story = TrackerWithCachedStates.from_events(
"test story",
[
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(UTTER_GREET_ACTION),
SlotSet("slot_1", False),
ActionExecuted(UTTER_GREET_ACTION),
ActionExecuted(UTTER_GREET_ACTION),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(UTTER_GREET_ACTION),
SlotSet("slot_2", True),
ActionExecuted(UTTER_GREET_ACTION),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(UTTER_GREET_ACTION),
SlotSet("slot_3", True),
# ActionExecuted(UTTER_BYE_ACTION),
],
domain=domain,
slots=domain.slots,
)
policy.train([training_story], domain)
prediction = policy.predict_action_probabilities(test_story, domain)
assert (domain.action_names_or_texts[prediction.probabilities.index(
max(prediction.probabilities))] == UTTER_BYE_ACTION)
class TestTEDPolicy(PolicyTestCollection):
@staticmethod
def _policy_class_to_test() -> Type[TEDPolicy]:
return TEDPolicy
def test_train_model_checkpointing(self, tmp_path: Path,
tmp_path_factory: TempPathFactory):
train_core(
domain="data/test_domains/default.yml",
stories="data/test_yaml_stories/stories_defaultdomain.yml",
output=str(tmp_path),
fixed_model_name="my_model.tar.gz",
config="data/test_config/config_ted_policy_model_checkpointing.yml",
)
storage_dir = tmp_path_factory.mktemp("storage dir")
storage, _ = LocalModelStorage.from_model_archive(
storage_dir, tmp_path / "my_model.tar.gz")
checkpoint_dir = get_checkpoint_dir_path(storage_dir)
assert checkpoint_dir.is_dir()
def test_doesnt_checkpoint_with_no_checkpointing(
self, tmp_path: Path, tmp_path_factory: TempPathFactory):
train_core(
domain="data/test_domains/default.yml",
stories="data/test_yaml_stories/stories_defaultdomain.yml",
output=str(tmp_path),
fixed_model_name="my_model.tar.gz",
config=
"data/test_config/config_ted_policy_no_model_checkpointing.yml",
)
storage_dir = tmp_path_factory.mktemp("storage dir")
storage, _ = LocalModelStorage.from_model_archive(
storage_dir, tmp_path / "my_model.tar.gz")
checkpoint_dir = get_checkpoint_dir_path(storage_dir)
assert not checkpoint_dir.is_dir()
def test_doesnt_checkpoint_with_zero_eval_num_examples(
self, tmp_path: Path, tmp_path_factory: TempPathFactory):
checkpoint_dir = get_checkpoint_dir_path(tmp_path)
assert not checkpoint_dir.is_dir()
config_file = "config_ted_policy_model_checkpointing_zero_eval_num_examples.yml"
with pytest.warns(UserWarning) as warning:
train_core(
domain="data/test_domains/default.yml",
stories="data/test_yaml_stories/stories_defaultdomain.yml",
output=str(tmp_path),
fixed_model_name="my_model.tar.gz",
config=f"data/test_config/{config_file}",
)
warn_text = (
f"You have opted to save the best model, but the value of "
f"'{EVAL_NUM_EXAMPLES}' is not greater than 0. No checkpoint model will be "
f"saved.")
assert len([w for w in warning if warn_text in str(w.message)]) == 1
storage_dir = tmp_path_factory.mktemp("storage dir")
storage, _ = LocalModelStorage.from_model_archive(
storage_dir, tmp_path / "my_model.tar.gz")
checkpoint_dir = get_checkpoint_dir_path(storage_dir)
assert not checkpoint_dir.is_dir()
@pytest.mark.parametrize(
"should_finetune, epoch_override, expected_epoch_value",
[
(
True,
TEDPolicy.get_default_config()[EPOCHS] + 1,
TEDPolicy.get_default_config()[EPOCHS] + 1,
),
(
False,
TEDPolicy.get_default_config()[EPOCHS] + 1,
TEDPolicy.get_default_config()[EPOCHS],
), # trained_policy uses default epochs during training
],
)
def test_epoch_override_when_loaded(
self,
trained_policy: TEDPolicy,
should_finetune: bool,
epoch_override: int,
expected_epoch_value: int,
resource: Resource,
model_storage: ModelStorage,
execution_context: ExecutionContext,
):
execution_context.is_finetuning = should_finetune
loaded_policy = trained_policy.__class__.load(
{
**self._config(), EPOCH_OVERRIDE: epoch_override
},
model_storage,
resource,
execution_context,
)
assert loaded_policy.config[EPOCHS] == expected_epoch_value
def test_train_fails_with_checkpoint_zero_eval_num_epochs(
self, tmp_path: Path):
config_file = "config_ted_policy_model_checkpointing_zero_every_num_epochs.yml"
match_string = ("Only values either equal to -1 or greater"
" than 0 are allowed for this parameter.")
with pytest.raises(
InvalidConfigException,
match=match_string,
):
train_core(
domain="data/test_domains/default.yml",
stories="data/test_yaml_stories/stories_defaultdomain.yml",
output=str(tmp_path),
config=f"data/test_config/{config_file}",
)
assert not (tmp_path / "my_model.tar.gz").is_file()
def test_training_with_no_intent(
self,
featurizer: Optional[TrackerFeaturizer],
default_domain: Domain,
tmp_path: Path,
caplog: LogCaptureFixture,
model_storage: ModelStorage,
resource: Resource,
execution_context: ExecutionContext,
):
stories = tmp_path / "stories.yml"
stories.write_text("""
version: "3.0"
stories:
- story: test path
steps:
- action: utter_greet
""")
policy = self.create_policy(
featurizer=featurizer,
model_storage=model_storage,
resource=resource,
execution_context=execution_context,
)
import tests.core.test_policies
training_trackers = tests.core.test_policies.train_trackers(
default_domain, str(stories), augmentation_factor=20)
with pytest.raises(RasaException) as e:
policy.train(training_trackers,
default_domain,
precomputations=None)
assert "No user features specified. Cannot train 'TED' model." == str(
e.value)
def test_similarity_type(self, trained_policy: TEDPolicy):
assert trained_policy.config[SIMILARITY_TYPE] == "inner"
def test_ranking_length(self, trained_policy: TEDPolicy):
assert trained_policy.config[RANKING_LENGTH] == 0
def test_ranking_length_and_renormalization(
self,
trained_policy: TEDPolicy,
tracker: DialogueStateTracker,
default_domain: Domain,
monkeypatch: MonkeyPatch,
):
precomputations = None
prediction = trained_policy.predict_action_probabilities(
tracker,
default_domain,
precomputations,
)
# first check the output is what we expect
assert not prediction.is_end_to_end_prediction
# check that ranking length is applied - without normalization
if trained_policy.config[RANKING_LENGTH] == 0:
assert sum([confidence for confidence in prediction.probabilities
]) == pytest.approx(1)
assert all(confidence > 0
for confidence in prediction.probabilities)
else:
assert (sum([
confidence > 0 for confidence in prediction.probabilities
]) == trained_policy.config[RANKING_LENGTH])
assert sum([confidence for confidence in prediction.probabilities
]) != pytest.approx(1)
def test_label_data_assembly(self, trained_policy: TEDPolicy,
default_domain: Domain):
state_featurizer = trained_policy.featurizer.state_featurizer
encoded_all_labels = state_featurizer.encode_all_labels(
default_domain, precomputations=None)
attribute_data, _ = model_data_utils.convert_to_data_format(
encoded_all_labels)
assembled_label_data = trained_policy._assemble_label_data(
attribute_data, default_domain)
assembled_label_data_signature = assembled_label_data.get_signature()
assert list(assembled_label_data_signature.keys()) == [
f"{LABEL}_{ACTION_NAME}",
f"{LABEL}",
]
assert assembled_label_data.num_examples == default_domain.num_actions
assert list(assembled_label_data_signature[f"{LABEL}_{ACTION_NAME}"].
keys()) == [
MASK,
SENTENCE,
]
assert list(assembled_label_data_signature[LABEL].keys()) == [IDS]
assert (assembled_label_data_signature[f"{LABEL}_{ACTION_NAME}"]
[SENTENCE][0].units == default_domain.num_actions)
def test_gen_batch(self, trained_policy: TEDPolicy, default_domain: Domain,
stories_path: Path):
training_trackers = tests.core.test_policies.train_trackers(
default_domain, stories_path, augmentation_factor=0)
precomputations = None
training_data, label_ids, entity_tags = trained_policy._featurize_for_training(
training_trackers,
default_domain,
precomputations,
)
_, all_labels = trained_policy._create_label_data(
default_domain, precomputations)
model_data = trained_policy._create_model_data(training_data,
label_ids, entity_tags,
all_labels)
batch_size = 2
data_generator = RasaBatchDataGenerator(model_data,
batch_size=batch_size,
shuffle=False,
batch_strategy="sequence")
iterator = iter(data_generator)
# model data keys were sorted, so the order is alphabetical
(
(
batch_action_name_mask,
_,
_,
batch_action_name_sentence_shape,
batch_dialogue_length,
batch_entities_mask,
_,
_,
batch_entities_sentence_shape,
batch_intent_mask,
_,
_,
batch_intent_sentence_shape,
batch_label_ids,
batch_slots_mask,
_,
_,
batch_slots_sentence_shape,
),
_,
) = next(iterator)
assert (batch_label_ids.shape[0] == batch_size
and batch_dialogue_length.shape[0] == batch_size)
# batch and dialogue dimensions are NOT combined for masks
assert (batch_slots_mask.shape[0] == batch_size
and batch_intent_mask.shape[0] == batch_size
and batch_entities_mask.shape[0] == batch_size
and batch_action_name_mask.shape[0] == batch_size)
# some features might be "fake" so there sequence is `0`
seq_len = max([
batch_intent_sentence_shape[1],
batch_action_name_sentence_shape[1],
batch_entities_sentence_shape[1],
batch_slots_sentence_shape[1],
])
assert (batch_intent_sentence_shape[1] == seq_len
or batch_intent_sentence_shape[1] == 0)
assert (batch_action_name_sentence_shape[1] == seq_len
or batch_action_name_sentence_shape[1] == 0)
assert (batch_entities_sentence_shape[1] == seq_len
or batch_entities_sentence_shape[1] == 0)
assert (batch_slots_sentence_shape[1] == seq_len
or batch_slots_sentence_shape[1] == 0)
data_generator = RasaBatchDataGenerator(model_data,
batch_size=batch_size,
shuffle=True,
batch_strategy="balanced")
iterator = iter(data_generator)
(
(
batch_action_name_mask,
_,
_,
batch_action_name_sentence_shape,
batch_dialogue_length,
batch_entities_mask,
_,
_,
batch_entities_sentence_shape,
batch_intent_mask,
_,
_,
batch_intent_sentence_shape,
batch_label_ids,
batch_slots_mask,
_,
_,
batch_slots_sentence_shape,
),
_,
) = next(iterator)
assert (batch_label_ids.shape[0] == batch_size
and batch_dialogue_length.shape[0] == batch_size)
# some features might be "fake" so there sequence is `0`
seq_len = max([
batch_intent_sentence_shape[1],
batch_action_name_sentence_shape[1],
batch_entities_sentence_shape[1],
batch_slots_sentence_shape[1],
])
assert (batch_intent_sentence_shape[1] == seq_len
or batch_intent_sentence_shape[1] == 0)
assert (batch_action_name_sentence_shape[1] == seq_len
or batch_action_name_sentence_shape[1] == 0)
assert (batch_entities_sentence_shape[1] == seq_len
or batch_entities_sentence_shape[1] == 0)
assert (batch_slots_sentence_shape[1] == seq_len
or batch_slots_sentence_shape[1] == 0)
@pytest.mark.parametrize(
"tracker_events_with_action, tracker_events_without_action",
[
(
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="hello", intent={"name": "greet"}),
ActionExecuted(ACTION_UNLIKELY_INTENT_NAME),
],
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="hello", intent={"name": "greet"}),
],
),
(
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="hello", intent={"name": "greet"}),
EntitiesAdded(entities=[
{
"entity": "name",
"value": "Peter"
},
]),
ActionExecuted(ACTION_UNLIKELY_INTENT_NAME),
ActionExecuted("utter_greet"),
],
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="hello", intent={"name": "greet"}),
EntitiesAdded(entities=[
{
"entity": "name",
"value": "Peter"
},
]),
ActionExecuted("utter_greet"),
],
),
(
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="hello", intent={"name": "greet"}),
ActionExecuted(ACTION_UNLIKELY_INTENT_NAME),
ActionExecuted("some_form"),
ActiveLoop("some_form"),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="default", intent={"name": "default"}),
ActionExecuted(ACTION_UNLIKELY_INTENT_NAME),
],
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="hello", intent={"name": "greet"}),
ActionExecuted(ACTION_UNLIKELY_INTENT_NAME),
ActionExecuted("some_form"),
ActiveLoop("some_form"),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="default", intent={"name": "default"}),
],
),
],
)
def test_ignore_action_unlikely_intent(
self,
trained_policy: TEDPolicy,
default_domain: Domain,
tracker_events_with_action: List[Event],
tracker_events_without_action: List[Event],
):
precomputations = None
tracker_with_action = DialogueStateTracker.from_events(
"test 1", evts=tracker_events_with_action)
tracker_without_action = DialogueStateTracker.from_events(
"test 2", evts=tracker_events_without_action)
prediction_with_action = trained_policy.predict_action_probabilities(
tracker_with_action,
default_domain,
precomputations,
)
prediction_without_action = trained_policy.predict_action_probabilities(
tracker_without_action,
default_domain,
precomputations,
)
# If the weights didn't change then both trackers
# should result in same prediction.
assert (prediction_with_action.probabilities ==
prediction_without_action.probabilities)
@pytest.mark.parametrize(
"featurizer_config, tracker_featurizer, state_featurizer",
[
(None, MaxHistoryTrackerFeaturizer(), SingleStateFeaturizer),
([], MaxHistoryTrackerFeaturizer(), SingleStateFeaturizer),
],
)
def test_empty_featurizer_configs(
self,
featurizer_config: Optional[Dict[Text, Any]],
model_storage: ModelStorage,
resource: Resource,
execution_context: ExecutionContext,
tracker_featurizer: MaxHistoryTrackerFeaturizer,
state_featurizer: Type[SingleStateFeaturizer],
):
featurizer_config_override = ({
"featurizer": featurizer_config
} if featurizer_config else {})
policy = self.create_policy(
None,
model_storage=model_storage,
resource=resource,
execution_context=execution_context,
config=self._config(featurizer_config_override),
)
featurizer = policy.featurizer
assert isinstance(featurizer, tracker_featurizer.__class__)
if featurizer_config:
expected_max_history = featurizer_config[0].get(POLICY_MAX_HISTORY)
else:
expected_max_history = self._config().get(POLICY_MAX_HISTORY)
assert featurizer.max_history == expected_max_history
assert isinstance(featurizer.state_featurizer, state_featurizer)
def test_augmented_prediction_across_max_history_actions(
self,
max_history: Optional[int],
model_storage: ModelStorage,
resource: Resource,
execution_context: ExecutionContext,
):
"""Tests that the last user utterance is preserved in action states
even when the utterance occurs prior to `max_history` actions in the
past.
"""
policy = self.create_policy(
featurizer=MaxHistoryTrackerFeaturizer(max_history=max_history),
model_storage=model_storage,
resource=resource,
execution_context=execution_context,
config={POLICY_MAX_HISTORY: max_history},
)
GREET_INTENT_NAME = "greet"
UTTER_GREET_ACTION = "utter_greet"
UTTER_ACTION_1 = "utter_1"
UTTER_ACTION_2 = "utter_2"
UTTER_ACTION_3 = "utter_3"
UTTER_ACTION_4 = "utter_4"
UTTER_ACTION_5 = "utter_5"
UTTER_BYE_ACTION = "utter_goodbye"
domain = Domain.from_yaml(f"""
intents:
- {GREET_INTENT_NAME}
actions:
- {UTTER_GREET_ACTION}
- {UTTER_ACTION_1}
- {UTTER_ACTION_2}
- {UTTER_ACTION_3}
- {UTTER_ACTION_4}
- {UTTER_ACTION_5}
- {UTTER_BYE_ACTION}
""")
training_story = TrackerWithCachedStates.from_events(
"training story",
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(UTTER_ACTION_1),
ActionExecuted(UTTER_ACTION_2),
ActionExecuted(UTTER_ACTION_3),
ActionExecuted(UTTER_ACTION_4),
ActionExecuted(UTTER_ACTION_5),
ActionExecuted(UTTER_BYE_ACTION),
ActionExecuted(ACTION_LISTEN_NAME),
],
domain=domain,
slots=domain.slots,
)
test_story = TrackerWithCachedStates.from_events(
"test story",
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(UTTER_ACTION_1),
ActionExecuted(UTTER_ACTION_2),
ActionExecuted(UTTER_ACTION_3),
ActionExecuted(UTTER_ACTION_4),
ActionExecuted(UTTER_ACTION_5),
# ActionExecuted(UTTER_BYE_ACTION),
],
domain=domain,
slots=domain.slots,
)
policy.train([training_story], domain)
prediction = policy.predict_action_probabilities(test_story, domain)
assert (domain.action_names_or_texts[prediction.probabilities.index(
max(prediction.probabilities))] == UTTER_BYE_ACTION)
def _standard_featurizer(self) -> MaxHistoryTrackerFeaturizer:
# Memoization policy always uses MaxHistoryTrackerFeaturizer
# without state_featurizer
return MaxHistoryTrackerFeaturizer(
state_featurizer=None, max_history=self.config[POLICY_MAX_HISTORY])
class PolicyTestCollection:
"""Tests every policy needs to fulfill.
Each policy can declare further tests on its own."""
@staticmethod
def _policy_class_to_test() -> Type[PolicyGraphComponent]:
raise NotImplementedError
max_history = 3 # this is the amount of history we test on
@pytest.fixture(scope="class")
def resource(self, ) -> Resource:
return Resource(uuid.uuid4().hex)
@pytest.fixture(scope="class")
def model_storage(self, tmp_path_factory: TempPathFactory) -> ModelStorage:
return LocalModelStorage(tmp_path_factory.mktemp(uuid.uuid4().hex))
@pytest.fixture(scope="class")
def execution_context(self) -> ExecutionContext:
return ExecutionContext(GraphSchema({}), uuid.uuid4().hex)
def _config(
self,
config_override: Optional[Dict[Text,
Any]] = None) -> Dict[Text, Any]:
config_override = config_override or {}
config = self._policy_class_to_test().get_default_config()
return {**config, **config_override}
def create_policy(
self,
featurizer: Optional[TrackerFeaturizer],
model_storage: ModelStorage,
resource: Resource,
execution_context: ExecutionContext,
config: Optional[Dict[Text, Any]] = None,
) -> PolicyGraphComponent:
return self._policy_class_to_test()(
config=self._config(config),
model_storage=model_storage,
resource=resource,
execution_context=execution_context,
featurizer=featurizer,
)
@pytest.fixture(scope="class")
def featurizer(self) -> TrackerFeaturizer:
featurizer = MaxHistoryTrackerFeaturizer(SingleStateFeaturizer(),
max_history=self.max_history)
return featurizer
@pytest.fixture(scope="class")
def default_domain(self, domain_path: Text) -> Domain:
return Domain.load(domain_path)
@pytest.fixture(scope="class")
def tracker(self, default_domain: Domain) -> DialogueStateTracker:
return DialogueStateTracker(DEFAULT_SENDER_ID, default_domain.slots)
@pytest.fixture(scope="class")
def trained_policy(
self,
featurizer: Optional[TrackerFeaturizer],
stories_path: Text,
default_domain: Domain,
model_storage: ModelStorage,
resource: Resource,
execution_context: ExecutionContext,
) -> PolicyGraphComponent:
policy = self.create_policy(featurizer, model_storage, resource,
execution_context)
training_trackers = train_trackers(default_domain,
stories_path,
augmentation_factor=20)
policy.train(training_trackers, default_domain)
return policy
def test_featurizer(
self,
trained_policy: PolicyGraphComponent,
resource: Resource,
model_storage: ModelStorage,
tmp_path: Path,
execution_context: ExecutionContext,
):
assert isinstance(trained_policy.featurizer,
MaxHistoryTrackerFeaturizer)
assert trained_policy.featurizer.max_history == self.max_history
assert isinstance(trained_policy.featurizer.state_featurizer,
SingleStateFeaturizer)
loaded = trained_policy.__class__.load(
self._config(trained_policy.config),
model_storage,
resource,
execution_context,
)
assert isinstance(loaded.featurizer, MaxHistoryTrackerFeaturizer)
assert loaded.featurizer.max_history == self.max_history
assert isinstance(loaded.featurizer.state_featurizer,
SingleStateFeaturizer)
@pytest.mark.parametrize("should_finetune", [False, True])
def test_persist_and_load(
self,
trained_policy: PolicyGraphComponent,
default_domain: Domain,
should_finetune: bool,
stories_path: Text,
model_storage: ModelStorage,
resource: Resource,
execution_context: ExecutionContext,
):
loaded = trained_policy.__class__.load(
self._config(trained_policy.config),
model_storage,
resource,
dataclasses.replace(execution_context,
is_finetuning=should_finetune),
)
assert loaded.finetune_mode == should_finetune
trackers = train_trackers(default_domain,
stories_path,
augmentation_factor=20)
for tracker in trackers:
predicted_probabilities = loaded.predict_action_probabilities(
tracker, default_domain)
actual_probabilities = trained_policy.predict_action_probabilities(
tracker, default_domain)
assert predicted_probabilities == actual_probabilities
def test_prediction_on_empty_tracker(self, trained_policy: Policy,
default_domain: Domain):
tracker = DialogueStateTracker(DEFAULT_SENDER_ID, default_domain.slots)
prediction = trained_policy.predict_action_probabilities(
tracker,
default_domain,
)
assert not prediction.is_end_to_end_prediction
assert len(prediction.probabilities) == default_domain.num_actions
assert max(prediction.probabilities) <= 1.0
assert min(prediction.probabilities) >= 0.0
@pytest.mark.filterwarnings(
"ignore:.*without a trained model present.*:UserWarning")
def test_persist_and_load_empty_policy(
self,
default_domain: Domain,
default_model_storage: ModelStorage,
execution_context: ExecutionContext,
):
resource = Resource(uuid.uuid4().hex)
empty_policy = self.create_policy(
None,
default_model_storage,
resource,
execution_context,
)
empty_policy.train([], default_domain)
loaded = empty_policy.__class__.load(
self._config(),
default_model_storage,
resource,
execution_context,
)
assert loaded is not None
@staticmethod
def _get_next_action(policy: PolicyGraphComponent, events: List[Event],
domain: Domain) -> Text:
tracker = get_tracker(events)
scores = policy.predict_action_probabilities(
tracker,
domain,
).probabilities
index = scores.index(max(scores))
return domain.action_names_or_texts[index]
@pytest.mark.parametrize(
"featurizer_config, tracker_featurizer, state_featurizer",
[
(
[{
# TODO: remove "2" when migration of policies is done
"name": "MaxHistoryTrackerFeaturizer2",
"max_history": 12,
"state_featurizer": [],
}],
MaxHistoryTrackerFeaturizer(max_history=12),
type(None),
),
(
# TODO: remove "2" when migration of policies is done
[{
"name": "MaxHistoryTrackerFeaturizer2",
"max_history": 12
}],
MaxHistoryTrackerFeaturizer(max_history=12),
type(None),
),
(
[{
# TODO: remove "2" when migration of policies is done
"name":
"IntentMaxHistoryTrackerFeaturizer2",
"max_history":
12,
"state_featurizer": [{
"name":
"IntentTokenizerSingleStateFeaturizer2"
}],
}],
IntentMaxHistoryTrackerFeaturizer(max_history=12),
IntentTokenizerSingleStateFeaturizer,
),
],
)
def test_different_featurizer_configs(
self,
featurizer_config: Optional[Dict[Text, Any]],
model_storage: ModelStorage,
resource: Resource,
execution_context: ExecutionContext,
tracker_featurizer: MaxHistoryTrackerFeaturizer,
state_featurizer: Type[SingleStateFeaturizer],
):
featurizer_config_override = ({
"featurizer": featurizer_config
} if featurizer_config else {})
policy = self.create_policy(
None,
model_storage=model_storage,
resource=resource,
execution_context=execution_context,
config=self._config(featurizer_config_override),
)
featurizer = policy.featurizer
assert isinstance(featurizer, tracker_featurizer.__class__)
if featurizer_config:
expected_max_history = featurizer_config[0].get(POLICY_MAX_HISTORY)
else:
expected_max_history = self._config().get(POLICY_MAX_HISTORY)
assert featurizer.max_history == expected_max_history
assert isinstance(featurizer.state_featurizer, state_featurizer)
@pytest.mark.parametrize(
"featurizer_config",
[
[
# TODO: remove "2" when migration of policies is done
{
"name": "MaxHistoryTrackerFeaturizer2",
"max_history": 12
},
{
"name": "MaxHistoryTrackerFeaturizer2",
"max_history": 12
},
],
[{
# TODO: remove "2" when migration of policies is done
"name":
"IntentMaxHistoryTrackerFeaturizer2",
"max_history":
12,
"state_featurizer": [
{
"name": "IntentTokenizerSingleStateFeaturizer2"
},
{
"name": "IntentTokenizerSingleStateFeaturizer2"
},
],
}],
],
)
def test_different_invalid_featurizer_configs(
self,
trained_policy: PolicyGraphComponent,
featurizer_config: Optional[Dict[Text, Any]],
model_storage: ModelStorage,
resource: Resource,
execution_context: ExecutionContext,
):
with pytest.raises(InvalidPolicyConfig):
self.create_policy(
None,
model_storage=model_storage,
resource=resource,
execution_context=execution_context,
config={"featurizer": featurizer_config},
)
def _standard_featurizer(self) -> MaxHistoryTrackerFeaturizer:
"""Initializes the standard featurizer for this policy."""
return MaxHistoryTrackerFeaturizer(
SingleStateFeaturizer(), self.config.get(POLICY_MAX_HISTORY)
)
def featurizer(self) -> TrackerFeaturizer:
featurizer = MaxHistoryTrackerFeaturizer(
SingleStateFeaturizer(), max_history=self.max_history
)
return featurizer
def _standard_featurizer() -> MaxHistoryTrackerFeaturizer:
return MaxHistoryTrackerFeaturizer(SingleStateFeaturizer())
def test_aug_pred_sensitive_to_intent_across_max_history_actions(
self,
max_history: Optional[int],
model_storage: ModelStorage,
resource: Resource,
execution_context: ExecutionContext,
):
"""Tests that only the most recent user utterance propagates to state
creation of following actions.
"""
policy = self.create_policy(
featurizer=MaxHistoryTrackerFeaturizer(max_history=max_history),
model_storage=model_storage,
resource=resource,
execution_context=execution_context,
config={POLICY_MAX_HISTORY: max_history},
)
GREET_INTENT_NAME = "greet"
GOODBYE_INTENT_NAME = "goodbye"
UTTER_GREET_ACTION = "utter_greet"
UTTER_ACTION_1 = "utter_1"
UTTER_ACTION_2 = "utter_2"
UTTER_ACTION_3 = "utter_3"
UTTER_ACTION_4 = "utter_4"
UTTER_ACTION_5 = "utter_5"
UTTER_BYE_ACTION = "utter_goodbye"
domain = Domain.from_yaml(f"""
intents:
- {GREET_INTENT_NAME}
- {GOODBYE_INTENT_NAME}
actions:
- {UTTER_GREET_ACTION}
- {UTTER_ACTION_1}
- {UTTER_ACTION_2}
- {UTTER_ACTION_3}
- {UTTER_ACTION_4}
- {UTTER_ACTION_5}
- {UTTER_BYE_ACTION}
""")
training_story = TrackerWithCachedStates.from_events(
"training story",
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(UTTER_ACTION_1),
ActionExecuted(UTTER_ACTION_2),
ActionExecuted(UTTER_ACTION_3),
ActionExecuted(UTTER_ACTION_4),
ActionExecuted(UTTER_ACTION_5),
ActionExecuted(UTTER_BYE_ACTION),
ActionExecuted(ACTION_LISTEN_NAME),
],
domain=domain,
slots=domain.slots,
)
test_story1 = TrackerWithCachedStates.from_events(
"test story",
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": GOODBYE_INTENT_NAME}),
ActionExecuted(UTTER_BYE_ACTION),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(UTTER_ACTION_1),
ActionExecuted(UTTER_ACTION_2),
ActionExecuted(UTTER_ACTION_3),
ActionExecuted(UTTER_ACTION_4),
ActionExecuted(UTTER_ACTION_5),
# ActionExecuted(UTTER_BYE_ACTION),
],
domain=domain,
slots=domain.slots,
)
policy.train([training_story], domain)
prediction1 = policy.predict_action_probabilities(test_story1, domain)
assert (domain.action_names_or_texts[prediction1.probabilities.index(
max(prediction1.probabilities))] == UTTER_BYE_ACTION)
test_story2_no_match_expected = TrackerWithCachedStates.from_events(
"test story",
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(UTTER_BYE_ACTION),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": GOODBYE_INTENT_NAME}),
ActionExecuted(UTTER_ACTION_1),
ActionExecuted(UTTER_ACTION_2),
ActionExecuted(UTTER_ACTION_3),
ActionExecuted(UTTER_ACTION_4),
ActionExecuted(UTTER_ACTION_5),
# No prediction should be made here.
],
domain=domain,
slots=domain.slots,
)
prediction2 = policy.predict_action_probabilities(
test_story2_no_match_expected,
domain,
)
assert all([prob == 0.0 for prob in prediction2.probabilities])
