async def test_create_train_data_with_history(default_domain): featurizer = MaxHistoryTrackerFeaturizer(max_history=4) training_trackers = await training.load_data(DEFAULT_STORIES_FILE, default_domain, augmentation_factor=0) assert len(training_trackers) == 3 (decoded, _) = featurizer.training_states_and_actions(training_trackers, default_domain) # decoded needs to be sorted hashed = [] for states in decoded: hashed.append(json.dumps(states, sort_keys=True)) hashed = sorted(hashed) assert hashed == [ '[{"prev_action": {"action_name": "action_listen"}, "slots": {"name": [1.0]}, "user": {"entities": ["name"], "intent": "greet"}}, {"prev_action": {"action_name": "utter_greet"}, "slots": {"name": [1.0]}, "user": {"entities": ["name"], "intent": "greet"}}, {"prev_action": {"action_name": "action_listen"}, "slots": {"name": [1.0]}, "user": {"intent": "default"}}, {"prev_action": {"action_name": "utter_default"}, "slots": {"name": [1.0]}, "user": {"intent": "default"}}]', '[{"prev_action": {"action_name": "action_listen"}, "user": {"intent": "default"}}, {"prev_action": {"action_name": "utter_default"}, "user": {"intent": "default"}}, {"prev_action": {"action_name": "action_listen"}, "user": {"intent": "goodbye"}}, {"prev_action": {"action_name": "utter_goodbye"}, "user": {"intent": "goodbye"}}]', '[{"prev_action": {"action_name": "action_listen"}, "user": {"intent": "greet"}}, {"prev_action": {"action_name": "utter_greet"}, "user": {"intent": "greet"}}, {"prev_action": {"action_name": "action_listen"}, "user": {"intent": "default"}}, {"prev_action": {"action_name": "utter_default"}, "user": {"intent": "default"}}]', '[{"prev_action": {"action_name": "utter_greet"}, "user": {"intent": "greet"}}, {"prev_action": {"action_name": "action_listen"}, "user": {"intent": "default"}}, {"prev_action": {"action_name": "utter_default"}, "user": {"intent": "default"}}, {"prev_action": {"action_name": "action_listen"}, "user": {"intent": "goodbye"}}]', '[{}, {"prev_action": {"action_name": "action_listen"}, "slots": {"name": [1.0]}, "user": {"entities": ["name"], "intent": "greet"}}, {"prev_action": {"action_name": "utter_greet"}, "slots": {"name": [1.0]}, "user": {"entities": ["name"], "intent": "greet"}}, {"prev_action": {"action_name": "action_listen"}, "slots": {"name": [1.0]}, "user": {"intent": "default"}}]', '[{}, {"prev_action": {"action_name": "action_listen"}, "slots": {"name": [1.0]}, "user": {"entities": ["name"], "intent": "greet"}}, {"prev_action": {"action_name": "utter_greet"}, "slots": {"name": [1.0]}, "user": {"entities": ["name"], "intent": "greet"}}]', '[{}, {"prev_action": {"action_name": "action_listen"}, "slots": {"name": [1.0]}, "user": {"entities": ["name"], "intent": "greet"}}]', '[{}, {"prev_action": {"action_name": "action_listen"}, "user": {"intent": "greet"}}, {"prev_action": {"action_name": "utter_greet"}, "user": {"intent": "greet"}}, {"prev_action": {"action_name": "action_listen"}, "user": {"intent": "default"}}]', '[{}, {"prev_action": {"action_name": "action_listen"}, "user": {"intent": "greet"}}, {"prev_action": {"action_name": "utter_greet"}, "user": {"intent": "greet"}}]', '[{}, {"prev_action": {"action_name": "action_listen"}, "user": {"intent": "greet"}}]', "[{}]", ]
async def test_create_train_data_no_history(domain: Domain, stories_path: Text): featurizer = MaxHistoryTrackerFeaturizer(max_history=1) training_trackers = await training.load_data(stories_path, domain, augmentation_factor=0) assert len(training_trackers) == 4 (decoded, _) = featurizer.training_states_and_actions(training_trackers, domain) # decoded needs to be sorted hashed = [] for states in decoded: hashed.append(json.dumps(states, sort_keys=True)) hashed = sorted(hashed, reverse=True) assert hashed == [ "[{}]", '[{"prev_action": {"action_name": "utter_greet"}, "user": {"intent": "greet"}}]', '[{"prev_action": {"action_name": "utter_greet"}, "slots": {"name": [1.0]}, "user": {"entities": ["name"], "intent": "greet"}}]', '[{"prev_action": {"action_name": "utter_goodbye"}, "user": {"intent": "goodbye"}}]', '[{"prev_action": {"action_name": "utter_default"}, "user": {"intent": "default"}}]', '[{"prev_action": {"action_name": "utter_default"}, "slots": {"name": [1.0]}, "user": {"intent": "default"}}]', '[{"prev_action": {"action_name": "action_listen"}, "user": {"intent": "greet"}}]', '[{"prev_action": {"action_name": "action_listen"}, "user": {"intent": "goodbye"}}]', '[{"prev_action": {"action_name": "action_listen"}, "user": {"intent": "default"}}]', '[{"prev_action": {"action_name": "action_listen"}, "slots": {"name": [1.0]}, "user": {"intent": "default"}}]', '[{"prev_action": {"action_name": "action_listen"}, "slots": {"name": [1.0]}, "user": {"entities": ["name"], "intent": "greet"}}]', ]
def test_persist_and_load_tracker_featurizer(tmp_path: Text, moodbot_domain: Domain): state_featurizer = SingleStateFeaturizer() state_featurizer.prepare_for_training(moodbot_domain, RegexInterpreter()) tracker_featurizer = MaxHistoryTrackerFeaturizer(state_featurizer) tracker_featurizer.persist(tmp_path) loaded_tracker_featurizer = TrackerFeaturizer.load(tmp_path) assert loaded_tracker_featurizer is not None assert loaded_tracker_featurizer.state_featurizer is not None
def test_slots_states_before_user_utterance(default_domain): featurizer = MaxHistoryTrackerFeaturizer() tracker = DialogueStateTracker.from_events( "bla", evts=[ SlotSet(default_domain.slots[0].name, "some_value"), ActionExecuted("utter_default"), ], slots=default_domain.slots, ) trackers_as_states, _ = featurizer.training_states_and_actions( [tracker], default_domain) expected_states = [[{"slots": {"name": (1.0, )}}]] assert trackers_as_states == expected_states
def test_load_multi_file_training_data(domain: Domain): featurizer = MaxHistoryTrackerFeaturizer(SingleStateFeaturizer(), max_history=2) trackers = training.load_data( "data/test_yaml_stories/stories.yml", domain, augmentation_factor=0 ) trackers = sorted(trackers, key=lambda t: t.sender_id) (tr_as_sts, tr_as_acts) = featurizer.training_states_and_labels(trackers, domain) hashed = [] for sts, acts in zip(tr_as_sts, tr_as_acts): hashed.append(json.dumps(sts + acts, sort_keys=True)) hashed = sorted(hashed, reverse=True) data, label_ids, _ = featurizer.featurize_trackers( trackers, domain, precomputations=None ) featurizer_mul = MaxHistoryTrackerFeaturizer(SingleStateFeaturizer(), max_history=2) trackers_mul = training.load_data( "data/test_multifile_yaml_stories", domain, augmentation_factor=0 ) trackers_mul = sorted(trackers_mul, key=lambda t: t.sender_id) (tr_as_sts_mul, tr_as_acts_mul) = featurizer.training_states_and_labels( trackers_mul, domain ) hashed_mul = [] for sts_mul, acts_mul in zip(tr_as_sts_mul, tr_as_acts_mul): hashed_mul.append(json.dumps(sts_mul + acts_mul, sort_keys=True)) hashed_mul = sorted(hashed_mul, reverse=True) data_mul, label_ids_mul, _ = featurizer_mul.featurize_trackers( trackers_mul, domain, precomputations=None ) assert hashed == hashed_mul # we check for intents, action names and entities -- the features which # are included in the story files data = _surface_attributes(data) data_mul = _surface_attributes(data_mul) for attribute in [INTENT, ACTION_NAME, ENTITIES]: if attribute not in data or attribute not in data_mul: continue assert len(data.get(attribute)) == len(data_mul.get(attribute)) for idx_tracker in range(len(data.get(attribute))): for idx_dialogue in range(len(data.get(attribute)[idx_tracker])): f1 = data.get(attribute)[idx_tracker][idx_dialogue] f2 = data_mul.get(attribute)[idx_tracker][idx_dialogue] if f1 is None or f2 is None: assert f1 == f2 continue for idx_turn in range(len(f1)): f1 = data.get(attribute)[idx_tracker][idx_dialogue][idx_turn] f2 = data_mul.get(attribute)[idx_tracker][idx_dialogue][idx_turn] assert np.all((f1 == f2).data) assert np.all(label_ids == label_ids_mul)
def test_featurize_trackers_with_max_history_tracker_featurizer( moodbot_domain: Domain): state_featurizer = SingleStateFeaturizer() tracker_featurizer = MaxHistoryTrackerFeaturizer(state_featurizer) tracker = tracker_from_dialogue_file("data/test_dialogues/moodbot.json", moodbot_domain) state_features, labels, entity_tags = tracker_featurizer.featurize_trackers( [tracker], moodbot_domain, RegexInterpreter()) assert state_features is not None assert len(state_features) == 7 assert labels is not None assert len(labels) == 7 # moodbot doesn't contain e2e entities assert not any([any(turn_tags) for turn_tags in entity_tags])
def _standard_featurizer( max_history: int = DEFAULT_MAX_HISTORY, ) -> MaxHistoryTrackerFeaturizer: # Sklearn policy always uses MaxHistoryTrackerFeaturizer return MaxHistoryTrackerFeaturizer( state_featurizer=SingleStateFeaturizer(), max_history=5 )
def _sliced_states_iterator( trackers: List[TrackerWithCachedStates], domain: Domain, max_history: Optional[int], tokenizer: Optional[Tokenizer], ) -> Generator[TrackerEventStateTuple, None, None]: """Creates an iterator over sliced states. Iterate over all given trackers and all sliced states within each tracker, where the slicing is based on `max_history`. Args: trackers: List of trackers. domain: Domain (used for tracker.past_states). max_history: Assumed `max_history` value for slicing. tokenizer: A tokenizer to tokenize the user messages. Yields: A (tracker, event, sliced_states) triplet. """ for tracker in trackers: states = tracker.past_states(domain) idx = 0 for event in tracker.events: if isinstance(event, ActionExecuted): sliced_states = MaxHistoryTrackerFeaturizer.slice_state_history( states[:idx + 1], max_history) if tokenizer: _apply_tokenizer_to_states(tokenizer, sliced_states) # TODO: deal with oov (different tokens can lead to identical features # if some of those tokens are out of vocabulary for all featurizers) yield TrackerEventStateTuple(tracker, event, sliced_states) idx += 1
def _sliced_states_iterator( trackers: List[TrackerWithCachedStates], domain: Domain, max_history: int ) -> Generator[TrackerEventStateTuple, None, None]: """Creates an iterator over sliced states. Iterate over all given trackers and all sliced states within each tracker, where the slicing is based on `max_history`. Args: trackers: List of trackers. domain: Domain (used for tracker.past_states). max_history: Assumed `max_history` value for slicing. Yields: A (tracker, event, sliced_states) triplet. """ for tracker in trackers: states = tracker.past_states(domain) idx = 0 for event in tracker.events: if isinstance(event, ActionExecuted): sliced_states = MaxHistoryTrackerFeaturizer.slice_state_history( states[: idx + 1], max_history ) yield TrackerEventStateTuple(tracker, event, sliced_states) idx += 1
def _standard_featurizer( max_history: Optional[int] = None, ) -> MaxHistoryTrackerFeaturizer: # Memoization policy always uses MaxHistoryTrackerFeaturizer # without state_featurizer return MaxHistoryTrackerFeaturizer( state_featurizer=None, max_history=max_history )
async def test_create_train_data_unfeaturized_entities(): import copy domain_file = "data/test_domains/default_unfeaturized_entities.yml" stories_file = "data/test_stories/stories_unfeaturized_entities.md" domain = Domain.load(domain_file) featurizer = MaxHistoryTrackerFeaturizer(max_history=1) training_trackers = await training.load_data(stories_file, domain, augmentation_factor=0) assert len(training_trackers) == 2 (decoded, _) = featurizer.training_states_and_actions(training_trackers, domain) # decoded needs to be sorted hashed = [] for states in decoded: new_states = [ check_for_too_many_entities_and_remove_them(state) for state in states ] hashed.append(json.dumps(new_states, sort_keys=True)) hashed = sorted(hashed, reverse=True) assert hashed == [ "[{}]", '[{"prev_action": {"action_name": "utter_greet"}, "user": {"intent": "greet"}}]', '[{"prev_action": {"action_name": "utter_greet"}, "user": {"entities": ["name"], "intent": "greet"}}]', '[{"prev_action": {"action_name": "utter_goodbye"}, "user": {"intent": "goodbye"}}]', '[{"prev_action": {"action_name": "utter_default"}, "user": {"intent": "why"}}]', '[{"prev_action": {"action_name": "utter_default"}, "user": {"intent": "thank"}}]', '[{"prev_action": {"action_name": "utter_default"}, "user": {"entities": [], "intent": "default"}}]', '[{"prev_action": {"action_name": "utter_default"}, "user": {"entities": [], "intent": "ask"}}]', '[{"prev_action": {"action_name": "action_listen"}, "user": {"intent": "why"}}]', '[{"prev_action": {"action_name": "action_listen"}, "user": {"intent": "thank"}}]', '[{"prev_action": {"action_name": "action_listen"}, "user": {"intent": "greet"}}]', '[{"prev_action": {"action_name": "action_listen"}, "user": {"intent": "goodbye"}}]', '[{"prev_action": {"action_name": "action_listen"}, "user": {"entities": [], "intent": "default"}}]', '[{"prev_action": {"action_name": "action_listen"}, "user": {"entities": [], "intent": "ask"}}]', '[{"prev_action": {"action_name": "action_listen"}, "user": {"entities": ["name"], "intent": "greet"}}]', ]
def test_generate_training_data_with_cycles(domain: Domain): featurizer = MaxHistoryTrackerFeaturizer(SingleStateFeaturizer(), max_history=4) training_trackers = training.load_data( "data/test_yaml_stories/stories_with_cycle.yml", domain, augmentation_factor=0, ) _, label_ids, _ = featurizer.featurize_trackers( training_trackers, domain, precomputations=None ) # how many there are depends on the graph which is not created in a # deterministic way but should always be 3 or 4 assert len(training_trackers) == 3 or len(training_trackers) == 4 # if we have 4 trackers, there is going to be one example more for label 10 num_tens = len(training_trackers) - 1 # if new default actions are added the keys of the actions will be changed all_label_ids = [id for ids in label_ids for id in ids] assert Counter(all_label_ids) == {0: 6, 15: 3, 14: num_tens, 1: 2, 16: 1}
async def test_generate_training_data_with_cycles(stories_file: Text, default_domain: Domain): featurizer = MaxHistoryTrackerFeaturizer(SingleStateFeaturizer(), max_history=4) training_trackers = await training.load_data(stories_file, default_domain, augmentation_factor=0) training_data, label_ids = featurizer.featurize_trackers( training_trackers, default_domain, interpreter=RegexInterpreter()) # how many there are depends on the graph which is not created in a # deterministic way but should always be 3 or 4 assert len(training_trackers) == 3 or len(training_trackers) == 4 # if we have 4 trackers, there is going to be one example more for label 10 num_tens = len(training_trackers) - 1 # if new default actions are added the keys of the actions will be changed all_label_ids = [id for ids in label_ids for id in ids] assert Counter(all_label_ids) == {0: 6, 12: num_tens, 14: 1, 1: 2, 13: 3}
def test_prediction( self, max_history: Optional[int], model_storage: ModelStorage, resource: Resource, execution_context: ExecutionContext, ): policy = self.create_policy( featurizer=MaxHistoryTrackerFeaturizer(max_history=max_history), model_storage=model_storage, resource=resource, execution_context=execution_context, config={POLICY_MAX_HISTORY: max_history}, ) GREET_INTENT_NAME = "greet" UTTER_GREET_ACTION = "utter_greet" UTTER_BYE_ACTION = "utter_goodbye" domain = Domain.from_yaml(f""" intents: - {GREET_INTENT_NAME} actions: - {UTTER_GREET_ACTION} - {UTTER_BYE_ACTION} slots: slot_1: type: bool mappings: - type: from_text slot_2: type: bool mappings: - type: from_text slot_3: type: bool mappings: - type: from_text slot_4: type: bool mappings: - type: from_text """) events = [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(UTTER_GREET_ACTION), SlotSet("slot_1", True), ActionExecuted(UTTER_GREET_ACTION), SlotSet("slot_2", True), SlotSet("slot_3", True), ActionExecuted(UTTER_GREET_ACTION), ActionExecuted(UTTER_GREET_ACTION), ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(UTTER_GREET_ACTION), SlotSet("slot_4", True), ActionExecuted(UTTER_BYE_ACTION), ActionExecuted(ACTION_LISTEN_NAME), ] training_story = TrackerWithCachedStates.from_events( "training story", evts=events, domain=domain, slots=domain.slots) test_story = TrackerWithCachedStates.from_events("training story", events[:-2], domain=domain, slots=domain.slots) policy.train([training_story], domain) prediction = policy.predict_action_probabilities(test_story, domain) assert (domain.action_names_or_texts[prediction.probabilities.index( max(prediction.probabilities))] == UTTER_BYE_ACTION)
class TestMemoizationPolicy(PolicyTestCollection): @staticmethod def _policy_class_to_test() -> Type[PolicyGraphComponent]: return MemoizationPolicy @pytest.fixture(scope="class") def featurizer(self) -> TrackerFeaturizer: featurizer = MaxHistoryTrackerFeaturizer(None, max_history=self.max_history) return featurizer def test_featurizer( self, trained_policy: PolicyGraphComponent, resource: Resource, model_storage: ModelStorage, tmp_path: Path, execution_context: ExecutionContext, ) -> None: assert isinstance(trained_policy.featurizer, MaxHistoryTrackerFeaturizer) assert trained_policy.featurizer.state_featurizer is None loaded = trained_policy.__class__.load( self._config(trained_policy.config), model_storage, resource, execution_context, ) assert isinstance(loaded.featurizer, MaxHistoryTrackerFeaturizer) assert loaded.featurizer.state_featurizer is None def test_memorise( self, trained_policy: MemoizationPolicy, default_domain: Domain, stories_path: Text, ): trackers = train_trackers(default_domain, stories_path, augmentation_factor=20) trained_policy.train(trackers, default_domain) lookup_with_augmentation = trained_policy.lookup trackers = [ t for t in trackers if not hasattr(t, "is_augmented") or not t.is_augmented ] ( all_states, all_actions, ) = trained_policy.featurizer.training_states_and_labels( trackers, default_domain) for tracker, states, actions in zip(trackers, all_states, all_actions): recalled = trained_policy.recall(states, tracker, default_domain, None) assert recalled == actions[0] nums = np.random.randn(default_domain.num_states) random_states = [{ f: num for f, num in zip(default_domain.input_states, nums) }] assert trained_policy._recall_states(random_states) is None # compare augmentation for augmentation_factor of 0 and 20: trackers_no_augmentation = train_trackers(default_domain, stories_path, augmentation_factor=0) trained_policy.train(trackers_no_augmentation, default_domain) lookup_no_augmentation = trained_policy.lookup assert lookup_no_augmentation == lookup_with_augmentation def test_memorise_with_nlu(self, trained_policy: MemoizationPolicy, default_domain: Domain): tracker = tracker_from_dialogue(TEST_DEFAULT_DIALOGUE, default_domain) states = trained_policy._prediction_states(tracker, default_domain) recalled = trained_policy.recall(states, tracker, default_domain, None) assert recalled is not None def test_finetune_after_load( self, trained_policy: MemoizationPolicy, resource: Resource, model_storage: ModelStorage, execution_context: ExecutionContext, default_domain: Domain, stories_path: Text, ): execution_context = dataclasses.replace(execution_context, is_finetuning=True) loaded_policy = MemoizationPolicy.load(trained_policy.config, model_storage, resource, execution_context) assert loaded_policy.finetune_mode new_story = TrackerWithCachedStates.from_events( "channel", domain=default_domain, slots=default_domain.slots, evts=[ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": "why"}), ActionExecuted("utter_channel"), ActionExecuted(ACTION_LISTEN_NAME), ], ) original_train_data = train_trackers(default_domain, stories_path, augmentation_factor=20) loaded_policy.train( original_train_data + [new_story], default_domain, ) # Get the hash of the tracker state of new story new_story_states, _ = loaded_policy.featurizer.training_states_and_labels( [new_story], default_domain) # Feature keys for each new state should be present in the lookup for states in new_story_states: state_key = loaded_policy._create_feature_key(states) assert state_key in loaded_policy.lookup @pytest.mark.parametrize( "tracker_events_with_action, tracker_events_without_action", [ ( [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(text="hello", intent={"name": "greet"}), ActionExecuted(ACTION_UNLIKELY_INTENT_NAME), ], [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(text="hello", intent={"name": "greet"}), ], ), ( [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(text="hello", intent={"name": "greet"}), EntitiesAdded(entities=[{ "entity": "name", "value": "Peter" }]), SlotSet("name", "Peter"), ActionExecuted(ACTION_UNLIKELY_INTENT_NAME), ], [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(text="hello", intent={"name": "greet"}), SlotSet("name", "Peter"), EntitiesAdded(entities=[{ "entity": "name", "value": "Peter" }]), ], ), ], ) def test_ignore_action_unlikely_intent( self, trained_policy: MemoizationPolicy, default_domain: Domain, tracker_events_with_action: List[Event], tracker_events_without_action: List[Event], ): tracker_with_action = DialogueStateTracker.from_events( "test 1", evts=tracker_events_with_action, slots=default_domain.slots) tracker_without_action = DialogueStateTracker.from_events( "test 2", evts=tracker_events_without_action, slots=default_domain.slots) prediction_with_action = trained_policy.predict_action_probabilities( tracker_with_action, default_domain, ) prediction_without_action = trained_policy.predict_action_probabilities( tracker_without_action, default_domain, ) # Memoization shouldn't be affected with the # presence of action_unlikely_intent. assert (prediction_with_action.probabilities == prediction_without_action.probabilities) @pytest.mark.parametrize( "featurizer_config, tracker_featurizer, state_featurizer", [ (None, MaxHistoryTrackerFeaturizer(), type(None)), ([], MaxHistoryTrackerFeaturizer(), type(None)), ], ) def test_empty_featurizer_configs( self, featurizer_config: Optional[Dict[Text, Any]], model_storage: ModelStorage, resource: Resource, execution_context: ExecutionContext, tracker_featurizer: MaxHistoryTrackerFeaturizer, state_featurizer: Type[SingleStateFeaturizer], ): featurizer_config_override = ({ "featurizer": featurizer_config } if featurizer_config else {}) policy = self.create_policy( None, model_storage=model_storage, resource=resource, execution_context=execution_context, config=self._config(featurizer_config_override), ) featurizer = policy.featurizer assert isinstance(featurizer, tracker_featurizer.__class__) if featurizer_config: expected_max_history = featurizer_config[0].get(POLICY_MAX_HISTORY) else: expected_max_history = self._config().get(POLICY_MAX_HISTORY) assert featurizer.max_history == expected_max_history assert isinstance(featurizer.state_featurizer, state_featurizer) @pytest.mark.parametrize("max_history", [1, 2, 3, 4, None]) def test_prediction( self, max_history: Optional[int], model_storage: ModelStorage, resource: Resource, execution_context: ExecutionContext, ): policy = self.create_policy( featurizer=MaxHistoryTrackerFeaturizer(max_history=max_history), model_storage=model_storage, resource=resource, execution_context=execution_context, ) GREET_INTENT_NAME = "greet" UTTER_GREET_ACTION = "utter_greet" UTTER_BYE_ACTION = "utter_goodbye" domain = Domain.from_yaml(f""" intents: - {GREET_INTENT_NAME} actions: - {UTTER_GREET_ACTION} - {UTTER_BYE_ACTION} slots: slot_1: type: bool slot_2: type: bool slot_3: type: bool slot_4: type: bool """) events = [ UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(UTTER_GREET_ACTION), SlotSet("slot_1", True), ActionExecuted(UTTER_GREET_ACTION), SlotSet("slot_2", True), SlotSet("slot_3", True), ActionExecuted(UTTER_GREET_ACTION), ActionExecuted(UTTER_GREET_ACTION), UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(UTTER_GREET_ACTION), SlotSet("slot_4", True), ActionExecuted(UTTER_BYE_ACTION), ] training_story = TrackerWithCachedStates.from_events( "training story", evts=events, domain=domain, slots=domain.slots, ) test_story = TrackerWithCachedStates.from_events( "training story", events[:-1], domain=domain, slots=domain.slots, ) policy.train([training_story], domain) prediction = policy.predict_action_probabilities(test_story, domain) assert (domain.action_names_or_texts[prediction.probabilities.index( max(prediction.probabilities))] == UTTER_BYE_ACTION)
def _standard_featurizer(max_history: Optional[int] = None) -> TrackerFeaturizer: return MaxHistoryTrackerFeaturizer( SingleStateFeaturizer(), max_history=max_history )
def featurizer(self) -> TrackerFeaturizer: featurizer = MaxHistoryTrackerFeaturizer(None, max_history=self.max_history) return featurizer
def test_aug_pred_without_intent( self, max_history: Optional[int], model_storage: ModelStorage, resource: Resource, execution_context: ExecutionContext, ): """Tests memoization works for a memoized state sequence that does not have a user utterance. """ policy = self.create_policy( featurizer=MaxHistoryTrackerFeaturizer(max_history=max_history), model_storage=model_storage, resource=resource, execution_context=execution_context, config={POLICY_MAX_HISTORY: max_history}, ) GREET_INTENT_NAME = "greet" GOODBYE_INTENT_NAME = "goodbye" UTTER_GREET_ACTION = "utter_greet" UTTER_ACTION_1 = "utter_1" UTTER_ACTION_2 = "utter_2" UTTER_ACTION_3 = "utter_3" UTTER_ACTION_4 = "utter_4" domain = Domain.from_yaml(f""" intents: - {GREET_INTENT_NAME} - {GOODBYE_INTENT_NAME} actions: - {UTTER_GREET_ACTION} - {UTTER_ACTION_1} - {UTTER_ACTION_2} - {UTTER_ACTION_3} - {UTTER_ACTION_4} """) training_story = TrackerWithCachedStates.from_events( "training story", [ ActionExecuted(UTTER_ACTION_3), ActionExecuted(UTTER_ACTION_4), ActionExecuted(ACTION_LISTEN_NAME), ], domain=domain, slots=domain.slots, ) policy.train([training_story], domain) test_story = TrackerWithCachedStates.from_events( "test story", [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(UTTER_ACTION_1), ActionExecuted(UTTER_ACTION_2), ActionExecuted(UTTER_ACTION_3), # ActionExecuted(UTTER_ACTION_4), ], domain=domain, slots=domain.slots, ) prediction = policy.predict_action_probabilities(test_story, domain) assert (domain.action_names_or_texts[prediction.probabilities.index( max(prediction.probabilities))] == UTTER_ACTION_4)
async def test_load_multi_file_training_data( stories_resources: List, default_domain: Domain ): # the stories file in `data/test_multifile_stories` is the same as in # `data/test_stories/stories.md`, but split across multiple files featurizer = MaxHistoryTrackerFeaturizer(SingleStateFeaturizer(), max_history=2) trackers = await training.load_data( stories_resources[0], default_domain, augmentation_factor=0 ) (tr_as_sts, tr_as_acts) = featurizer.training_states_and_actions( trackers, default_domain ) hashed = [] for sts, acts in zip(tr_as_sts, tr_as_acts): hashed.append(json.dumps(sts + acts, sort_keys=True)) hashed = sorted(hashed, reverse=True) data, label_ids = featurizer.featurize_trackers( trackers, default_domain, interpreter=RegexInterpreter() ) featurizer_mul = MaxHistoryTrackerFeaturizer(SingleStateFeaturizer(), max_history=2) trackers_mul = await training.load_data( stories_resources[1], default_domain, augmentation_factor=0 ) (tr_as_sts_mul, tr_as_acts_mul) = featurizer.training_states_and_actions( trackers_mul, default_domain ) hashed_mul = [] for sts_mul, acts_mul in zip(tr_as_sts_mul, tr_as_acts_mul): hashed_mul.append(json.dumps(sts_mul + acts_mul, sort_keys=True)) hashed_mul = sorted(hashed_mul, reverse=True) data_mul, label_ids_mul = featurizer_mul.featurize_trackers( trackers_mul, default_domain, interpreter=RegexInterpreter() ) assert hashed == hashed_mul # we check for intents, action names and entities -- the features which # are included in the story files data = surface_attributes(data) data_mul = surface_attributes(data_mul) for attribute in [INTENT, ACTION_NAME, ENTITIES]: if attribute not in data or attribute not in data_mul: continue assert len(data.get(attribute)) == len(data_mul.get(attribute)) for idx_tracker in range(len(data.get(attribute))): for idx_dialogue in range(len(data.get(attribute)[idx_tracker])): f1 = data.get(attribute)[idx_tracker][idx_dialogue] f2 = data_mul.get(attribute)[idx_tracker][idx_dialogue] if f1 is None or f2 is None: assert f1 == f2 continue for idx_turn in range(len(f1)): f1 = data.get(attribute)[idx_tracker][idx_dialogue][idx_turn] f2 = data_mul.get(attribute)[idx_tracker][idx_dialogue][idx_turn] assert np.all((f1 == f2).data) assert np.all(label_ids == label_ids_mul)
def test_augmented_prediction( self, max_history: Optional[int], model_storage: ModelStorage, resource: Resource, execution_context: ExecutionContext, ): policy = self.create_policy( featurizer=MaxHistoryTrackerFeaturizer(max_history=max_history), model_storage=model_storage, resource=resource, execution_context=execution_context, ) GREET_INTENT_NAME = "greet" UTTER_GREET_ACTION = "utter_greet" UTTER_BYE_ACTION = "utter_goodbye" domain = Domain.from_yaml(f""" intents: - {GREET_INTENT_NAME} actions: - {UTTER_GREET_ACTION} - {UTTER_BYE_ACTION} slots: slot_1: type: bool initial_value: true slot_2: type: bool slot_3: type: bool """) training_story = TrackerWithCachedStates.from_events( "training story", [ ActionExecuted(UTTER_GREET_ACTION), UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(UTTER_GREET_ACTION), SlotSet("slot_3", True), ActionExecuted(UTTER_BYE_ACTION), ], domain=domain, slots=domain.slots, ) test_story = TrackerWithCachedStates.from_events( "test story", [ UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(UTTER_GREET_ACTION), SlotSet("slot_1", False), ActionExecuted(UTTER_GREET_ACTION), ActionExecuted(UTTER_GREET_ACTION), UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(UTTER_GREET_ACTION), SlotSet("slot_2", True), ActionExecuted(UTTER_GREET_ACTION), UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(UTTER_GREET_ACTION), SlotSet("slot_3", True), # ActionExecuted(UTTER_BYE_ACTION), ], domain=domain, slots=domain.slots, ) policy.train([training_story], domain) prediction = policy.predict_action_probabilities(test_story, domain) assert (domain.action_names_or_texts[prediction.probabilities.index( max(prediction.probabilities))] == UTTER_BYE_ACTION)
class TestTEDPolicy(PolicyTestCollection): @staticmethod def _policy_class_to_test() -> Type[TEDPolicy]: return TEDPolicy def test_train_model_checkpointing(self, tmp_path: Path, tmp_path_factory: TempPathFactory): train_core( domain="data/test_domains/default.yml", stories="data/test_yaml_stories/stories_defaultdomain.yml", output=str(tmp_path), fixed_model_name="my_model.tar.gz", config="data/test_config/config_ted_policy_model_checkpointing.yml", ) storage_dir = tmp_path_factory.mktemp("storage dir") storage, _ = LocalModelStorage.from_model_archive( storage_dir, tmp_path / "my_model.tar.gz") checkpoint_dir = get_checkpoint_dir_path(storage_dir) assert checkpoint_dir.is_dir() def test_doesnt_checkpoint_with_no_checkpointing( self, tmp_path: Path, tmp_path_factory: TempPathFactory): train_core( domain="data/test_domains/default.yml", stories="data/test_yaml_stories/stories_defaultdomain.yml", output=str(tmp_path), fixed_model_name="my_model.tar.gz", config= "data/test_config/config_ted_policy_no_model_checkpointing.yml", ) storage_dir = tmp_path_factory.mktemp("storage dir") storage, _ = LocalModelStorage.from_model_archive( storage_dir, tmp_path / "my_model.tar.gz") checkpoint_dir = get_checkpoint_dir_path(storage_dir) assert not checkpoint_dir.is_dir() def test_doesnt_checkpoint_with_zero_eval_num_examples( self, tmp_path: Path, tmp_path_factory: TempPathFactory): checkpoint_dir = get_checkpoint_dir_path(tmp_path) assert not checkpoint_dir.is_dir() config_file = "config_ted_policy_model_checkpointing_zero_eval_num_examples.yml" with pytest.warns(UserWarning) as warning: train_core( domain="data/test_domains/default.yml", stories="data/test_yaml_stories/stories_defaultdomain.yml", output=str(tmp_path), fixed_model_name="my_model.tar.gz", config=f"data/test_config/{config_file}", ) warn_text = ( f"You have opted to save the best model, but the value of " f"'{EVAL_NUM_EXAMPLES}' is not greater than 0. No checkpoint model will be " f"saved.") assert len([w for w in warning if warn_text in str(w.message)]) == 1 storage_dir = tmp_path_factory.mktemp("storage dir") storage, _ = LocalModelStorage.from_model_archive( storage_dir, tmp_path / "my_model.tar.gz") checkpoint_dir = get_checkpoint_dir_path(storage_dir) assert not checkpoint_dir.is_dir() @pytest.mark.parametrize( "should_finetune, epoch_override, expected_epoch_value", [ ( True, TEDPolicy.get_default_config()[EPOCHS] + 1, TEDPolicy.get_default_config()[EPOCHS] + 1, ), ( False, TEDPolicy.get_default_config()[EPOCHS] + 1, TEDPolicy.get_default_config()[EPOCHS], ), # trained_policy uses default epochs during training ], ) def test_epoch_override_when_loaded( self, trained_policy: TEDPolicy, should_finetune: bool, epoch_override: int, expected_epoch_value: int, resource: Resource, model_storage: ModelStorage, execution_context: ExecutionContext, ): execution_context.is_finetuning = should_finetune loaded_policy = trained_policy.__class__.load( { **self._config(), EPOCH_OVERRIDE: epoch_override }, model_storage, resource, execution_context, ) assert loaded_policy.config[EPOCHS] == expected_epoch_value def test_train_fails_with_checkpoint_zero_eval_num_epochs( self, tmp_path: Path): config_file = "config_ted_policy_model_checkpointing_zero_every_num_epochs.yml" match_string = ("Only values either equal to -1 or greater" " than 0 are allowed for this parameter.") with pytest.raises( InvalidConfigException, match=match_string, ): train_core( domain="data/test_domains/default.yml", stories="data/test_yaml_stories/stories_defaultdomain.yml", output=str(tmp_path), config=f"data/test_config/{config_file}", ) assert not (tmp_path / "my_model.tar.gz").is_file() def test_training_with_no_intent( self, featurizer: Optional[TrackerFeaturizer], default_domain: Domain, tmp_path: Path, caplog: LogCaptureFixture, model_storage: ModelStorage, resource: Resource, execution_context: ExecutionContext, ): stories = tmp_path / "stories.yml" stories.write_text(""" version: "3.0" stories: - story: test path steps: - action: utter_greet """) policy = self.create_policy( featurizer=featurizer, model_storage=model_storage, resource=resource, execution_context=execution_context, ) import tests.core.test_policies training_trackers = tests.core.test_policies.train_trackers( default_domain, str(stories), augmentation_factor=20) with pytest.raises(RasaException) as e: policy.train(training_trackers, default_domain, precomputations=None) assert "No user features specified. Cannot train 'TED' model." == str( e.value) def test_similarity_type(self, trained_policy: TEDPolicy): assert trained_policy.config[SIMILARITY_TYPE] == "inner" def test_ranking_length(self, trained_policy: TEDPolicy): assert trained_policy.config[RANKING_LENGTH] == 0 def test_ranking_length_and_renormalization( self, trained_policy: TEDPolicy, tracker: DialogueStateTracker, default_domain: Domain, monkeypatch: MonkeyPatch, ): precomputations = None prediction = trained_policy.predict_action_probabilities( tracker, default_domain, precomputations, ) # first check the output is what we expect assert not prediction.is_end_to_end_prediction # check that ranking length is applied - without normalization if trained_policy.config[RANKING_LENGTH] == 0: assert sum([confidence for confidence in prediction.probabilities ]) == pytest.approx(1) assert all(confidence > 0 for confidence in prediction.probabilities) else: assert (sum([ confidence > 0 for confidence in prediction.probabilities ]) == trained_policy.config[RANKING_LENGTH]) assert sum([confidence for confidence in prediction.probabilities ]) != pytest.approx(1) def test_label_data_assembly(self, trained_policy: TEDPolicy, default_domain: Domain): state_featurizer = trained_policy.featurizer.state_featurizer encoded_all_labels = state_featurizer.encode_all_labels( default_domain, precomputations=None) attribute_data, _ = model_data_utils.convert_to_data_format( encoded_all_labels) assembled_label_data = trained_policy._assemble_label_data( attribute_data, default_domain) assembled_label_data_signature = assembled_label_data.get_signature() assert list(assembled_label_data_signature.keys()) == [ f"{LABEL}_{ACTION_NAME}", f"{LABEL}", ] assert assembled_label_data.num_examples == default_domain.num_actions assert list(assembled_label_data_signature[f"{LABEL}_{ACTION_NAME}"]. keys()) == [ MASK, SENTENCE, ] assert list(assembled_label_data_signature[LABEL].keys()) == [IDS] assert (assembled_label_data_signature[f"{LABEL}_{ACTION_NAME}"] [SENTENCE][0].units == default_domain.num_actions) def test_gen_batch(self, trained_policy: TEDPolicy, default_domain: Domain, stories_path: Path): training_trackers = tests.core.test_policies.train_trackers( default_domain, stories_path, augmentation_factor=0) precomputations = None training_data, label_ids, entity_tags = trained_policy._featurize_for_training( training_trackers, default_domain, precomputations, ) _, all_labels = trained_policy._create_label_data( default_domain, precomputations) model_data = trained_policy._create_model_data(training_data, label_ids, entity_tags, all_labels) batch_size = 2 data_generator = RasaBatchDataGenerator(model_data, batch_size=batch_size, shuffle=False, batch_strategy="sequence") iterator = iter(data_generator) # model data keys were sorted, so the order is alphabetical ( ( batch_action_name_mask, _, _, batch_action_name_sentence_shape, batch_dialogue_length, batch_entities_mask, _, _, batch_entities_sentence_shape, batch_intent_mask, _, _, batch_intent_sentence_shape, batch_label_ids, batch_slots_mask, _, _, batch_slots_sentence_shape, ), _, ) = next(iterator) assert (batch_label_ids.shape[0] == batch_size and batch_dialogue_length.shape[0] == batch_size) # batch and dialogue dimensions are NOT combined for masks assert (batch_slots_mask.shape[0] == batch_size and batch_intent_mask.shape[0] == batch_size and batch_entities_mask.shape[0] == batch_size and batch_action_name_mask.shape[0] == batch_size) # some features might be "fake" so there sequence is `0` seq_len = max([ batch_intent_sentence_shape[1], batch_action_name_sentence_shape[1], batch_entities_sentence_shape[1], batch_slots_sentence_shape[1], ]) assert (batch_intent_sentence_shape[1] == seq_len or batch_intent_sentence_shape[1] == 0) assert (batch_action_name_sentence_shape[1] == seq_len or batch_action_name_sentence_shape[1] == 0) assert (batch_entities_sentence_shape[1] == seq_len or batch_entities_sentence_shape[1] == 0) assert (batch_slots_sentence_shape[1] == seq_len or batch_slots_sentence_shape[1] == 0) data_generator = RasaBatchDataGenerator(model_data, batch_size=batch_size, shuffle=True, batch_strategy="balanced") iterator = iter(data_generator) ( ( batch_action_name_mask, _, _, batch_action_name_sentence_shape, batch_dialogue_length, batch_entities_mask, _, _, batch_entities_sentence_shape, batch_intent_mask, _, _, batch_intent_sentence_shape, batch_label_ids, batch_slots_mask, _, _, batch_slots_sentence_shape, ), _, ) = next(iterator) assert (batch_label_ids.shape[0] == batch_size and batch_dialogue_length.shape[0] == batch_size) # some features might be "fake" so there sequence is `0` seq_len = max([ batch_intent_sentence_shape[1], batch_action_name_sentence_shape[1], batch_entities_sentence_shape[1], batch_slots_sentence_shape[1], ]) assert (batch_intent_sentence_shape[1] == seq_len or batch_intent_sentence_shape[1] == 0) assert (batch_action_name_sentence_shape[1] == seq_len or batch_action_name_sentence_shape[1] == 0) assert (batch_entities_sentence_shape[1] == seq_len or batch_entities_sentence_shape[1] == 0) assert (batch_slots_sentence_shape[1] == seq_len or batch_slots_sentence_shape[1] == 0) @pytest.mark.parametrize( "tracker_events_with_action, tracker_events_without_action", [ ( [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(text="hello", intent={"name": "greet"}), ActionExecuted(ACTION_UNLIKELY_INTENT_NAME), ], [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(text="hello", intent={"name": "greet"}), ], ), ( [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(text="hello", intent={"name": "greet"}), EntitiesAdded(entities=[ { "entity": "name", "value": "Peter" }, ]), ActionExecuted(ACTION_UNLIKELY_INTENT_NAME), ActionExecuted("utter_greet"), ], [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(text="hello", intent={"name": "greet"}), EntitiesAdded(entities=[ { "entity": "name", "value": "Peter" }, ]), ActionExecuted("utter_greet"), ], ), ( [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(text="hello", intent={"name": "greet"}), ActionExecuted(ACTION_UNLIKELY_INTENT_NAME), ActionExecuted("some_form"), ActiveLoop("some_form"), ActionExecuted(ACTION_LISTEN_NAME), UserUttered(text="default", intent={"name": "default"}), ActionExecuted(ACTION_UNLIKELY_INTENT_NAME), ], [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(text="hello", intent={"name": "greet"}), ActionExecuted(ACTION_UNLIKELY_INTENT_NAME), ActionExecuted("some_form"), ActiveLoop("some_form"), ActionExecuted(ACTION_LISTEN_NAME), UserUttered(text="default", intent={"name": "default"}), ], ), ], ) def test_ignore_action_unlikely_intent( self, trained_policy: TEDPolicy, default_domain: Domain, tracker_events_with_action: List[Event], tracker_events_without_action: List[Event], ): precomputations = None tracker_with_action = DialogueStateTracker.from_events( "test 1", evts=tracker_events_with_action) tracker_without_action = DialogueStateTracker.from_events( "test 2", evts=tracker_events_without_action) prediction_with_action = trained_policy.predict_action_probabilities( tracker_with_action, default_domain, precomputations, ) prediction_without_action = trained_policy.predict_action_probabilities( tracker_without_action, default_domain, precomputations, ) # If the weights didn't change then both trackers # should result in same prediction. assert (prediction_with_action.probabilities == prediction_without_action.probabilities) @pytest.mark.parametrize( "featurizer_config, tracker_featurizer, state_featurizer", [ (None, MaxHistoryTrackerFeaturizer(), SingleStateFeaturizer), ([], MaxHistoryTrackerFeaturizer(), SingleStateFeaturizer), ], ) def test_empty_featurizer_configs( self, featurizer_config: Optional[Dict[Text, Any]], model_storage: ModelStorage, resource: Resource, execution_context: ExecutionContext, tracker_featurizer: MaxHistoryTrackerFeaturizer, state_featurizer: Type[SingleStateFeaturizer], ): featurizer_config_override = ({ "featurizer": featurizer_config } if featurizer_config else {}) policy = self.create_policy( None, model_storage=model_storage, resource=resource, execution_context=execution_context, config=self._config(featurizer_config_override), ) featurizer = policy.featurizer assert isinstance(featurizer, tracker_featurizer.__class__) if featurizer_config: expected_max_history = featurizer_config[0].get(POLICY_MAX_HISTORY) else: expected_max_history = self._config().get(POLICY_MAX_HISTORY) assert featurizer.max_history == expected_max_history assert isinstance(featurizer.state_featurizer, state_featurizer)
def test_augmented_prediction_across_max_history_actions( self, max_history: Optional[int], model_storage: ModelStorage, resource: Resource, execution_context: ExecutionContext, ): """Tests that the last user utterance is preserved in action states even when the utterance occurs prior to `max_history` actions in the past. """ policy = self.create_policy( featurizer=MaxHistoryTrackerFeaturizer(max_history=max_history), model_storage=model_storage, resource=resource, execution_context=execution_context, config={POLICY_MAX_HISTORY: max_history}, ) GREET_INTENT_NAME = "greet" UTTER_GREET_ACTION = "utter_greet" UTTER_ACTION_1 = "utter_1" UTTER_ACTION_2 = "utter_2" UTTER_ACTION_3 = "utter_3" UTTER_ACTION_4 = "utter_4" UTTER_ACTION_5 = "utter_5" UTTER_BYE_ACTION = "utter_goodbye" domain = Domain.from_yaml(f""" intents: - {GREET_INTENT_NAME} actions: - {UTTER_GREET_ACTION} - {UTTER_ACTION_1} - {UTTER_ACTION_2} - {UTTER_ACTION_3} - {UTTER_ACTION_4} - {UTTER_ACTION_5} - {UTTER_BYE_ACTION} """) training_story = TrackerWithCachedStates.from_events( "training story", [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(UTTER_ACTION_1), ActionExecuted(UTTER_ACTION_2), ActionExecuted(UTTER_ACTION_3), ActionExecuted(UTTER_ACTION_4), ActionExecuted(UTTER_ACTION_5), ActionExecuted(UTTER_BYE_ACTION), ActionExecuted(ACTION_LISTEN_NAME), ], domain=domain, slots=domain.slots, ) test_story = TrackerWithCachedStates.from_events( "test story", [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(UTTER_ACTION_1), ActionExecuted(UTTER_ACTION_2), ActionExecuted(UTTER_ACTION_3), ActionExecuted(UTTER_ACTION_4), ActionExecuted(UTTER_ACTION_5), # ActionExecuted(UTTER_BYE_ACTION), ], domain=domain, slots=domain.slots, ) policy.train([training_story], domain) prediction = policy.predict_action_probabilities(test_story, domain) assert (domain.action_names_or_texts[prediction.probabilities.index( max(prediction.probabilities))] == UTTER_BYE_ACTION)
def _standard_featurizer(self) -> MaxHistoryTrackerFeaturizer: # Memoization policy always uses MaxHistoryTrackerFeaturizer # without state_featurizer return MaxHistoryTrackerFeaturizer( state_featurizer=None, max_history=self.config[POLICY_MAX_HISTORY])
class PolicyTestCollection: """Tests every policy needs to fulfill. Each policy can declare further tests on its own.""" @staticmethod def _policy_class_to_test() -> Type[PolicyGraphComponent]: raise NotImplementedError max_history = 3 # this is the amount of history we test on @pytest.fixture(scope="class") def resource(self, ) -> Resource: return Resource(uuid.uuid4().hex) @pytest.fixture(scope="class") def model_storage(self, tmp_path_factory: TempPathFactory) -> ModelStorage: return LocalModelStorage(tmp_path_factory.mktemp(uuid.uuid4().hex)) @pytest.fixture(scope="class") def execution_context(self) -> ExecutionContext: return ExecutionContext(GraphSchema({}), uuid.uuid4().hex) def _config( self, config_override: Optional[Dict[Text, Any]] = None) -> Dict[Text, Any]: config_override = config_override or {} config = self._policy_class_to_test().get_default_config() return {**config, **config_override} def create_policy( self, featurizer: Optional[TrackerFeaturizer], model_storage: ModelStorage, resource: Resource, execution_context: ExecutionContext, config: Optional[Dict[Text, Any]] = None, ) -> PolicyGraphComponent: return self._policy_class_to_test()( config=self._config(config), model_storage=model_storage, resource=resource, execution_context=execution_context, featurizer=featurizer, ) @pytest.fixture(scope="class") def featurizer(self) -> TrackerFeaturizer: featurizer = MaxHistoryTrackerFeaturizer(SingleStateFeaturizer(), max_history=self.max_history) return featurizer @pytest.fixture(scope="class") def default_domain(self, domain_path: Text) -> Domain: return Domain.load(domain_path) @pytest.fixture(scope="class") def tracker(self, default_domain: Domain) -> DialogueStateTracker: return DialogueStateTracker(DEFAULT_SENDER_ID, default_domain.slots) @pytest.fixture(scope="class") def trained_policy( self, featurizer: Optional[TrackerFeaturizer], stories_path: Text, default_domain: Domain, model_storage: ModelStorage, resource: Resource, execution_context: ExecutionContext, ) -> PolicyGraphComponent: policy = self.create_policy(featurizer, model_storage, resource, execution_context) training_trackers = train_trackers(default_domain, stories_path, augmentation_factor=20) policy.train(training_trackers, default_domain) return policy def test_featurizer( self, trained_policy: PolicyGraphComponent, resource: Resource, model_storage: ModelStorage, tmp_path: Path, execution_context: ExecutionContext, ): assert isinstance(trained_policy.featurizer, MaxHistoryTrackerFeaturizer) assert trained_policy.featurizer.max_history == self.max_history assert isinstance(trained_policy.featurizer.state_featurizer, SingleStateFeaturizer) loaded = trained_policy.__class__.load( self._config(trained_policy.config), model_storage, resource, execution_context, ) assert isinstance(loaded.featurizer, MaxHistoryTrackerFeaturizer) assert loaded.featurizer.max_history == self.max_history assert isinstance(loaded.featurizer.state_featurizer, SingleStateFeaturizer) @pytest.mark.parametrize("should_finetune", [False, True]) def test_persist_and_load( self, trained_policy: PolicyGraphComponent, default_domain: Domain, should_finetune: bool, stories_path: Text, model_storage: ModelStorage, resource: Resource, execution_context: ExecutionContext, ): loaded = trained_policy.__class__.load( self._config(trained_policy.config), model_storage, resource, dataclasses.replace(execution_context, is_finetuning=should_finetune), ) assert loaded.finetune_mode == should_finetune trackers = train_trackers(default_domain, stories_path, augmentation_factor=20) for tracker in trackers: predicted_probabilities = loaded.predict_action_probabilities( tracker, default_domain) actual_probabilities = trained_policy.predict_action_probabilities( tracker, default_domain) assert predicted_probabilities == actual_probabilities def test_prediction_on_empty_tracker(self, trained_policy: Policy, default_domain: Domain): tracker = DialogueStateTracker(DEFAULT_SENDER_ID, default_domain.slots) prediction = trained_policy.predict_action_probabilities( tracker, default_domain, ) assert not prediction.is_end_to_end_prediction assert len(prediction.probabilities) == default_domain.num_actions assert max(prediction.probabilities) <= 1.0 assert min(prediction.probabilities) >= 0.0 @pytest.mark.filterwarnings( "ignore:.*without a trained model present.*:UserWarning") def test_persist_and_load_empty_policy( self, default_domain: Domain, default_model_storage: ModelStorage, execution_context: ExecutionContext, ): resource = Resource(uuid.uuid4().hex) empty_policy = self.create_policy( None, default_model_storage, resource, execution_context, ) empty_policy.train([], default_domain) loaded = empty_policy.__class__.load( self._config(), default_model_storage, resource, execution_context, ) assert loaded is not None @staticmethod def _get_next_action(policy: PolicyGraphComponent, events: List[Event], domain: Domain) -> Text: tracker = get_tracker(events) scores = policy.predict_action_probabilities( tracker, domain, ).probabilities index = scores.index(max(scores)) return domain.action_names_or_texts[index] @pytest.mark.parametrize( "featurizer_config, tracker_featurizer, state_featurizer", [ ( [{ # TODO: remove "2" when migration of policies is done "name": "MaxHistoryTrackerFeaturizer2", "max_history": 12, "state_featurizer": [], }], MaxHistoryTrackerFeaturizer(max_history=12), type(None), ), ( # TODO: remove "2" when migration of policies is done [{ "name": "MaxHistoryTrackerFeaturizer2", "max_history": 12 }], MaxHistoryTrackerFeaturizer(max_history=12), type(None), ), ( [{ # TODO: remove "2" when migration of policies is done "name": "IntentMaxHistoryTrackerFeaturizer2", "max_history": 12, "state_featurizer": [{ "name": "IntentTokenizerSingleStateFeaturizer2" }], }], IntentMaxHistoryTrackerFeaturizer(max_history=12), IntentTokenizerSingleStateFeaturizer, ), ], ) def test_different_featurizer_configs( self, featurizer_config: Optional[Dict[Text, Any]], model_storage: ModelStorage, resource: Resource, execution_context: ExecutionContext, tracker_featurizer: MaxHistoryTrackerFeaturizer, state_featurizer: Type[SingleStateFeaturizer], ): featurizer_config_override = ({ "featurizer": featurizer_config } if featurizer_config else {}) policy = self.create_policy( None, model_storage=model_storage, resource=resource, execution_context=execution_context, config=self._config(featurizer_config_override), ) featurizer = policy.featurizer assert isinstance(featurizer, tracker_featurizer.__class__) if featurizer_config: expected_max_history = featurizer_config[0].get(POLICY_MAX_HISTORY) else: expected_max_history = self._config().get(POLICY_MAX_HISTORY) assert featurizer.max_history == expected_max_history assert isinstance(featurizer.state_featurizer, state_featurizer) @pytest.mark.parametrize( "featurizer_config", [ [ # TODO: remove "2" when migration of policies is done { "name": "MaxHistoryTrackerFeaturizer2", "max_history": 12 }, { "name": "MaxHistoryTrackerFeaturizer2", "max_history": 12 }, ], [{ # TODO: remove "2" when migration of policies is done "name": "IntentMaxHistoryTrackerFeaturizer2", "max_history": 12, "state_featurizer": [ { "name": "IntentTokenizerSingleStateFeaturizer2" }, { "name": "IntentTokenizerSingleStateFeaturizer2" }, ], }], ], ) def test_different_invalid_featurizer_configs( self, trained_policy: PolicyGraphComponent, featurizer_config: Optional[Dict[Text, Any]], model_storage: ModelStorage, resource: Resource, execution_context: ExecutionContext, ): with pytest.raises(InvalidPolicyConfig): self.create_policy( None, model_storage=model_storage, resource=resource, execution_context=execution_context, config={"featurizer": featurizer_config}, )
def _standard_featurizer(self) -> MaxHistoryTrackerFeaturizer: """Initializes the standard featurizer for this policy.""" return MaxHistoryTrackerFeaturizer( SingleStateFeaturizer(), self.config.get(POLICY_MAX_HISTORY) )
def featurizer(self) -> TrackerFeaturizer: featurizer = MaxHistoryTrackerFeaturizer( SingleStateFeaturizer(), max_history=self.max_history ) return featurizer
def _standard_featurizer() -> MaxHistoryTrackerFeaturizer: return MaxHistoryTrackerFeaturizer(SingleStateFeaturizer())
def test_aug_pred_sensitive_to_intent_across_max_history_actions( self, max_history: Optional[int], model_storage: ModelStorage, resource: Resource, execution_context: ExecutionContext, ): """Tests that only the most recent user utterance propagates to state creation of following actions. """ policy = self.create_policy( featurizer=MaxHistoryTrackerFeaturizer(max_history=max_history), model_storage=model_storage, resource=resource, execution_context=execution_context, config={POLICY_MAX_HISTORY: max_history}, ) GREET_INTENT_NAME = "greet" GOODBYE_INTENT_NAME = "goodbye" UTTER_GREET_ACTION = "utter_greet" UTTER_ACTION_1 = "utter_1" UTTER_ACTION_2 = "utter_2" UTTER_ACTION_3 = "utter_3" UTTER_ACTION_4 = "utter_4" UTTER_ACTION_5 = "utter_5" UTTER_BYE_ACTION = "utter_goodbye" domain = Domain.from_yaml(f""" intents: - {GREET_INTENT_NAME} - {GOODBYE_INTENT_NAME} actions: - {UTTER_GREET_ACTION} - {UTTER_ACTION_1} - {UTTER_ACTION_2} - {UTTER_ACTION_3} - {UTTER_ACTION_4} - {UTTER_ACTION_5} - {UTTER_BYE_ACTION} """) training_story = TrackerWithCachedStates.from_events( "training story", [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(UTTER_ACTION_1), ActionExecuted(UTTER_ACTION_2), ActionExecuted(UTTER_ACTION_3), ActionExecuted(UTTER_ACTION_4), ActionExecuted(UTTER_ACTION_5), ActionExecuted(UTTER_BYE_ACTION), ActionExecuted(ACTION_LISTEN_NAME), ], domain=domain, slots=domain.slots, ) test_story1 = TrackerWithCachedStates.from_events( "test story", [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": GOODBYE_INTENT_NAME}), ActionExecuted(UTTER_BYE_ACTION), ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(UTTER_ACTION_1), ActionExecuted(UTTER_ACTION_2), ActionExecuted(UTTER_ACTION_3), ActionExecuted(UTTER_ACTION_4), ActionExecuted(UTTER_ACTION_5), # ActionExecuted(UTTER_BYE_ACTION), ], domain=domain, slots=domain.slots, ) policy.train([training_story], domain) prediction1 = policy.predict_action_probabilities(test_story1, domain) assert (domain.action_names_or_texts[prediction1.probabilities.index( max(prediction1.probabilities))] == UTTER_BYE_ACTION) test_story2_no_match_expected = TrackerWithCachedStates.from_events( "test story", [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(UTTER_BYE_ACTION), ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": GOODBYE_INTENT_NAME}), ActionExecuted(UTTER_ACTION_1), ActionExecuted(UTTER_ACTION_2), ActionExecuted(UTTER_ACTION_3), ActionExecuted(UTTER_ACTION_4), ActionExecuted(UTTER_ACTION_5), # No prediction should be made here. ], domain=domain, slots=domain.slots, ) prediction2 = policy.predict_action_probabilities( test_story2_no_match_expected, domain, ) assert all([prob == 0.0 for prob in prediction2.probabilities])