def test_get_next_action_probabilities_pass_policy_predictions_without_interpreter_arg( predict_function: Callable, ): policy = TEDPolicy() policy.predict_action_probabilities = predict_function ensemble = SimplePolicyEnsemble(policies=[policy]) interpreter = Mock() domain = Domain.empty() processor = MessageProcessor( interpreter, ensemble, domain, InMemoryTrackerStore(domain), InMemoryLockStore(), Mock(), ) with pytest.warns(DeprecationWarning): processor._get_next_action_probabilities( DialogueStateTracker.from_events( "lala", [ActionExecuted(ACTION_LISTEN_NAME)] ) )
def test_ignore_action_unlikely_intent( self, trained_policy: TEDPolicy, default_domain: Domain, tracker_events_with_action: List[Event], tracker_events_without_action: List[Event], ): precomputations = None tracker_with_action = DialogueStateTracker.from_events( "test 1", evts=tracker_events_with_action) tracker_without_action = DialogueStateTracker.from_events( "test 2", evts=tracker_events_without_action) prediction_with_action = trained_policy.predict_action_probabilities( tracker_with_action, default_domain, precomputations, ) prediction_without_action = trained_policy.predict_action_probabilities( tracker_without_action, default_domain, precomputations, ) # If the weights didn't change then both trackers # should result in same prediction. assert (prediction_with_action.probabilities == prediction_without_action.probabilities)
def test_normalization( self, trained_policy: TEDPolicy, tracker: DialogueStateTracker, default_domain: Domain, monkeypatch: MonkeyPatch, ): # first check the output is what we expect prediction = trained_policy.predict_action_probabilities( tracker, default_domain, RegexInterpreter()) assert not prediction.is_end_to_end_prediction # count number of non-zero confidences assert (sum([ confidence > 0 for confidence in prediction.probabilities ]) == trained_policy.config[RANKING_LENGTH]) # check that the norm is still 1 assert sum(prediction.probabilities) == pytest.approx(1) # also check our function is called mock = Mock() monkeypatch.setattr(train_utils, "normalize", mock.normalize) trained_policy.predict_action_probabilities(tracker, default_domain, RegexInterpreter()) mock.normalize.assert_called_once()
def test_get_next_action_probabilities_passes_interpreter_to_policies( monkeypatch: MonkeyPatch, ): policy = TEDPolicy() test_interpreter = Mock() def predict_action_probabilities( tracker: DialogueStateTracker, domain: Domain, interpreter: NaturalLanguageInterpreter, **kwargs, ) -> List[float]: assert interpreter == test_interpreter return [1, 0] policy.predict_action_probabilities = predict_action_probabilities ensemble = SimplePolicyEnsemble(policies=[policy]) domain = Domain.empty() processor = MessageProcessor(test_interpreter, ensemble, domain, InMemoryTrackerStore(domain), Mock()) # This should not raise processor._get_next_action_probabilities( DialogueStateTracker.from_events("lala", [ActionExecuted(ACTION_LISTEN_NAME)]))
def test_fingerprint_stays_same(): key1 = fingerprinting.calculate_fingerprint_key( TEDPolicy, TEDPolicy.get_default_config(), {"input": FingerprintableText("Hi")}) key2 = fingerprinting.calculate_fingerprint_key( TEDPolicy, TEDPolicy.get_default_config(), {"input": FingerprintableText("Hi")}) assert key1 == key2
def test_fingerprint_changes_due_to_class(): key1 = fingerprinting.calculate_fingerprint_key( DIETClassifier, TEDPolicy.get_default_config(), {"input": FingerprintableText("Hi")}, ) key2 = fingerprinting.calculate_fingerprint_key( ResponseSelector, TEDPolicy.get_default_config(), {"input": FingerprintableText("Hi")}, ) assert key1 != key2
def test_label_data_assembly(self, trained_policy: TEDPolicy, default_domain: Domain): state_featurizer = trained_policy.featurizer.state_featurizer encoded_all_labels = state_featurizer.encode_all_labels( default_domain, precomputations=None) attribute_data, _ = model_data_utils.convert_to_data_format( encoded_all_labels) assembled_label_data = trained_policy._assemble_label_data( attribute_data, default_domain) assembled_label_data_signature = assembled_label_data.get_signature() assert list(assembled_label_data_signature.keys()) == [ f"{LABEL}_{ACTION_NAME}", f"{LABEL}", ] assert assembled_label_data.num_examples == default_domain.num_actions assert list(assembled_label_data_signature[f"{LABEL}_{ACTION_NAME}"]. keys()) == [ MASK, SENTENCE, ] assert list(assembled_label_data_signature[LABEL].keys()) == [IDS] assert (assembled_label_data_signature[f"{LABEL}_{ACTION_NAME}"] [SENTENCE][0].units == default_domain.num_actions)
def test_ranking_length_and_renormalization( self, trained_policy: TEDPolicy, tracker: DialogueStateTracker, default_domain: Domain, monkeypatch: MonkeyPatch, ): precomputations = None prediction = trained_policy.predict_action_probabilities( tracker, default_domain, precomputations, ) # first check the output is what we expect assert not prediction.is_end_to_end_prediction # check that ranking length is applied - without normalization if trained_policy.config[RANKING_LENGTH] == 0: assert sum([confidence for confidence in prediction.probabilities ]) == pytest.approx(1) assert all(confidence > 0 for confidence in prediction.probabilities) else: assert (sum([ confidence > 0 for confidence in prediction.probabilities ]) == trained_policy.config[RANKING_LENGTH]) assert sum([confidence for confidence in prediction.probabilities ]) != pytest.approx(1)
def test_diagnostics(default_model_storage: ModelStorage, default_execution_context: ExecutionContext): domain = Domain.from_yaml(DOMAIN_YAML) policy = TEDPolicy( TEDPolicy.get_default_config(), default_model_storage, Resource("TEDPolicy"), default_execution_context, ) GREET_RULE = DialogueStateTracker.from_events( "greet rule", evts=[ UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(UTTER_GREET_ACTION), ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(ACTION_LISTEN_NAME), ], ) precomputations = None policy.train([GREET_RULE], domain, precomputations) prediction = policy.predict_action_probabilities( GREET_RULE, domain, precomputations, ) assert prediction.diagnostic_data assert "attention_weights" in prediction.diagnostic_data assert isinstance(prediction.diagnostic_data.get("attention_weights"), np.ndarray)
def trained_ted(tmp_path_factory: TempPathFactory, moodbot_domain_path: Path) -> TEDPolicy: training_files = "data/test_moodbot/data/stories.yml" domain = Domain.load(moodbot_domain_path) trackers = training.load_data(str(training_files), domain) policy = TEDPolicy.create( { **TEDPolicy.get_default_config(), EPOCHS: 1 }, LocalModelStorage.create(tmp_path_factory.mktemp("storage")), Resource("ted"), ExecutionContext(GraphSchema({})), ) policy.train(trackers, domain) return policy
def create_policy(self, featurizer, priority): p = TEDPolicy( featurizer=featurizer, priority=priority, **{SCALE_LOSS: False, EVAL_NUM_EXAMPLES: 4}, ) return p
def create_policy( self, featurizer: Optional[TrackerFeaturizer], priority: int ) -> Policy: # use standard featurizer from TEDPolicy, # since it is using MaxHistoryTrackerFeaturizer # if max_history is specified return TEDPolicy(priority=priority, max_history=self.max_history)
def create_policy( self, featurizer: Optional[TrackerFeaturizer], priority: int ) -> Policy: return TEDPolicy( featurizer=featurizer, priority=priority, **{SCALE_LOSS: False, EVAL_NUM_EXAMPLES: 4}, )
def _config( self, config_override: Optional[Dict[Text, Any]] = None) -> Dict[Text, Any]: config_override = config_override or {} return { **TEDPolicy.get_default_config(), **config_override, }
def _config( self, config_override: Optional[Dict[Text, Any]] = None) -> Dict[Text, Any]: config_override = config_override or {} return { **TEDPolicy.get_default_config(), POLICY_MAX_HISTORY: self.max_history, **config_override, }
def create_policy(self, featurizer, priority): p = TEDPolicy( featurizer=featurizer, priority=priority, **{ KEY_RELATIVE_ATTENTION: True, VALUE_RELATIVE_ATTENTION: True, MAX_RELATIVE_POSITION: 5, }, ) return p
def _config( self, config_override: Optional[Dict[Text, Any]] = None) -> Dict[Text, Any]: config_override = config_override or {} return { **TEDPolicy.get_default_config(), SCALE_LOSS: False, EVAL_NUM_EXAMPLES: 4, **config_override, }
def create_policy(self, featurizer: Optional[TrackerFeaturizer], priority: int) -> Policy: return TEDPolicy( featurizer=featurizer, priority=priority, **{ KEY_RELATIVE_ATTENTION: True, VALUE_RELATIVE_ATTENTION: True, MAX_RELATIVE_POSITION: 5, }, )
def _config( self, config_override: Optional[Dict[Text, Any]] = None) -> Dict[Text, Any]: config_override = config_override or {} return { **TEDPolicy.get_default_config(), RANKING_LENGTH: 4, RENORMALIZE_CONFIDENCES: True, **config_override, }
def test_diagnostics(): domain = Domain.from_yaml(DOMAIN_YAML) policy = TEDPolicy() GREET_RULE = DialogueStateTracker.from_events( "greet rule", evts=[ UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(UTTER_GREET_ACTION), ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(ACTION_LISTEN_NAME), ], ) policy.train([GREET_RULE], domain, RegexInterpreter()) prediction = policy.predict_action_probabilities(GREET_RULE, domain, RegexInterpreter()) assert prediction.diagnostic_data assert "attention_weights" in prediction.diagnostic_data assert isinstance(prediction.diagnostic_data.get("attention_weights"), np.ndarray)
def _config( self, config_override: Optional[Dict[Text, Any]] = None) -> Dict[Text, Any]: config_override = config_override or {} return { **TEDPolicy.get_default_config(), KEY_RELATIVE_ATTENTION: True, VALUE_RELATIVE_ATTENTION: True, MAX_RELATIVE_POSITION: 5, **config_override, }
async def test_infer(): default_domain = Domain.load(DEFAULT_DOMAIN_PATH_WITH_SLOTS) print("action names: {}".format(default_domain.action_names)) trackers = await training.load_data(DEFAULT_STORIES_FILE, default_domain, augmentation_factor=0, debug_plots=True) policy = TEDPolicy.load("{}/models/ted".format(prj_dir)) for tracker in trackers: y_pred = policy.predict_action_probabilities(tracker, default_domain) index = y_pred.index(max(y_pred)) print(default_domain.action_names[index])
async def test_train_keras_policy(): default_domain = Domain.load(DEFAULT_DOMAIN_PATH_WITH_SLOTS) trackers = await training.load_data(DEFAULT_STORIES_FILE, default_domain, augmentation_factor=0, debug_plots=False) config = {"epochs": 100} policy = TEDPolicy(featurizer=featurizer(), priority=1, **config) policy.train(trackers, default_domain) policy.persist("{}/models/ted".format(prj_dir))
def create_policy( self, featurizer: Optional[TrackerFeaturizer], model_storage: ModelStorage, resource: Resource, execution_context: ExecutionContext, config: Optional[Dict[Text, Any]] = None, ) -> Policy: # use standard featurizer from TEDPolicy, # since it is using MaxHistoryTrackerFeaturizer # if max_history is specified return TEDPolicy( config=self._config(config), model_storage=model_storage, resource=resource, execution_context=execution_context, )
def create_policy(self, featurizer, priority): # use standard featurizer from TEDPolicy, # since it is using MaxHistoryTrackerFeaturizer # if max_history is specified p = TEDPolicy(priority=priority, max_history=self.max_history) return p
def create_policy(self, featurizer, priority): # use standard featurizer from TEDPolicy, # since it is using FullDialogueTrackerFeaturizer # if max_history is not specified p = TEDPolicy(priority=priority) return p
def create_policy(self, featurizer, priority): p = TEDPolicy(featurizer=featurizer, priority=priority, **{RANKING_LENGTH: 11}) return p
def create_policy(self, featurizer, priority): p = TEDPolicy(featurizer=featurizer, priority=priority, **{LOSS_TYPE: "margin"}) return p
def create_policy(self, featurizer, priority): p = TEDPolicy(featurizer=featurizer, priority=priority) return p
) def test_supported_data(policy: Type[Policy], supported_data: SupportedData): assert policy.supported_data() == supported_data class OnlyRulePolicy(Policy): """Test policy that supports both rule-based and ML-based training data.""" @staticmethod def supported_data() -> SupportedData: return SupportedData.RULE_DATA @pytest.mark.parametrize( "policy,n_rule_trackers,n_ml_trackers", [ (TEDPolicy(), 0, 3), (RulePolicy(), 2, 3), (OnlyRulePolicy, 2, 0), # policy can be passed as a `type` as well ], ) def test_get_training_trackers_for_policy(policy: Policy, n_rule_trackers: int, n_ml_trackers): # create five trackers (two rule-based and three ML trackers) trackers = [ DialogueStateTracker("id1", slots=[], is_rule_tracker=True), DialogueStateTracker("id2", slots=[], is_rule_tracker=False), DialogueStateTracker("id3", slots=[], is_rule_tracker=False), DialogueStateTracker("id4", slots=[], is_rule_tracker=True), DialogueStateTracker("id5", slots=[], is_rule_tracker=False), ]