def test_train_with_e2e_data(
default_model_storage: ModelStorage,
default_execution_context: ExecutionContext,
tracker_events: List[List[Event]],
skip_training: bool,
domain: Domain,
):
policy = UnexpecTEDIntentPolicy(
UnexpecTEDIntentPolicy.get_default_config(),
default_model_storage,
Resource("UnexpecTEDIntentPolicy"),
default_execution_context,
featurizer=IntentMaxHistoryTrackerFeaturizer(
IntentTokenizerSingleStateFeaturizer()
),
)
trackers_for_training = [
TrackerWithCachedStates.from_events(
sender_id=f"{tracker_index}", evts=events, domain=domain
)
for tracker_index, events in enumerate(tracker_events)
]
if skip_training:
with pytest.warns(UserWarning):
policy.train(trackers_for_training, domain, precomputations=None)
else:
policy.train(trackers_for_training, domain, precomputations=None)
def _standard_featurizer(self) -> TrackerFeaturizer:
return IntentMaxHistoryTrackerFeaturizer(
IntentTokenizerSingleStateFeaturizer(),
max_history=self.config.get(POLICY_MAX_HISTORY),
)
class PolicyTestCollection:
"""Tests every policy needs to fulfill.
Each policy can declare further tests on its own."""
@staticmethod
def _policy_class_to_test() -> Type[PolicyGraphComponent]:
raise NotImplementedError
max_history = 3 # this is the amount of history we test on
@pytest.fixture(scope="class")
def resource(self, ) -> Resource:
return Resource(uuid.uuid4().hex)
@pytest.fixture(scope="class")
def model_storage(self, tmp_path_factory: TempPathFactory) -> ModelStorage:
return LocalModelStorage(tmp_path_factory.mktemp(uuid.uuid4().hex))
@pytest.fixture(scope="class")
def execution_context(self) -> ExecutionContext:
return ExecutionContext(GraphSchema({}), uuid.uuid4().hex)
def _config(
self,
config_override: Optional[Dict[Text,
Any]] = None) -> Dict[Text, Any]:
config_override = config_override or {}
config = self._policy_class_to_test().get_default_config()
return {**config, **config_override}
def create_policy(
self,
featurizer: Optional[TrackerFeaturizer],
model_storage: ModelStorage,
resource: Resource,
execution_context: ExecutionContext,
config: Optional[Dict[Text, Any]] = None,
) -> PolicyGraphComponent:
return self._policy_class_to_test()(
config=self._config(config),
model_storage=model_storage,
resource=resource,
execution_context=execution_context,
featurizer=featurizer,
)
@pytest.fixture(scope="class")
def featurizer(self) -> TrackerFeaturizer:
featurizer = MaxHistoryTrackerFeaturizer(SingleStateFeaturizer(),
max_history=self.max_history)
return featurizer
@pytest.fixture(scope="class")
def default_domain(self, domain_path: Text) -> Domain:
return Domain.load(domain_path)
@pytest.fixture(scope="class")
def tracker(self, default_domain: Domain) -> DialogueStateTracker:
return DialogueStateTracker(DEFAULT_SENDER_ID, default_domain.slots)
@pytest.fixture(scope="class")
def trained_policy(
self,
featurizer: Optional[TrackerFeaturizer],
stories_path: Text,
default_domain: Domain,
model_storage: ModelStorage,
resource: Resource,
execution_context: ExecutionContext,
) -> PolicyGraphComponent:
policy = self.create_policy(featurizer, model_storage, resource,
execution_context)
training_trackers = train_trackers(default_domain,
stories_path,
augmentation_factor=20)
policy.train(training_trackers, default_domain)
return policy
def test_featurizer(
self,
trained_policy: PolicyGraphComponent,
resource: Resource,
model_storage: ModelStorage,
tmp_path: Path,
execution_context: ExecutionContext,
):
assert isinstance(trained_policy.featurizer,
MaxHistoryTrackerFeaturizer)
assert trained_policy.featurizer.max_history == self.max_history
assert isinstance(trained_policy.featurizer.state_featurizer,
SingleStateFeaturizer)
loaded = trained_policy.__class__.load(
self._config(trained_policy.config),
model_storage,
resource,
execution_context,
)
assert isinstance(loaded.featurizer, MaxHistoryTrackerFeaturizer)
assert loaded.featurizer.max_history == self.max_history
assert isinstance(loaded.featurizer.state_featurizer,
SingleStateFeaturizer)
@pytest.mark.parametrize("should_finetune", [False, True])
def test_persist_and_load(
self,
trained_policy: PolicyGraphComponent,
default_domain: Domain,
should_finetune: bool,
stories_path: Text,
model_storage: ModelStorage,
resource: Resource,
execution_context: ExecutionContext,
):
loaded = trained_policy.__class__.load(
self._config(trained_policy.config),
model_storage,
resource,
dataclasses.replace(execution_context,
is_finetuning=should_finetune),
)
assert loaded.finetune_mode == should_finetune
trackers = train_trackers(default_domain,
stories_path,
augmentation_factor=20)
for tracker in trackers:
predicted_probabilities = loaded.predict_action_probabilities(
tracker, default_domain)
actual_probabilities = trained_policy.predict_action_probabilities(
tracker, default_domain)
assert predicted_probabilities == actual_probabilities
def test_prediction_on_empty_tracker(self, trained_policy: Policy,
default_domain: Domain):
tracker = DialogueStateTracker(DEFAULT_SENDER_ID, default_domain.slots)
prediction = trained_policy.predict_action_probabilities(
tracker,
default_domain,
)
assert not prediction.is_end_to_end_prediction
assert len(prediction.probabilities) == default_domain.num_actions
assert max(prediction.probabilities) <= 1.0
assert min(prediction.probabilities) >= 0.0
@pytest.mark.filterwarnings(
"ignore:.*without a trained model present.*:UserWarning")
def test_persist_and_load_empty_policy(
self,
default_domain: Domain,
default_model_storage: ModelStorage,
execution_context: ExecutionContext,
):
resource = Resource(uuid.uuid4().hex)
empty_policy = self.create_policy(
None,
default_model_storage,
resource,
execution_context,
)
empty_policy.train([], default_domain)
loaded = empty_policy.__class__.load(
self._config(),
default_model_storage,
resource,
execution_context,
)
assert loaded is not None
@staticmethod
def _get_next_action(policy: PolicyGraphComponent, events: List[Event],
domain: Domain) -> Text:
tracker = get_tracker(events)
scores = policy.predict_action_probabilities(
tracker,
domain,
).probabilities
index = scores.index(max(scores))
return domain.action_names_or_texts[index]
@pytest.mark.parametrize(
"featurizer_config, tracker_featurizer, state_featurizer",
[
(
[{
# TODO: remove "2" when migration of policies is done
"name": "MaxHistoryTrackerFeaturizer2",
"max_history": 12,
"state_featurizer": [],
}],
MaxHistoryTrackerFeaturizer(max_history=12),
type(None),
),
(
# TODO: remove "2" when migration of policies is done
[{
"name": "MaxHistoryTrackerFeaturizer2",
"max_history": 12
}],
MaxHistoryTrackerFeaturizer(max_history=12),
type(None),
),
(
[{
# TODO: remove "2" when migration of policies is done
"name":
"IntentMaxHistoryTrackerFeaturizer2",
"max_history":
12,
"state_featurizer": [{
"name":
"IntentTokenizerSingleStateFeaturizer2"
}],
}],
IntentMaxHistoryTrackerFeaturizer(max_history=12),
IntentTokenizerSingleStateFeaturizer,
),
],
)
def test_different_featurizer_configs(
self,
featurizer_config: Optional[Dict[Text, Any]],
model_storage: ModelStorage,
resource: Resource,
execution_context: ExecutionContext,
tracker_featurizer: MaxHistoryTrackerFeaturizer,
state_featurizer: Type[SingleStateFeaturizer],
):
featurizer_config_override = ({
"featurizer": featurizer_config
} if featurizer_config else {})
policy = self.create_policy(
None,
model_storage=model_storage,
resource=resource,
execution_context=execution_context,
config=self._config(featurizer_config_override),
)
featurizer = policy.featurizer
assert isinstance(featurizer, tracker_featurizer.__class__)
if featurizer_config:
expected_max_history = featurizer_config[0].get(POLICY_MAX_HISTORY)
else:
expected_max_history = self._config().get(POLICY_MAX_HISTORY)
assert featurizer.max_history == expected_max_history
assert isinstance(featurizer.state_featurizer, state_featurizer)
@pytest.mark.parametrize(
"featurizer_config",
[
[
# TODO: remove "2" when migration of policies is done
{
"name": "MaxHistoryTrackerFeaturizer2",
"max_history": 12
},
{
"name": "MaxHistoryTrackerFeaturizer2",
"max_history": 12
},
],
[{
# TODO: remove "2" when migration of policies is done
"name":
"IntentMaxHistoryTrackerFeaturizer2",
"max_history":
12,
"state_featurizer": [
{
"name": "IntentTokenizerSingleStateFeaturizer2"
},
{
"name": "IntentTokenizerSingleStateFeaturizer2"
},
],
}],
],
)
def test_different_invalid_featurizer_configs(
self,
trained_policy: PolicyGraphComponent,
featurizer_config: Optional[Dict[Text, Any]],
model_storage: ModelStorage,
resource: Resource,
execution_context: ExecutionContext,
):
with pytest.raises(InvalidPolicyConfig):
self.create_policy(
None,
model_storage=model_storage,
resource=resource,
execution_context=execution_context,
config={"featurizer": featurizer_config},
)
def featurizer(self) -> TrackerFeaturizer:
featurizer = IntentMaxHistoryTrackerFeaturizer(
IntentTokenizerSingleStateFeaturizer(), max_history=self.max_history
)
return featurizer
def _standard_featurizer(
max_history: Optional[int] = None) -> TrackerFeaturizer:
return IntentMaxHistoryTrackerFeaturizer(
IntentTokenizerSingleStateFeaturizer(), max_history=max_history)