async def test_form_unhappy_path_from_in_form_rule():
form_name = "some_form"
handle_rejection_action_name = "utter_handle_rejection"
domain = Domain.from_yaml(
f"""
intents:
- {GREET_INTENT_NAME}
actions:
- {UTTER_GREET_ACTION}
- {handle_rejection_action_name}
- some-action
slots:
{REQUESTED_SLOT}:
type: unfeaturized
forms:
- {form_name}
"""
)
unhappy_rule = TrackerWithCachedStates.from_events(
"bla",
domain=domain,
slots=domain.slots,
evts=[
# We are in an active form
ActiveLoop(form_name),
SlotSet(REQUESTED_SLOT, "bla"),
ActionExecuted(RULE_SNIPPET_ACTION_NAME),
ActionExecuted(ACTION_LISTEN_NAME),
# When a user says "hi", and the form is unhappy,
# we want to run a specific action
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(handle_rejection_action_name),
ActionExecuted(form_name),
ActionExecuted(ACTION_LISTEN_NAME),
],
is_rule_tracker=True,
)
policy = RulePolicy()
# RulePolicy should memorize that unhappy_rule overrides GREET_RULE
policy.train([GREET_RULE, unhappy_rule], domain, RegexInterpreter())
# Check that RulePolicy predicts action to handle unhappy path
conversation_events = [
ActionExecuted(form_name),
ActiveLoop(form_name),
SlotSet(REQUESTED_SLOT, "some value"),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered("haha", {"name": GREET_INTENT_NAME}),
ActionExecutionRejected(form_name),
]
prediction = policy.predict_action_probabilities(
DialogueStateTracker.from_events(
"casd", evts=conversation_events, slots=domain.slots
),
domain,
RegexInterpreter(),
)
assert_predicted_action(prediction, domain, handle_rejection_action_name)
# Check that RulePolicy triggers form again after handling unhappy path
conversation_events.append(ActionExecuted(handle_rejection_action_name))
prediction = policy.predict_action_probabilities(
DialogueStateTracker.from_events(
"casd", evts=conversation_events, slots=domain.slots
),
domain,
RegexInterpreter(),
)
assert_predicted_action(prediction, domain, form_name)
# assert that the initial slot has not been affected
assert slot.value == initial_value
@pytest.mark.parametrize(
"events, expected_applied_events",
[
(
[
# Form gets triggered.
ActionExecuted(ACTION_LISTEN_NAME),
user_uttered("fill_whole_form"),
# Form executes and fills slots.
ActionExecuted("loop"),
ActiveLoop("loop"),
SlotSet("slot1", "value"),
SlotSet("slot2", "value2"),
],
[
ActionExecuted(ACTION_LISTEN_NAME),
user_uttered("fill_whole_form"),
ActionExecuted("loop"),
ActiveLoop("loop"),
SlotSet("slot1", "value"),
SlotSet("slot2", "value2"),
],
),
(
[
# Form gets triggered.
def test_incomplete_rules_due_to_loops():
some_form = "some_form"
domain = Domain.from_yaml(
f"""
intents:
- {GREET_INTENT_NAME}
forms:
- {some_form}
"""
)
policy = RulePolicy()
complete_rule = TrackerWithCachedStates.from_events(
"complete_rule",
domain=domain,
slots=domain.slots,
evts=[
ActionExecuted(RULE_SNIPPET_ACTION_NAME),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(some_form),
ActiveLoop(some_form),
ActionExecuted(ACTION_LISTEN_NAME),
],
is_rule_tracker=True,
)
incomplete_rule = TrackerWithCachedStates.from_events(
"incomplete_rule",
domain=domain,
slots=domain.slots,
evts=[
ActionExecuted(RULE_SNIPPET_ACTION_NAME),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(some_form),
ActionExecuted(ACTION_LISTEN_NAME),
],
is_rule_tracker=True,
)
with pytest.raises(InvalidRule) as execinfo:
policy.train([complete_rule, incomplete_rule], domain, RegexInterpreter())
assert all(
name in execinfo.value.message
for name in {some_form, incomplete_rule.sender_id}
)
fixed_incomplete_rule = TrackerWithCachedStates.from_events(
"fixed_incomplete_rule",
domain=domain,
slots=domain.slots,
evts=[
ActionExecuted(RULE_SNIPPET_ACTION_NAME),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(some_form),
ActionExecuted(RULE_SNIPPET_ACTION_NAME),
ActionExecuted(ACTION_LISTEN_NAME),
],
is_rule_tracker=True,
)
policy.train([complete_rule, fixed_incomplete_rule], domain, RegexInterpreter())
async def test_switch_forms_with_same_slot(default_agent: Agent):
"""Tests switching of forms, where the first slot is the same in both forms.
Tests the fix for issue 7710"""
# Define two forms in the domain, with same first slot
slot_a = "my_slot_a"
form_1 = "my_form_1"
utter_ask_form_1 = f"Please provide the value for {slot_a} of form 1"
form_2 = "my_form_2"
utter_ask_form_2 = f"Please provide the value for {slot_a} of form 2"
domain = f"""
version: "2.0"
nlu:
- intent: order_status
examples: |
- check status of my order
- when are my shoes coming in
- intent: return
examples: |
- start a return
- I don't want my shoes anymore
forms:
{form_1}:
{slot_a}:
- type: from_entity
entity: number
{form_2}:
{slot_a}:
- type: from_entity
entity: number
responses:
utter_ask_{form_1}_{slot_a}:
- text: {utter_ask_form_1}
utter_ask_{form_2}_{slot_a}:
- text: {utter_ask_form_2}
"""
domain = Domain.from_yaml(domain)
# Driving it like rasa/core/processor
processor = MessageProcessor(
default_agent.interpreter,
default_agent.policy_ensemble,
domain,
InMemoryTrackerStore(domain),
TemplatedNaturalLanguageGenerator(domain.templates),
)
# activate the first form
tracker = DialogueStateTracker.from_events(
"some-sender",
evts=[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered("order status", {
"name": "form_1",
"confidence": 1.0
}),
DefinePrevUserUtteredFeaturization(False),
],
)
# rasa/core/processor.predict_next_action
prediction = PolicyPrediction([], "some_policy")
action_1 = FormAction(form_1, None)
await processor._run_action(
action_1,
tracker,
CollectingOutputChannel(),
TemplatedNaturalLanguageGenerator(domain.templates),
prediction,
)
events_expected = [
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered("order status", {
"name": "form_1",
"confidence": 1.0
}),
DefinePrevUserUtteredFeaturization(False),
ActionExecuted(form_1),
ActiveLoop(form_1),
SlotSet(REQUESTED_SLOT, slot_a),
BotUttered(
text=utter_ask_form_1,
metadata={"template_name": f"utter_ask_{form_1}_{slot_a}"},
),
]
assert tracker.applied_events() == events_expected
next_events = [
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered("return my shoes", {
"name": "form_2",
"confidence": 1.0
}),
DefinePrevUserUtteredFeaturization(False),
]
tracker.update_with_events(
next_events,
domain,
)
events_expected.extend(next_events)
# form_1 is still active, and bot will first validate if the user utterance
# provides valid data for the requested slot, which is rejected
await processor._run_action(
action_1,
tracker,
CollectingOutputChannel(),
TemplatedNaturalLanguageGenerator(domain.templates),
prediction,
)
events_expected.extend([ActionExecutionRejected(action_name=form_1)])
assert tracker.applied_events() == events_expected
# Next, bot predicts form_2
action_2 = FormAction(form_2, None)
await processor._run_action(
action_2,
tracker,
CollectingOutputChannel(),
TemplatedNaturalLanguageGenerator(domain.templates),
prediction,
)
events_expected.extend([
ActionExecuted(form_2),
ActiveLoop(form_2),
SlotSet(REQUESTED_SLOT, slot_a),
BotUttered(
text=utter_ask_form_2,
metadata={"template_name": f"utter_ask_{form_2}_{slot_a}"},
),
])
assert tracker.applied_events() == events_expected
class TestUnexpecTEDIntentPolicy(TestTEDPolicy):
@staticmethod
def _policy_class_to_test() -> Type[UnexpecTEDIntentPolicy]:
return UnexpecTEDIntentPolicy
@pytest.fixture(scope="class")
def featurizer(self) -> TrackerFeaturizer:
featurizer = IntentMaxHistoryTrackerFeaturizer(
IntentTokenizerSingleStateFeaturizer(),
max_history=self.max_history)
return featurizer
@staticmethod
def persist_and_load_policy(
trained_policy: UnexpecTEDIntentPolicy,
model_storage: ModelStorage,
resource: Resource,
execution_context: ExecutionContext,
):
return trained_policy.__class__.load(trained_policy.config,
model_storage, resource,
execution_context)
def test_ranking_length(self, trained_policy: UnexpecTEDIntentPolicy):
assert trained_policy.config[RANKING_LENGTH] == LABEL_RANKING_LENGTH
def test_ranking_length_and_renormalization(
self,
trained_policy: UnexpecTEDIntentPolicy,
tracker: DialogueStateTracker,
default_domain: Domain,
):
precomputations = None
prediction_metadata = trained_policy.predict_action_probabilities(
tracker, default_domain, precomputations).action_metadata
assert (prediction_metadata is None
or len(prediction_metadata[RANKING_KEY])
== trained_policy.config[RANKING_LENGTH])
def test_label_data_assembly(self, trained_policy: UnexpecTEDIntentPolicy,
default_domain: Domain):
# Construct input data
state_featurizer = trained_policy.featurizer.state_featurizer
encoded_all_labels = state_featurizer.encode_all_labels(
default_domain, precomputations=None)
attribute_data, _ = model_data_utils.convert_to_data_format(
encoded_all_labels)
assembled_label_data = trained_policy._assemble_label_data(
attribute_data, default_domain)
assembled_label_data_signature = assembled_label_data.get_signature()
assert list(assembled_label_data_signature.keys()) == [
f"{LABEL}_{INTENT}",
LABEL,
]
assert assembled_label_data.num_examples == len(default_domain.intents)
assert list(
assembled_label_data_signature[f"{LABEL}_{INTENT}"].keys()) == [
MASK,
SENTENCE,
]
assert list(assembled_label_data_signature[LABEL].keys()) == [IDS]
assert assembled_label_data_signature[f"{LABEL}_{INTENT}"][SENTENCE][
0].units == len(default_domain.intents)
def test_training_with_no_intent(
self,
featurizer: Optional[TrackerFeaturizer],
default_domain: Domain,
tmp_path: Path,
caplog: LogCaptureFixture,
model_storage: ModelStorage,
resource: Resource,
execution_context: ExecutionContext,
):
stories = tmp_path / "stories.yml"
stories.write_text("""
version: "3.0"
stories:
- story: test path
steps:
- action: utter_greet
""")
policy = self.create_policy(
featurizer=featurizer,
model_storage=model_storage,
resource=resource,
execution_context=execution_context,
)
import tests.core.test_policies
training_trackers = tests.core.test_policies.train_trackers(
default_domain, str(stories), augmentation_factor=20)
with pytest.warns(UserWarning):
policy.train(training_trackers,
default_domain,
precomputations=None)
def test_prepared_data_for_threshold_prediction(
self,
trained_policy: UnexpecTEDIntentPolicy,
default_domain: Domain,
stories_path: Path,
):
training_trackers = train_trackers(default_domain,
stories_path,
augmentation_factor=0)
training_model_data, _ = trained_policy._prepare_for_training(
training_trackers, default_domain, precomputations=None)
data_for_prediction = trained_policy._prepare_data_for_prediction(
training_model_data)
assert set(
data_for_prediction.data.keys()).issubset(PREDICTION_FEATURES)
def test_similarities_collection_for_label_ids(self):
label_ids = np.array([[0, 1], [1, -1], [2, -1]])
outputs = {
"similarities":
np.array([[[1.2, 0.3, 0.2]], [[0.5, 0.2, 1.6]], [[0.01, 0.1,
1.7]]])
}
label_id_similarities = UnexpecTEDIntentPolicy._collect_label_id_grouped_scores(
outputs, label_ids)
# Should contain similarities for all label ids except padding token.
assert sorted(list(label_id_similarities.keys())) == [0, 1, 2]
# Cross-check that the collected similarities are correct for each label id.
assert label_id_similarities[0] == {
POSITIVE_SCORES_KEY: [1.2],
NEGATIVE_SCORES_KEY: [0.5, 0.01],
}
assert label_id_similarities[1] == {
POSITIVE_SCORES_KEY: [0.3, 0.2],
NEGATIVE_SCORES_KEY: [0.1],
}
assert label_id_similarities[2] == {
POSITIVE_SCORES_KEY: [1.7],
NEGATIVE_SCORES_KEY: [0.2, 1.6],
}
def test_label_quantiles_computation(self):
label_id_scores = {
0: {
POSITIVE_SCORES_KEY: [1.3, 0.2],
NEGATIVE_SCORES_KEY: [
-0.1,
-1.2,
-2.3,
-4.1,
-0.5,
0.2,
0.8,
0.9,
-3.2,
-2.7,
],
},
3: {
POSITIVE_SCORES_KEY: [1.3, 0.2],
NEGATIVE_SCORES_KEY: [-0.1]
},
6: {
POSITIVE_SCORES_KEY: [1.3, 0.2],
NEGATIVE_SCORES_KEY: []
},
}
expected_thresholds = {
0: [
0.2,
0.2,
0.2,
0.2,
0.2,
-0.1,
-0.1,
-0.5,
-0.5,
-1.2,
-1.2,
-1.2,
-2.3,
-2.3,
-2.7,
-2.7,
-3.2,
-3.2,
-4.1,
-4.1,
],
3: [
-0.1,
-0.1,
-0.1,
-0.1,
-0.1,
-0.1,
-0.1,
-0.1,
-0.1,
-0.1,
-0.1,
-0.1,
-0.1,
-0.1,
-0.1,
-0.1,
-0.1,
-0.1,
-0.1,
-0.1,
],
6: [
0.2,
0.2,
0.2,
0.2,
0.2,
0.2,
0.2,
0.2,
0.2,
0.2,
0.2,
0.2,
0.2,
0.2,
0.2,
0.2,
0.2,
0.2,
0.2,
0.2,
],
}
thresholds = UnexpecTEDIntentPolicy._compute_label_quantiles(
label_id_scores)
assert sorted(list(thresholds.keys())) == sorted(
list(expected_thresholds.keys()))
for label_id, tolerance_thresholds in thresholds.items():
assert expected_thresholds[label_id] == tolerance_thresholds
def test_post_training_threshold_computation(
self,
trained_policy: UnexpecTEDIntentPolicy,
default_domain: Domain,
stories_path: Path,
):
training_trackers = train_trackers(default_domain,
stories_path,
augmentation_factor=0)
training_model_data, label_ids = trained_policy._prepare_for_training(
training_trackers, default_domain, precomputations=None)
trained_policy.compute_label_quantiles_post_training(
training_model_data, label_ids)
computed_thresholds = trained_policy.label_quantiles
# -1 is used for padding and hence is not expected in the keys
expected_keys = list(np.unique(label_ids))
expected_keys.remove(-1)
assert sorted(list(
computed_thresholds.keys())) == sorted(expected_keys)
@pytest.mark.parametrize(
"tolerance, expected_thresholds",
[
(0.0, [0.2, -0.1, 0.2]),
(0.75, [-2.9, -0.1, -4.3]),
(0.72, [-2.7, -0.1, -4.0]),
(0.78, [-2.9, -0.1, -4.3]),
(1.0, [-4.1, -0.1, -5.5]),
],
)
def test_pick_thresholds_for_labels(self, tolerance: float,
expected_thresholds: List[float]):
label_id_tolerance_thresholds = {
0: [
0.2,
0.2,
0.2,
0.2,
0.2,
0.2,
-0.1,
-0.1,
-0.5,
-0.5,
-1.2,
-1.2,
-2.3,
-2.3,
-2.7,
-2.9,
-3.2,
-3.2,
-4.1,
-4.1,
],
3: [
-0.1,
-0.1,
-0.1,
-0.1,
-0.1,
-0.1,
-0.1,
-0.1,
-0.1,
-0.1,
-0.1,
-0.1,
-0.1,
-0.1,
-0.1,
-0.1,
-0.1,
-0.1,
-0.1,
-0.1,
],
4: [0.2 - (index * 0.3) for index in range(20)],
}
thresholds = UnexpecTEDIntentPolicy._pick_thresholds(
label_id_tolerance_thresholds, tolerance)
assert sorted(list(thresholds.keys())) == sorted(
list(label_id_tolerance_thresholds.keys()))
computed_values = list(thresholds.values())
assert expected_thresholds == computed_values
@pytest.mark.parametrize(
"predicted_similarity, threshold_value, is_unlikely",
[(1.2, 0.2, False), (0.3, -0.1, False), (-1.5, 0.03, True)],
)
def test_unlikely_intent_check(
self,
trained_policy: UnexpecTEDIntentPolicy,
model_storage: ModelStorage,
resource: Resource,
execution_context: ExecutionContext,
default_domain: Domain,
predicted_similarity: float,
threshold_value: float,
is_unlikely: bool,
tmp_path: Path,
):
loaded_policy = self.persist_and_load_policy(trained_policy,
model_storage, resource,
execution_context)
# Construct dummy similarities
similarities = np.array([[0.0] * len(default_domain.intents)])
dummy_intent_index = 4
similarities[0, dummy_intent_index] = predicted_similarity
loaded_policy.label_thresholds[dummy_intent_index] = threshold_value
query_intent = default_domain.intents[dummy_intent_index]
unlikely_intent_prediction = loaded_policy._check_unlikely_intent(
default_domain, similarities, query_intent)
assert is_unlikely == unlikely_intent_prediction
def test_should_check_for_intent(
self,
trained_policy: UnexpecTEDIntentPolicy,
model_storage: ModelStorage,
resource: Resource,
execution_context: ExecutionContext,
default_domain: Domain,
tmp_path: Path,
):
loaded_policy = self.persist_and_load_policy(trained_policy,
model_storage, resource,
execution_context)
intent_index = 0
assert (loaded_policy._should_check_for_intent(
default_domain.intents[intent_index], default_domain) is False)
intent_index = 4
assert loaded_policy._should_check_for_intent(
default_domain.intents[intent_index], default_domain)
loaded_policy.config[IGNORE_INTENTS_LIST] = [
default_domain.intents[intent_index]
]
assert (loaded_policy._should_check_for_intent(
default_domain.intents[intent_index], default_domain) is False)
def test_no_action_unlikely_intent_prediction(
self,
trained_policy: UnexpecTEDIntentPolicy,
model_storage: ModelStorage,
resource: Resource,
execution_context: ExecutionContext,
default_domain: Domain,
tmp_path: Path,
):
loaded_policy = self.persist_and_load_policy(trained_policy,
model_storage, resource,
execution_context)
expected_probabilities = [0] * default_domain.num_actions
precomputations = None
tracker = DialogueStateTracker(sender_id="init",
slots=default_domain.slots)
prediction = loaded_policy.predict_action_probabilities(
tracker, default_domain, precomputations)
assert prediction.probabilities == expected_probabilities
tracker.update_with_events(
[
UserUttered(text="hello", intent={"name": "greet"}),
ActionExecuted(action_name="utter_greet"),
],
default_domain,
)
prediction = loaded_policy.predict_action_probabilities(
tracker, default_domain, precomputations)
assert prediction.probabilities == expected_probabilities
loaded_policy.model = None
prediction = loaded_policy.predict_action_probabilities(
tracker, default_domain, precomputations)
assert prediction.probabilities == expected_probabilities
@pytest.mark.parametrize(
"predicted_similarity, threshold_value, is_unlikely",
[(1.2, 0.2, False), (0.3, -0.1, False), (-1.5, 0.03, True)],
)
def test_action_unlikely_intent_prediction(
self,
trained_policy: UnexpecTEDIntentPolicy,
model_storage: ModelStorage,
resource: Resource,
execution_context: ExecutionContext,
default_domain: Domain,
predicted_similarity: float,
threshold_value: float,
is_unlikely: bool,
monkeypatch: MonkeyPatch,
tmp_path: Path,
):
loaded_policy = self.persist_and_load_policy(trained_policy,
model_storage, resource,
execution_context)
similarities = np.array([[[0.0] * len(default_domain.intents)]])
dummy_intent_index = 4
similarities[0, 0, dummy_intent_index] = predicted_similarity
query_intent = default_domain.intents[dummy_intent_index]
loaded_policy.label_thresholds[dummy_intent_index] = threshold_value
precomputations = None
tracker = DialogueStateTracker(sender_id="init",
slots=default_domain.slots)
tracker.update_with_events(
[UserUttered(text="hello", intent={"name": query_intent})],
default_domain)
# Preset the model predictions to the similarity values
# so that we don't need to hardcode for particular model predictions.
monkeypatch.setattr(
loaded_policy.model,
"run_inference",
lambda data: {"similarities": similarities},
)
prediction = loaded_policy.predict_action_probabilities(
tracker, default_domain, precomputations)
if not is_unlikely:
assert prediction.probabilities == [0.0
] * default_domain.num_actions
else:
assert (prediction.probabilities[default_domain.index_for_action(
ACTION_UNLIKELY_INTENT_NAME)] == 1.0)
# Make sure metadata is set. The exact structure
# of the metadata is tested separately and
# not as part of this test.
assert prediction.action_metadata is not None
# Assert metadata is serializable
assert json.dumps(prediction.action_metadata)
@pytest.mark.parametrize(
"tracker_events, should_skip",
[
([], True),
([ActionExecuted("action_listen")], True),
(
[
ActionExecuted("action_listen"),
UserUttered("hi", intent={"name": "greet"}),
],
False,
),
(
[
ActionExecuted("action_listen"),
UserUttered("hi", intent={"name": "greet"}),
EntitiesAdded([{
"name": "dummy"
}]),
],
False,
),
(
[
ActionExecuted("action_listen"),
UserUttered("hi", intent={"name": "greet"}),
SlotSet("name"),
],
False,
),
(
[
ActiveLoop("loop"),
ActionExecuted("action_listen"),
UserUttered("hi", intent={"name": "greet"}),
ActionExecutionRejected("loop"),
],
False,
),
(
[
ActionExecuted("action_listen"),
UserUttered("hi", intent={"name": "greet"}),
ActionExecuted("utter_greet"),
],
True,
),
],
)
def test_skip_predictions_to_prevent_loop(
self,
trained_policy: UnexpecTEDIntentPolicy,
model_storage: ModelStorage,
resource: Resource,
execution_context: ExecutionContext,
default_domain: Domain,
caplog: LogCaptureFixture,
tracker_events: List[Event],
should_skip: bool,
tmp_path: Path,
):
"""Skips predictions to prevent loop."""
loaded_policy = self.persist_and_load_policy(trained_policy,
model_storage, resource,
execution_context)
precomputations = None
tracker = DialogueStateTracker(sender_id="init",
slots=default_domain.slots)
tracker.update_with_events(tracker_events, default_domain)
with caplog.at_level(logging.DEBUG):
prediction = loaded_policy.predict_action_probabilities(
tracker, default_domain, precomputations)
assert ("Skipping predictions for UnexpecTEDIntentPolicy"
in caplog.text) == should_skip
if should_skip:
assert prediction.probabilities == loaded_policy._default_predictions(
default_domain)
@pytest.mark.parametrize(
"tracker_events",
[
[
ActionExecuted("action_listen"),
UserUttered("hi", intent={"name": "inexistent_intent"}),
],
[
ActionExecuted("action_listen"),
UserUttered("hi", intent={"name": "inexistent_intent"}),
EntitiesAdded([{
"name": "dummy"
}]),
],
[
ActionExecuted("action_listen"),
UserUttered("hi", intent={"name": "inexistent_intent"}),
SlotSet("name"),
],
[
ActiveLoop("loop"),
ActionExecuted("action_listen"),
UserUttered("hi", intent={"name": "inexistent_intent"}),
ActionExecutionRejected("loop"),
],
],
)
def test_skip_predictions_if_new_intent(
self,
trained_policy: UnexpecTEDIntentPolicy,
model_storage: ModelStorage,
resource: Resource,
execution_context: ExecutionContext,
default_domain: Domain,
caplog: LogCaptureFixture,
tracker_events: List[Event],
):
"""Skips predictions if there's a new intent created."""
loaded_policy = self.persist_and_load_policy(trained_policy,
model_storage, resource,
execution_context)
tracker = DialogueStateTracker(sender_id="init",
slots=default_domain.slots)
tracker.update_with_events(tracker_events, default_domain)
with caplog.at_level(logging.DEBUG):
prediction = loaded_policy.predict_action_probabilities(
tracker, default_domain, precomputations=None)
assert "Skipping predictions for UnexpecTEDIntentPolicy" in caplog.text
assert prediction.probabilities == loaded_policy._default_predictions(
default_domain)
@pytest.mark.parametrize(
"tracker_events_with_action, tracker_events_without_action",
[
(
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="hello", intent={"name": "greet"}),
ActionExecuted(ACTION_UNLIKELY_INTENT_NAME),
ActionExecuted("utter_greet"),
UserUttered(text="sad", intent={"name": "thank_you"}),
],
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="hello", intent={"name": "greet"}),
ActionExecuted("utter_greet"),
UserUttered(text="sad", intent={"name": "thank_you"}),
],
),
(
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="hello", intent={"name": "greet"}),
EntitiesAdded(entities=[{
"entity": "name",
"value": "Peter"
}]),
ActionExecuted(ACTION_UNLIKELY_INTENT_NAME),
ActionExecuted("utter_greet"),
UserUttered(text="sad", intent={"name": "thank_you"}),
],
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="hello", intent={"name": "greet"}),
EntitiesAdded(entities=[{
"entity": "name",
"value": "Peter"
}]),
ActionExecuted("utter_greet"),
UserUttered(text="sad", intent={"name": "thank_you"}),
],
),
(
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="hello", intent={"name": "greet"}),
ActionExecuted(ACTION_UNLIKELY_INTENT_NAME),
ActionExecuted("some_form"),
ActiveLoop("some_form"),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="default", intent={"name": "default"}),
ActionExecuted(ACTION_UNLIKELY_INTENT_NAME),
UserUttered(text="sad", intent={"name": "thank_you"}),
],
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="hello", intent={"name": "greet"}),
ActionExecuted(ACTION_UNLIKELY_INTENT_NAME),
ActionExecuted("some_form"),
ActiveLoop("some_form"),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="default", intent={"name": "default"}),
UserUttered(text="sad", intent={"name": "thank_you"}),
],
),
],
)
def test_ignore_action_unlikely_intent(
self,
trained_policy: UnexpecTEDIntentPolicy,
model_storage: ModelStorage,
resource: Resource,
execution_context: ExecutionContext,
default_domain: Domain,
tracker_events_with_action: List[Event],
tracker_events_without_action: List[Event],
tmp_path: Path,
):
loaded_policy = self.persist_and_load_policy(trained_policy,
model_storage, resource,
execution_context)
precomputations = None
tracker_with_action = DialogueStateTracker.from_events(
"test 1", evts=tracker_events_with_action)
tracker_without_action = DialogueStateTracker.from_events(
"test 2", evts=tracker_events_without_action)
prediction_with_action = loaded_policy.predict_action_probabilities(
tracker_with_action, default_domain, precomputations)
prediction_without_action = loaded_policy.predict_action_probabilities(
tracker_without_action, default_domain, precomputations)
# If the weights didn't change then both trackers
# should result in same prediction. For `UnexpecTEDIntentPolicy`, the real
# prediction is inside action metadata.
assert (prediction_with_action.action_metadata ==
prediction_without_action.action_metadata)
def test_label_embedding_collection(
self, trained_policy: UnexpecTEDIntentPolicy):
label_ids = tf.constant([[[2], [-1]], [[1], [2]], [[0], [-1]]],
dtype=tf.int32)
all_label_embeddings = np.random.random((10, 20))
# `-1` is used as padding label id. The embedding for it
# will be the same as `label_id=0`
expected_extracted_label_embeddings = tf.constant(
np.concatenate([
all_label_embeddings[2],
all_label_embeddings[0],
all_label_embeddings[1],
all_label_embeddings[2],
all_label_embeddings[0],
all_label_embeddings[0],
]).reshape((3, 2, 20)),
dtype=tf.float32,
)
actual_extracted_label_embeddings = trained_policy.model._get_labels_embed(
label_ids, tf.constant(all_label_embeddings, dtype=tf.float32))
assert np.all(expected_extracted_label_embeddings ==
actual_extracted_label_embeddings)
@pytest.mark.parametrize("query_intent_index, ranking_length", [(0, 0),
(1, 3),
(2, 1),
(5, 0)])
def test_collect_action_metadata(
self,
trained_policy: UnexpecTEDIntentPolicy,
model_storage: ModelStorage,
resource: Resource,
execution_context: ExecutionContext,
default_domain: Domain,
tmp_path: Path,
query_intent_index: int,
ranking_length: int,
):
loaded_policy = self.persist_and_load_policy(trained_policy,
model_storage, resource,
execution_context)
def test_individual_label_metadata(
label_metadata: Dict[Text, Optional[float]],
all_thresholds: Dict[int, float],
all_similarities: np.array,
label_index: int,
):
expected_score = all_similarities[0][label_index]
expected_threshold = (all_thresholds[label_index]
if label_index in all_thresholds else None)
expected_severity = (expected_threshold - expected_score
if expected_threshold else None)
assert label_metadata.get(SCORE_KEY) == expected_score
assert label_metadata.get(THRESHOLD_KEY) == expected_threshold
assert label_metadata.get(SEVERITY_KEY) == expected_severity
# Monkey-patch certain attributes of the policy to make the testing easier.
label_thresholds = {0: 1.2, 1: -0.3, 4: -2.3, 5: 0.2}
loaded_policy.label_thresholds = label_thresholds
loaded_policy.config[RANKING_LENGTH] = ranking_length
# Some dummy similarities
similarities = np.array([[3.2, 0.2, -1.2, -4.3, -5.1, 2.3]])
query_intent = default_domain.intents[query_intent_index]
metadata = loaded_policy._collect_action_metadata(
default_domain, similarities, query_intent=query_intent)
# Expected outer-most keys
assert sorted(list(metadata.keys())) == sorted(
[QUERY_INTENT_KEY, RANKING_KEY])
# Schema validation for query intent key
assert sorted(list(metadata[QUERY_INTENT_KEY].keys())) == sorted(
[NAME, SCORE_KEY, THRESHOLD_KEY, SEVERITY_KEY])
# Test all elements of metadata for query intent
assert metadata[QUERY_INTENT_KEY].get(NAME) == query_intent
test_individual_label_metadata(
metadata.get(QUERY_INTENT_KEY),
label_thresholds,
similarities,
query_intent_index,
)
# Check if ranking is sorted correctly and truncated to `ranking_length`
sorted_label_similarities = sorted(
[(index, score) for index, score in enumerate(similarities[0])],
key=lambda x: -x[1],
)
sorted_label_similarities = (sorted_label_similarities[:ranking_length]
if ranking_length else
sorted_label_similarities)
expected_label_rankings = [
default_domain.intents[index]
for index, _ in sorted_label_similarities
]
collected_label_rankings = [
label_metadata.get(NAME)
for label_metadata in metadata.get(RANKING_KEY)
]
assert collected_label_rankings == expected_label_rankings
# Test all elements of metadata for all labels in ranking
for label_metadata in metadata.get(RANKING_KEY):
label_index = default_domain.intents.index(
label_metadata.get(NAME))
test_individual_label_metadata(label_metadata, label_thresholds,
similarities, label_index)
@pytest.mark.parametrize(
"tracker_events_for_training, expected_trackers_with_events",
[
# Filter because of no intent and action name
(
[
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="hello", intent={"name": "greet"}),
ActionExecuted("utter_greet"),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="happy to make it work",
intent={"name": "goodbye"}),
ActionExecuted("utter_goodbye"),
ActionExecuted(ACTION_LISTEN_NAME),
],
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="hello"),
ActionExecuted("utter_greet"),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="happy to make it work"),
ActionExecuted(action_text="Great!"),
ActionExecuted(ACTION_LISTEN_NAME),
],
],
[[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="hello", intent={"name": "greet"}),
ActionExecuted("utter_greet"),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="happy to make it work",
intent={"name": "goodbye"}),
ActionExecuted("utter_goodbye"),
ActionExecuted(ACTION_LISTEN_NAME),
]],
),
# Filter because of no action name
(
[
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="hello", intent={"name": "greet"}),
ActionExecuted("utter_greet"),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="happy to make it work",
intent={"name": "goodbye"}),
ActionExecuted("utter_goodbye"),
ActionExecuted(ACTION_LISTEN_NAME),
],
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="hello"),
ActionExecuted("utter_greet"),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="happy to make it work",
intent={"name": "goodbye"}),
ActionExecuted(action_text="Great!"),
ActionExecuted(ACTION_LISTEN_NAME),
],
],
[[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="hello", intent={"name": "greet"}),
ActionExecuted("utter_greet"),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="happy to make it work",
intent={"name": "goodbye"}),
ActionExecuted("utter_goodbye"),
ActionExecuted(ACTION_LISTEN_NAME),
]],
),
# Filter because of no intent
(
[
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="hello", intent={"name": "greet"}),
ActionExecuted("utter_greet"),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="happy to make it work",
intent={"name": "goodbye"}),
ActionExecuted("utter_goodbye"),
ActionExecuted(ACTION_LISTEN_NAME),
],
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="hello"),
ActionExecuted("utter_greet"),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="happy to make it work"),
ActionExecuted("utter_goodbye"),
ActionExecuted(ACTION_LISTEN_NAME),
],
],
[[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="hello", intent={"name": "greet"}),
ActionExecuted("utter_greet"),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="happy to make it work",
intent={"name": "goodbye"}),
ActionExecuted("utter_goodbye"),
ActionExecuted(ACTION_LISTEN_NAME),
]],
),
# No filter needed
(
[[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="hello", intent={"name": "greet"}),
ActionExecuted("utter_greet"),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="happy to make it work",
intent={"name": "goodbye"}),
ActionExecuted("utter_goodbye"),
ActionExecuted(ACTION_LISTEN_NAME),
]],
[[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="hello", intent={"name": "greet"}),
ActionExecuted("utter_greet"),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="happy to make it work",
intent={"name": "goodbye"}),
ActionExecuted("utter_goodbye"),
ActionExecuted(ACTION_LISTEN_NAME),
]],
),
# Filter to return empty list of trackers
(
[[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="hello", intent={"name": "greet"}),
ActionExecuted("utter_greet"),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="happy to make it work",
intent={"name": "goodbye"}),
ActionExecuted(action_text="Great!"),
ActionExecuted(ACTION_LISTEN_NAME),
]],
[],
),
],
)
def test_filter_training_trackers(
self,
tracker_events_for_training: List[List[Event]],
expected_trackers_with_events: List[List[Event]],
domain: Domain,
):
trackers_for_training = [
TrackerWithCachedStates.from_events(sender_id=f"{tracker_index}",
evts=events,
domain=domain)
for tracker_index, events in enumerate(tracker_events_for_training)
]
filtered_trackers = UnexpecTEDIntentPolicy._get_trackers_for_training(
trackers_for_training)
assert len(filtered_trackers) == len(expected_trackers_with_events)
for collected_tracker, expected_tracker_events in zip(
filtered_trackers, expected_trackers_with_events):
collected_tracker_events = list(collected_tracker.events)
assert collected_tracker_events == expected_tracker_events
async def test_form_unhappy_path_no_validation_from_story():
form_name = "some_form"
handle_rejection_action_name = "utter_handle_rejection"
domain = Domain.from_yaml(
f"""
intents:
- {GREET_INTENT_NAME}
actions:
- {UTTER_GREET_ACTION}
- {handle_rejection_action_name}
- some-action
slots:
{REQUESTED_SLOT}:
type: unfeaturized
forms:
- {form_name}
"""
)
unhappy_story = TrackerWithCachedStates.from_events(
"bla",
domain=domain,
slots=domain.slots,
evts=[
# We are in an active form
ActionExecuted(form_name),
ActiveLoop(form_name),
ActionExecuted(ACTION_LISTEN_NAME),
# When a user says "hi", and the form is unhappy,
# we want to run a specific action
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(handle_rejection_action_name),
ActionExecuted(ACTION_LISTEN_NAME),
# Next user utterance is an answer to the previous question
# and shouldn't be validated by the form
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(form_name),
ActionExecuted(ACTION_LISTEN_NAME),
],
)
policy = RulePolicy()
policy.train([unhappy_story], domain, RegexInterpreter())
# Check that RulePolicy predicts no validation to handle unhappy path
conversation_events = [
ActionExecuted(form_name),
ActiveLoop(form_name),
SlotSet(REQUESTED_SLOT, "some value"),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered("haha", {"name": GREET_INTENT_NAME}),
ActionExecutionRejected(form_name),
ActionExecuted(handle_rejection_action_name),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered("haha", {"name": GREET_INTENT_NAME}),
]
tracker = DialogueStateTracker.from_events(
"casd", evts=conversation_events, slots=domain.slots
)
prediction = policy.predict_action_probabilities(
tracker, domain, RegexInterpreter()
)
# there is no rule for next action
assert prediction.max_confidence == policy._core_fallback_threshold
# check that RulePolicy entered unhappy path based on the training story
assert prediction.events == [LoopInterrupted(True)]
parse_data={
"entities": [],
"intent": {
"confidence": 0.2287036031484604,
"name": "affirm"
},
"text": "Let's just pretend everything went correctly",
},
text="Let's just pretend everything went correctly",
timestamp=1551884208.092693,
),
ActionExecuted(
action_name="action_deactivate_loop",
timestamp=1551884214.951055,
),
ActiveLoop(name=None, timestamp=1551884214.9510589),
SlotSet(key="requested_slot", timestamp=1551884214.951062, value=None),
ActionExecuted(
action_name="action_listen",
confidence=0.7680902069097734,
policy="policy_0_TEDPolicy",
timestamp=1551884216.705635,
),
],
)
TEST_MOODBOT_DIALOGUE = Dialogue(
name="moodbot",
events=[
ActionExecuted(
action_name="action_listen",
timestamp=1551883958.346432,
async def test_persist_form_story():
domain = Domain.load("data/test_domains/form.yml")
tracker = DialogueStateTracker("", domain.slots)
story = (
"* greet\n"
" - utter_greet\n"
"* start_form\n"
" - some_form\n"
' - active_loop{"name": "some_form"}\n'
"* default\n"
" - utter_default\n"
" - some_form\n"
"* stop\n"
" - utter_ask_continue\n"
"* affirm\n"
" - some_form\n"
"* stop\n"
" - utter_ask_continue\n"
"* inform\n"
" - some_form\n"
' - active_loop{"name": null}\n'
"* goodbye\n"
" - utter_goodbye\n"
)
# simulate talking to the form
events = [
UserUttered(intent={"name": "greet"}),
ActionExecuted("utter_greet"),
ActionExecuted("action_listen"),
# start the form
UserUttered(intent={"name": "start_form"}),
ActionExecuted("some_form"),
ActiveLoop("some_form"),
ActionExecuted("action_listen"),
# out of form input
UserUttered(intent={"name": "default"}),
ActionExecutionRejected("some_form"),
ActionExecuted("utter_default"),
ActionExecuted("some_form"),
ActionExecuted("action_listen"),
# out of form input
UserUttered(intent={"name": "stop"}),
ActionExecutionRejected("some_form"),
ActionExecuted("utter_ask_continue"),
ActionExecuted("action_listen"),
# out of form input but continue with the form
UserUttered(intent={"name": "affirm"}),
LoopInterrupted(True),
ActionExecuted("some_form"),
ActionExecuted("action_listen"),
# out of form input
UserUttered(intent={"name": "stop"}),
ActionExecutionRejected("some_form"),
ActionExecuted("utter_ask_continue"),
ActionExecuted("action_listen"),
# form input
UserUttered(intent={"name": "inform"}),
LoopInterrupted(False),
ActionExecuted("some_form"),
ActionExecuted("action_listen"),
ActiveLoop(None),
UserUttered(intent={"name": "goodbye"}),
ActionExecuted("utter_goodbye"),
ActionExecuted("action_listen"),
]
for event in events:
tracker.update(event)
assert story in tracker.export_stories(MarkdownStoryWriter())
EntitiesAdded(entities=[]),
EntitiesAdded(entities=[{
"entity": "name",
"value": "John",
"role": "contact",
"group": "test"
}]),
DefinePrevUserUtteredFeaturization(
use_text_for_featurization=False, timestamp=None, metadata=None),
ReminderCancelled(timestamp=1621590172.3872123),
ReminderScheduled(
timestamp=None, trigger_date_time=datetime.now(), intent="greet"),
ActionExecutionRejected(action_name="my_action"),
LegacyFormValidation(validate=True, timestamp=None),
LoopInterrupted(timestamp=None, is_interrupted=False),
ActiveLoop(name="loop"),
LegacyForm(name="my_form"),
AllSlotsReset(),
SlotSet(key="my_slot", value={}),
SlotSet(key="my slot", value=[]),
SlotSet(key="test", value=1),
SlotSet(key="test", value="text"),
ConversationResumed(),
ConversationPaused(),
FollowupAction(name="test"),
StoryExported(),
Restarted(),
ActionReverted(),
UserUtteranceReverted(),
BotUttered(text="Test bot utterance"),
UserUttered(
prediction = mapping_fallback_ensemble.probabilities_using_best_policy(
tracker, domain, RegexInterpreter())
index_of_mapping_policy = 1
next_action = rasa.core.actions.action.action_for_index(
prediction.max_confidence_index, domain, None)
assert (prediction.policy_name ==
f"policy_{index_of_mapping_policy}_{MappingPolicy.__name__}")
assert next_action.name() == ACTION_RESTART_NAME
@pytest.mark.parametrize(
"events",
[
[
ActiveLoop("test-form"),
ActionExecuted(ACTION_LISTEN_NAME),
utilities.user_uttered(USER_INTENT_RESTART, 1),
],
[
ActionExecuted(ACTION_LISTEN_NAME),
utilities.user_uttered(USER_INTENT_RESTART, 1),
],
],
)
def test_mapping_wins_over_form(events: List[Event]):
domain = """
forms:
- test-form
"""
domain = Domain.from_yaml(domain)
class TestTEDPolicy(PolicyTestCollection):
@staticmethod
def _policy_class_to_test() -> Type[TEDPolicy]:
return TEDPolicy
def test_train_model_checkpointing(self, tmp_path: Path):
checkpoint_dir = get_checkpoint_dir_path(tmp_path)
assert not checkpoint_dir.is_dir()
train_core(
domain="data/test_domains/default.yml",
stories="data/test_yaml_stories/stories_defaultdomain.yml",
train_path=str(tmp_path),
output=str(tmp_path),
config="data/test_config/config_ted_policy_model_checkpointing.yml",
)
assert checkpoint_dir.is_dir()
def test_doesnt_checkpoint_with_no_checkpointing(self, tmp_path: Path):
checkpoint_dir = get_checkpoint_dir_path(tmp_path)
assert not checkpoint_dir.is_dir()
train_core(
domain="data/test_domains/default.yml",
stories="data/test_yaml_stories/stories_defaultdomain.yml",
train_path=str(tmp_path),
output=str(tmp_path),
config=
"data/test_config/config_ted_policy_no_model_checkpointing.yml",
)
assert not checkpoint_dir.is_dir()
def test_doesnt_checkpoint_with_zero_eval_num_examples(
self, tmp_path: Path):
checkpoint_dir = get_checkpoint_dir_path(tmp_path)
assert not checkpoint_dir.is_dir()
config_file = "config_ted_policy_model_checkpointing_zero_eval_num_examples.yml"
with pytest.warns(UserWarning) as warning:
train_core(
domain="data/test_domains/default.yml",
stories="data/test_yaml_stories/stories_defaultdomain.yml",
train_path=str(tmp_path),
output=str(tmp_path),
config=f"data/test_config/{config_file}",
)
warn_text = (
f"You have opted to save the best model, but the value of "
f"'{EVAL_NUM_EXAMPLES}' is not greater than 0. No checkpoint model will be "
f"saved.")
assert not checkpoint_dir.is_dir()
assert len([w for w in warning if warn_text in str(w.message)]) == 1
@pytest.mark.parametrize(
"should_finetune, epoch_override, expected_epoch_value",
[
(
True,
TEDPolicy.get_default_config()[EPOCHS] + 1,
TEDPolicy.get_default_config()[EPOCHS] + 1,
),
(
False,
TEDPolicy.get_default_config()[EPOCHS] + 1,
TEDPolicy.get_default_config()[EPOCHS],
), # trained_policy uses default epochs during training
],
)
def test_epoch_override_when_loaded(
self,
trained_policy: TEDPolicy,
should_finetune: bool,
epoch_override: int,
expected_epoch_value: int,
resource: Resource,
model_storage: ModelStorage,
execution_context: ExecutionContext,
):
execution_context.is_finetuning = should_finetune
loaded_policy = trained_policy.__class__.load(
{
**self._config(), EPOCH_OVERRIDE: epoch_override
},
model_storage,
resource,
execution_context,
)
assert loaded_policy.config[EPOCHS] == expected_epoch_value
def test_train_fails_with_checkpoint_zero_eval_num_epochs(
self, tmp_path: Path):
checkpoint_dir = get_checkpoint_dir_path(tmp_path)
assert not checkpoint_dir.is_dir()
config_file = "config_ted_policy_model_checkpointing_zero_every_num_epochs.yml"
with pytest.raises(InvalidConfigException):
with pytest.warns(UserWarning) as warning:
train_core(
domain="data/test_domains/default.yml",
stories="data/test_yaml_stories/stories_defaultdomain.yml",
train_path=str(tmp_path),
output=str(tmp_path),
config=f"data/test_config/{config_file}",
)
warn_text = (
f"You have opted to save the best model, but the value of "
f"'{EVAL_NUM_EPOCHS}' is not -1 or greater than 0. Training will fail."
)
assert len([w for w in warning if warn_text in str(w.message)]) == 1
assert not checkpoint_dir.is_dir()
def test_training_with_no_intent(
self,
featurizer: Optional[TrackerFeaturizer],
default_domain: Domain,
tmp_path: Path,
caplog: LogCaptureFixture,
model_storage: ModelStorage,
resource: Resource,
execution_context: ExecutionContext,
):
stories = tmp_path / "stories.yml"
stories.write_text("""
version: "2.0"
stories:
- story: test path
steps:
- action: utter_greet
""")
policy = self.create_policy(
featurizer=featurizer,
model_storage=model_storage,
resource=resource,
execution_context=execution_context,
)
import tests.core.test_policies
training_trackers = tests.core.test_policies.train_trackers(
default_domain, str(stories), augmentation_factor=20)
with pytest.raises(RasaException) as e:
policy.train(training_trackers,
default_domain,
precomputations=None)
assert "No user features specified. Cannot train 'TED' model." == str(
e.value)
def test_similarity_type(self, trained_policy: TEDPolicy):
assert trained_policy.config[SIMILARITY_TYPE] == "inner"
def test_ranking_length(self, trained_policy: TEDPolicy):
assert trained_policy.config[RANKING_LENGTH] == 10
def test_normalization(
self,
trained_policy: TEDPolicy,
tracker: DialogueStateTracker,
default_domain: Domain,
monkeypatch: MonkeyPatch,
):
precomputations = None
# first check the output is what we expect
prediction = trained_policy.predict_action_probabilities(
tracker,
default_domain,
precomputations,
)
assert not prediction.is_end_to_end_prediction
# count number of non-zero confidences
assert (sum([
confidence > 0 for confidence in prediction.probabilities
]) == trained_policy.config[RANKING_LENGTH])
# check that the norm is still 1
assert sum(prediction.probabilities) == pytest.approx(1)
# also check our function is called
mock = Mock()
monkeypatch.setattr(train_utils, "normalize", mock.normalize)
trained_policy.predict_action_probabilities(
tracker,
default_domain,
precomputations,
)
mock.normalize.assert_called_once()
def test_label_data_assembly(self, trained_policy: TEDPolicy,
default_domain: Domain):
state_featurizer = trained_policy.featurizer.state_featurizer
encoded_all_labels = state_featurizer.encode_all_labels(
default_domain, precomputations=None)
attribute_data, _ = model_data_utils.convert_to_data_format(
encoded_all_labels)
assembled_label_data = trained_policy._assemble_label_data(
attribute_data, default_domain)
assembled_label_data_signature = assembled_label_data.get_signature()
assert list(assembled_label_data_signature.keys()) == [
f"{LABEL}_{ACTION_NAME}",
f"{LABEL}",
]
assert assembled_label_data.num_examples == default_domain.num_actions
assert list(assembled_label_data_signature[f"{LABEL}_{ACTION_NAME}"].
keys()) == [
MASK,
SENTENCE,
]
assert list(assembled_label_data_signature[LABEL].keys()) == [IDS]
assert (assembled_label_data_signature[f"{LABEL}_{ACTION_NAME}"]
[SENTENCE][0].units == default_domain.num_actions)
def test_gen_batch(self, trained_policy: TEDPolicy, default_domain: Domain,
stories_path: Path):
training_trackers = tests.core.test_policies.train_trackers(
default_domain, stories_path, augmentation_factor=0)
precomputations = None
training_data, label_ids, entity_tags = trained_policy._featurize_for_training(
training_trackers,
default_domain,
precomputations,
)
_, all_labels = trained_policy._create_label_data(
default_domain, precomputations)
model_data = trained_policy._create_model_data(training_data,
label_ids, entity_tags,
all_labels)
batch_size = 2
data_generator = RasaBatchDataGenerator(model_data,
batch_size=batch_size,
shuffle=False,
batch_strategy="sequence")
iterator = iter(data_generator)
# model data keys were sorted, so the order is alphabetical
(
(
batch_action_name_mask,
_,
_,
batch_action_name_sentence_shape,
batch_dialogue_length,
batch_entities_mask,
_,
_,
batch_entities_sentence_shape,
batch_intent_mask,
_,
_,
batch_intent_sentence_shape,
batch_label_ids,
batch_slots_mask,
_,
_,
batch_slots_sentence_shape,
),
_,
) = next(iterator)
assert (batch_label_ids.shape[0] == batch_size
and batch_dialogue_length.shape[0] == batch_size)
# batch and dialogue dimensions are NOT combined for masks
assert (batch_slots_mask.shape[0] == batch_size
and batch_intent_mask.shape[0] == batch_size
and batch_entities_mask.shape[0] == batch_size
and batch_action_name_mask.shape[0] == batch_size)
# some features might be "fake" so there sequence is `0`
seq_len = max([
batch_intent_sentence_shape[1],
batch_action_name_sentence_shape[1],
batch_entities_sentence_shape[1],
batch_slots_sentence_shape[1],
])
assert (batch_intent_sentence_shape[1] == seq_len
or batch_intent_sentence_shape[1] == 0)
assert (batch_action_name_sentence_shape[1] == seq_len
or batch_action_name_sentence_shape[1] == 0)
assert (batch_entities_sentence_shape[1] == seq_len
or batch_entities_sentence_shape[1] == 0)
assert (batch_slots_sentence_shape[1] == seq_len
or batch_slots_sentence_shape[1] == 0)
data_generator = RasaBatchDataGenerator(model_data,
batch_size=batch_size,
shuffle=True,
batch_strategy="balanced")
iterator = iter(data_generator)
(
(
batch_action_name_mask,
_,
_,
batch_action_name_sentence_shape,
batch_dialogue_length,
batch_entities_mask,
_,
_,
batch_entities_sentence_shape,
batch_intent_mask,
_,
_,
batch_intent_sentence_shape,
batch_label_ids,
batch_slots_mask,
_,
_,
batch_slots_sentence_shape,
),
_,
) = next(iterator)
assert (batch_label_ids.shape[0] == batch_size
and batch_dialogue_length.shape[0] == batch_size)
# some features might be "fake" so there sequence is `0`
seq_len = max([
batch_intent_sentence_shape[1],
batch_action_name_sentence_shape[1],
batch_entities_sentence_shape[1],
batch_slots_sentence_shape[1],
])
assert (batch_intent_sentence_shape[1] == seq_len
or batch_intent_sentence_shape[1] == 0)
assert (batch_action_name_sentence_shape[1] == seq_len
or batch_action_name_sentence_shape[1] == 0)
assert (batch_entities_sentence_shape[1] == seq_len
or batch_entities_sentence_shape[1] == 0)
assert (batch_slots_sentence_shape[1] == seq_len
or batch_slots_sentence_shape[1] == 0)
@pytest.mark.parametrize(
"tracker_events_with_action, tracker_events_without_action",
[
(
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="hello", intent={"name": "greet"}),
ActionExecuted(ACTION_UNLIKELY_INTENT_NAME),
],
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="hello", intent={"name": "greet"}),
],
),
(
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="hello", intent={"name": "greet"}),
EntitiesAdded(entities=[
{
"entity": "name",
"value": "Peter"
},
]),
ActionExecuted(ACTION_UNLIKELY_INTENT_NAME),
ActionExecuted("utter_greet"),
],
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="hello", intent={"name": "greet"}),
EntitiesAdded(entities=[
{
"entity": "name",
"value": "Peter"
},
]),
ActionExecuted("utter_greet"),
],
),
(
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="hello", intent={"name": "greet"}),
ActionExecuted(ACTION_UNLIKELY_INTENT_NAME),
ActionExecuted("some_form"),
ActiveLoop("some_form"),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="default", intent={"name": "default"}),
ActionExecuted(ACTION_UNLIKELY_INTENT_NAME),
],
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="hello", intent={"name": "greet"}),
ActionExecuted(ACTION_UNLIKELY_INTENT_NAME),
ActionExecuted("some_form"),
ActiveLoop("some_form"),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(text="default", intent={"name": "default"}),
],
),
],
)
def test_ignore_action_unlikely_intent(
self,
trained_policy: TEDPolicy,
default_domain: Domain,
tracker_events_with_action: List[Event],
tracker_events_without_action: List[Event],
):
precomputations = None
tracker_with_action = DialogueStateTracker.from_events(
"test 1", evts=tracker_events_with_action)
tracker_without_action = DialogueStateTracker.from_events(
"test 2", evts=tracker_events_without_action)
prediction_with_action = trained_policy.predict_action_probabilities(
tracker_with_action,
default_domain,
precomputations,
)
prediction_without_action = trained_policy.predict_action_probabilities(
tracker_without_action,
default_domain,
precomputations,
)
# If the weights didn't change then both trackers
# should result in same prediction.
assert (prediction_with_action.probabilities ==
prediction_without_action.probabilities)
@pytest.mark.parametrize(
"featurizer_config, tracker_featurizer, state_featurizer",
[
(None, MaxHistoryTrackerFeaturizer(), SingleStateFeaturizer),
([], MaxHistoryTrackerFeaturizer(), SingleStateFeaturizer),
],
)
def test_empty_featurizer_configs(
self,
featurizer_config: Optional[Dict[Text, Any]],
model_storage: ModelStorage,
resource: Resource,
execution_context: ExecutionContext,
tracker_featurizer: MaxHistoryTrackerFeaturizer,
state_featurizer: Type[SingleStateFeaturizer],
):
featurizer_config_override = ({
"featurizer": featurizer_config
} if featurizer_config else {})
policy = self.create_policy(
None,
model_storage=model_storage,
resource=resource,
execution_context=execution_context,
config=self._config(featurizer_config_override),
)
featurizer = policy.featurizer
assert isinstance(featurizer, tracker_featurizer.__class__)
if featurizer_config:
expected_max_history = featurizer_config[0].get(POLICY_MAX_HISTORY)
else:
expected_max_history = self._config().get(POLICY_MAX_HISTORY)
assert featurizer.max_history == expected_max_history
assert isinstance(featurizer.state_featurizer, state_featurizer)
async def test_whole_loop():
expected_activation_events = [
ActionExecutionRejected("tada"),
ActionExecuted("test"),
]
expected_do_events = [ActionExecuted("do")]
expected_deactivation_events = [SlotSet("deactivated")]
form_name = "my form"
class MyLoop(LoopAction):
def name(self) -> Text:
return form_name
async def activate(self, *args: Any) -> List[Event]:
return expected_activation_events
async def do(self, *args: Any) -> List[Event]:
events_so_far = args[-1]
assert events_so_far == [
ActiveLoop(form_name), *expected_activation_events
]
return expected_do_events
async def deactivate(self, *args) -> List[Event]:
events_so_far = args[-1]
assert events_so_far == [
ActiveLoop(form_name),
*expected_activation_events,
*expected_do_events,
ActiveLoop(None),
]
return expected_deactivation_events
async def is_done(self, *args) -> bool:
events_so_far = args[-1]
return events_so_far == [
ActiveLoop(form_name),
*expected_activation_events,
*expected_do_events,
]
tracker = DialogueStateTracker.from_events("some sender", [])
domain = Domain.empty()
action = MyLoop()
actual = await action.run(
CollectingOutputChannel(),
TemplatedNaturalLanguageGenerator(domain.templates),
tracker,
domain,
)
assert actual == [
ActiveLoop(form_name),
*expected_activation_events,
*expected_do_events,
ActiveLoop(None),
*expected_deactivation_events,
]
def _default_deactivation_events(self) -> List[Event]:
return [ActiveLoop(None)]
def _default_activation_events(self) -> List[Event]:
return [ActiveLoop(self.name())] # pytype: disable=attribute-error
async def test_form_unhappy_path_from_story():
form_name = "some_form"
handle_rejection_action_name = "utter_handle_rejection"
domain = Domain.from_yaml(
f"""
intents:
- {GREET_INTENT_NAME}
actions:
- {UTTER_GREET_ACTION}
- {handle_rejection_action_name}
- some-action
slots:
{REQUESTED_SLOT}:
type: unfeaturized
forms:
- {form_name}
"""
)
unhappy_story = TrackerWithCachedStates.from_events(
"bla",
domain=domain,
slots=domain.slots,
evts=[
# We are in an active form
ActionExecuted(form_name),
ActiveLoop(form_name),
ActionExecuted(ACTION_LISTEN_NAME),
# in training stories there is either intent or text, never both
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(UTTER_GREET_ACTION),
# After our bot says "hi", we want to run a specific action
ActionExecuted(handle_rejection_action_name),
ActionExecuted(form_name),
ActionExecuted(ACTION_LISTEN_NAME),
],
)
policy = RulePolicy()
policy.train([GREET_RULE, unhappy_story], domain, RegexInterpreter())
# Check that RulePolicy predicts action to handle unhappy path
conversation_events = [
ActionExecuted(form_name),
ActiveLoop(form_name),
SlotSet(REQUESTED_SLOT, "some value"),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered("haha", {"name": GREET_INTENT_NAME}),
ActionExecutionRejected(form_name),
]
prediction = policy.predict_action_probabilities(
DialogueStateTracker.from_events(
"casd", evts=conversation_events, slots=domain.slots
),
domain,
RegexInterpreter(),
)
assert_predicted_action(prediction, domain, UTTER_GREET_ACTION)
# Check that RulePolicy doesn't trigger form or action_listen
# after handling unhappy path
conversation_events.append(ActionExecuted(handle_rejection_action_name))
prediction = policy.predict_action_probabilities(
DialogueStateTracker.from_events(
"casd", evts=conversation_events, slots=domain.slots
),
domain,
RegexInterpreter(),
)
assert prediction.max_confidence == policy._core_fallback_threshold
{
"event": "slot",
"name": "num_people",
"value": "so_clean"
},
{
"event": "slot",
"name": "num_tables",
"value": 5
},
],
[
SlotSet("num_people", "so_clean"),
SlotSet("num_tables", 5),
SlotSet(REQUESTED_SLOT, None),
ActiveLoop(None),
],
),
# Validate function returns extra Slot Event
(
[
{
"event": "slot",
"name": "num_people",
"value": "so_clean"
},
{
"event": "slot",
"name": "some_other_slot",
"value": 2
},
async def test_form_unhappy_path_no_validation_from_rule():
form_name = "some_form"
handle_rejection_action_name = "utter_handle_rejection"
domain = Domain.from_yaml(
f"""
intents:
- {GREET_INTENT_NAME}
actions:
- {UTTER_GREET_ACTION}
- {handle_rejection_action_name}
- some-action
slots:
{REQUESTED_SLOT}:
type: unfeaturized
forms:
- {form_name}
"""
)
unhappy_rule = TrackerWithCachedStates.from_events(
"bla",
domain=domain,
slots=domain.slots,
evts=[
# We are in an active form
ActiveLoop(form_name),
SlotSet(REQUESTED_SLOT, "bla"),
ActionExecuted(RULE_SNIPPET_ACTION_NAME),
ActionExecuted(ACTION_LISTEN_NAME),
# When a user says "hi", and the form is unhappy,
# we want to run a specific action
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(handle_rejection_action_name),
# Next user utterance is an answer to the previous question
# and shouldn't be validated by the form
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(form_name),
ActionExecuted(ACTION_LISTEN_NAME),
],
is_rule_tracker=True,
)
# unhappy rule is multi user turn rule, therefore remove restriction for policy
policy = RulePolicy(restrict_rules=False)
# RulePolicy should memorize that unhappy_rule overrides GREET_RULE
policy.train([GREET_RULE, unhappy_rule], domain, RegexInterpreter())
# Check that RulePolicy predicts action to handle unhappy path
conversation_events = [
ActionExecuted(form_name),
ActiveLoop(form_name),
SlotSet(REQUESTED_SLOT, "some value"),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered("haha", {"name": GREET_INTENT_NAME}),
ActionExecutionRejected(form_name),
]
prediction = policy.predict_action_probabilities(
DialogueStateTracker.from_events(
"casd", evts=conversation_events, slots=domain.slots
),
domain,
RegexInterpreter(),
)
assert_predicted_action(prediction, domain, handle_rejection_action_name)
# Check that RulePolicy predicts action_listen
conversation_events.append(ActionExecuted(handle_rejection_action_name))
prediction = policy.predict_action_probabilities(
DialogueStateTracker.from_events(
"casd", evts=conversation_events, slots=domain.slots
),
domain,
RegexInterpreter(),
)
assert_predicted_action(prediction, domain, ACTION_LISTEN_NAME)
# Check that RulePolicy triggers form again after handling unhappy path
conversation_events.append(ActionExecuted(ACTION_LISTEN_NAME))
tracker = DialogueStateTracker.from_events(
"casd", evts=conversation_events, slots=domain.slots
)
prediction = policy.predict_action_probabilities(
tracker, domain, RegexInterpreter()
)
assert_predicted_action(prediction, domain, form_name)
# check that RulePolicy entered unhappy path based on the training story
assert prediction.events == [LoopInterrupted(True)]
async def test_request_correct_slots_after_unhappy_path_with_custom_required_slots(
):
form_name = "some_form"
slot_name_1 = "slot_1"
slot_name_2 = "slot_2"
domain = f"""
slots:
{slot_name_1}:
type: any
{slot_name_2}:
type: any
forms:
{form_name}:
{slot_name_1}:
- type: from_intent
intent: some_intent
value: some_value
{slot_name_2}:
- type: from_intent
intent: some_intent
value: some_value
actions:
- validate_{form_name}
"""
domain = Domain.from_yaml(domain)
tracker = DialogueStateTracker.from_events(
"default",
[
ActiveLoop(form_name),
SlotSet(REQUESTED_SLOT, "slot_2"),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(
"hello",
intent={
"name": "greet",
"confidence": 1.0
},
),
ActionExecutionRejected(form_name),
ActionExecuted("utter_greet"),
],
)
action_server_url = "http://my-action-server:5055/webhook"
# Custom form validation action changes the order of the requested slots
validate_return_events = [
{
"event": "slot",
"name": REQUESTED_SLOT,
"value": slot_name_2
},
]
# The form should ask the same slot again when coming back after unhappy path
expected_events = [SlotSet(REQUESTED_SLOT, slot_name_2)]
with aioresponses() as mocked:
mocked.post(action_server_url,
payload={"events": validate_return_events})
action_server = EndpointConfig(action_server_url)
action = FormAction(form_name, action_server)
events = await action.run(
CollectingOutputChannel(),
TemplatedNaturalLanguageGenerator(domain.templates),
tracker,
domain,
)
assert events == expected_events
async def test_persist_legacy_form_story():
domain = Domain.load("data/test_domains/form.yml")
tracker = DialogueStateTracker("", domain.slots)
story = ("* greet\n"
" - utter_greet\n"
"* start_form\n"
" - some_form\n"
' - form{"name": "some_form"}\n'
"* default\n"
" - utter_default\n"
" - some_form\n"
"* stop\n"
" - utter_ask_continue\n"
"* affirm\n"
" - some_form\n"
"* stop\n"
" - utter_ask_continue\n"
"* inform\n"
" - some_form\n"
' - form{"name": null}\n'
"* goodbye\n"
" - utter_goodbye\n")
# simulate talking to the form
events = [
UserUttered(intent={"name": "greet"}),
ActionExecuted("utter_greet"),
ActionExecuted("action_listen"),
# start the form
UserUttered(intent={"name": "start_form"}),
ActionExecuted("some_form"),
ActiveLoop("some_form"),
ActionExecuted("action_listen"),
# out of form input
UserUttered(intent={"name": "default"}),
ActionExecutionRejected("some_form"),
ActionExecuted("utter_default"),
ActionExecuted("some_form"),
ActionExecuted("action_listen"),
# out of form input
UserUttered(intent={"name": "stop"}),
ActionExecutionRejected("some_form"),
ActionExecuted("utter_ask_continue"),
ActionExecuted("action_listen"),
# out of form input but continue with the form
UserUttered(intent={"name": "affirm"}),
FormValidation(False),
ActionExecuted("some_form"),
ActionExecuted("action_listen"),
# out of form input
UserUttered(intent={"name": "stop"}),
ActionExecutionRejected("some_form"),
ActionExecuted("utter_ask_continue"),
ActionExecuted("action_listen"),
# form input
UserUttered(intent={"name": "inform"}),
FormValidation(True),
ActionExecuted("some_form"),
ActionExecuted("action_listen"),
ActiveLoop(None),
UserUttered(intent={"name": "goodbye"}),
ActionExecuted("utter_goodbye"),
ActionExecuted("action_listen"),
]
[tracker.update(e) for e in events]
story = story.replace(f"- {LegacyForm.type_name}",
f"- {ActiveLoop.type_name}")
assert story in tracker.export_stories()
def deactivate(self) -> List[Event]:
logger.debug(f"Deactivating the form '{self.name()}'")
return [ActiveLoop(None), SlotSet(REQUESTED_SLOT, None)]
