def test_no_incomplete_rules_due_to_slots_after_listen():
some_action = "some_action"
some_slot = "some_slot"
domain = Domain.from_yaml(f"""
intents:
- {GREET_INTENT_NAME}
actions:
- {some_action}
entities:
- {some_slot}
slots:
{some_slot}:
type: text
""")
policy = RulePolicy()
complete_rule = TrackerWithCachedStates.from_events(
"complete_rule",
domain=domain,
slots=domain.slots,
evts=[
ActionExecuted(RULE_SNIPPET_ACTION_NAME),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(
intent={"name": GREET_INTENT_NAME},
entities=[{
"entity": some_slot,
"value": "bla"
}],
),
SlotSet(some_slot, "bla"),
ActionExecuted(some_action),
ActionExecuted(ACTION_LISTEN_NAME),
],
is_rule_tracker=True,
)
potentially_incomplete_rule = TrackerWithCachedStates.from_events(
"potentially_incomplete_rule",
domain=domain,
slots=domain.slots,
evts=[
ActionExecuted(RULE_SNIPPET_ACTION_NAME),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(some_action),
ActionExecuted(ACTION_LISTEN_NAME),
],
is_rule_tracker=True,
)
policy.train([complete_rule, potentially_incomplete_rule], domain,
RegexInterpreter())
def test_train_with_e2e_data(
default_model_storage: ModelStorage,
default_execution_context: ExecutionContext,
tracker_events: List[List[Event]],
skip_training: bool,
domain: Domain,
):
policy = UnexpecTEDIntentPolicy(
UnexpecTEDIntentPolicy.get_default_config(),
default_model_storage,
Resource("UnexpecTEDIntentPolicy"),
default_execution_context,
featurizer=IntentMaxHistoryTrackerFeaturizer(
IntentTokenizerSingleStateFeaturizer()
),
)
trackers_for_training = [
TrackerWithCachedStates.from_events(
sender_id=f"{tracker_index}", evts=events, domain=domain
)
for tracker_index, events in enumerate(tracker_events)
]
if skip_training:
with pytest.warns(UserWarning):
policy.train(trackers_for_training, domain, precomputations=None)
else:
policy.train(trackers_for_training, domain, precomputations=None)
def test_contradicting_rules_and_stories():
utter_anti_greet_action = "utter_anti_greet"
domain = Domain.from_yaml(f"""
intents:
- {GREET_INTENT_NAME}
actions:
- {UTTER_GREET_ACTION}
- {utter_anti_greet_action}
""")
policy = RulePolicy()
anti_greet_story = TrackerWithCachedStates.from_events(
"anti greet story",
domain=domain,
slots=domain.slots,
evts=[
ActionExecuted(RULE_SNIPPET_ACTION_NAME),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(utter_anti_greet_action),
ActionExecuted(ACTION_LISTEN_NAME),
],
)
with pytest.raises(InvalidRule) as execinfo:
policy.train([GREET_RULE, anti_greet_story], domain,
RegexInterpreter())
assert all(
name in execinfo.value.message
for name in {utter_anti_greet_action, anti_greet_story.sender_id})
async def test_finetune_after_load(self, trained_policy: MemoizationPolicy,
default_domain: Domain, tmp_path: Path):
trained_policy.persist(tmp_path)
loaded_policy = MemoizationPolicy.load(tmp_path, should_finetune=True)
assert loaded_policy.finetune_mode
new_story = TrackerWithCachedStates.from_events(
"channel",
domain=default_domain,
slots=default_domain.slots,
evts=[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": "why"}),
ActionExecuted("utter_channel"),
ActionExecuted(ACTION_LISTEN_NAME),
],
)
original_train_data = await train_trackers(default_domain,
augmentation_factor=20)
loaded_policy.train(original_train_data + [new_story], default_domain,
RegexInterpreter())
# Get the hash of the tracker state of new story
new_story_states, _ = loaded_policy.featurizer.training_states_and_actions(
[new_story], default_domain)
# Feature keys for each new state should be present in the lookup
for states in new_story_states:
state_key = loaded_policy._create_feature_key(states)
assert state_key in loaded_policy.lookup
def test_finetune_after_load(
self,
trained_policy: MemoizationPolicy,
resource: Resource,
model_storage: ModelStorage,
execution_context: ExecutionContext,
default_domain: Domain,
stories_path: Text,
):
execution_context = dataclasses.replace(execution_context,
is_finetuning=True)
loaded_policy = MemoizationPolicy.load(trained_policy.config,
model_storage, resource,
execution_context)
assert loaded_policy.finetune_mode
new_story = TrackerWithCachedStates.from_events(
"channel",
domain=default_domain,
slots=default_domain.slots,
evts=[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": "why"}),
ActionExecuted("utter_channel"),
ActionExecuted(ACTION_LISTEN_NAME),
],
)
original_train_data = train_trackers(default_domain,
stories_path,
augmentation_factor=20)
loaded_policy.train(
original_train_data + [new_story],
default_domain,
)
# Get the hash of the tracker state of new story
new_story_states, _ = loaded_policy.featurizer.training_states_and_labels(
[new_story], default_domain)
# Feature keys for each new state should be present in the lookup
for states in new_story_states:
state_key = loaded_policy._create_feature_key(states)
assert state_key in loaded_policy.lookup
def _form_activation_rule(
domain: Domain, form_name: Text,
activation_intent_name: Text) -> TrackerWithCachedStates:
return TrackerWithCachedStates.from_events(
"form activation rule",
domain=domain,
slots=domain.slots,
evts=[
ActionExecuted(RULE_SNIPPET_ACTION_NAME),
# The intent `other_intent` activates the form
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": activation_intent_name}),
ActionExecuted(form_name),
ActiveLoop(form_name),
ActionExecuted(ACTION_LISTEN_NAME),
],
is_rule_tracker=True,
)
def _form_submit_rule(domain: Domain, submit_action_name: Text,
form_name: Text) -> TrackerWithCachedStates:
return TrackerWithCachedStates.from_events(
"form submit rule",
domain=domain,
slots=domain.slots,
evts=[
ActiveLoop(form_name),
# Any events in between
ActionExecuted(RULE_SNIPPET_ACTION_NAME),
# Form runs and deactivates itself
ActionExecuted(form_name),
ActiveLoop(None),
SlotSet(REQUESTED_SLOT, None),
ActionExecuted(submit_action_name),
ActionExecuted(ACTION_LISTEN_NAME),
],
is_rule_tracker=True,
)
def test_filter_training_trackers(
self,
tracker_events_for_training: List[List[Event]],
expected_trackers_with_events: List[List[Event]],
domain: Domain,
):
trackers_for_training = [
TrackerWithCachedStates.from_events(sender_id=f"{tracker_index}",
evts=events,
domain=domain)
for tracker_index, events in enumerate(tracker_events_for_training)
]
filtered_trackers = UnexpecTEDIntentPolicy._get_trackers_for_training(
trackers_for_training)
assert len(filtered_trackers) == len(expected_trackers_with_events)
for collected_tracker, expected_tracker_events in zip(
filtered_trackers, expected_trackers_with_events):
collected_tracker_events = list(collected_tracker.events)
assert collected_tracker_events == expected_tracker_events
async def test_form_unhappy_path_no_validation_from_story():
form_name = "some_form"
handle_rejection_action_name = "utter_handle_rejection"
domain = Domain.from_yaml(f"""
intents:
- {GREET_INTENT_NAME}
actions:
- {UTTER_GREET_ACTION}
- {handle_rejection_action_name}
- some-action
slots:
{REQUESTED_SLOT}:
type: unfeaturized
forms:
- {form_name}
""")
unhappy_story = TrackerWithCachedStates.from_events(
"bla",
domain=domain,
slots=domain.slots,
evts=[
# We are in an active form
ActionExecuted(form_name),
ActiveLoop(form_name),
ActionExecuted(ACTION_LISTEN_NAME),
# When a user says "hi", and the form is unhappy,
# we want to run a specific action
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(handle_rejection_action_name),
ActionExecuted(ACTION_LISTEN_NAME),
# Next user utterance is an answer to the previous question
# and shouldn't be validated by the form
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(form_name),
ActionExecuted(ACTION_LISTEN_NAME),
],
)
policy = RulePolicy()
policy.train([unhappy_story], domain, RegexInterpreter())
# Check that RulePolicy predicts no validation to handle unhappy path
conversation_events = [
ActionExecuted(form_name),
ActiveLoop(form_name),
SlotSet(REQUESTED_SLOT, "some value"),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered("haha", {"name": GREET_INTENT_NAME}),
ActionExecutionRejected(form_name),
ActionExecuted(handle_rejection_action_name),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered("haha", {"name": GREET_INTENT_NAME}),
]
tracker = DialogueStateTracker.from_events("casd",
evts=conversation_events,
slots=domain.slots)
action_probabilities = policy.predict_action_probabilities(
tracker, domain, RegexInterpreter())
# there is no rule for next action
assert max(action_probabilities) == policy._core_fallback_threshold
# check that RulePolicy entered unhappy path based on the training story
assert tracker.events[-1] == LoopInterrupted(True)
async def test_form_unhappy_path_from_story():
form_name = "some_form"
handle_rejection_action_name = "utter_handle_rejection"
domain = Domain.from_yaml(f"""
intents:
- {GREET_INTENT_NAME}
actions:
- {UTTER_GREET_ACTION}
- {handle_rejection_action_name}
- some-action
slots:
{REQUESTED_SLOT}:
type: unfeaturized
forms:
- {form_name}
""")
unhappy_story = TrackerWithCachedStates.from_events(
"bla",
domain=domain,
slots=domain.slots,
evts=[
# We are in an active form
ActionExecuted(form_name),
ActiveLoop(form_name),
ActionExecuted(ACTION_LISTEN_NAME),
# in training stories there is either intent or text, never both
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(UTTER_GREET_ACTION),
# After our bot says "hi", we want to run a specific action
ActionExecuted(handle_rejection_action_name),
ActionExecuted(form_name),
ActionExecuted(ACTION_LISTEN_NAME),
],
)
policy = RulePolicy()
policy.train([GREET_RULE, unhappy_story], domain, RegexInterpreter())
# Check that RulePolicy predicts action to handle unhappy path
conversation_events = [
ActionExecuted(form_name),
ActiveLoop(form_name),
SlotSet(REQUESTED_SLOT, "some value"),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered("haha", {"name": GREET_INTENT_NAME}),
ActionExecutionRejected(form_name),
]
action_probabilities = policy.predict_action_probabilities(
DialogueStateTracker.from_events("casd",
evts=conversation_events,
slots=domain.slots),
domain,
RegexInterpreter(),
)
assert_predicted_action(action_probabilities, domain, UTTER_GREET_ACTION)
# Check that RulePolicy doesn't trigger form or action_listen
# after handling unhappy path
conversation_events.append(ActionExecuted(handle_rejection_action_name))
action_probabilities = policy.predict_action_probabilities(
DialogueStateTracker.from_events("casd",
evts=conversation_events,
slots=domain.slots),
domain,
RegexInterpreter(),
)
assert max(action_probabilities) == policy._core_fallback_threshold
async def test_form_unhappy_path_no_validation_from_rule():
form_name = "some_form"
handle_rejection_action_name = "utter_handle_rejection"
domain = Domain.from_yaml(f"""
intents:
- {GREET_INTENT_NAME}
actions:
- {UTTER_GREET_ACTION}
- {handle_rejection_action_name}
- some-action
slots:
{REQUESTED_SLOT}:
type: unfeaturized
forms:
- {form_name}
""")
unhappy_rule = TrackerWithCachedStates.from_events(
"bla",
domain=domain,
slots=domain.slots,
evts=[
# We are in an active form
ActiveLoop(form_name),
SlotSet(REQUESTED_SLOT, "bla"),
ActionExecuted(RULE_SNIPPET_ACTION_NAME),
ActionExecuted(ACTION_LISTEN_NAME),
# When a user says "hi", and the form is unhappy,
# we want to run a specific action
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(handle_rejection_action_name),
# Next user utterance is an answer to the previous question
# and shouldn't be validated by the form
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(form_name),
ActionExecuted(ACTION_LISTEN_NAME),
],
is_rule_tracker=True,
)
# unhappy rule is multi user turn rule, therefore remove restriction for policy
policy = RulePolicy(restrict_rules=False)
# RulePolicy should memorize that unhappy_rule overrides GREET_RULE
policy.train([GREET_RULE, unhappy_rule], domain, RegexInterpreter())
# Check that RulePolicy predicts action to handle unhappy path
conversation_events = [
ActionExecuted(form_name),
ActiveLoop(form_name),
SlotSet(REQUESTED_SLOT, "some value"),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered("haha", {"name": GREET_INTENT_NAME}),
ActionExecutionRejected(form_name),
]
action_probabilities = policy.predict_action_probabilities(
DialogueStateTracker.from_events("casd",
evts=conversation_events,
slots=domain.slots),
domain,
RegexInterpreter(),
)
assert_predicted_action(action_probabilities, domain,
handle_rejection_action_name)
# Check that RulePolicy predicts action_listen
conversation_events.append(ActionExecuted(handle_rejection_action_name))
action_probabilities = policy.predict_action_probabilities(
DialogueStateTracker.from_events("casd",
evts=conversation_events,
slots=domain.slots),
domain,
RegexInterpreter(),
)
assert_predicted_action(action_probabilities, domain, ACTION_LISTEN_NAME)
# Check that RulePolicy triggers form again after handling unhappy path
conversation_events.append(ActionExecuted(ACTION_LISTEN_NAME))
tracker = DialogueStateTracker.from_events("casd",
evts=conversation_events,
slots=domain.slots)
action_probabilities = policy.predict_action_probabilities(
tracker, domain, RegexInterpreter())
assert_predicted_action(action_probabilities, domain, form_name)
# check that RulePolicy entered unhappy path based on the training story
assert tracker.events[-1] == LoopInterrupted(True)
async def test_form_unhappy_path_from_in_form_rule():
form_name = "some_form"
handle_rejection_action_name = "utter_handle_rejection"
domain = Domain.from_yaml(f"""
intents:
- {GREET_INTENT_NAME}
actions:
- {UTTER_GREET_ACTION}
- {handle_rejection_action_name}
- some-action
slots:
{REQUESTED_SLOT}:
type: unfeaturized
forms:
- {form_name}
""")
unhappy_rule = TrackerWithCachedStates.from_events(
"bla",
domain=domain,
slots=domain.slots,
evts=[
# We are in an active form
ActiveLoop(form_name),
SlotSet(REQUESTED_SLOT, "bla"),
ActionExecuted(RULE_SNIPPET_ACTION_NAME),
ActionExecuted(ACTION_LISTEN_NAME),
# When a user says "hi", and the form is unhappy,
# we want to run a specific action
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(handle_rejection_action_name),
ActionExecuted(form_name),
ActionExecuted(ACTION_LISTEN_NAME),
],
is_rule_tracker=True,
)
policy = RulePolicy()
# RulePolicy should memorize that unhappy_rule overrides GREET_RULE
policy.train([GREET_RULE, unhappy_rule], domain, RegexInterpreter())
# Check that RulePolicy predicts action to handle unhappy path
conversation_events = [
ActionExecuted(form_name),
ActiveLoop(form_name),
SlotSet(REQUESTED_SLOT, "some value"),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered("haha", {"name": GREET_INTENT_NAME}),
ActionExecutionRejected(form_name),
]
action_probabilities = policy.predict_action_probabilities(
DialogueStateTracker.from_events("casd",
evts=conversation_events,
slots=domain.slots),
domain,
RegexInterpreter(),
)
assert_predicted_action(action_probabilities, domain,
handle_rejection_action_name)
# Check that RulePolicy triggers form again after handling unhappy path
conversation_events.append(ActionExecuted(handle_rejection_action_name))
action_probabilities = policy.predict_action_probabilities(
DialogueStateTracker.from_events("casd",
evts=conversation_events,
slots=domain.slots),
domain,
RegexInterpreter(),
)
assert_predicted_action(action_probabilities, domain, form_name)
def test_prediction(
self,
max_history: Optional[int],
model_storage: ModelStorage,
resource: Resource,
execution_context: ExecutionContext,
):
policy = self.create_policy(
featurizer=MaxHistoryTrackerFeaturizer(max_history=max_history),
model_storage=model_storage,
resource=resource,
execution_context=execution_context,
config={POLICY_MAX_HISTORY: max_history},
)
GREET_INTENT_NAME = "greet"
UTTER_GREET_ACTION = "utter_greet"
UTTER_BYE_ACTION = "utter_goodbye"
domain = Domain.from_yaml(f"""
intents:
- {GREET_INTENT_NAME}
actions:
- {UTTER_GREET_ACTION}
- {UTTER_BYE_ACTION}
slots:
slot_1:
type: bool
mappings:
- type: from_text
slot_2:
type: bool
mappings:
- type: from_text
slot_3:
type: bool
mappings:
- type: from_text
slot_4:
type: bool
mappings:
- type: from_text
""")
events = [
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(UTTER_GREET_ACTION),
SlotSet("slot_1", True),
ActionExecuted(UTTER_GREET_ACTION),
SlotSet("slot_2", True),
SlotSet("slot_3", True),
ActionExecuted(UTTER_GREET_ACTION),
ActionExecuted(UTTER_GREET_ACTION),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(UTTER_GREET_ACTION),
SlotSet("slot_4", True),
ActionExecuted(UTTER_BYE_ACTION),
ActionExecuted(ACTION_LISTEN_NAME),
]
training_story = TrackerWithCachedStates.from_events(
"training story", evts=events, domain=domain, slots=domain.slots)
test_story = TrackerWithCachedStates.from_events("training story",
events[:-2],
domain=domain,
slots=domain.slots)
policy.train([training_story], domain)
prediction = policy.predict_action_probabilities(test_story, domain)
assert (domain.action_names_or_texts[prediction.probabilities.index(
max(prediction.probabilities))] == UTTER_BYE_ACTION)
def test_augmented_prediction_across_max_history_actions(
self,
max_history: Optional[int],
model_storage: ModelStorage,
resource: Resource,
execution_context: ExecutionContext,
):
"""Tests that the last user utterance is preserved in action states
even when the utterance occurs prior to `max_history` actions in the
past.
"""
policy = self.create_policy(
featurizer=MaxHistoryTrackerFeaturizer(max_history=max_history),
model_storage=model_storage,
resource=resource,
execution_context=execution_context,
config={POLICY_MAX_HISTORY: max_history},
)
GREET_INTENT_NAME = "greet"
UTTER_GREET_ACTION = "utter_greet"
UTTER_ACTION_1 = "utter_1"
UTTER_ACTION_2 = "utter_2"
UTTER_ACTION_3 = "utter_3"
UTTER_ACTION_4 = "utter_4"
UTTER_ACTION_5 = "utter_5"
UTTER_BYE_ACTION = "utter_goodbye"
domain = Domain.from_yaml(f"""
intents:
- {GREET_INTENT_NAME}
actions:
- {UTTER_GREET_ACTION}
- {UTTER_ACTION_1}
- {UTTER_ACTION_2}
- {UTTER_ACTION_3}
- {UTTER_ACTION_4}
- {UTTER_ACTION_5}
- {UTTER_BYE_ACTION}
""")
training_story = TrackerWithCachedStates.from_events(
"training story",
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(UTTER_ACTION_1),
ActionExecuted(UTTER_ACTION_2),
ActionExecuted(UTTER_ACTION_3),
ActionExecuted(UTTER_ACTION_4),
ActionExecuted(UTTER_ACTION_5),
ActionExecuted(UTTER_BYE_ACTION),
ActionExecuted(ACTION_LISTEN_NAME),
],
domain=domain,
slots=domain.slots,
)
test_story = TrackerWithCachedStates.from_events(
"test story",
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(UTTER_ACTION_1),
ActionExecuted(UTTER_ACTION_2),
ActionExecuted(UTTER_ACTION_3),
ActionExecuted(UTTER_ACTION_4),
ActionExecuted(UTTER_ACTION_5),
# ActionExecuted(UTTER_BYE_ACTION),
],
domain=domain,
slots=domain.slots,
)
policy.train([training_story], domain)
prediction = policy.predict_action_probabilities(test_story, domain)
assert (domain.action_names_or_texts[prediction.probabilities.index(
max(prediction.probabilities))] == UTTER_BYE_ACTION)
def test_aug_pred_sensitive_to_intent_across_max_history_actions(
self,
max_history: Optional[int],
model_storage: ModelStorage,
resource: Resource,
execution_context: ExecutionContext,
):
"""Tests that only the most recent user utterance propagates to state
creation of following actions.
"""
policy = self.create_policy(
featurizer=MaxHistoryTrackerFeaturizer(max_history=max_history),
model_storage=model_storage,
resource=resource,
execution_context=execution_context,
config={POLICY_MAX_HISTORY: max_history},
)
GREET_INTENT_NAME = "greet"
GOODBYE_INTENT_NAME = "goodbye"
UTTER_GREET_ACTION = "utter_greet"
UTTER_ACTION_1 = "utter_1"
UTTER_ACTION_2 = "utter_2"
UTTER_ACTION_3 = "utter_3"
UTTER_ACTION_4 = "utter_4"
UTTER_ACTION_5 = "utter_5"
UTTER_BYE_ACTION = "utter_goodbye"
domain = Domain.from_yaml(f"""
intents:
- {GREET_INTENT_NAME}
- {GOODBYE_INTENT_NAME}
actions:
- {UTTER_GREET_ACTION}
- {UTTER_ACTION_1}
- {UTTER_ACTION_2}
- {UTTER_ACTION_3}
- {UTTER_ACTION_4}
- {UTTER_ACTION_5}
- {UTTER_BYE_ACTION}
""")
training_story = TrackerWithCachedStates.from_events(
"training story",
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(UTTER_ACTION_1),
ActionExecuted(UTTER_ACTION_2),
ActionExecuted(UTTER_ACTION_3),
ActionExecuted(UTTER_ACTION_4),
ActionExecuted(UTTER_ACTION_5),
ActionExecuted(UTTER_BYE_ACTION),
ActionExecuted(ACTION_LISTEN_NAME),
],
domain=domain,
slots=domain.slots,
)
test_story1 = TrackerWithCachedStates.from_events(
"test story",
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": GOODBYE_INTENT_NAME}),
ActionExecuted(UTTER_BYE_ACTION),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(UTTER_ACTION_1),
ActionExecuted(UTTER_ACTION_2),
ActionExecuted(UTTER_ACTION_3),
ActionExecuted(UTTER_ACTION_4),
ActionExecuted(UTTER_ACTION_5),
# ActionExecuted(UTTER_BYE_ACTION),
],
domain=domain,
slots=domain.slots,
)
policy.train([training_story], domain)
prediction1 = policy.predict_action_probabilities(test_story1, domain)
assert (domain.action_names_or_texts[prediction1.probabilities.index(
max(prediction1.probabilities))] == UTTER_BYE_ACTION)
test_story2_no_match_expected = TrackerWithCachedStates.from_events(
"test story",
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(UTTER_BYE_ACTION),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": GOODBYE_INTENT_NAME}),
ActionExecuted(UTTER_ACTION_1),
ActionExecuted(UTTER_ACTION_2),
ActionExecuted(UTTER_ACTION_3),
ActionExecuted(UTTER_ACTION_4),
ActionExecuted(UTTER_ACTION_5),
# No prediction should be made here.
],
domain=domain,
slots=domain.slots,
)
prediction2 = policy.predict_action_probabilities(
test_story2_no_match_expected,
domain,
)
assert all([prob == 0.0 for prob in prediction2.probabilities])
def test_aug_pred_without_intent(
self,
max_history: Optional[int],
model_storage: ModelStorage,
resource: Resource,
execution_context: ExecutionContext,
):
"""Tests memoization works for a memoized state sequence that does
not have a user utterance.
"""
policy = self.create_policy(
featurizer=MaxHistoryTrackerFeaturizer(max_history=max_history),
model_storage=model_storage,
resource=resource,
execution_context=execution_context,
config={POLICY_MAX_HISTORY: max_history},
)
GREET_INTENT_NAME = "greet"
GOODBYE_INTENT_NAME = "goodbye"
UTTER_GREET_ACTION = "utter_greet"
UTTER_ACTION_1 = "utter_1"
UTTER_ACTION_2 = "utter_2"
UTTER_ACTION_3 = "utter_3"
UTTER_ACTION_4 = "utter_4"
domain = Domain.from_yaml(f"""
intents:
- {GREET_INTENT_NAME}
- {GOODBYE_INTENT_NAME}
actions:
- {UTTER_GREET_ACTION}
- {UTTER_ACTION_1}
- {UTTER_ACTION_2}
- {UTTER_ACTION_3}
- {UTTER_ACTION_4}
""")
training_story = TrackerWithCachedStates.from_events(
"training story",
[
ActionExecuted(UTTER_ACTION_3),
ActionExecuted(UTTER_ACTION_4),
ActionExecuted(ACTION_LISTEN_NAME),
],
domain=domain,
slots=domain.slots,
)
policy.train([training_story], domain)
test_story = TrackerWithCachedStates.from_events(
"test story",
[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(UTTER_ACTION_1),
ActionExecuted(UTTER_ACTION_2),
ActionExecuted(UTTER_ACTION_3),
# ActionExecuted(UTTER_ACTION_4),
],
domain=domain,
slots=domain.slots,
)
prediction = policy.predict_action_probabilities(test_story, domain)
assert (domain.action_names_or_texts[prediction.probabilities.index(
max(prediction.probabilities))] == UTTER_ACTION_4)
def test_augmented_prediction(
self,
max_history: Optional[int],
model_storage: ModelStorage,
resource: Resource,
execution_context: ExecutionContext,
):
policy = self.create_policy(
featurizer=MaxHistoryTrackerFeaturizer(max_history=max_history),
model_storage=model_storage,
resource=resource,
execution_context=execution_context,
)
GREET_INTENT_NAME = "greet"
UTTER_GREET_ACTION = "utter_greet"
UTTER_BYE_ACTION = "utter_goodbye"
domain = Domain.from_yaml(f"""
intents:
- {GREET_INTENT_NAME}
actions:
- {UTTER_GREET_ACTION}
- {UTTER_BYE_ACTION}
slots:
slot_1:
type: bool
initial_value: true
slot_2:
type: bool
slot_3:
type: bool
""")
training_story = TrackerWithCachedStates.from_events(
"training story",
[
ActionExecuted(UTTER_GREET_ACTION),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(UTTER_GREET_ACTION),
SlotSet("slot_3", True),
ActionExecuted(UTTER_BYE_ACTION),
],
domain=domain,
slots=domain.slots,
)
test_story = TrackerWithCachedStates.from_events(
"test story",
[
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(UTTER_GREET_ACTION),
SlotSet("slot_1", False),
ActionExecuted(UTTER_GREET_ACTION),
ActionExecuted(UTTER_GREET_ACTION),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(UTTER_GREET_ACTION),
SlotSet("slot_2", True),
ActionExecuted(UTTER_GREET_ACTION),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(UTTER_GREET_ACTION),
SlotSet("slot_3", True),
# ActionExecuted(UTTER_BYE_ACTION),
],
domain=domain,
slots=domain.slots,
)
policy.train([training_story], domain)
prediction = policy.predict_action_probabilities(test_story, domain)
assert (domain.action_names_or_texts[prediction.probabilities.index(
max(prediction.probabilities))] == UTTER_BYE_ACTION)
async def test_one_stage_fallback_rule():
domain = Domain.from_yaml(f"""
intents:
- {GREET_INTENT_NAME}
- {DEFAULT_NLU_FALLBACK_INTENT_NAME}
actions:
- {UTTER_GREET_ACTION}
""")
fallback_recover_rule = TrackerWithCachedStates.from_events(
"bla",
domain=domain,
slots=domain.slots,
evts=[
ActionExecuted(RULE_SNIPPET_ACTION_NAME),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": DEFAULT_NLU_FALLBACK_INTENT_NAME}),
ActionExecuted(ACTION_DEFAULT_FALLBACK_NAME),
ActionExecuted(ACTION_LISTEN_NAME),
],
is_rule_tracker=True,
)
greet_rule_which_only_applies_at_start = TrackerWithCachedStates.from_events(
"bla",
domain=domain,
evts=[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(UTTER_GREET_ACTION),
ActionExecuted(ACTION_LISTEN_NAME),
],
is_rule_tracker=True,
)
policy = RulePolicy()
policy.train(
[greet_rule_which_only_applies_at_start, fallback_recover_rule],
domain,
RegexInterpreter(),
)
# RulePolicy predicts fallback action
conversation_events = [
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered("dasdakl;fkasd",
{"name": DEFAULT_NLU_FALLBACK_INTENT_NAME}),
]
tracker = DialogueStateTracker.from_events("casd",
evts=conversation_events,
slots=domain.slots)
action_probabilities = policy.predict_action_probabilities(
tracker, domain, RegexInterpreter())
assert_predicted_action(action_probabilities, domain,
ACTION_DEFAULT_FALLBACK_NAME)
# Fallback action reverts fallback events, next action is `ACTION_LISTEN`
conversation_events += await ActionDefaultFallback().run(
CollectingOutputChannel(),
TemplatedNaturalLanguageGenerator(domain.templates),
tracker,
domain,
)
# Rasa is back on track when user rephrased intent
conversation_events += [
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered("haha", {"name": GREET_INTENT_NAME}),
]
tracker = DialogueStateTracker.from_events("casd",
evts=conversation_events,
slots=domain.slots)
action_probabilities = policy.predict_action_probabilities(
tracker, domain, RegexInterpreter())
assert_predicted_action(action_probabilities, domain, UTTER_GREET_ACTION)
def test_predict_next_action_with_hidden_rules():
rule_intent = "rule_intent"
rule_action = "rule_action"
story_intent = "story_intent"
story_action = "story_action"
rule_slot = "rule_slot"
story_slot = "story_slot"
domain = Domain.from_yaml(f"""
version: "2.0"
intents:
- {rule_intent}
- {story_intent}
actions:
- {rule_action}
- {story_action}
slots:
{rule_slot}:
type: text
{story_slot}:
type: text
""")
rule = TrackerWithCachedStates.from_events(
"rule",
domain=domain,
slots=domain.slots,
evts=[
ActionExecuted(RULE_SNIPPET_ACTION_NAME),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": rule_intent}),
ActionExecuted(rule_action),
SlotSet(rule_slot, rule_slot),
ActionExecuted(ACTION_LISTEN_NAME),
],
is_rule_tracker=True,
)
story = TrackerWithCachedStates.from_events(
"story",
domain=domain,
slots=domain.slots,
evts=[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": story_intent}),
ActionExecuted(story_action),
SlotSet(story_slot, story_slot),
ActionExecuted(ACTION_LISTEN_NAME),
],
)
interpreter = RegexInterpreter()
ensemble = SimplePolicyEnsemble(
policies=[RulePolicy(), MemoizationPolicy()])
ensemble.train([rule, story], domain, interpreter)
tracker_store = InMemoryTrackerStore(domain)
lock_store = InMemoryLockStore()
processor = MessageProcessor(
interpreter,
ensemble,
domain,
tracker_store,
lock_store,
TemplatedNaturalLanguageGenerator(domain.responses),
)
tracker = DialogueStateTracker.from_events(
"casd",
evts=[
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": rule_intent}),
],
slots=domain.slots,
)
action, prediction = processor.predict_next_action(tracker)
assert action._name == rule_action
assert prediction.hide_rule_turn
processor._log_action_on_tracker(tracker, action,
[SlotSet(rule_slot, rule_slot)],
prediction)
action, prediction = processor.predict_next_action(tracker)
assert isinstance(action, ActionListen)
assert prediction.hide_rule_turn
processor._log_action_on_tracker(tracker, action, None, prediction)
tracker.events.append(UserUttered(intent={"name": story_intent}))
# rules are hidden correctly if memo policy predicts next actions correctly
action, prediction = processor.predict_next_action(tracker)
assert action._name == story_action
assert not prediction.hide_rule_turn
processor._log_action_on_tracker(tracker, action,
[SlotSet(story_slot, story_slot)],
prediction)
action, prediction = processor.predict_next_action(tracker)
assert isinstance(action, ActionListen)
assert not prediction.hide_rule_turn
def test_incomplete_rules_due_to_loops():
some_form = "some_form"
domain = Domain.from_yaml(f"""
intents:
- {GREET_INTENT_NAME}
forms:
- {some_form}
""")
policy = RulePolicy()
complete_rule = TrackerWithCachedStates.from_events(
"complete_rule",
domain=domain,
slots=domain.slots,
evts=[
ActionExecuted(RULE_SNIPPET_ACTION_NAME),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(some_form),
ActiveLoop(some_form),
ActionExecuted(ACTION_LISTEN_NAME),
],
is_rule_tracker=True,
)
incomplete_rule = TrackerWithCachedStates.from_events(
"incomplete_rule",
domain=domain,
slots=domain.slots,
evts=[
ActionExecuted(RULE_SNIPPET_ACTION_NAME),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(some_form),
ActionExecuted(ACTION_LISTEN_NAME),
],
is_rule_tracker=True,
)
with pytest.raises(InvalidRule) as execinfo:
policy.train([complete_rule, incomplete_rule], domain,
RegexInterpreter())
assert all(name in execinfo.value.message for name in {
some_form,
incomplete_rule.sender_id,
})
fixed_incomplete_rule = TrackerWithCachedStates.from_events(
"fixed_incomplete_rule",
domain=domain,
slots=domain.slots,
evts=[
ActionExecuted(RULE_SNIPPET_ACTION_NAME),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(some_form),
ActionExecuted(RULE_SNIPPET_ACTION_NAME),
ActionExecuted(ACTION_LISTEN_NAME),
],
is_rule_tracker=True,
)
policy.train([complete_rule, fixed_incomplete_rule], domain,
RegexInterpreter())