async def _run_action(
self, action, tracker, output_channel, nlg, policy=None, confidence=None
):
# events and return values are used to update
# the tracker state after an action has been taken
try:
events = await action.run(output_channel, nlg, tracker, self.domain)
except ActionExecutionRejection:
events = [ActionExecutionRejected(action.name(), policy, confidence)]
tracker.update(events[0])
return self.should_predict_another_action(action.name(), events)
except Exception as e:
logger.error(
"Encountered an exception while running action '{}'. "
"Bot will continue, but the actions events are lost. "
"Please check the logs of your action server for "
"more information.".format(action.name())
)
logger.debug(e, exc_info=True)
events = []
self._log_action_on_tracker(tracker, action.name(), events, policy, confidence)
if action.name() != ACTION_LISTEN_NAME and not action.name().startswith(
UTTER_PREFIX
):
self._log_slots(tracker)
await self._send_bot_messages(events, tracker, output_channel)
await self._schedule_reminders(events, tracker, output_channel, nlg)
await self._cancel_reminders(events, tracker)
return self.should_predict_another_action(action.name(), events)
async def _run_action(
self, action, tracker, dispatcher, policy=None, confidence=None
):
# events and return values are used to update
# the tracker state after an action has been taken
try:
events = await action.run(dispatcher, tracker, self.domain)
except ActionExecutionRejection:
events = [ActionExecutionRejected(action.name(), policy, confidence)]
tracker.update(events[0])
return self.should_predict_another_action(action.name(), events)
except Exception as e:
logger.error(
"Encountered an exception while running action '{}'. "
"Bot will continue, but the actions events are lost. "
"Make sure to fix the exception in your custom "
"code.".format(action.name())
)
logger.debug(e, exc_info=True)
events = []
self._log_action_on_tracker(tracker, action.name(), events, policy, confidence)
if action.name() != "action_listen" and "utter_" not in action.name():
self._log_slots(tracker)
self.log_bot_utterances_on_tracker(tracker, dispatcher)
await self._schedule_reminders(events, tracker, dispatcher)
await self._cancel_reminders(events, tracker)
return self.should_predict_another_action(action.name(), events)
async def test_form_unhappy_path_from_general_rule():
form_name = "some_form"
domain = Domain.from_yaml(f"""
intents:
- {GREET_INTENT_NAME}
actions:
- {UTTER_GREET_ACTION}
- some-action
slots:
{REQUESTED_SLOT}:
type: unfeaturized
forms:
- {form_name}
""")
policy = RulePolicy()
# RulePolicy should memorize that unhappy_rule overrides GREET_RULE
policy.train([GREET_RULE], domain, RegexInterpreter())
# Check that RulePolicy predicts action to handle unhappy path
conversation_events = [
ActionExecuted(form_name),
ActiveLoop(form_name),
SlotSet(REQUESTED_SLOT, "some value"),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered("haha", {"name": GREET_INTENT_NAME}),
ActionExecutionRejected(form_name),
]
action_probabilities = policy.predict_action_probabilities(
DialogueStateTracker.from_events("casd",
evts=conversation_events,
slots=domain.slots),
domain,
RegexInterpreter(),
)
# check that general rule action is predicted
assert_predicted_action(action_probabilities, domain, UTTER_GREET_ACTION)
# Check that RulePolicy triggers form again after handling unhappy path
conversation_events.append(ActionExecuted(UTTER_GREET_ACTION))
action_probabilities = policy.predict_action_probabilities(
DialogueStateTracker.from_events("casd",
evts=conversation_events,
slots=domain.slots),
domain,
RegexInterpreter(),
)
# check that action_listen from general rule is overwritten by form action
assert_predicted_action(action_probabilities, domain, form_name)
def _emulate_form_rejection(processor, partial_tracker):
from rasa.core.policies import FormPolicy
if partial_tracker.active_form.get("name"):
for p in processor.policy_ensemble.policies:
if isinstance(p, FormPolicy):
# emulate form rejection
partial_tracker.update(
ActionExecutionRejected(
partial_tracker.active_form["name"]))
# check if unhappy path is covered by the train stories
if not p.state_is_unhappy(partial_tracker, processor.domain):
# this state is not covered by the stories
del partial_tracker.events[-1]
partial_tracker.active_form['rejected'] = False
async def _run_action(
self,
action,
tracker,
output_channel,
nlg,
policy=None,
confidence=None,
metadata: Optional[Dict[Text, Any]] = None,
) -> bool:
# events and return values are used to update
# the tracker state after an action has been taken
try:
# Here we set optional metadata to the ActionSessionStart, which will then
# be passed to the SessionStart event. Otherwise the metadata will be lost.
if action.name() == ACTION_SESSION_START_NAME:
action.metadata = metadata
events = await action.run(output_channel, nlg, tracker,
self.domain)
except ActionExecutionRejection:
events = [
ActionExecutionRejected(action.name(), policy, confidence)
]
tracker.update(events[0])
return self.should_predict_another_action(action.name())
except Exception as e:
logger.error(
f"Encountered an exception while running action '{action.name()}'. "
"Bot will continue, but the actions events are lost. "
"Please check the logs of your action server for "
"more information.")
logger.debug(e, exc_info=True)
events = []
self._log_action_on_tracker(tracker, action.name(), events, policy,
confidence)
if action.name() != ACTION_LISTEN_NAME and not action.name(
).startswith(UTTER_PREFIX):
self._log_slots(tracker)
await self._send_bot_messages(events, tracker, output_channel)
await self._schedule_reminders(events, tracker, output_channel, nlg)
await self._cancel_reminders(events, tracker)
return self.should_predict_another_action(action.name())
async def test_form_unhappy_path():
form_name = "some_form"
domain = Domain.from_yaml(
f"""
intents:
- {GREET_INTENT_NAME}
actions:
- {UTTER_GREET_ACTION}
- some-action
slots:
{REQUESTED_SLOT}:
type: unfeaturized
forms:
- {form_name}
"""
)
policy = RulePolicy()
policy.train([GREET_RULE], domain, RegexInterpreter())
unhappy_form_conversation = DialogueStateTracker.from_events(
"in a form",
evts=[
# We are in an active form
ActionExecuted(form_name),
Form(form_name),
SlotSet(REQUESTED_SLOT, "some value"),
# User responds to slot request
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered("haha", {"name": GREET_INTENT_NAME}),
# Form isn't happy with the answer and rejects execution
ActionExecutionRejected(form_name),
],
slots=domain.slots,
)
# RulePolicy doesn't trigger form but FAQ
action_probabilities = policy.predict_action_probabilities(
unhappy_form_conversation, domain
)
assert_predicted_action(action_probabilities, domain, UTTER_GREET_ACTION)
async def test_form_unhappy_path_without_rule():
form_name = "some_form"
other_intent = "bye"
domain = Domain.from_yaml(
f"""
intents:
- {GREET_INTENT_NAME}
- {other_intent}
actions:
- {UTTER_GREET_ACTION}
- some-action
slots:
{REQUESTED_SLOT}:
type: unfeaturized
forms:
- {form_name}
"""
)
policy = RulePolicy()
policy.train([GREET_RULE], domain, RegexInterpreter())
conversation_events = [
ActionExecuted(form_name),
Form(form_name),
SlotSet(REQUESTED_SLOT, "some value"),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered("haha", {"name": other_intent}),
Form(form_name),
ActionExecutionRejected(form_name),
]
# Unhappy path is not handled. No rule matches. Let's hope ML fixes our problems 🤞
action_probabilities = policy.predict_action_probabilities(
DialogueStateTracker.from_events(
"casd", evts=conversation_events, slots=domain.slots
),
domain,
)
assert max(action_probabilities) == policy._core_fallback_threshold
async def test_loop_without_deactivate():
expected_activation_events = [
ActionExecutionRejected("tada"),
ActionExecuted("test"),
]
expected_do_events = [ActionExecuted("do")]
form_name = "my form"
class MyLoop(LoopAction):
def name(self) -> Text:
return form_name
async def activate(self, *args: Any) -> List[Event]:
return expected_activation_events
async def do(self, *args: Any) -> List[Event]:
return expected_do_events
async def deactivate(self, *args) -> List[Event]:
raise ValueError("this shouldn't be called")
async def is_done(self, *args) -> bool:
return False
tracker = DialogueStateTracker.from_events("some sender", [])
domain = Domain.empty()
action = MyLoop()
actual = await action.run(
CollectingOutputChannel(),
TemplatedNaturalLanguageGenerator(domain.templates),
tracker,
domain,
)
assert actual == [
ActiveLoop(form_name),
*expected_activation_events,
*expected_do_events,
]
async def test_form_unhappy_path_no_validation_from_story():
form_name = "some_form"
handle_rejection_action_name = "utter_handle_rejection"
domain = Domain.from_yaml(f"""
intents:
- {GREET_INTENT_NAME}
actions:
- {UTTER_GREET_ACTION}
- {handle_rejection_action_name}
- some-action
slots:
{REQUESTED_SLOT}:
type: unfeaturized
forms:
- {form_name}
""")
unhappy_story = TrackerWithCachedStates.from_events(
"bla",
domain=domain,
slots=domain.slots,
evts=[
# We are in an active form
ActionExecuted(form_name),
ActiveLoop(form_name),
# When a user says "hi", and the form is unhappy,
# we want to run a specific action
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(handle_rejection_action_name),
ActionExecuted(ACTION_LISTEN_NAME),
# Next user utterance is an answer to the previous question
# and shouldn't be validated by the form
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(form_name),
ActionExecuted(ACTION_LISTEN_NAME),
],
)
policy = RulePolicy()
policy.train([unhappy_story], domain, RegexInterpreter())
# Check that RulePolicy predicts no validation to handle unhappy path
conversation_events = [
ActionExecuted(form_name),
ActiveLoop(form_name),
SlotSet(REQUESTED_SLOT, "some value"),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered("haha", {"name": GREET_INTENT_NAME}),
ActionExecutionRejected(form_name),
ActionExecuted(handle_rejection_action_name),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered("haha", {"name": GREET_INTENT_NAME}),
]
tracker = DialogueStateTracker.from_events("casd",
evts=conversation_events,
slots=domain.slots)
action_probabilities = policy.predict_action_probabilities(
tracker, domain, RegexInterpreter())
# there is no rule for next action
assert max(action_probabilities) == policy._core_fallback_threshold
# check that RulePolicy added FormValidation False event based on the training story
assert tracker.events[-1] == FormValidation(False)
async def test_form_unhappy_path_no_validation_from_rule():
form_name = "some_form"
handle_rejection_action_name = "utter_handle_rejection"
domain = Domain.from_yaml(f"""
intents:
- {GREET_INTENT_NAME}
actions:
- {UTTER_GREET_ACTION}
- {handle_rejection_action_name}
- some-action
slots:
{REQUESTED_SLOT}:
type: unfeaturized
forms:
- {form_name}
""")
unhappy_rule = TrackerWithCachedStates.from_events(
"bla",
domain=domain,
slots=domain.slots,
evts=[
# We are in an active form
ActiveLoop(form_name),
SlotSet(REQUESTED_SLOT, "bla"),
ActionExecuted(RULE_SNIPPET_ACTION_NAME),
ActionExecuted(ACTION_LISTEN_NAME),
# When a user says "hi", and the form is unhappy,
# we want to run a specific action
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(handle_rejection_action_name),
# Next user utterance is an answer to the previous question
# and shouldn't be validated by the form
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered(intent={"name": GREET_INTENT_NAME}),
ActionExecuted(form_name),
ActionExecuted(ACTION_LISTEN_NAME),
],
is_rule_tracker=True,
)
policy = RulePolicy()
# RulePolicy should memorize that unhappy_rule overrides GREET_RULE
policy.train([GREET_RULE, unhappy_rule], domain, RegexInterpreter())
# Check that RulePolicy predicts action to handle unhappy path
conversation_events = [
ActionExecuted(form_name),
ActiveLoop(form_name),
SlotSet(REQUESTED_SLOT, "some value"),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered("haha", {"name": GREET_INTENT_NAME}),
ActionExecutionRejected(form_name),
]
action_probabilities = policy.predict_action_probabilities(
DialogueStateTracker.from_events("casd",
evts=conversation_events,
slots=domain.slots),
domain,
RegexInterpreter(),
)
assert_predicted_action(action_probabilities, domain,
handle_rejection_action_name)
# Check that RulePolicy predicts action_listen
conversation_events.append(ActionExecuted(handle_rejection_action_name))
action_probabilities = policy.predict_action_probabilities(
DialogueStateTracker.from_events("casd",
evts=conversation_events,
slots=domain.slots),
domain,
RegexInterpreter(),
)
assert_predicted_action(action_probabilities, domain, ACTION_LISTEN_NAME)
# Check that RulePolicy triggers form again after handling unhappy path
conversation_events.append(ActionExecuted(ACTION_LISTEN_NAME))
tracker = DialogueStateTracker.from_events("casd",
evts=conversation_events,
slots=domain.slots)
action_probabilities = policy.predict_action_probabilities(
tracker, domain, RegexInterpreter())
assert_predicted_action(action_probabilities, domain, form_name)
# check that RulePolicy added FormValidation False event based on the training rule
assert tracker.events[-1] == FormValidation(False)
async def test_form_unhappy_path_from_story():
form_name = "some_form"
handle_rejection_action_name = "utter_handle_rejection"
domain = Domain.from_yaml(f"""
intents:
- {GREET_INTENT_NAME}
actions:
- {UTTER_GREET_ACTION}
- {handle_rejection_action_name}
- some-action
slots:
{REQUESTED_SLOT}:
type: unfeaturized
forms:
- {form_name}
""")
unhappy_story = TrackerWithCachedStates.from_events(
"bla",
domain=domain,
slots=domain.slots,
evts=[
# We are in an active form
ActionExecuted(form_name),
ActiveLoop(form_name),
UserUttered("haha", {"name": GREET_INTENT_NAME}),
ActionExecuted(UTTER_GREET_ACTION),
# After our bot says "hi", we want to run a specific action
ActionExecuted(handle_rejection_action_name),
ActionExecuted(form_name),
ActionExecuted(ACTION_LISTEN_NAME),
],
)
policy = RulePolicy()
policy.train([GREET_RULE, unhappy_story], domain, RegexInterpreter())
# Check that RulePolicy predicts action to handle unhappy path
conversation_events = [
ActionExecuted(form_name),
ActiveLoop(form_name),
SlotSet(REQUESTED_SLOT, "some value"),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered("haha", {"name": GREET_INTENT_NAME}),
ActionExecutionRejected(form_name),
]
action_probabilities = policy.predict_action_probabilities(
DialogueStateTracker.from_events("casd",
evts=conversation_events,
slots=domain.slots),
domain,
RegexInterpreter(),
)
assert_predicted_action(action_probabilities, domain, UTTER_GREET_ACTION)
# Check that RulePolicy doesn't trigger form or action_listen
# after handling unhappy path
conversation_events.append(ActionExecuted(handle_rejection_action_name))
action_probabilities = policy.predict_action_probabilities(
DialogueStateTracker.from_events("casd",
evts=conversation_events,
slots=domain.slots),
domain,
RegexInterpreter(),
)
assert max(action_probabilities) == policy._core_fallback_threshold
@pytest.mark.parametrize(
"events, expected_applied_events",
[
(
[
# Form is triggered and requests slot.
ActionExecuted(ACTION_LISTEN_NAME),
user_uttered("greet"),
ActionExecuted("loop"),
ActiveLoop("loop"),
SlotSet(REQUESTED_SLOT, "bla"),
# User sends chitchat instead of answering form.
ActionExecuted(ACTION_LISTEN_NAME),
user_uttered("chitchat"),
# Form rejected execution.
ActionExecutionRejected("loop"),
# Action which deals with unhappy path.
ActionExecuted("handling chitchat"),
# We immediately return to form after executing an action to handle it.
ActionExecuted("loop"),
# Form happy path continues until all slots are filled.
SlotSet(REQUESTED_SLOT, "bla"),
ActionExecuted(ACTION_LISTEN_NAME),
user_uttered("fill slots"),
ActionExecuted("loop"),
SlotSet("slot", "value"),
SlotSet(REQUESTED_SLOT, None),
ActiveLoop(None),
],
[
ActionExecuted(ACTION_LISTEN_NAME),
def _emulate_form_rejection(partial_tracker: DialogueStateTracker) -> None:
from rasa.core.events import ActionExecutionRejected
rejected_action_name: Text = partial_tracker.active_loop["name"]
partial_tracker.update(ActionExecutionRejected(rejected_action_name))
async def test_persist_form_story(tmpdir):
domain = Domain.load("data/test_domains/form.yml")
tracker = DialogueStateTracker("", domain.slots)
story = ("* greet\n"
" - utter_greet\n"
"* start_form\n"
" - some_form\n"
' - form{"name": "some_form"}\n'
"* default\n"
" - utter_default\n"
" - some_form\n"
"* stop\n"
" - utter_ask_continue\n"
"* affirm\n"
" - some_form\n"
"* stop\n"
" - utter_ask_continue\n"
" - action_listen\n"
"* form: inform\n"
" - some_form\n"
' - form{"name": null}\n'
"* goodbye\n"
" - utter_goodbye\n")
# simulate talking to the form
events = [
UserUttered(intent={"name": "greet"}),
ActionExecuted("utter_greet"),
ActionExecuted("action_listen"),
# start the form
UserUttered(intent={"name": "start_form"}),
ActionExecuted("some_form"),
Form("some_form"),
ActionExecuted("action_listen"),
# out of form input
UserUttered(intent={"name": "default"}),
ActionExecutionRejected("some_form"),
ActionExecuted("utter_default"),
ActionExecuted("some_form"),
ActionExecuted("action_listen"),
# out of form input
UserUttered(intent={"name": "stop"}),
ActionExecutionRejected("some_form"),
ActionExecuted("utter_ask_continue"),
ActionExecuted("action_listen"),
# out of form input but continue with the form
UserUttered(intent={"name": "affirm"}),
FormValidation(False),
ActionExecuted("some_form"),
ActionExecuted("action_listen"),
# out of form input
UserUttered(intent={"name": "stop"}),
ActionExecutionRejected("some_form"),
ActionExecuted("utter_ask_continue"),
ActionExecuted("action_listen"),
# form input
UserUttered(intent={"name": "inform"}),
FormValidation(True),
ActionExecuted("some_form"),
ActionExecuted("action_listen"),
Form(None),
UserUttered(intent={"name": "goodbye"}),
ActionExecuted("utter_goodbye"),
ActionExecuted("action_listen"),
]
[tracker.update(e) for e in events]
assert story in tracker.export_stories()
async def test_whole_loop():
expected_activation_events = [
ActionExecutionRejected("tada"),
ActionExecuted("test"),
]
expected_do_events = [ActionExecuted("do")]
expected_deactivation_events = [SlotSet("deactivated")]
form_name = "my form"
class MyLoop(LoopAction):
def name(self) -> Text:
return form_name
async def activate(self, *args: Any) -> List[Event]:
return expected_activation_events
async def do(self, *args: Any) -> List[Event]:
events_so_far = args[-1]
assert events_so_far == [
ActiveLoop(form_name), *expected_activation_events
]
return expected_do_events
async def deactivate(self, *args) -> List[Event]:
events_so_far = args[-1]
assert events_so_far == [
ActiveLoop(form_name),
*expected_activation_events,
*expected_do_events,
ActiveLoop(None),
]
return expected_deactivation_events
async def is_done(self, *args) -> bool:
events_so_far = args[-1]
return events_so_far == [
ActiveLoop(form_name),
*expected_activation_events,
*expected_do_events,
]
tracker = DialogueStateTracker.from_events("some sender", [])
domain = Domain.empty()
action = MyLoop()
actual = await action.run(
CollectingOutputChannel(),
TemplatedNaturalLanguageGenerator(domain.templates),
tracker,
domain,
)
assert actual == [
ActiveLoop(form_name),
*expected_activation_events,
*expected_do_events,
ActiveLoop(None),
*expected_deactivation_events,
]
async def test_form_unhappy_path_triggering_form_again():
form_name = "some_form"
handle_rejection_action_name = "utter_handle_rejection"
domain = Domain.from_yaml(f"""
intents:
- {GREET_INTENT_NAME}
actions:
- {UTTER_GREET_ACTION}
- {handle_rejection_action_name}
- some-action
slots:
{REQUESTED_SLOT}:
type: unfeaturized
forms:
- {form_name}
""")
unhappy_rule = TrackerWithCachedStates.from_events(
"bla",
domain=domain,
slots=domain.slots,
evts=[
# We are in an active form
Form(form_name),
SlotSet(REQUESTED_SLOT, "bla"),
ActionExecuted(RULE_SNIPPET_ACTION_NAME),
ActionExecuted(ACTION_LISTEN_NAME),
# When a user says "hi", and the form is unhappy, we want to run a specific
# action
UserUttered("haha", {"name": GREET_INTENT_NAME}),
ActionExecuted(handle_rejection_action_name),
ActionExecuted(form_name),
ActionExecuted(ACTION_LISTEN_NAME),
],
is_rule_tracker=True,
)
policy = RulePolicy()
policy.train([unhappy_rule], domain, RegexInterpreter())
# Check that RulePolicy predicts action to handle unhappy path
conversation_events = [
ActionExecuted(form_name),
Form(form_name),
SlotSet(REQUESTED_SLOT, "some value"),
ActionExecuted(ACTION_LISTEN_NAME),
UserUttered("haha", {"name": GREET_INTENT_NAME}),
ActionExecutionRejected(form_name),
]
action_probabilities = policy.predict_action_probabilities(
DialogueStateTracker.from_events("casd",
evts=conversation_events,
slots=domain.slots),
domain,
)
assert_predicted_action(action_probabilities, domain,
handle_rejection_action_name)
# Check that RulePolicy triggers form again after handling unhappy path
conversation_events.append(ActionExecuted(handle_rejection_action_name))
action_probabilities = policy.predict_action_probabilities(
DialogueStateTracker.from_events("casd",
evts=conversation_events,
slots=domain.slots),
domain,
)
assert_predicted_action(action_probabilities, domain, form_name)
