def _collect_action_executed_predictions(
processor: "MessageProcessor",
partial_tracker: DialogueStateTracker,
event: ActionExecuted,
fail_on_prediction_errors: bool,
circuit_breaker_tripped: bool,
) -> Tuple[EvaluationStore, PolicyPrediction,
Optional[EntityEvaluationResult]]:
from rasa.core.policies.form_policy import FormPolicy
action_executed_eval_store = EvaluationStore()
gold_action_name = event.action_name
gold_action_text = event.action_text
gold = gold_action_name or gold_action_text
policy_entity_result = None
if circuit_breaker_tripped:
prediction = PolicyPrediction([], policy_name=None)
predicted = "circuit breaker tripped"
else:
action, prediction = processor.predict_next_action(partial_tracker)
predicted = action.name()
policy_entity_result = _get_e2e_entity_evaluation_result(
processor, partial_tracker, prediction)
if (prediction.policy_name
and predicted != gold and _form_might_have_been_rejected(
processor.domain, partial_tracker, predicted)):
# Wrong action was predicted,
# but it might be Ok if form action is rejected.
emulate_loop_rejection(partial_tracker)
# try again
action, prediction = processor.predict_next_action(partial_tracker)
# Even if the prediction is also wrong, we don't have to undo the emulation
# of the action rejection as we know that the user explicitly specified
# that something else than the form was supposed to run.
predicted = action.name()
action_executed_eval_store.add_to_store(action_predictions=[predicted],
action_targets=[gold])
if action_executed_eval_store.has_prediction_target_mismatch():
partial_tracker.update(
WronglyPredictedAction(
gold_action_name,
gold_action_text,
predicted,
prediction.policy_name,
prediction.max_confidence,
event.timestamp,
))
if fail_on_prediction_errors:
story_dump = YAMLStoryWriter().dumps(
partial_tracker.as_story().story_steps)
error_msg = (f"Model predicted a wrong action. Failed Story: "
f"\n\n{story_dump}")
if FormPolicy.__name__ in prediction.policy_name:
error_msg += ("FormAction is not run during "
"evaluation therefore it is impossible to know "
"if validation failed or this story is wrong. "
"If the story is correct, add it to the "
"training stories and retrain.")
raise WrongPredictionException(error_msg)
else:
partial_tracker.update(
ActionExecuted(
predicted,
prediction.policy_name,
prediction.max_confidence,
event.timestamp,
))
return action_executed_eval_store, prediction, policy_entity_result
def _message_clarification(tracker: DialogueStateTracker) -> List[Event]:
clarification = copy.deepcopy(tracker.latest_message)
clarification.parse_data["intent"]["confidence"] = 1.0
clarification.timestamp = time.time()
return [ActionExecuted(ACTION_LISTEN_NAME), clarification]
events = await ActionSessionStart().run(
default_channel, template_nlg, template_sender_tracker, default_domain
)
assert events == [SessionStarted(), ActionExecuted(ACTION_LISTEN_NAME)]
@pytest.mark.parametrize(
"session_config, expected_events",
[
(
SessionConfig(123, True),
[
SessionStarted(),
SlotSet("my_slot", "value"),
SlotSet("another-slot", "value2"),
ActionExecuted(action_name=ACTION_LISTEN_NAME),
],
),
(
SessionConfig(123, False),
[SessionStarted(), ActionExecuted(action_name=ACTION_LISTEN_NAME)],
),
],
)
async def test_action_session_start_with_slots(
default_channel: CollectingOutputChannel,
template_nlg: TemplatedNaturalLanguageGenerator,
template_sender_tracker: DialogueStateTracker,
default_domain: Domain,
session_config: SessionConfig,
expected_events: List[Event],
async def test_import_nlu_training_data_from_e2e_stories(project: Text):
config_path = os.path.join(project, DEFAULT_CONFIG_PATH)
domain_path = os.path.join(project, DEFAULT_DOMAIN_PATH)
default_data_path = os.path.join(project, DEFAULT_DATA_PATH)
importer = TrainingDataImporter.load_from_dict({}, config_path,
domain_path,
[default_data_path])
# The `E2EImporter` correctly wraps the underlying `CombinedDataImporter`
assert isinstance(importer, E2EImporter)
importer_without_e2e = importer.importer
stories = StoryGraph([
StoryStep(events=[
SlotSet("some slot", "doesn't matter"),
UserUttered("greet_from_stories", {"name": "greet_from_stories"}),
ActionExecuted("utter_greet_from_stories"),
]),
StoryStep(events=[
UserUttered("how are you doing?"),
ActionExecuted("utter_greet_from_stories", action_text="Hi Joey."),
]),
])
# Patch to return our test stories
importer_without_e2e.get_stories = asyncio.coroutine(lambda *args: stories)
# The wrapping `E2EImporter` simply forwards these method calls
assert (await importer_without_e2e.get_stories()).as_story_string() == (
await importer.get_stories()).as_story_string()
assert (await importer_without_e2e.get_config()) == (await
importer.get_config())
# Check additional NLU training data from stories was added
nlu_data = await importer.get_nlu_data()
# The `E2EImporter` adds NLU training data based on our training stories
assert len(nlu_data.training_examples) > len(
(await importer_without_e2e.get_nlu_data()).training_examples)
# Check if the NLU training data was added correctly from the story training data
expected_additional_messages = [
Message(data={
TEXT: "greet_from_stories",
INTENT_NAME: "greet_from_stories"
}),
Message(data={
ACTION_NAME: "utter_greet_from_stories",
ACTION_TEXT: ""
}),
Message(data={
TEXT: "how are you doing?",
INTENT_NAME: None
}),
Message(data={
ACTION_NAME: "utter_greet_from_stories",
ACTION_TEXT: "Hi Joey."
}),
]
assert all(m in nlu_data.training_examples
for m in expected_additional_messages)
def _messages_from_action(event: ActionExecuted) -> Message:
# sub state correctly encodes action_name vs action_text
return Message(data=event.as_sub_state())
assert len(message.data) == 1
def test_container_message_lookup_fails_if_text_cannot_be_looked_up():
table = MessageContainerForCoreFeaturization()
with pytest.raises(ValueError, match="Expected a message with key"):
table.lookup_message(user_text="a text not included in the table")
@pytest.mark.parametrize(
"events,expected_num_entries",
[
(
[
UserUttered(intent={INTENT_NAME_KEY: "greet"}),
ActionExecuted(action_name="utter_greet"),
ActionExecuted(action_name="utter_greet"),
],
2,
),
(
[
UserUttered(text="text", intent={INTENT_NAME_KEY: "greet"}),
ActionExecuted(action_name="utter_greet"),
],
3,
),
],
)
def test_container_derive_messages_from_events_and_add(
events: List[Event], expected_num_entries: int):
):
events = await ActionSessionStart().run(default_channel, template_nlg,
template_sender_tracker, domain)
assert events == [SessionStarted(), ActionExecuted(ACTION_LISTEN_NAME)]
@pytest.mark.parametrize(
"session_config, expected_events",
[
(
SessionConfig(123, True),
[
SessionStarted(),
SlotSet("my_slot", "value"),
SlotSet("another-slot", "value2"),
ActionExecuted(action_name=ACTION_LISTEN_NAME),
],
),
(
SessionConfig(123, False),
[SessionStarted(),
ActionExecuted(action_name=ACTION_LISTEN_NAME)],
),
],
)
async def test_action_session_start_with_slots(
default_channel: CollectingOutputChannel,
template_nlg: TemplatedNaturalLanguageGenerator,
template_sender_tracker: DialogueStateTracker,
domain: Domain,
session_config: SessionConfig,
async def test_persist_legacy_form_story():
domain = Domain.load("data/test_domains/form.yml")
tracker = DialogueStateTracker("", domain.slots)
story = ("* greet\n"
" - utter_greet\n"
"* start_form\n"
" - some_form\n"
' - form{"name": "some_form"}\n'
"* default\n"
" - utter_default\n"
" - some_form\n"
"* stop\n"
" - utter_ask_continue\n"
"* affirm\n"
" - some_form\n"
"* stop\n"
" - utter_ask_continue\n"
"* inform\n"
" - some_form\n"
' - form{"name": null}\n'
"* goodbye\n"
" - utter_goodbye\n")
# simulate talking to the form
events = [
UserUttered(intent={"name": "greet"}),
ActionExecuted("utter_greet"),
ActionExecuted("action_listen"),
# start the form
UserUttered(intent={"name": "start_form"}),
ActionExecuted("some_form"),
ActiveLoop("some_form"),
ActionExecuted("action_listen"),
# out of form input
UserUttered(intent={"name": "default"}),
ActionExecutionRejected("some_form"),
ActionExecuted("utter_default"),
ActionExecuted("some_form"),
ActionExecuted("action_listen"),
# out of form input
UserUttered(intent={"name": "stop"}),
ActionExecutionRejected("some_form"),
ActionExecuted("utter_ask_continue"),
ActionExecuted("action_listen"),
# out of form input but continue with the form
UserUttered(intent={"name": "affirm"}),
LoopInterrupted(True),
ActionExecuted("some_form"),
ActionExecuted("action_listen"),
# out of form input
UserUttered(intent={"name": "stop"}),
ActionExecutionRejected("some_form"),
ActionExecuted("utter_ask_continue"),
ActionExecuted("action_listen"),
# form input
UserUttered(intent={"name": "inform"}),
LoopInterrupted(False),
ActionExecuted("some_form"),
ActionExecuted("action_listen"),
ActiveLoop(None),
UserUttered(intent={"name": "goodbye"}),
ActionExecuted("utter_goodbye"),
ActionExecuted("action_listen"),
]
[tracker.update(e) for e in events]
story = story.replace(f"- {LegacyForm.type_name}",
f"- {ActiveLoop.type_name}")
assert story in tracker.export_stories(MarkdownStoryWriter())
