async def test_remote_action_runs(default_dispatcher_collecting, default_domain):
tracker = DialogueStateTracker("default", default_domain.slots)
endpoint = EndpointConfig("https://example.com/webhooks/actions")
remote_action = action.RemoteAction("my_action", endpoint)
with aioresponses() as mocked:
mocked.post(
"https://example.com/webhooks/actions",
payload={"events": [], "responses": []},
)
await remote_action.run(default_dispatcher_collecting, tracker, default_domain)
r = latest_request(mocked, "post", "https://example.com/webhooks/actions")
assert r
assert json_of_latest_request(r) == {
"domain": default_domain.as_dict(),
"next_action": "my_action",
"sender_id": "default",
"version": rasa.__version__,
"tracker": {
"latest_message": {"entities": [], "intent": {}, "text": None},
"active_form": {},
"latest_action_name": None,
"sender_id": "default",
"paused": False,
"latest_event_time": None,
"followup_action": "action_listen",
"slots": {"name": None},
"events": [],
"latest_input_channel": None,
},
}
async def replace_events(request: Request, conversation_id: Text):
"""Use a list of events to set a conversations tracker to a state."""
validate_request_body(
request,
"You must provide events in the request body to set the sate of the "
"conversation tracker.",
)
verbosity = event_verbosity_parameter(request,
EventVerbosity.AFTER_RESTART)
try:
async with app.agent.lock_store.lock(conversation_id):
tracker = DialogueStateTracker.from_dict(
conversation_id, request.json, app.agent.domain.slots)
# will override an existing tracker with the same id!
app.agent.tracker_store.save(tracker)
return response.json(tracker.current_state(verbosity))
except Exception as e:
logger.debug(traceback.format_exc())
raise ErrorResponse(500, "ConversationError",
f"An unexpected error occurred. Error: {e}")
def test_form_wins_over_everything_else(ensemble: SimplePolicyEnsemble):
form_name = "test-form"
domain = f"""
forms:
- {form_name}
"""
domain = Domain.from_yaml(domain)
events = [
Form("test-form"),
ActionExecuted(ACTION_LISTEN_NAME),
utilities.user_uttered("test", 1),
]
tracker = DialogueStateTracker.from_events("test", events, [])
result, best_policy = ensemble.probabilities_using_best_policy(
tracker, domain, RegexInterpreter()
)
max_confidence_index = result.index(max(result))
next_action = domain.action_for_index(max_confidence_index, None)
index_of_form_policy = 0
assert best_policy == f"policy_{index_of_form_policy}_{FormPolicy.__name__}"
assert next_action.name() == form_name
async def tracker_predict(request: Request):
""" Given a list of events, predicts the next action"""
sender_id = UserMessage.DEFAULT_SENDER_ID
request_params = request.json
verbosity = event_verbosity_parameter(request, EventVerbosity.AFTER_RESTART)
try:
tracker = DialogueStateTracker.from_dict(
sender_id, request_params, app.agent.domain.slots
)
except Exception as e:
raise ErrorResponse(
400,
"InvalidParameter",
"Supplied events are not valid. {}".format(e),
{"parameter": "", "in": "body"},
)
policy_ensemble = app.agent.policy_ensemble
probabilities, policy = policy_ensemble.probabilities_using_best_policy(
tracker, app.agent.domain
)
scores = [
{"action": a, "score": p}
for a, p in zip(app.agent.domain.action_names, probabilities)
]
return response.json(
{
"scores": scores,
"policy": policy,
"tracker": tracker.current_state(verbosity),
}
)
async def _update_tracker_session(self, tracker: DialogueStateTracker,
output_channel: OutputChannel) -> None:
"""Check the current session in `tracker` and update it if expired.
An 'action_session_start' is run if the latest tracker session has expired,
or if the tracker does not yet contain any events (only those after the last
restart are considered).
Args:
tracker: Tracker to inspect.
output_channel: Output channel for potential utterances in a custom
`ActionSessionStart`.
"""
if not tracker.applied_events() or self._has_session_expired(tracker):
logger.debug(
f"Starting a new session for conversation ID '{tracker.sender_id}'."
)
await self._run_action(
action=self._get_action(ACTION_SESSION_START_NAME),
tracker=tracker,
output_channel=output_channel,
nlg=self.nlg,
)
async def replay_events(tracker: DialogueStateTracker, agent: "Agent") -> None:
"""Take a tracker and replay the logged user utterances against an agent.
During replaying of the user utterances, the executed actions and events
created by the agent are compared to the logged ones of the tracker that
is getting replayed. If they differ, a warning is logged.
At the end, the tracker stored in the agent's tracker store for the
same sender id will have quite the same state as the one
that got replayed."""
actions_between_utterances = []
last_prediction = [ACTION_LISTEN_NAME]
for i, event in enumerate(tracker.events_after_latest_restart()):
if isinstance(event, UserUttered):
_check_prediction_aligns_with_story(last_prediction,
actions_between_utterances)
actions_between_utterances = []
cliutils.print_success(event.text)
out = CollectingOutputChannel()
await agent.handle_text(event.text,
sender_id=tracker.sender_id,
output_channel=out)
for m in out.messages:
console.print_bot_output(m)
tracker = agent.tracker_store.retrieve(tracker.sender_id)
last_prediction = actions_since_last_utterance(tracker)
elif isinstance(event, ActionExecuted):
actions_between_utterances.append(event.action_name)
_check_prediction_aligns_with_story(last_prediction,
actions_between_utterances)
def test_temporary_tracker():
extra_slot = "some_slot"
sender_id = "test"
domain = Domain.from_yaml(
f""" slots:
{extra_slot}:
type: unfeaturized
"""
)
previous_events = [ActionExecuted(ACTION_LISTEN_NAME)]
old_tracker = DialogueStateTracker.from_events(
sender_id, previous_events, slots=domain.slots
)
new_events = [Restarted()]
form_action = FormAction("some name", None)
temp_tracker = form_action._temporary_tracker(old_tracker, new_events, domain)
assert extra_slot in temp_tracker.slots.keys()
assert list(temp_tracker.events) == [
*previous_events,
ActionExecuted(form_action.name()),
*new_events,
]
def test_policy_priority():
domain = Domain.load("data/test_domains/default.yml")
tracker = DialogueStateTracker.from_events("test", [UserUttered("hi")], [])
priority_1 = ConstantPolicy(priority=1, predict_index=0)
priority_2 = ConstantPolicy(priority=2, predict_index=1)
policy_ensemble_0 = SimplePolicyEnsemble([priority_1, priority_2])
policy_ensemble_1 = SimplePolicyEnsemble([priority_2, priority_1])
priority_2_result = priority_2.predict_action_probabilities(
tracker, domain)
i = 1 # index of priority_2 in ensemble_0
result, best_policy = policy_ensemble_0.probabilities_using_best_policy(
tracker, domain, RegexInterpreter())
assert best_policy == "policy_{}_{}".format(i, type(priority_2).__name__)
assert result == priority_2_result
i = 0 # index of priority_2 in ensemble_1
result, best_policy = policy_ensemble_1.probabilities_using_best_policy(
tracker, domain, RegexInterpreter())
assert best_policy == "policy_{}_{}".format(i, type(priority_2).__name__)
assert result == priority_2_result
def test_yaml_writer_dumps_user_messages():
events = [
UserUttered("Hello", {"name": "greet"}),
ActionExecuted("utter_greet"),
]
tracker = DialogueStateTracker.from_events("default", events)
dump = YAMLStoryWriter().dumps(tracker.as_story().story_steps)
assert (
dump.strip()
== textwrap.dedent(
"""
version: "2.0"
stories:
- story: default
steps:
- intent: greet
user: |-
Hello
- action: utter_greet
"""
).strip()
)
async def generate(
self,
template_name: Text,
tracker: DialogueStateTracker,
output_channel: Text,
**kwargs: Any,
) -> Optional[Dict[Text, Any]]:
"""Generate a response for the requested template."""
filled_slots = tracker.current_slot_values()
fallback_language_slot = tracker.slots.get("fallback_language")
fallback_language = fallback_language_slot.initial_value if fallback_language_slot else None
language = tracker.latest_message.metadata.get(
"language") or fallback_language
return self.generate_from_slots(
template_name,
filled_slots,
output_channel,
**kwargs,
language=language,
fallback_language=fallback_language,
)
def retrieve(self, sender_id: Text) -> Optional[DialogueStateTracker]:
"""Create a tracker from all previously stored events."""
import sqlalchemy as sa
from rasa.core.events import SessionStarted
with self.session_scope() as session:
serialised_events = self._event_query(session, sender_id).all()
events = [json.loads(event.data) for event in serialised_events]
if self.domain and len(events) > 0:
logger.debug(f"Recreating tracker from sender id '{sender_id}'")
return DialogueStateTracker.from_dict(
sender_id, events, self.domain.slots
)
else:
logger.debug(
f"Can't retrieve tracker matching "
f"sender id '{sender_id}' from SQL storage. "
f"Returning `None` instead."
)
return None
async def test_give_it_up_after_low_confidence_after_affirm_request():
tracker = DialogueStateTracker.from_events(
"some-sender",
evts=[
# User sends message with low NLU confidence
*_message_requiring_fallback(),
ActiveLoop(ACTION_TWO_STAGE_FALLBACK_NAME),
# Action asks user to affirm
*_two_stage_clarification_request(),
# User's affirms with low NLU confidence again
*_message_requiring_fallback(),
],
)
domain = Domain.empty()
action = TwoStageFallbackAction()
events = await action.run(
CollectingOutputChannel(),
TemplatedNaturalLanguageGenerator(domain.templates),
tracker,
domain,
)
assert events == [ActiveLoop(None), UserUtteranceReverted()]
def test_tracker_entity_retrieval(default_domain: Domain):
tracker = DialogueStateTracker("default", default_domain.slots)
# the retrieved tracker should be empty
assert len(tracker.events) == 0
assert list(tracker.get_latest_entity_values("entity_name")) == []
intent = {"name": "greet", "confidence": 1.0}
tracker.update(
UserUttered(
"/greet",
intent,
[
{
"start": 1,
"end": 5,
"value": "greet",
"entity": "entity_name",
"extractor": "manual",
}
],
)
)
assert list(tracker.get_latest_entity_values("entity_name")) == ["greet"]
assert list(tracker.get_latest_entity_values("unknown")) == []
def test_get_latest_entity_values(
entities: List[Dict[Text, Any]], expected_values: List[Text], default_domain: Domain
):
entity_type = entities[0].get("entity")
entity_role = entities[0].get("role")
entity_group = entities[0].get("group")
tracker = DialogueStateTracker("default", default_domain.slots)
# the retrieved tracker should be empty
assert len(tracker.events) == 0
assert list(tracker.get_latest_entity_values(entity_type)) == []
intent = {"name": "greet", "confidence": 1.0}
tracker.update(UserUttered("/greet", intent, entities))
assert (
list(
tracker.get_latest_entity_values(
entity_type, entity_role=entity_role, entity_group=entity_group
)
)
== expected_values
)
assert list(tracker.get_latest_entity_values("unknown")) == []
def test_tracker_duplicate():
filename = "{}/data/test_dialogues/moodbot.json".format(PRJ_DIR)
dialogue = read_dialogue_file(filename)
tracker = DialogueStateTracker(dialogue.name, domain.slots)
tracker.recreate_from_dialogue(dialogue)
num_actions = len(
[event for event in dialogue.events if isinstance(event, ActionExecuted)]
)
events = [event for event in dialogue.events if isinstance(event, ActionExecuted)]
viz_events(dialogue.events)
# print(type(events[0]).__name__)
# exit()
# viz_events(tracker.events)
# viz_tracker(tracker, v_domain=True)
# There is always one duplicated tracker more than we have actions,
# as the tracker also gets duplicated for the
# action that would be next (but isn't part of the operations)
assert len(list(tracker.generate_all_prior_trackers())) == num_actions + 1
# print(list(tracker.generate_all_prior_trackers()))
viz_trackers(list(tracker.generate_all_prior_trackers()))
def test_missing_classes_filled_correctly(
self, default_domain, trackers, tracker, featurizer, priority
):
# Pretend that a couple of classes are missing and check that
# those classes are predicted as 0, while the other class
# probabilities are predicted normally.
policy = self.create_policy(featurizer=featurizer, priority=priority, cv=None)
classes = [1, 3]
new_trackers = []
for tr in trackers:
new_tracker = DialogueStateTracker(
UserMessage.DEFAULT_SENDER_ID, default_domain.slots
)
for e in tr.applied_events():
if isinstance(e, ActionExecuted):
new_action = default_domain.action_for_index(
np.random.choice(classes), action_endpoint=None
).name()
new_tracker.update(ActionExecuted(new_action))
else:
new_tracker.update(e)
new_trackers.append(new_tracker)
policy.train(
new_trackers, domain=default_domain, interpreter=RegexInterpreter()
)
predicted_probabilities = policy.predict_action_probabilities(
tracker, default_domain
)
assert len(predicted_probabilities) == default_domain.num_actions
assert np.allclose(sum(predicted_probabilities), 1.0)
for i, prob in enumerate(predicted_probabilities):
if i in classes:
assert prob >= 0.0
else:
assert prob == 0.0
def default_tracker(default_domain):
return DialogueStateTracker("my-sender", default_domain.slots)
def _should_handle_message(tracker: DialogueStateTracker):
return (not tracker.is_paused()
or tracker.latest_message.intent.get("name")
== USER_INTENT_RESTART)
async def test_restart(default_dispatcher_collecting, default_domain):
tracker = DialogueStateTracker("default", default_domain.slots)
events = await ActionRestart().run(default_dispatcher_collecting, tracker,
default_domain)
assert events == [Restarted()]
async def test_remote_action_logs_events(default_dispatcher_collecting,
default_domain):
tracker = DialogueStateTracker("default", default_domain.slots)
endpoint = EndpointConfig("https://example.com/webhooks/actions")
remote_action = action.RemoteAction("my_action", endpoint)
response = {
"events": [{
"event": "slot",
"value": "rasa",
"name": "name"
}],
"responses": [
{
"text": "test text",
"buttons": [{
"title": "cheap",
"payload": "cheap"
}]
},
{
"template": "utter_greet"
},
],
}
with aioresponses() as mocked:
mocked.post("https://example.com/webhooks/actions", payload=response)
events = await remote_action.run(default_dispatcher_collecting,
tracker, default_domain)
r = latest_request(mocked, "post",
"https://example.com/webhooks/actions")
assert r
assert json_of_latest_request(r) == {
"domain": default_domain.as_dict(),
"next_action": "my_action",
"sender_id": "default",
"version": rasa.__version__,
"tracker": {
"latest_message": {
"entities": [],
"intent": {},
"text": None
},
"active_form": {},
"latest_action_name": None,
"sender_id": "default",
"paused": False,
"followup_action": "action_listen",
"latest_event_time": None,
"slots": {
"name": None
},
"events": [],
"latest_input_channel": None,
},
}
assert events == [SlotSet("name", "rasa")]
channel = default_dispatcher_collecting.output_channel
assert channel.messages == [
{
"text": "test text",
"recipient_id": "my-sender",
"buttons": [{
"title": "cheap",
"payload": "cheap"
}],
},
{
"text": "hey there None!",
"recipient_id": "my-sender"
},
]
def get_tracker(events: List[Event]) -> DialogueStateTracker:
return DialogueStateTracker.from_events("sender", events, [], 20)
def has_user_rephrased(tracker: DialogueStateTracker) -> bool:
return tracker.last_executed_action_has(ACTION_DEFAULT_ASK_REPHRASE_NAME)
def _has_user_denied(self, last_intent: Text,
tracker: DialogueStateTracker) -> bool:
return (tracker.last_executed_action_has(
ACTION_DEFAULT_ASK_AFFIRMATION_NAME)
and last_intent == self.deny_suggestion_intent_name)
def predict_action_probabilities(self, tracker: DialogueStateTracker,
domain: Domain) -> List[float]:
"""Predicts the assigned action.
If the current intent is assigned to an action that action will be
predicted with the highest probability of all policies. If it is not
the policy will predict zero for every action."""
result = self._default_predictions(domain)
intent = tracker.latest_message.intent.get("name")
if intent == USER_INTENT_RESTART:
action = ACTION_RESTART_NAME
elif intent == USER_INTENT_BACK:
action = ACTION_BACK_NAME
elif intent == USER_INTENT_SESSION_START:
action = ACTION_SESSION_START_NAME
else:
action = domain.intent_properties.get(intent, {}).get("triggers")
if tracker.latest_action_name == ACTION_LISTEN_NAME:
# predict mapped action
if action:
idx = domain.index_for_action(action)
if idx is None:
raise_warning(
f"MappingPolicy tried to predict unknown "
f"action '{action}'. Make sure all mapped actions are "
f"listed in the domain.",
docs=DOCS_URL_POLICIES + "#mapping-policy",
)
else:
result[idx] = 1
if any(result):
logger.debug("The predicted intent '{}' is mapped to "
" action '{}' in the domain."
"".format(intent, action))
elif tracker.latest_action_name == action and action is not None:
# predict next action_listen after mapped action
latest_action = tracker.get_last_event_for(ActionExecuted)
assert latest_action.action_name == action
if latest_action.policy and latest_action.policy.endswith(
type(self).__name__):
# this ensures that we only predict listen,
# if we predicted the mapped action
logger.debug(
"The mapped action, '{}', for this intent, '{}', was "
"executed last so MappingPolicy is returning to "
"action_listen.".format(action, intent))
idx = domain.index_for_action(ACTION_LISTEN_NAME)
result[idx] = 1
else:
logger.debug(
"The mapped action, '{}', for the intent, '{}', was "
"executed last, but it was predicted by another policy, '{}', "
"so MappingPolicy is not predicting any action.".format(
action, intent, latest_action.policy))
elif action == ACTION_RESTART_NAME:
logger.debug("Restarting the conversation with action_restart.")
idx = domain.index_for_action(ACTION_RESTART_NAME)
result[idx] = 1
else:
logger.debug("There is no mapped action for the predicted intent, "
"'{}'.".format(intent))
return result
def serialise_tracker(tracker: DialogueStateTracker) -> Text:
"""Serializes the tracker, returns representation of the tracker."""
dialogue = tracker.as_dialogue()
return json.dumps(dialogue.as_dict())
def _have_options_been_suggested(self,
tracker: DialogueStateTracker) -> bool:
return tracker.last_executed_action_has(self.disambiguation_action)
def test_traveling_back_in_time(default_domain):
tracker = DialogueStateTracker("default", default_domain.slots)
# the retrieved tracker should be empty
assert len(tracker.events) == 0
intent = {"name": "greet", "confidence": 1.0}
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
tracker.update(UserUttered("/greet", intent, []))
import time
time.sleep(1)
time_for_timemachine = time.time()
time.sleep(1)
tracker.update(ActionExecuted("my_action"))
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
# Expecting count of 4:
# +3 executed actions
# +1 final state
assert tracker.latest_action_name == ACTION_LISTEN_NAME
assert len(tracker.events) == 4
assert len(list(tracker.generate_all_prior_trackers())) == 4
tracker = tracker.travel_back_in_time(time_for_timemachine)
# Expecting count of 2:
# +1 executed actions
# +1 final state
assert tracker.latest_action_name == ACTION_LISTEN_NAME
assert len(tracker.events) == 2
assert len(list(tracker.generate_all_prior_trackers())) == 2
def test_revert_user_utterance_event(default_domain):
tracker = DialogueStateTracker("default", default_domain.slots)
# the retrieved tracker should be empty
assert len(tracker.events) == 0
intent1 = {"name": "greet", "confidence": 1.0}
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
tracker.update(UserUttered("/greet", intent1, []))
tracker.update(ActionExecuted("my_action_1"))
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
intent2 = {"name": "goodbye", "confidence": 1.0}
tracker.update(UserUttered("/goodbye", intent2, []))
tracker.update(ActionExecuted("my_action_2"))
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
# Expecting count of 6:
# +5 executed actions
# +1 final state
assert tracker.latest_action_name == ACTION_LISTEN_NAME
assert len(list(tracker.generate_all_prior_trackers())) == 6
tracker.update(UserUtteranceReverted())
# Expecting count of 3:
# +5 executed actions
# +1 final state
# -2 rewound actions associated with the /goodbye
# -1 rewound action from the listen right before /goodbye
assert tracker.latest_action_name == "my_action_1"
assert len(list(tracker.generate_all_prior_trackers())) == 3
dialogue = tracker.as_dialogue()
recovered = DialogueStateTracker("default", default_domain.slots)
recovered.recreate_from_dialogue(dialogue)
assert recovered.current_state() == tracker.current_state()
assert tracker.latest_action_name == "my_action_1"
assert len(list(tracker.generate_all_prior_trackers())) == 3
def test_revert_action_event(default_domain):
tracker = DialogueStateTracker("default", default_domain.slots)
# the retrieved tracker should be empty
assert len(tracker.events) == 0
intent = {"name": "greet", "confidence": 1.0}
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
tracker.update(UserUttered("/greet", intent, []))
tracker.update(ActionExecuted("my_action"))
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
# Expecting count of 4:
# +3 executed actions
# +1 final state
assert tracker.latest_action_name == ACTION_LISTEN_NAME
assert len(list(tracker.generate_all_prior_trackers())) == 4
tracker.update(ActionReverted())
# Expecting count of 3:
# +3 executed actions
# +1 final state
# -1 reverted action
assert tracker.latest_action_name == "my_action"
assert len(list(tracker.generate_all_prior_trackers())) == 3
dialogue = tracker.as_dialogue()
recovered = DialogueStateTracker("default", default_domain.slots)
recovered.recreate_from_dialogue(dialogue)
assert recovered.current_state() == tracker.current_state()
assert tracker.latest_action_name == "my_action"
assert len(list(tracker.generate_all_prior_trackers())) == 3
def test_restart_event(default_domain):
tracker = DialogueStateTracker("default", default_domain.slots)
# the retrieved tracker should be empty
assert len(tracker.events) == 0
intent = {"name": "greet", "confidence": 1.0}
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
tracker.update(UserUttered("/greet", intent, []))
tracker.update(ActionExecuted("my_action"))
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
assert len(tracker.events) == 4
assert tracker.latest_message.text == "/greet"
assert len(list(tracker.generate_all_prior_trackers())) == 4
tracker.update(Restarted())
assert len(tracker.events) == 5
assert tracker.followup_action is not None
assert tracker.followup_action == ACTION_LISTEN_NAME
assert tracker.latest_message.text is None
assert len(list(tracker.generate_all_prior_trackers())) == 1
dialogue = tracker.as_dialogue()
recovered = DialogueStateTracker("default", default_domain.slots)
recovered.recreate_from_dialogue(dialogue)
assert recovered.current_state() == tracker.current_state()
assert len(recovered.events) == 5
assert recovered.latest_message.text is None
assert len(list(recovered.generate_all_prior_trackers())) == 1
