def test_session_start_is_not_serialised(domain: Domain):
tracker = DialogueStateTracker("default", domain.slots)
# the retrieved tracker should be empty
assert len(tracker.events) == 0
# add SlotSet event
tracker.update(SlotSet("slot", "value"))
# add the two SessionStarted events and a user event
tracker.update(ActionExecuted(ACTION_SESSION_START_NAME))
tracker.update(SessionStarted())
tracker.update(
UserUttered("say something", intent={INTENT_NAME_KEY: "some_intent"}))
tracker.update(DefinePrevUserUtteredFeaturization(False))
YAMLStoryWriter().dumps(
Story.from_events(tracker.events, "some-story01").story_steps)
expected = """version: "2.0"
stories:
- story: some-story01
steps:
- slot_was_set:
- slot: value
- intent: some_intent
"""
actual = YAMLStoryWriter().dumps(
Story.from_events(tracker.events, "some-story01").story_steps)
assert actual == expected
def _log_action_on_tracker(
self,
tracker: DialogueStateTracker,
action_name: Text,
events: Optional[List[Event]],
prediction: PolicyPrediction,
) -> None:
# Ensures that the code still works even if a lazy programmer missed
# to type `return []` at the end of an action or the run method
# returns `None` for some other reason.
if events is None:
events = []
self._warn_about_new_slots(tracker, action_name, events)
action_was_rejected_manually = any(
isinstance(event, ActionExecutionRejected) for event in events)
if action_name is not None and not action_was_rejected_manually:
logger.debug(
f"Policy prediction ended with events '{prediction.events}'.")
tracker.update_with_events(prediction.events, self.domain)
# log the action and its produced events
tracker.update(
ActionExecuted(action_name, prediction.policy_name,
prediction.max_confidence))
logger.debug(f"Action '{action_name}' ended with events '{events}'.")
tracker.update_with_events(events, self.domain)
def _log_action_on_tracker(
self,
tracker: DialogueStateTracker,
action_name: Text,
events: Optional[List[Event]],
policy: Optional[Text],
confidence: Optional[float],
) -> None:
# Ensures that the code still works even if a lazy programmer missed
# to type `return []` at the end of an action or the run method
# returns `None` for some other reason.
if events is None:
events = []
logger.debug(
f"Action '{action_name}' ended with events '{[e for e in events]}'."
)
self._warn_about_new_slots(tracker, action_name, events)
action_was_rejected_manually = any(
isinstance(event, ActionExecutionRejected) for event in events)
if action_name is not None and not action_was_rejected_manually:
# log the action and its produced events
tracker.update(ActionExecuted(action_name, policy, confidence))
for e in events:
# this makes sure the events are ordered by timestamp -
# since the event objects are created somewhere else,
# the timestamp would indicate a time before the time
# of the action executed
e.timestamp = time.time()
tracker.update(e, self.domain)
def test_revert_action_event(default_domain: Domain):
tracker = DialogueStateTracker("default", default_domain.slots)
# the retrieved tracker should be empty
assert len(tracker.events) == 0
intent = {"name": "greet", "confidence": 1.0}
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
tracker.update(UserUttered("/greet", intent, []))
tracker.update(ActionExecuted("my_action"))
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
# Expecting count of 4:
# +3 executed actions
# +1 final state
assert tracker.latest_action.get(ACTION_NAME) == ACTION_LISTEN_NAME
assert len(list(tracker.generate_all_prior_trackers())) == 4
tracker.update(ActionReverted())
# Expecting count of 3:
# +3 executed actions
# +1 final state
# -1 reverted action
assert tracker.latest_action.get(ACTION_NAME) == "my_action"
assert len(list(tracker.generate_all_prior_trackers())) == 3
dialogue = tracker.as_dialogue()
recovered = DialogueStateTracker("default", default_domain.slots)
recovered.recreate_from_dialogue(dialogue)
assert recovered.current_state() == tracker.current_state()
assert tracker.latest_action.get(ACTION_NAME) == "my_action"
assert len(list(tracker.generate_all_prior_trackers())) == 3
def test_restart_event(default_domain: Domain):
tracker = DialogueStateTracker("default", default_domain.slots)
# the retrieved tracker should be empty
assert len(tracker.events) == 0
intent = {"name": "greet", "confidence": 1.0}
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
tracker.update(UserUttered("/greet", intent, []))
tracker.update(ActionExecuted("my_action"))
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
assert len(tracker.events) == 4
assert tracker.latest_message.text == "/greet"
assert len(list(tracker.generate_all_prior_trackers())) == 4
tracker.update(Restarted())
assert len(tracker.events) == 5
assert tracker.followup_action is not None
assert tracker.followup_action == ACTION_LISTEN_NAME
assert tracker.latest_message.text is None
assert len(list(tracker.generate_all_prior_trackers())) == 1
dialogue = tracker.as_dialogue()
recovered = DialogueStateTracker("default", default_domain.slots)
recovered.recreate_from_dialogue(dialogue)
assert recovered.current_state() == tracker.current_state()
assert len(recovered.events) == 5
assert recovered.latest_message.text is None
assert len(list(recovered.generate_all_prior_trackers())) == 1
async def _handle_message_with_tracker(
self, message: UserMessage, tracker: DialogueStateTracker) -> None:
if message.parse_data:
parse_data = message.parse_data
else:
parse_data = await self.parse_message(message)
# don't ever directly mutate the tracker
# - instead pass its events to log
tracker.update(
UserUttered(
message.text,
parse_data["intent"],
parse_data["entities"],
parse_data,
input_channel=message.input_channel,
message_id=message.message_id,
metadata=message.metadata,
),
self.domain,
)
if parse_data["entities"]:
self._log_slots(tracker)
logger.debug(
f"Logged UserUtterance - tracker now has {len(tracker.events)} events."
)
def _log_action_on_tracker(
self,
tracker: DialogueStateTracker,
action: Action,
events: Optional[List[Event]],
prediction: PolicyPrediction,
) -> None:
# Ensures that the code still works even if a lazy programmer missed
# to type `return []` at the end of an action or the run method
# returns `None` for some other reason.
if events is None:
events = []
action_was_rejected_manually = any(
isinstance(event, ActionExecutionRejected) for event in events)
if not action_was_rejected_manually:
logger.debug(
f"Policy prediction ended with events '{prediction.events}'.")
tracker.update_with_events(prediction.events, self.domain)
# log the action and its produced events
tracker.update(action.event_for_successful_execution(prediction))
logger.debug(f"Action '{action.name()}' ended with events '{events}'.")
tracker.update_with_events(events, self.domain)
def test_tracker_update_slots_with_entity(default_domain: Domain):
tracker = DialogueStateTracker("default", default_domain.slots)
test_entity = default_domain.entities[0]
expected_slot_value = "test user"
intent = {"name": "greet", PREDICTED_CONFIDENCE_KEY: 1.0}
tracker.update(
UserUttered(
"/greet",
intent,
[
{
"start": 1,
"end": 5,
"value": expected_slot_value,
"entity": test_entity,
"extractor": "manual",
}
],
),
default_domain,
)
assert tracker.get_slot(test_entity) == expected_slot_value
def predict_action_probabilities(
self,
tracker: DialogueStateTracker,
domain: Domain,
interpreter: NaturalLanguageInterpreter,
**kwargs: Any,
) -> List[float]:
"""Predicts the corresponding form action if there is an active form"""
result = self._default_predictions(domain)
if tracker.active_loop_name:
logger.debug(
"There is an active form '{}'".format(tracker.active_loop_name)
)
if tracker.latest_action_name == ACTION_LISTEN_NAME:
# predict form action after user utterance
if tracker.active_loop.get(LOOP_REJECTED):
if self.state_is_unhappy(tracker, domain):
tracker.update(LoopInterrupted(True))
return result
result = self._prediction_result(
tracker.active_loop_name, tracker, domain
)
elif tracker.latest_action_name == tracker.active_loop_name:
# predict action_listen after form action
result = self._prediction_result(ACTION_LISTEN_NAME, tracker, domain)
else:
logger.debug("There is no active form")
return result
def _collect_action_executed_predictions(
processor: "MessageProcessor",
partial_tracker: DialogueStateTracker,
event: ActionExecuted,
fail_on_prediction_errors: bool,
circuit_breaker_tripped: bool,
) -> Tuple[EvaluationStore, Optional[Text], Optional[float]]:
from rasa.core.policies.form_policy import FormPolicy
action_executed_eval_store = EvaluationStore()
gold = event.action_name or event.action_text
if circuit_breaker_tripped:
predicted = "circuit breaker tripped"
policy = None
confidence = None
else:
action, policy, confidence = processor.predict_next_action(
partial_tracker)
predicted = action.name()
if (policy and predicted != gold and _form_might_have_been_rejected(
processor.domain, partial_tracker, predicted)):
# Wrong action was predicted,
# but it might be Ok if form action is rejected.
emulate_loop_rejection(partial_tracker)
# try again
action, policy, confidence = processor.predict_next_action(
partial_tracker)
# Even if the prediction is also wrong, we don't have to undo the emulation
# of the action rejection as we know that the user explicitly specified
# that something else than the form was supposed to run.
predicted = action.name()
action_executed_eval_store.add_to_store(action_predictions=[predicted],
action_targets=[gold])
if action_executed_eval_store.has_prediction_target_mismatch():
partial_tracker.update(
WronglyPredictedAction(gold, predicted, policy, confidence,
event.timestamp))
if fail_on_prediction_errors:
story_dump = YAMLStoryWriter().dumps(
partial_tracker.as_story().story_steps)
error_msg = (f"Model predicted a wrong action. Failed Story: "
f"\n\n{story_dump}")
if FormPolicy.__name__ in policy:
error_msg += ("FormAction is not run during "
"evaluation therefore it is impossible to know "
"if validation failed or this story is wrong. "
"If the story is correct, add it to the "
"training stories and retrain.")
raise WrongPredictionException(error_msg)
else:
partial_tracker.update(
ActionExecuted(predicted, policy, confidence, event.timestamp))
return action_executed_eval_store, policy, confidence
def _collect_user_uttered_predictions(
event: UserUttered,
predicted: Dict[Text, Any],
partial_tracker: DialogueStateTracker,
fail_on_prediction_errors: bool,
) -> EvaluationStore:
user_uttered_eval_store = EvaluationStore()
# intent from the test story, may either be base intent or full retrieval intent
base_intent = event.intent.get(INTENT_NAME_KEY)
full_retrieval_intent = event.intent.get(FULL_RETRIEVAL_INTENT_NAME_KEY)
intent_gold = full_retrieval_intent if full_retrieval_intent else base_intent
# predicted intent: note that this is only the base intent at this point
predicted_base_intent = predicted.get(INTENT, {}).get(INTENT_NAME_KEY)
# if the test story only provides the base intent AND the prediction was correct,
# we are not interested in full retrieval intents and skip this section.
# In any other case we are interested in the full retrieval intent (e.g. for report)
if intent_gold != predicted_base_intent:
predicted_base_intent = _get_full_retrieval_intent(predicted)
user_uttered_eval_store.add_to_store(
intent_targets=[intent_gold],
intent_predictions=[predicted_base_intent])
entity_gold = event.entities
predicted_entities = predicted.get(ENTITIES)
if entity_gold or predicted_entities:
user_uttered_eval_store.add_to_store(
entity_targets=_clean_entity_results(event.text, entity_gold),
entity_predictions=_clean_entity_results(event.text,
predicted_entities),
)
if user_uttered_eval_store.check_prediction_target_mismatch():
partial_tracker.update(
WronglyClassifiedUserUtterance(event, user_uttered_eval_store))
if fail_on_prediction_errors:
story_dump = YAMLStoryWriter().dumps(
partial_tracker.as_story().story_steps)
raise WrongPredictionException(
f"NLU model predicted a wrong intent or entities. Failed Story:"
f" \n\n{story_dump}")
else:
response_selector_info = ({
RESPONSE_SELECTOR_PROPERTY_NAME:
predicted[RESPONSE_SELECTOR_PROPERTY_NAME]
} if RESPONSE_SELECTOR_PROPERTY_NAME in predicted else None)
end_to_end_user_utterance = EndToEndUserUtterance(
text=event.text,
intent=event.intent,
entities=event.entities,
parse_data=response_selector_info,
)
partial_tracker.update(end_to_end_user_utterance)
return user_uttered_eval_store
def test_session_start(default_domain: Domain):
tracker = DialogueStateTracker("default", default_domain.slots)
# the retrieved tracker should be empty
assert len(tracker.events) == 0
# add a SessionStarted event
tracker.update(SessionStarted())
# tracker has one event
assert len(tracker.events) == 1
def test_slot_mapping_entity_is_desired(slot_name: Text, expected: bool):
domain = Domain.from_file("data/test_domains/travel_form.yml")
tracker = DialogueStateTracker("test_id", slots=domain.slots)
event = UserUttered(
text="I'm travelling to Vancouver.",
intent={"name": "inform", "confidence": 0.9604260921478271},
entities=[{"entity": "GPE", "value": "Vancouver", "role": "destination"}],
)
tracker.update(event, domain)
slot_mappings = domain.as_dict().get("slots").get(slot_name).get("mappings")
assert SlotMapping.entity_is_desired(slot_mappings[0], tracker) is expected
def emulate_loop_rejection(partial_tracker: DialogueStateTracker) -> None:
"""Add `ActionExecutionRejected` event to the tracker.
During evaluation, we don't run action server, therefore in order to correctly
test unhappy paths of the loops, we need to emulate loop rejection.
Args:
partial_tracker: a :class:`rasa.core.trackers.DialogueStateTracker`
"""
from rasa.shared.core.events import ActionExecutionRejected
rejected_action_name: Text = partial_tracker.active_loop_name
partial_tracker.update(ActionExecutionRejected(rejected_action_name))
async def trigger_external_user_uttered(
self,
intent_name: Text,
entities: Optional[Union[List[Dict[Text, Any]], Dict[Text, Text]]],
tracker: DialogueStateTracker,
output_channel: OutputChannel,
) -> None:
"""Triggers an external message.
Triggers an external message (like a user message, but invisible;
used, e.g., by a reminder or the trigger_intent endpoint).
Args:
intent_name: Name of the intent to be triggered.
entities: Entities to be passed on.
tracker: The tracker to which the event should be added.
output_channel: The output channel.
"""
if isinstance(entities, list):
entity_list = entities
elif isinstance(entities, dict):
# Allow for a short-hand notation {"ent1": "val1", "ent2": "val2", ...}.
# Useful if properties like 'start', 'end', or 'extractor' are not given,
# e.g. for external events.
entity_list = [{
"entity": ent,
"value": val
} for ent, val in entities.items()]
elif not entities:
entity_list = []
else:
rasa.shared.utils.io.raise_warning(
f"Invalid entity specification: {entities}. Assuming no entities."
)
entity_list = []
# Set the new event's input channel to the latest input channel, so
# that we don't lose this property.
input_channel = tracker.get_latest_input_channel()
tracker.update(
UserUttered.create_external(intent_name, entity_list,
input_channel),
self.domain,
)
tracker = await self.run_action_extract_slots(output_channel, tracker)
await self._run_prediction_loop(output_channel, tracker)
# save tracker state to continue conversation from this state
self.save_tracker(tracker)
def test_traveling_back_in_time(default_domain: Domain):
tracker = DialogueStateTracker("default", default_domain.slots)
# the retrieved tracker should be empty
assert len(tracker.events) == 0
intent = {"name": "greet", "confidence": 1.0}
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
tracker.update(UserUttered("/greet", intent, []))
import time
time.sleep(1)
time_for_timemachine = time.time()
time.sleep(1)
tracker.update(ActionExecuted("my_action"))
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
# Expecting count of 4:
# +3 executed actions
# +1 final state
assert tracker.latest_action.get(ACTION_NAME) == ACTION_LISTEN_NAME
assert len(tracker.events) == 4
assert len(list(tracker.generate_all_prior_trackers())) == 4
tracker = tracker.travel_back_in_time(time_for_timemachine)
# Expecting count of 2:
# +1 executed actions
# +1 final state
assert tracker.latest_action.get(ACTION_NAME) == ACTION_LISTEN_NAME
assert len(tracker.events) == 2
assert len(list(tracker.generate_all_prior_trackers())) == 2
def marker_sqlite_tracker(tmp_path: Path) -> Tuple[SQLTrackerStore, Text]:
domain = Domain.empty()
db_path = str(tmp_path / "rasa.db")
tracker_store = SQLTrackerStore(dialect="sqlite", db=db_path)
for i in range(5):
tracker = DialogueStateTracker(str(i), None)
tracker.update_with_events([SlotSet(str(j), "slot") for j in range(5)],
domain)
tracker.update(ActionExecuted(ACTION_SESSION_START_NAME))
tracker.update(UserUttered("hello"))
tracker.update_with_events(
[SlotSet(str(5 + j), "slot") for j in range(5)], domain)
tracker_store.save(tracker)
return tracker_store, db_path
async def _run_action(
self,
action: rasa.core.actions.action.Action,
tracker: DialogueStateTracker,
output_channel: OutputChannel,
nlg: NaturalLanguageGenerator,
prediction: PolicyPrediction,
metadata: Optional[Dict[Text, Any]] = None,
) -> bool:
# events and return values are used to update
# the tracker state after an action has been taken
try:
# Here we set optional metadata to the ActionSessionStart, which will then
# be passed to the SessionStart event. Otherwise the metadata will be lost.
if action.name() == ACTION_SESSION_START_NAME:
action.metadata = metadata
# Use temporary tracker as we might need to discard the policy events in
# case of a rejection.
temporary_tracker = tracker.copy()
temporary_tracker.update_with_events(prediction.events,
self.domain)
events = await action.run(output_channel, nlg, temporary_tracker,
self.domain)
except rasa.core.actions.action.ActionExecutionRejection:
events = [
ActionExecutionRejected(action.name(), prediction.policy_name,
prediction.max_confidence)
]
tracker.update(events[0])
return self.should_predict_another_action(action.name())
except Exception:
logger.exception(
f"Encountered an exception while running action '{action.name()}'."
"Bot will continue, but the actions events are lost. "
"Please check the logs of your action server for "
"more information.")
events = []
self._log_action_on_tracker(tracker, action.name(), events, prediction)
if action.name() != ACTION_LISTEN_NAME and not action.name(
).startswith(UTTER_PREFIX):
self._log_slots(tracker)
await self.execute_side_effects(events, tracker, output_channel)
return self.should_predict_another_action(action.name())
async def _update_tracker_session(
self,
tracker: DialogueStateTracker,
output_channel: OutputChannel,
metadata: Optional[Dict] = None,
) -> None:
"""Check the current session in `tracker` and update it if expired.
An 'action_session_start' is run if the latest tracker session has expired,
or if the tracker does not yet contain any events (only those after the last
restart are considered).
Args:
metadata: Data sent from client associated with the incoming user message.
tracker: Tracker to inspect.
output_channel: Output channel for potential utterances in a custom
`ActionSessionStart`.
"""
if not tracker.applied_events() or self._has_session_expired(tracker):
logger.debug(
f"Starting a new session for conversation ID '{tracker.sender_id}'."
)
action_session_start = self._get_action(ACTION_SESSION_START_NAME)
# TODO: Remove in 3.0.0 and describe migration to `session_start_metadata`
# slot in migration guide.
if isinstance(
action_session_start, rasa.core.actions.action.ActionSessionStart
):
# Here we set optional metadata to the ActionSessionStart, which will
# then be passed to the SessionStart event.
action_session_start.metadata = metadata
if metadata:
tracker.update(
SlotSet(SESSION_START_METADATA_SLOT, metadata), self.domain
)
await self._run_action(
action=action_session_start,
tracker=tracker,
output_channel=output_channel,
nlg=self.nlg,
prediction=PolicyPrediction.for_action_name(
self.domain, ACTION_SESSION_START_NAME
),
)
async def cancel_reminder_and_check(
tracker: DialogueStateTracker,
default_processor: MessageProcessor,
reminder_canceled_event: ReminderCancelled,
num_jobs_before: int,
num_jobs_after: int,
) -> None:
# cancel the sixth reminder
tracker.update(reminder_canceled_event)
# check that the jobs were added
assert len((await jobs.scheduler()).get_jobs()) == num_jobs_before
await default_processor._cancel_reminders(tracker.events, tracker)
# check that only one job was removed
assert len((await jobs.scheduler()).get_jobs()) == num_jobs_after
def test_missing_classes_filled_correctly(
self,
default_domain: Domain,
trackers: List[TrackerWithCachedStates],
tracker: DialogueStateTracker,
featurizer: Optional[TrackerFeaturizer],
priority: int,
):
# Pretend that a couple of classes are missing and check that
# those classes are predicted as 0, while the other class
# probabilities are predicted normally.
policy = self.create_policy(featurizer=featurizer,
priority=priority,
cv=None)
classes = [1, 3]
new_trackers = []
for tr in trackers:
new_tracker = DialogueStateTracker(DEFAULT_SENDER_ID,
default_domain.slots)
for e in tr.applied_events():
if isinstance(e, ActionExecuted):
new_action = rasa.core.actions.action.action_for_index(
np.random.choice(classes),
default_domain,
action_endpoint=None).name()
new_tracker.update(ActionExecuted(new_action))
else:
new_tracker.update(e)
new_trackers.append(new_tracker)
policy.train(new_trackers,
domain=default_domain,
interpreter=RegexInterpreter())
prediction = policy.predict_action_probabilities(
tracker, default_domain, RegexInterpreter())
assert not prediction.is_end_to_end_prediction
assert len(prediction.probabilities) == default_domain.num_actions
assert np.allclose(sum(prediction.probabilities), 1.0)
for i, prob in enumerate(prediction.probabilities):
if i in classes:
assert prob >= 0.0
else:
assert prob == 0.0
def test_get_latest_entity_values(entities: List[Dict[Text, Any]],
expected_values: List[Text], domain: Domain):
entity_type = entities[0].get("entity")
entity_role = entities[0].get("role")
entity_group = entities[0].get("group")
tracker = DialogueStateTracker("default", domain.slots)
# the retrieved tracker should be empty
assert len(tracker.events) == 0
assert list(tracker.get_latest_entity_values(entity_type)) == []
intent = {"name": "greet", PREDICTED_CONFIDENCE_KEY: 1.0}
tracker.update(UserUttered("/greet", intent, entities))
assert (list(
tracker.get_latest_entity_values(
entity_type, entity_role=entity_role,
entity_group=entity_group)) == expected_values)
assert list(tracker.get_latest_entity_values("unknown")) == []
def _collect_user_uttered_predictions(
event: UserUttered,
predicted: Dict[Text, Any],
partial_tracker: DialogueStateTracker,
fail_on_prediction_errors: bool,
) -> EvaluationStore:
user_uttered_eval_store = EvaluationStore()
intent_gold = event.intent.get("name")
predicted_intent = predicted.get(INTENT, {}).get("name")
user_uttered_eval_store.add_to_store(
intent_predictions=[predicted_intent], intent_targets=[intent_gold]
)
entity_gold = event.entities
predicted_entities = predicted.get(ENTITIES)
if entity_gold or predicted_entities:
user_uttered_eval_store.add_to_store(
entity_targets=_clean_entity_results(event.text, entity_gold),
entity_predictions=_clean_entity_results(event.text, predicted_entities),
)
if user_uttered_eval_store.has_prediction_target_mismatch():
partial_tracker.update(
WronglyClassifiedUserUtterance(event, user_uttered_eval_store)
)
if fail_on_prediction_errors:
raise ValueError(
"NLU model predicted a wrong intent. Failed Story:"
" \n\n{}".format(
YAMLStoryWriter().dumps(partial_tracker.as_story().story_steps)
)
)
else:
end_to_end_user_utterance = EndToEndUserUtterance(
event.text, event.intent, event.entities
)
partial_tracker.update(end_to_end_user_utterance)
return user_uttered_eval_store
def add(
self,
predictions: List[Message],
tracker: DialogueStateTracker,
domain: Domain,
original_message: UserMessage,
) -> DialogueStateTracker:
"""Adds NLU predictions to the tracker.
Args:
predictions: A list of NLU predictions wrapped as Messages
tracker: The tracker the predictions should be attached to
domain: The domain of the model.
original_message: An original message from the user with
extra metadata to annotate the predictions (e.g. channel)
Returns:
The original tracker updated with events created from the predictions
"""
for message in predictions:
user_event = UserUttered(
message.data.get(TEXT),
message.data.get(INTENT),
message.data.get(ENTITIES),
input_channel=original_message.input_channel,
message_id=message.data.get("message_id"),
metadata=original_message.metadata,
)
tracker.update(user_event, domain)
if user_event.entities:
# Log currently set slots
slot_values = "\n".join(
[f"\t{s.name}: {s.value}" for s in tracker.slots.values()])
if slot_values.strip():
logger.debug(f"Current slot values: \n{slot_values}")
logger.debug(f"Logged {len(predictions)} UserUtterance(s) - \
tracker now has {len(tracker.events)} events.")
return tracker
async def test_markers_cli_results_save_correctly(tmp_path: Path):
domain = Domain.empty()
store = InMemoryTrackerStore(domain)
for i in range(5):
tracker = DialogueStateTracker(str(i), None)
tracker.update_with_events([SlotSet(str(j), "slot") for j in range(5)],
domain)
tracker.update(ActionExecuted(ACTION_SESSION_START_NAME))
tracker.update(UserUttered("hello"))
tracker.update_with_events(
[SlotSet(str(5 + j), "slot") for j in range(5)], domain)
await store.save(tracker)
tracker_loader = MarkerTrackerLoader(store, "all")
results_path = tmp_path / "results.csv"
markers = OrMarker(markers=[
SlotSetMarker("2", name="marker1"),
SlotSetMarker("7", name="marker2")
])
await markers.evaluate_trackers(tracker_loader.load(), results_path)
with open(results_path, "r") as results:
result_reader = csv.DictReader(results)
senders = set()
for row in result_reader:
senders.add(row["sender_id"])
if row["marker"] == "marker1":
assert row["session_idx"] == "0"
assert int(row["event_idx"]) >= 2
assert row["num_preceding_user_turns"] == "0"
if row["marker"] == "marker2":
assert row["session_idx"] == "1"
assert int(row["event_idx"]) >= 3
assert row["num_preceding_user_turns"] == "1"
assert len(senders) == 5
async def test_action_session_start_with_slots(
default_channel: CollectingOutputChannel,
template_nlg: TemplatedNaturalLanguageGenerator,
template_sender_tracker: DialogueStateTracker,
domain: Domain,
session_config: SessionConfig,
expected_events: List[Event],
):
# set a few slots on tracker
slot_set_event_1 = SlotSet("my_slot", "value")
slot_set_event_2 = SlotSet("another-slot", "value2")
for event in [slot_set_event_1, slot_set_event_2]:
template_sender_tracker.update(event)
domain.session_config = session_config
events = await ActionSessionStart().run(default_channel, template_nlg,
template_sender_tracker, domain)
assert events == expected_events
# make sure that the list of events has ascending timestamps
assert sorted(events, key=lambda x: x.timestamp) == events
def test_session_start_is_not_serialised(default_domain: Domain):
tracker = DialogueStateTracker("default", default_domain.slots)
# the retrieved tracker should be empty
assert len(tracker.events) == 0
# add SlotSet event
tracker.update(SlotSet("slot", "value"))
# add the two SessionStarted events and a user event
tracker.update(ActionExecuted(ACTION_SESSION_START_NAME))
tracker.update(SessionStarted())
tracker.update(UserUttered("say something"))
# make sure session start is not serialised
story = Story.from_events(tracker.events, "some-story01")
expected = """## some-story01
- slot{"slot": "value"}
* say something
"""
assert story.as_story_string(flat=True) == expected
def test_slot_mapping_intent_is_desired(domain: Domain):
domain = Domain.from_file("examples/formbot/domain.yml")
tracker = DialogueStateTracker("sender_id_test", slots=domain.slots)
event1 = UserUttered(
text="I'd like to book a restaurant for 2 people.",
intent={
"name": "request_restaurant",
"confidence": 0.9604260921478271
},
entities=[{
"entity": "number",
"value": 2
}],
)
tracker.update(event1, domain)
mappings_for_num_people = (
domain.as_dict().get("slots").get("num_people").get("mappings"))
assert SlotMapping.intent_is_desired(mappings_for_num_people[0], tracker,
domain)
event2 = UserUttered(
text="Yes, 2 please",
intent={
"name": "affirm",
"confidence": 0.9604260921478271
},
entities=[{
"entity": "number",
"value": 2
}],
)
tracker.update(event2, domain)
assert (SlotMapping.intent_is_desired(mappings_for_num_people[0], tracker,
domain) is False)
event3 = UserUttered(
text="Yes, please",
intent={
"name": "affirm",
"confidence": 0.9604260921478271
},
entities=[],
)
tracker.update(event3, domain)
mappings_for_preferences = (
domain.as_dict().get("slots").get("preferences").get("mappings"))
assert (SlotMapping.intent_is_desired(mappings_for_preferences[0], tracker,
domain) is False)
def _collect_action_executed_predictions(
processor: "MessageProcessor",
partial_tracker: DialogueStateTracker,
event: ActionExecuted,
fail_on_prediction_errors: bool,
) -> Tuple[EvaluationStore, PolicyPrediction,
Optional[EntityEvaluationResult]]:
action_executed_eval_store = EvaluationStore()
expected_action_name = event.action_name
expected_action_text = event.action_text
expected_action = expected_action_name or expected_action_text
policy_entity_result = None
prev_action_unlikely_intent = False
try:
predicted_action, prediction, policy_entity_result = _run_action_prediction(
processor, partial_tracker, expected_action)
except ActionLimitReached:
prediction = PolicyPrediction([], policy_name=None)
predicted_action = "circuit breaker tripped"
predicted_action_unlikely_intent = predicted_action == ACTION_UNLIKELY_INTENT_NAME
if predicted_action_unlikely_intent and predicted_action != expected_action:
partial_tracker.update(
WronglyPredictedAction(
predicted_action,
expected_action_text,
predicted_action,
prediction.policy_name,
prediction.max_confidence,
event.timestamp,
metadata=prediction.action_metadata,
))
prev_action_unlikely_intent = True
try:
predicted_action, prediction, policy_entity_result = _run_action_prediction(
processor, partial_tracker, expected_action)
except ActionLimitReached:
prediction = PolicyPrediction([], policy_name=None)
predicted_action = "circuit breaker tripped"
action_executed_eval_store.add_to_store(
action_predictions=[predicted_action],
action_targets=[expected_action])
if action_executed_eval_store.has_prediction_target_mismatch():
partial_tracker.update(
WronglyPredictedAction(
expected_action_name,
expected_action_text,
predicted_action,
prediction.policy_name,
prediction.max_confidence,
event.timestamp,
metadata=prediction.action_metadata,
predicted_action_unlikely_intent=prev_action_unlikely_intent,
))
if (fail_on_prediction_errors
and predicted_action != ACTION_UNLIKELY_INTENT_NAME
and predicted_action != expected_action):
story_dump = YAMLStoryWriter().dumps(
partial_tracker.as_story().story_steps)
error_msg = (f"Model predicted a wrong action. Failed Story: "
f"\n\n{story_dump}")
raise WrongPredictionException(error_msg)
elif prev_action_unlikely_intent:
partial_tracker.update(
WarningPredictedAction(
ACTION_UNLIKELY_INTENT_NAME,
predicted_action,
prediction.policy_name,
prediction.max_confidence,
event.timestamp,
prediction.action_metadata,
))
else:
partial_tracker.update(
ActionExecuted(
predicted_action,
prediction.policy_name,
prediction.max_confidence,
event.timestamp,
metadata=prediction.action_metadata,
))
return action_executed_eval_store, prediction, policy_entity_result
def test_revert_user_utterance_event(default_domain: Domain):
tracker = DialogueStateTracker("default", default_domain.slots)
# the retrieved tracker should be empty
assert len(tracker.events) == 0
intent1 = {"name": "greet", "confidence": 1.0}
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
tracker.update(UserUttered("/greet", intent1, []))
tracker.update(ActionExecuted("my_action_1"))
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
intent2 = {"name": "goodbye", "confidence": 1.0}
tracker.update(UserUttered("/goodbye", intent2, []))
tracker.update(ActionExecuted("my_action_2"))
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
# Expecting count of 6:
# +5 executed actions
# +1 final state
assert tracker.latest_action.get(ACTION_NAME) == ACTION_LISTEN_NAME
assert len(list(tracker.generate_all_prior_trackers())) == 6
tracker.update(UserUtteranceReverted())
# Expecting count of 3:
# +5 executed actions
# +1 final state
# -2 rewound actions associated with the /goodbye
# -1 rewound action from the listen right before /goodbye
assert tracker.latest_action.get(ACTION_NAME) == "my_action_1"
assert len(list(tracker.generate_all_prior_trackers())) == 3
dialogue = tracker.as_dialogue()
recovered = DialogueStateTracker("default", default_domain.slots)
recovered.recreate_from_dialogue(dialogue)
assert recovered.current_state() == tracker.current_state()
assert tracker.latest_action.get(ACTION_NAME) == "my_action_1"
assert len(list(tracker.generate_all_prior_trackers())) == 3
