def test_revert_action_event(default_domain):
tracker = DialogueStateTracker("nlu", default_domain.slots,
default_domain.topics,
default_domain.default_topic)
# the retrieved tracker should be empty
assert len(tracker.events) == 0
intent = {"name": "greet", "confidence": 1.0}
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
tracker.update(UserUttered("_greet", intent, []))
tracker.update(ActionExecuted("my_action"))
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
assert tracker.latest_action_name == ACTION_LISTEN_NAME
assert len(list(tracker.generate_all_prior_states())) == 4
tracker.update(ActionReverted())
assert tracker.latest_action_name == "my_action"
assert len(list(tracker.generate_all_prior_states())) == 3
dialogue = tracker.as_dialogue()
recovered = DialogueStateTracker("nlu", default_domain.slots,
default_domain.topics,
default_domain.default_topic)
recovered.recreate_from_dialogue(dialogue)
assert recovered.current_state() == tracker.current_state()
assert tracker.latest_action_name == "my_action"
assert len(list(tracker.generate_all_prior_states())) == 3
def explicit_events(self,
domain,
interpreter,
should_append_final_listen=True):
# type: (Domain, NaturalLanguageInterpreter) -> List[Event]
"""Returns events contained in the story step including implicit events.
Not all events are always listed in the story dsl. This
includes listen actions as well as implicitly
set slots. This functions makes these events explicit and
returns them with the rest of the steps events."""
events = []
for e in self.events:
if isinstance(e, UserUttered):
parse_data = interpreter.parse(e.text)
updated_utterance = UserUttered(e.text, parse_data["intent"],
parse_data["entities"],
parse_data)
events.append(ActionExecuted(ActionListen().name()))
events.append(updated_utterance)
events.extend(domain.slots_for_entities(
parse_data["entities"]))
else:
events.append(e)
if self.end_checkpoint is None and should_append_final_listen:
events.append(ActionExecuted(ActionListen().name()))
return events
def _prepare_events(self, step):
# type: (StoryStep) -> List[Event]
"""Returns events contained in the story step inserting implicit events.
Not all events are always listed in the story dsl.
This includes listen actions as well as implicitly
set slots. This functions makes these events explicit
and returns them with the rest of the steps events."""
events = []
for e in step.events:
if isinstance(e, UserUttered):
parse_data = self.interpreter.parse(e.text)
updated_utterance = UserUttered(e.text, parse_data["intent"],
parse_data["entities"],
parse_data)
events.append(ActionExecuted(ActionListen().name()))
events.append(updated_utterance)
events.extend(
self.domain.slots_for_entities(parse_data["entities"]))
else:
events.append(e)
if step.end_checkpoint is None:
events.append(ActionExecuted(ActionListen().name()))
return events
def test_revert_action_event(default_domain):
tracker = DialogueStateTracker("default", default_domain.slots)
# the retrieved tracker should be empty
assert len(tracker.events) == 0
intent = {"name": "greet", "confidence": 1.0}
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
tracker.update(UserUttered("/greet", intent, []))
tracker.update(ActionExecuted("my_action"))
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
# Expecting count of 4:
# +3 executed actions
# +1 final state
assert tracker.latest_action_name == ACTION_LISTEN_NAME
assert len(list(tracker.generate_all_prior_trackers())) == 4
tracker.update(ActionReverted())
# Expecting count of 3:
# +3 executed actions
# +1 final state
# -1 reverted action
assert tracker.latest_action_name == "my_action"
assert len(list(tracker.generate_all_prior_trackers())) == 3
dialogue = tracker.as_dialogue()
recovered = DialogueStateTracker("default", default_domain.slots)
recovered.recreate_from_dialogue(dialogue)
assert recovered.current_state() == tracker.current_state()
assert tracker.latest_action_name == "my_action"
assert len(list(tracker.generate_all_prior_trackers())) == 3
def send_action(
endpoint, # type: EndpointConfig
sender_id, # type: Text
action_name, # type: Text
policy=None, # type: Optional[Text]
confidence=None, # type: Optional[float]
is_new_action=False # bool
):
# type: (...) -> Dict[Text, Any]
"""Log an action to a conversation."""
payload = ActionExecuted(action_name, policy, confidence).as_dict()
subpath = "/conversations/{}/execute".format(sender_id)
try:
r = endpoint.request(json=payload, method="post", subpath=subpath)
return _response_as_json(r)
except requests.exceptions.HTTPError:
if is_new_action:
logger.warning("You have created a new action: {} "
"which was not successfully executed. \n"
"If this action does not return any events, "
"you do not need to do anything. \n"
"If this is a custom action which returns events, "
"you are recommended to implement this action "
"in your action server and try again."
"".format(action_name))
payload = ActionExecuted(action_name).as_dict()
return send_event(endpoint, sender_id, payload)
else:
logger.error("failed to execute action!")
raise
def test_can_read_test_story(default_domain):
trackers = training.load_data(
"data/test_stories/stories.md",
default_domain,
use_story_concatenation=False,
tracker_limit=1000,
remove_duplicates=False
)
assert len(trackers) == 7
# this should be the story simple_story_with_only_end -> show_it_all
# the generated stories are in a non stable order - therefore we need to
# do some trickery to find the one we want to test
tracker = [t for t in trackers if len(t.events) == 5][0]
assert tracker.events[0] == ActionExecuted("action_listen")
assert tracker.events[1] == UserUttered(
"simple",
intent={"name": "simple", "confidence": 1.0},
parse_data={'text': '/simple',
'intent_ranking': [{'confidence': 1.0,
'name': 'simple'}],
'intent': {'confidence': 1.0, 'name': 'simple'},
'entities': []})
assert tracker.events[2] == ActionExecuted("utter_default")
assert tracker.events[3] == ActionExecuted("utter_greet")
assert tracker.events[4] == ActionExecuted("action_listen")
def test_restart_event(default_domain):
tracker = DialogueStateTracker("default", default_domain.slots)
# the retrieved tracker should be empty
assert len(tracker.events) == 0
intent = {"name": "greet", "confidence": 1.0}
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
tracker.update(UserUttered("/greet", intent, []))
tracker.update(ActionExecuted("my_action"))
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
assert len(tracker.events) == 4
assert tracker.latest_message.text == "/greet"
assert len(list(tracker.generate_all_prior_trackers())) == 4
tracker.update(Restarted())
assert len(tracker.events) == 5
assert tracker.followup_action is not None
assert tracker.followup_action == ACTION_LISTEN_NAME
assert tracker.latest_message.text is None
assert len(list(tracker.generate_all_prior_trackers())) == 1
dialogue = tracker.as_dialogue()
recovered = DialogueStateTracker("default", default_domain.slots)
recovered.recreate_from_dialogue(dialogue)
assert recovered.current_state() == tracker.current_state()
assert len(recovered.events) == 5
assert recovered.latest_message.text is None
assert len(list(recovered.generate_all_prior_trackers())) == 1
def test_traveling_back_in_time(default_domain):
tracker = DialogueStateTracker("default", default_domain.slots)
# the retrieved tracker should be empty
assert len(tracker.events) == 0
intent = {"name": "greet", "confidence": 1.0}
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
tracker.update(UserUttered("/greet", intent, []))
import time
time.sleep(1)
time_for_timemachine = time.time()
time.sleep(1)
tracker.update(ActionExecuted("my_action"))
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
# Expecting count of 4:
# +3 executed actions
# +1 final state
assert tracker.latest_action_name == ACTION_LISTEN_NAME
assert len(tracker.events) == 4
assert len(list(tracker.generate_all_prior_trackers())) == 4
tracker = tracker.travel_back_in_time(time_for_timemachine)
# Expecting count of 2:
# +1 executed actions
# +1 final state
assert tracker.latest_action_name == ACTION_LISTEN_NAME
assert len(tracker.events) == 2
assert len(list(tracker.generate_all_prior_trackers())) == 2
def send_action(endpoint: EndpointConfig,
sender_id: Text,
action_name: Text,
policy: Optional[Text] = None,
confidence: Optional[float] = None,
is_new_action: bool = False) -> Dict[Text, Any]:
"""Log an action to a conversation."""
payload = ActionExecuted(action_name, policy, confidence).as_dict()
subpath = "/conversations/{}/execute".format(sender_id)
try:
r = endpoint.request(json=payload, method="post", subpath=subpath)
return _response_as_json(r)
except requests.exceptions.HTTPError:
if is_new_action:
warning_questions = questionary.confirm(
"WARNING: You have created a new action: '{}', "
"which was not successfully executed. "
"If this action does not return any events, "
"you do not need to do anything. "
"If this is a custom action which returns events, "
"you are recommended to implement this action "
"in your action server and try again."
"".format(action_name))
_ask_or_abort(warning_questions, sender_id, endpoint)
payload = ActionExecuted(action_name).as_dict()
return send_event(endpoint, sender_id, payload)
else:
logger.error("failed to execute action!")
raise
def test_reminder_scheduled(default_processor):
out = CollectingOutputChannel()
sender_id = uuid.uuid4().hex
d = Dispatcher(sender_id, out, default_processor.nlg)
r = ReminderScheduled("utter_greet", datetime.datetime.now())
t = default_processor.tracker_store.get_or_create_tracker(sender_id)
t.update(UserUttered("test"))
t.update(ActionExecuted("action_reminder_reminder"))
t.update(r)
default_processor.tracker_store.save(t)
default_processor.handle_reminder(r, d)
# retrieve the updated tracker
t = default_processor.tracker_store.retrieve(sender_id)
assert t.events[-4] == UserUttered(None)
assert t.events[-3] == ActionExecuted("utter_greet")
assert t.events[-2] == BotUttered("hey there None!", {
'elements': None,
'buttons': None,
'attachment': None
})
assert t.events[-1] == ActionExecuted("action_listen")
def test_last_executed_has_not_name():
events = [
ActionExecuted('one'),
user_uttered('two', 1),
ActionExecuted(ACTION_LISTEN_NAME)
]
tracker = get_tracker(events)
assert tracker.last_executed_action_has('another') is False
def test_get_last_event_for_with_skip():
events = [
ActionExecuted('one'),
user_uttered('two', 1),
ActionExecuted('three')
]
tracker = get_tracker(events)
assert (tracker.get_last_event_for(ActionExecuted,
skip=1).action_name == 'one')
def test_ask_rephrase(self, trained_policy, default_domain):
events = [ActionExecuted(ACTION_LISTEN_NAME),
user_uttered("greet", 0.2),
ActionExecuted(ACTION_DEFAULT_ASK_AFFIRMATION_NAME),
ActionExecuted(ACTION_LISTEN_NAME),
user_uttered('deny', 1)]
next_action = self._get_next_action(trained_policy, events,
default_domain)
assert next_action == ACTION_DEFAULT_ASK_REPHRASE_NAME
def test_get_last_event_for_with_exclude():
events = [
ActionExecuted('one'),
user_uttered('two', 1),
ActionExecuted('three')
]
tracker = get_tracker(events)
assert (tracker.get_last_event_for(
ActionExecuted,
action_names_to_exclude=['three']).action_name == 'one')
def test_unknown_instead_affirmation(self, trained_policy, default_domain):
events = [ActionExecuted(ACTION_LISTEN_NAME),
user_uttered("greet", 0.2),
ActionExecuted(ACTION_DEFAULT_ASK_AFFIRMATION_NAME),
ActionExecuted(ACTION_LISTEN_NAME),
user_uttered("greet", 0.2),
]
next_action = self._get_next_action(trained_policy, events,
default_domain)
assert next_action == ACTION_DEFAULT_FALLBACK_NAME
def test_common_action_prefix_unequal():
this = [
ActionExecuted("action_listen"),
ActionExecuted("greet"),
UserUttered("hey"),
]
other = [
ActionExecuted("greet"),
ActionExecuted("action_listen"),
UserUttered("hey"),
]
num_common = visualization._length_of_common_action_prefix(this, other)
assert num_common == 0
def _log_action_on_tracker(self, tracker, action_name, events, policy,
policy_confidence):
# Ensures that the code still works even if a lazy programmer missed
# to type `return []` at the end of an action or the run method
# returns `None` for some other reason.
if events is None:
events = []
logger.debug("Action '{}' ended with events '{}'".format(
action_name, ['{}'.format(e) for e in events]))
self._warn_about_new_slots(tracker, action_name, events)
if action_name is not None:
# log the action and its produced events
tracker.update(
ActionExecuted(action_name, policy, policy_confidence))
for e in events:
# this makes sure the events are ordered by timestamp -
# since the event objects are created somewhere else,
# the timestamp would indicate a time before the time
# of the action executed
e.timestamp = time.time()
tracker.update(e)
def test_affirm_rephrased_intent(self, trained_policy, default_domain):
events = [
ActionExecuted(ACTION_LISTEN_NAME),
user_uttered("greet", 0.2),
ActionExecuted(ACTION_DEFAULT_ASK_AFFIRMATION_NAME),
ActionExecuted(ACTION_LISTEN_NAME),
user_uttered(USER_INTENT_DENY, 1),
ActionExecuted(ACTION_DEFAULT_ASK_REPHRASE_NAME),
ActionExecuted(ACTION_LISTEN_NAME),
user_uttered("greet", 0.2),
]
next_action = self._get_next_action(trained_policy, events,
default_domain)
assert next_action == ACTION_DEFAULT_ASK_AFFIRMATION_NAME
async def _predict_till_next_listen(endpoint: EndpointConfig, sender_id: Text,
finetune: bool, sender_ids: List[Text],
plot_file: Optional[Text]) -> None:
"""Predict and validate actions until we need to wait for a user msg."""
listen = False
while not listen:
result = await request_prediction(endpoint, sender_id)
predictions = result.get("scores")
probabilities = [prediction["score"] for prediction in predictions]
pred_out = int(np.argmax(probabilities))
action_name = predictions[pred_out].get("action")
policy = result.get("policy")
confidence = result.get("confidence")
await _print_history(sender_id, endpoint)
await _plot_trackers(sender_ids,
plot_file,
endpoint,
unconfirmed=[ActionExecuted(action_name)])
listen = await _validate_action(action_name,
policy,
confidence,
predictions,
endpoint,
sender_id,
finetune=finetune)
await _plot_trackers(sender_ids, plot_file, endpoint)
def test_listen_after_hand_off(self, trained_policy, default_domain):
events = [ActionExecuted(ACTION_DEFAULT_FALLBACK_NAME)]
next_action = self._get_next_action(trained_policy, events,
default_domain)
assert next_action == ACTION_LISTEN_NAME
def test_ask_affirmation(self, trained_policy, default_domain):
events = [ActionExecuted(ACTION_LISTEN_NAME), user_uttered("Hi", 0.2)]
next_action = self._get_next_action(trained_policy, events,
default_domain)
assert next_action == ACTION_DEFAULT_ASK_AFFIRMATION_NAME
def test_missing_classes_filled_correctly(self, default_domain, trackers,
tracker, featurizer):
# Pretend that a couple of classes are missing and check that
# those classes are predicted as 0, while the other class
# probabilities are predicted normally.
policy = self.create_policy(featurizer=featurizer, cv=None)
classes = [1, 3]
new_trackers = []
for tr in trackers:
new_tracker = DialogueStateTracker(UserMessage.DEFAULT_SENDER_ID,
default_domain.slots,
default_domain.topics,
default_domain.default_topic)
for e in tr.applied_events():
if isinstance(e, ActionExecuted):
new_action = default_domain.action_for_index(
np.random.choice(classes)).name()
new_tracker.update(ActionExecuted(new_action))
else:
new_tracker.update(e)
new_trackers.append(new_tracker)
policy.train(new_trackers, domain=default_domain)
predicted_probabilities = policy.predict_action_probabilities(
tracker, default_domain)
assert len(predicted_probabilities) == default_domain.num_actions
assert np.allclose(sum(predicted_probabilities), 1.0)
for i, prob in enumerate(predicted_probabilities):
if i in classes:
assert prob >= 0.0
else:
assert prob == 0.0
def _predict_till_next_listen(
endpoint, # type: EndpointConfig
sender_id, # type: Text
finetune, # type: bool
sender_ids, # type: List[Text]
plot_file # type: Optional[Text]
):
# type: (...) -> None
"""Predict and validate actions until we need to wait for a user msg."""
listen = False
while not listen:
response = request_prediction(endpoint, sender_id)
predictions = response.get("scores")
probabilities = [prediction["score"] for prediction in predictions]
pred_out = int(np.argmax(probabilities))
action_name = predictions[pred_out].get("action")
_print_history(sender_id, endpoint)
_plot_trackers(sender_ids,
plot_file,
endpoint,
unconfirmed=[ActionExecuted(action_name)])
listen = _validate_action(action_name,
predictions,
endpoint,
sender_id,
finetune=finetune)
_plot_trackers(sender_ids, plot_file, endpoint)
def _revert_single_affirmation_events() -> List[Event]:
return [
UserUtteranceReverted(), # revert affirmation and request
# revert original intent (has to be re-added later)
UserUtteranceReverted(),
# add action listen intent
ActionExecuted(action_name=ACTION_LISTEN_NAME)
]
def _revert_rephrasing_events() -> List[Event]:
return [UserUtteranceReverted(), # remove rephrasing
# remove feedback and rephrase request
UserUtteranceReverted(),
# remove affirmation request and false intent
UserUtteranceReverted(),
# replace action with action listen
ActionExecuted(action_name=ACTION_LISTEN_NAME)]
def as_dialogue(self, sender_id, domain):
events = []
for step in self.story_steps:
events.extend(
step.explicit_events(domain, should_append_final_listen=False))
events.append(ActionExecuted(ActionListen().name()))
return Dialogue(sender_id, events)
def test_json_parse_action():
# DOCS MARKER ActionExecuted
evt = \
{
'event': 'action',
'name': 'my_action'
}
# DOCS END
assert Event.from_parameters(evt) == ActionExecuted("my_action")
def test_revert_user_utterance_event(default_domain):
tracker = DialogueStateTracker("default", default_domain.slots,
default_domain.topics,
default_domain.default_topic)
# the retrieved tracker should be empty
assert len(tracker.events) == 0
intent1 = {"name": "greet", "confidence": 1.0}
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
tracker.update(UserUttered("/greet", intent1, []))
tracker.update(ActionExecuted("my_action_1"))
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
intent2 = {"name": "goodbye", "confidence": 1.0}
tracker.update(UserUttered("/goodbye", intent2, []))
tracker.update(ActionExecuted("my_action_2"))
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
# Expecting count of 6:
# +5 executed actions
# +1 final state
assert tracker.latest_action_name == ACTION_LISTEN_NAME
assert len(list(tracker.generate_all_prior_states())) == 6
tracker.update(UserUtteranceReverted())
# Expecting count of 3:
# +5 executed actions
# +1 final state
# -2 rewound actions associated with the /goodbye
# -1 rewound action from the listen right before /goodbye
assert tracker.latest_action_name == "my_action_1"
assert len(list(tracker.generate_all_prior_states())) == 3
dialogue = tracker.as_dialogue()
recovered = DialogueStateTracker("default", default_domain.slots,
default_domain.topics,
default_domain.default_topic)
recovered.recreate_from_dialogue(dialogue)
assert recovered.current_state() == tracker.current_state()
assert tracker.latest_action_name == "my_action_1"
assert len(list(tracker.generate_all_prior_states())) == 3
def test_successful_rephrasing(self, trained_policy,
default_dispatcher_collecting,
default_domain):
events = [
ActionExecuted(ACTION_LISTEN_NAME),
user_uttered("greet", 0.2),
ActionExecuted(ACTION_DEFAULT_ASK_AFFIRMATION_NAME),
ActionExecuted(ACTION_LISTEN_NAME),
user_uttered('deny', 1),
ActionExecuted(ACTION_DEFAULT_ASK_REPHRASE_NAME),
ActionExecuted(ACTION_LISTEN_NAME),
user_uttered("bye", 1),
]
tracker = self._get_tracker_after_reverts(
events, default_dispatcher_collecting, default_domain)
assert 'bye' == tracker.latest_message.parse_data['intent']['name']
assert tracker.export_stories() == "## sender\n* bye\n"
def test_affirmation(self, default_dispatcher_collecting, default_domain):
events = [ActionExecuted(ACTION_LISTEN_NAME),
user_uttered('greet', 1),
ActionExecuted('utter_hello'),
ActionExecuted(ACTION_LISTEN_NAME),
user_uttered('greet', 0.2),
ActionExecuted(ACTION_DEFAULT_ASK_AFFIRMATION_NAME),
ActionExecuted(ACTION_LISTEN_NAME),
user_uttered('greet', 1)]
tracker = self._get_tracker_after_reverts(events,
default_dispatcher_collecting,
default_domain)
assert 'greet' == tracker.latest_message.parse_data['intent']['name']
assert tracker.export_stories() == ("## sender\n"
"* greet\n"
" - utter_hello\n"
"* greet\n")