def predict_action_probabilities(self, tracker: DialogueStateTracker,
domain: Domain) -> List[float]:
"""Predicts the corresponding form action if there is an active form"""
result = [0.0] * domain.num_actions
if tracker.active_form.get('name'):
logger.debug("There is an active form '{}'"
"".format(tracker.active_form['name']))
if tracker.latest_action_name == ACTION_LISTEN_NAME:
# predict form action after user utterance
if tracker.active_form.get('rejected'):
if self.state_is_unhappy(tracker, domain):
tracker.update(FormValidation(False))
return result
idx = domain.index_for_action(tracker.active_form['name'])
result[idx] = FORM_SCORE
elif tracker.latest_action_name == tracker.active_form.get('name'):
# predict action_listen after form action
idx = domain.index_for_action(ACTION_LISTEN_NAME)
result[idx] = FORM_SCORE
else:
logger.debug("There is no active form")
return result
def test_missing_classes_filled_correctly(self, default_domain, trackers,
tracker, featurizer):
# Pretend that a couple of classes are missing and check that
# those classes are predicted as 0, while the other class
# probabilities are predicted normally.
policy = self.create_policy(featurizer=featurizer, cv=None)
classes = [1, 3]
new_trackers = []
for tr in trackers:
new_tracker = DialogueStateTracker(UserMessage.DEFAULT_SENDER_ID,
default_domain.slots,
default_domain.topics,
default_domain.default_topic)
for e in tr.applied_events():
if isinstance(e, ActionExecuted):
new_action = default_domain.action_for_index(
np.random.choice(classes)).name()
new_tracker.update(ActionExecuted(new_action))
else:
new_tracker.update(e)
new_trackers.append(new_tracker)
policy.train(new_trackers, domain=default_domain)
predicted_probabilities = policy.predict_action_probabilities(
tracker, default_domain)
assert len(predicted_probabilities) == default_domain.num_actions
assert np.allclose(sum(predicted_probabilities), 1.0)
for i, prob in enumerate(predicted_probabilities):
if i in classes:
assert prob >= 0.0
else:
assert prob == 0.0
def test_missing_classes_filled_correctly(
self, default_domain, trackers, tracker, featurizer):
# Pretend that a couple of classes are missing and check that
# those classes are predicted as 0, while the other class
# probabilities are predicted normally.
policy = self.create_policy(featurizer=featurizer, cv=None)
classes = [1, 3]
new_trackers = []
for tr in trackers:
new_tracker = DialogueStateTracker(UserMessage.DEFAULT_SENDER_ID,
default_domain.slots)
for e in tr.applied_events():
if isinstance(e, ActionExecuted):
new_action = default_domain.action_for_index(
np.random.choice(classes)).name()
new_tracker.update(ActionExecuted(new_action))
else:
new_tracker.update(e)
new_trackers.append(new_tracker)
policy.train(new_trackers, domain=default_domain)
predicted_probabilities = policy.predict_action_probabilities(
tracker, default_domain)
assert len(predicted_probabilities) == default_domain.num_actions
assert np.allclose(sum(predicted_probabilities), 1.0)
for i, prob in enumerate(predicted_probabilities):
if i in classes:
assert prob >= 0.0
else:
assert prob == 0.0
def test_revert_action_event(default_domain):
tracker = DialogueStateTracker("default", default_domain.slots)
# the retrieved tracker should be empty
assert len(tracker.events) == 0
intent = {"name": "greet", "confidence": 1.0}
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
tracker.update(UserUttered("/greet", intent, []))
tracker.update(ActionExecuted("my_action"))
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
# Expecting count of 4:
# +3 executed actions
# +1 final state
assert tracker.latest_action_name == ACTION_LISTEN_NAME
assert len(list(tracker.generate_all_prior_trackers())) == 4
tracker.update(ActionReverted())
# Expecting count of 3:
# +3 executed actions
# +1 final state
# -1 reverted action
assert tracker.latest_action_name == "my_action"
assert len(list(tracker.generate_all_prior_trackers())) == 3
dialogue = tracker.as_dialogue()
recovered = DialogueStateTracker("default", default_domain.slots)
recovered.recreate_from_dialogue(dialogue)
assert recovered.current_state() == tracker.current_state()
assert tracker.latest_action_name == "my_action"
assert len(list(tracker.generate_all_prior_trackers())) == 3
def test_revert_action_event(default_domain):
tracker = DialogueStateTracker("default", default_domain.slots)
# the retrieved tracker should be empty
assert len(tracker.events) == 0
intent = {"name": "greet", "confidence": 1.0}
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
tracker.update(UserUttered("/greet", intent, []))
tracker.update(ActionExecuted("my_action"))
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
# Expecting count of 4:
# +3 executed actions
# +1 final state
assert tracker.latest_action_name == ACTION_LISTEN_NAME
assert len(list(tracker.generate_all_prior_trackers())) == 4
tracker.update(ActionReverted())
# Expecting count of 3:
# +3 executed actions
# +1 final state
# -1 reverted action
assert tracker.latest_action_name == "my_action"
assert len(list(tracker.generate_all_prior_trackers())) == 3
dialogue = tracker.as_dialogue()
recovered = DialogueStateTracker("default", default_domain.slots)
recovered.recreate_from_dialogue(dialogue)
assert recovered.current_state() == tracker.current_state()
assert tracker.latest_action_name == "my_action"
assert len(list(tracker.generate_all_prior_trackers())) == 3
def test_restart_event(default_domain):
tracker = DialogueStateTracker("default", default_domain.slots)
# the retrieved tracker should be empty
assert len(tracker.events) == 0
intent = {"name": "greet", "confidence": 1.0}
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
tracker.update(UserUttered("/greet", intent, []))
tracker.update(ActionExecuted("my_action"))
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
assert len(tracker.events) == 4
assert tracker.latest_message.text == "/greet"
assert len(list(tracker.generate_all_prior_trackers())) == 4
tracker.update(Restarted())
assert len(tracker.events) == 5
assert tracker.followup_action is not None
assert tracker.followup_action == ACTION_LISTEN_NAME
assert tracker.latest_message.text is None
assert len(list(tracker.generate_all_prior_trackers())) == 1
dialogue = tracker.as_dialogue()
recovered = DialogueStateTracker("default", default_domain.slots)
recovered.recreate_from_dialogue(dialogue)
assert recovered.current_state() == tracker.current_state()
assert len(recovered.events) == 5
assert recovered.latest_message.text is None
assert len(list(recovered.generate_all_prior_trackers())) == 1
def test_revert_action_event(default_domain):
tracker = DialogueStateTracker("nlu", default_domain.slots,
default_domain.topics,
default_domain.default_topic)
# the retrieved tracker should be empty
assert len(tracker.events) == 0
intent = {"name": "greet", "confidence": 1.0}
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
tracker.update(UserUttered("_greet", intent, []))
tracker.update(ActionExecuted("my_action"))
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
assert tracker.latest_action_name == ACTION_LISTEN_NAME
assert len(list(tracker.generate_all_prior_states())) == 4
tracker.update(ActionReverted())
assert tracker.latest_action_name == "my_action"
assert len(list(tracker.generate_all_prior_states())) == 3
dialogue = tracker.as_dialogue()
recovered = DialogueStateTracker("nlu", default_domain.slots,
default_domain.topics,
default_domain.default_topic)
recovered.recreate_from_dialogue(dialogue)
assert recovered.current_state() == tracker.current_state()
assert tracker.latest_action_name == "my_action"
assert len(list(tracker.generate_all_prior_states())) == 3
def test_restart_event(default_domain):
tracker = DialogueStateTracker("default", default_domain.slots)
# the retrieved tracker should be empty
assert len(tracker.events) == 0
intent = {"name": "greet", "confidence": 1.0}
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
tracker.update(UserUttered("/greet", intent, []))
tracker.update(ActionExecuted("my_action"))
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
assert len(tracker.events) == 4
assert tracker.latest_message.text == "/greet"
assert len(list(tracker.generate_all_prior_trackers())) == 4
tracker.update(Restarted())
assert len(tracker.events) == 5
assert tracker.follow_up_action is not None
assert tracker.follow_up_action.name() == ACTION_LISTEN_NAME
assert tracker.latest_message.text is None
assert len(list(tracker.generate_all_prior_trackers())) == 1
dialogue = tracker.as_dialogue()
recovered = DialogueStateTracker("default", default_domain.slots)
recovered.recreate_from_dialogue(dialogue)
assert recovered.current_state() == tracker.current_state()
assert len(recovered.events) == 5
assert tracker.follow_up_action is not None
assert tracker.follow_up_action.name() == ACTION_LISTEN_NAME
assert recovered.latest_message.text is None
assert len(list(recovered.generate_all_prior_trackers())) == 1
def log_bot_utterances_on_tracker(tracker: DialogueStateTracker,
dispatcher: Dispatcher) -> None:
if dispatcher.latest_bot_messages:
for m in dispatcher.latest_bot_messages:
bot_utterance = BotUttered(text=m.text, data=m.data)
logger.debug("Bot utterance '{}'".format(bot_utterance))
tracker.update(bot_utterance)
dispatcher.latest_bot_messages = []
def test_traveling_back_in_time(default_domain):
tracker = DialogueStateTracker("default", default_domain.slots)
# the retrieved tracker should be empty
assert len(tracker.events) == 0
intent = {"name": "greet", "confidence": 1.0}
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
tracker.update(UserUttered("/greet", intent, []))
import time
time.sleep(1)
time_for_timemachine = time.time()
time.sleep(1)
tracker.update(ActionExecuted("my_action"))
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
# Expecting count of 4:
# +3 executed actions
# +1 final state
assert tracker.latest_action_name == ACTION_LISTEN_NAME
assert len(tracker.events) == 4
assert len(list(tracker.generate_all_prior_trackers())) == 4
tracker = tracker.travel_back_in_time(time_for_timemachine)
# Expecting count of 2:
# +1 executed actions
# +1 final state
assert tracker.latest_action_name == ACTION_LISTEN_NAME
assert len(tracker.events) == 2
assert len(list(tracker.generate_all_prior_trackers())) == 2
def test_traveling_back_in_time(default_domain):
tracker = DialogueStateTracker("default", default_domain.slots)
# the retrieved tracker should be empty
assert len(tracker.events) == 0
intent = {"name": "greet", "confidence": 1.0}
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
tracker.update(UserUttered("/greet", intent, []))
import time
time.sleep(1)
time_for_timemachine = time.time()
time.sleep(1)
tracker.update(ActionExecuted("my_action"))
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
# Expecting count of 4:
# +3 executed actions
# +1 final state
assert tracker.latest_action_name == ACTION_LISTEN_NAME
assert len(tracker.events) == 4
assert len(list(tracker.generate_all_prior_trackers())) == 4
tracker = tracker.travel_back_in_time(time_for_timemachine)
# Expecting count of 2:
# +1 executed actions
# +1 final state
assert tracker.latest_action_name == ACTION_LISTEN_NAME
assert len(tracker.events) == 2
assert len(list(tracker.generate_all_prior_trackers())) == 2
def test_tracker_entity_retrieval(default_domain):
tracker = DialogueStateTracker("default", default_domain.slots)
# the retrieved tracker should be empty
assert len(tracker.events) == 0
assert list(tracker.get_latest_entity_values("entity_name")) == []
intent = {"name": "greet", "confidence": 1.0}
tracker.update(UserUttered("/greet", intent, [{
"start": 1,
"end": 5,
"value": "greet",
"entity": "entity_name",
"extractor": "manual"
}]))
assert list(tracker.get_latest_entity_values("entity_name")) == ["greet"]
assert list(tracker.get_latest_entity_values("unknown")) == []
def test_tracker_entity_retrieval(default_domain):
tracker = DialogueStateTracker("default", default_domain.slots)
# the retrieved tracker should be empty
assert len(tracker.events) == 0
assert list(tracker.get_latest_entity_values("entity_name")) == []
intent = {"name": "greet", "confidence": 1.0}
tracker.update(UserUttered("/greet", intent, [{
"start": 1,
"end": 5,
"value": "greet",
"entity": "entity_name",
"extractor": "manual"
}]))
assert list(tracker.get_latest_entity_values("entity_name")) == ["greet"]
assert list(tracker.get_latest_entity_values("unknown")) == []
def _handle_message_with_tracker(self,
message: UserMessage,
tracker: DialogueStateTracker) -> None:
if message.parse_data:
parse_data = message.parse_data
else:
parse_data = self._parse_message(message)
# don't ever directly mutate the tracker
# - instead pass its events to log
tracker.update(UserUttered(message.text, parse_data["intent"],
parse_data["entities"], parse_data,
input_channel=message.input_channel))
# store all entities as slots
for e in self.domain.slots_for_entities(parse_data["entities"]):
tracker.update(e)
logger.debug("Logged UserUtterance - "
"tracker now has {} events".format(len(tracker.events)))
def predict_action_probabilities(self,
tracker: DialogueStateTracker,
domain: Domain) -> List[float]:
"""Predicts the corresponding form action if there is an active form"""
result = [0.0] * domain.num_actions
if tracker.active_form.get('name'):
logger.debug("There is an active form '{}'"
"".format(tracker.active_form['name']))
if tracker.latest_action_name == ACTION_LISTEN_NAME:
# predict form action after user utterance
if tracker.active_form.get('rejected'):
# since it is assumed that training stories contain
# only unhappy paths, notify the form that
# it should not be validated if predicted by other policy
tracker_as_states = self.featurizer.prediction_states(
[tracker], domain)
states = tracker_as_states[0]
memorized_form = self.recall(states, tracker, domain)
if memorized_form == tracker.active_form['name']:
logger.debug("There is a memorized tracker state {}, "
"added `FormValidation(False)` event"
"".format(self._modified_states(states)))
tracker.update(FormValidation(False))
return result
idx = domain.index_for_action(tracker.active_form['name'])
result[idx] = FORM_SCORE
elif tracker.latest_action_name == tracker.active_form.get('name'):
# predict action_listen after form action
idx = domain.index_for_action(ACTION_LISTEN_NAME)
result[idx] = FORM_SCORE
else:
logger.debug("There is no active form")
return result
def predict_action_probabilities(self, tracker: DialogueStateTracker,
domain: Domain) -> List[float]:
"""Predicts the next action the bot should take
after seeing the tracker.
Returns the list of probabilities for the next actions"""
if not self.connected:
# TODO: connection seems to expire,
# if expired we should set connected to false again.
self.connect_to_rabbit()
result = [0.0] * domain.num_actions
intent = tracker.latest_message.intent
if tracker.latest_action_name == self.custom_response_action_name:
result = [0.0] * domain.num_actions
idx = domain.index_for_action(ACTION_LISTEN_NAME)
result[idx] = 1.0
elif (intent.get("name") is None
and intent.get("confidence") < self.nlu_threshold):
text = tracker.latest_message.text or ""
answer = self.call(text)
logger.info("\n\n -- Answer Selected -- ")
logger.info("Bot: " + answer["bot"])
logger.info("Confidence: " + str(answer["intent_confidence"]))
logger.info("Confidence: " + str(answer["utter_confidence"]))
logger.info("Total Confidence: " + str(answer["total_confidence"]))
logger.info("Policy: " + str(answer["policy_name"]))
logger.info("Intent Name: " + answer["intent_name"])
set_answer_slot_event = SlotSet("bot_answers", answer["messages"])
tracker.update(set_answer_slot_event)
result = self.bottis_score(result, domain, self.core_threshold)
return result
def test_revert_user_utterance_event(default_domain):
tracker = DialogueStateTracker("default", default_domain.slots)
# the retrieved tracker should be empty
assert len(tracker.events) == 0
intent1 = {"name": "greet", "confidence": 1.0}
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
tracker.update(UserUttered("/greet", intent1, []))
tracker.update(ActionExecuted("my_action_1"))
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
intent2 = {"name": "goodbye", "confidence": 1.0}
tracker.update(UserUttered("/goodbye", intent2, []))
tracker.update(ActionExecuted("my_action_2"))
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
# Expecting count of 6:
# +5 executed actions
# +1 final state
assert tracker.latest_action_name == ACTION_LISTEN_NAME
assert len(list(tracker.generate_all_prior_trackers())) == 6
tracker.update(UserUtteranceReverted())
# Expecting count of 3:
# +5 executed actions
# +1 final state
# -2 rewound actions associated with the /goodbye
# -1 rewound action from the listen right before /goodbye
assert tracker.latest_action_name == "my_action_1"
assert len(list(tracker.generate_all_prior_trackers())) == 3
dialogue = tracker.as_dialogue()
recovered = DialogueStateTracker("default", default_domain.slots)
recovered.recreate_from_dialogue(dialogue)
assert recovered.current_state() == tracker.current_state()
assert tracker.latest_action_name == "my_action_1"
assert len(list(tracker.generate_all_prior_trackers())) == 3
def test_revert_user_utterance_event(default_domain):
tracker = DialogueStateTracker("default", default_domain.slots)
# the retrieved tracker should be empty
assert len(tracker.events) == 0
intent1 = {"name": "greet", "confidence": 1.0}
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
tracker.update(UserUttered("/greet", intent1, []))
tracker.update(ActionExecuted("my_action_1"))
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
intent2 = {"name": "goodbye", "confidence": 1.0}
tracker.update(UserUttered("/goodbye", intent2, []))
tracker.update(ActionExecuted("my_action_2"))
tracker.update(ActionExecuted(ACTION_LISTEN_NAME))
# Expecting count of 6:
# +5 executed actions
# +1 final state
assert tracker.latest_action_name == ACTION_LISTEN_NAME
assert len(list(tracker.generate_all_prior_trackers())) == 6
tracker.update(UserUtteranceReverted())
# Expecting count of 3:
# +5 executed actions
# +1 final state
# -2 rewound actions associated with the /goodbye
# -1 rewound action from the listen right before /goodbye
assert tracker.latest_action_name == "my_action_1"
assert len(list(tracker.generate_all_prior_trackers())) == 3
dialogue = tracker.as_dialogue()
recovered = DialogueStateTracker("default", default_domain.slots)
recovered.recreate_from_dialogue(dialogue)
assert recovered.current_state() == tracker.current_state()
assert tracker.latest_action_name == "my_action_1"
assert len(list(tracker.generate_all_prior_trackers())) == 3
