def predict_action_probabilities(self, tracker: DialogueStateTracker, domain: Domain) -> List[float]: """Predicts the corresponding form action if there is an active form""" result = [0.0] * domain.num_actions if tracker.active_form.get('name'): logger.debug("There is an active form '{}'" "".format(tracker.active_form['name'])) if tracker.latest_action_name == ACTION_LISTEN_NAME: # predict form action after user utterance if tracker.active_form.get('rejected'): if self.state_is_unhappy(tracker, domain): tracker.update(FormValidation(False)) return result idx = domain.index_for_action(tracker.active_form['name']) result[idx] = FORM_SCORE elif tracker.latest_action_name == tracker.active_form.get('name'): # predict action_listen after form action idx = domain.index_for_action(ACTION_LISTEN_NAME) result[idx] = FORM_SCORE else: logger.debug("There is no active form") return result
def test_missing_classes_filled_correctly(self, default_domain, trackers, tracker, featurizer): # Pretend that a couple of classes are missing and check that # those classes are predicted as 0, while the other class # probabilities are predicted normally. policy = self.create_policy(featurizer=featurizer, cv=None) classes = [1, 3] new_trackers = [] for tr in trackers: new_tracker = DialogueStateTracker(UserMessage.DEFAULT_SENDER_ID, default_domain.slots, default_domain.topics, default_domain.default_topic) for e in tr.applied_events(): if isinstance(e, ActionExecuted): new_action = default_domain.action_for_index( np.random.choice(classes)).name() new_tracker.update(ActionExecuted(new_action)) else: new_tracker.update(e) new_trackers.append(new_tracker) policy.train(new_trackers, domain=default_domain) predicted_probabilities = policy.predict_action_probabilities( tracker, default_domain) assert len(predicted_probabilities) == default_domain.num_actions assert np.allclose(sum(predicted_probabilities), 1.0) for i, prob in enumerate(predicted_probabilities): if i in classes: assert prob >= 0.0 else: assert prob == 0.0
def test_missing_classes_filled_correctly( self, default_domain, trackers, tracker, featurizer): # Pretend that a couple of classes are missing and check that # those classes are predicted as 0, while the other class # probabilities are predicted normally. policy = self.create_policy(featurizer=featurizer, cv=None) classes = [1, 3] new_trackers = [] for tr in trackers: new_tracker = DialogueStateTracker(UserMessage.DEFAULT_SENDER_ID, default_domain.slots) for e in tr.applied_events(): if isinstance(e, ActionExecuted): new_action = default_domain.action_for_index( np.random.choice(classes)).name() new_tracker.update(ActionExecuted(new_action)) else: new_tracker.update(e) new_trackers.append(new_tracker) policy.train(new_trackers, domain=default_domain) predicted_probabilities = policy.predict_action_probabilities( tracker, default_domain) assert len(predicted_probabilities) == default_domain.num_actions assert np.allclose(sum(predicted_probabilities), 1.0) for i, prob in enumerate(predicted_probabilities): if i in classes: assert prob >= 0.0 else: assert prob == 0.0
def test_revert_action_event(default_domain): tracker = DialogueStateTracker("default", default_domain.slots) # the retrieved tracker should be empty assert len(tracker.events) == 0 intent = {"name": "greet", "confidence": 1.0} tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) tracker.update(UserUttered("/greet", intent, [])) tracker.update(ActionExecuted("my_action")) tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) # Expecting count of 4: # +3 executed actions # +1 final state assert tracker.latest_action_name == ACTION_LISTEN_NAME assert len(list(tracker.generate_all_prior_trackers())) == 4 tracker.update(ActionReverted()) # Expecting count of 3: # +3 executed actions # +1 final state # -1 reverted action assert tracker.latest_action_name == "my_action" assert len(list(tracker.generate_all_prior_trackers())) == 3 dialogue = tracker.as_dialogue() recovered = DialogueStateTracker("default", default_domain.slots) recovered.recreate_from_dialogue(dialogue) assert recovered.current_state() == tracker.current_state() assert tracker.latest_action_name == "my_action" assert len(list(tracker.generate_all_prior_trackers())) == 3
def test_revert_action_event(default_domain): tracker = DialogueStateTracker("default", default_domain.slots) # the retrieved tracker should be empty assert len(tracker.events) == 0 intent = {"name": "greet", "confidence": 1.0} tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) tracker.update(UserUttered("/greet", intent, [])) tracker.update(ActionExecuted("my_action")) tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) # Expecting count of 4: # +3 executed actions # +1 final state assert tracker.latest_action_name == ACTION_LISTEN_NAME assert len(list(tracker.generate_all_prior_trackers())) == 4 tracker.update(ActionReverted()) # Expecting count of 3: # +3 executed actions # +1 final state # -1 reverted action assert tracker.latest_action_name == "my_action" assert len(list(tracker.generate_all_prior_trackers())) == 3 dialogue = tracker.as_dialogue() recovered = DialogueStateTracker("default", default_domain.slots) recovered.recreate_from_dialogue(dialogue) assert recovered.current_state() == tracker.current_state() assert tracker.latest_action_name == "my_action" assert len(list(tracker.generate_all_prior_trackers())) == 3
def test_restart_event(default_domain): tracker = DialogueStateTracker("default", default_domain.slots) # the retrieved tracker should be empty assert len(tracker.events) == 0 intent = {"name": "greet", "confidence": 1.0} tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) tracker.update(UserUttered("/greet", intent, [])) tracker.update(ActionExecuted("my_action")) tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) assert len(tracker.events) == 4 assert tracker.latest_message.text == "/greet" assert len(list(tracker.generate_all_prior_trackers())) == 4 tracker.update(Restarted()) assert len(tracker.events) == 5 assert tracker.followup_action is not None assert tracker.followup_action == ACTION_LISTEN_NAME assert tracker.latest_message.text is None assert len(list(tracker.generate_all_prior_trackers())) == 1 dialogue = tracker.as_dialogue() recovered = DialogueStateTracker("default", default_domain.slots) recovered.recreate_from_dialogue(dialogue) assert recovered.current_state() == tracker.current_state() assert len(recovered.events) == 5 assert recovered.latest_message.text is None assert len(list(recovered.generate_all_prior_trackers())) == 1
def test_revert_action_event(default_domain): tracker = DialogueStateTracker("nlu", default_domain.slots, default_domain.topics, default_domain.default_topic) # the retrieved tracker should be empty assert len(tracker.events) == 0 intent = {"name": "greet", "confidence": 1.0} tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) tracker.update(UserUttered("_greet", intent, [])) tracker.update(ActionExecuted("my_action")) tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) assert tracker.latest_action_name == ACTION_LISTEN_NAME assert len(list(tracker.generate_all_prior_states())) == 4 tracker.update(ActionReverted()) assert tracker.latest_action_name == "my_action" assert len(list(tracker.generate_all_prior_states())) == 3 dialogue = tracker.as_dialogue() recovered = DialogueStateTracker("nlu", default_domain.slots, default_domain.topics, default_domain.default_topic) recovered.recreate_from_dialogue(dialogue) assert recovered.current_state() == tracker.current_state() assert tracker.latest_action_name == "my_action" assert len(list(tracker.generate_all_prior_states())) == 3
def test_restart_event(default_domain): tracker = DialogueStateTracker("default", default_domain.slots) # the retrieved tracker should be empty assert len(tracker.events) == 0 intent = {"name": "greet", "confidence": 1.0} tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) tracker.update(UserUttered("/greet", intent, [])) tracker.update(ActionExecuted("my_action")) tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) assert len(tracker.events) == 4 assert tracker.latest_message.text == "/greet" assert len(list(tracker.generate_all_prior_trackers())) == 4 tracker.update(Restarted()) assert len(tracker.events) == 5 assert tracker.follow_up_action is not None assert tracker.follow_up_action.name() == ACTION_LISTEN_NAME assert tracker.latest_message.text is None assert len(list(tracker.generate_all_prior_trackers())) == 1 dialogue = tracker.as_dialogue() recovered = DialogueStateTracker("default", default_domain.slots) recovered.recreate_from_dialogue(dialogue) assert recovered.current_state() == tracker.current_state() assert len(recovered.events) == 5 assert tracker.follow_up_action is not None assert tracker.follow_up_action.name() == ACTION_LISTEN_NAME assert recovered.latest_message.text is None assert len(list(recovered.generate_all_prior_trackers())) == 1
def log_bot_utterances_on_tracker(tracker: DialogueStateTracker, dispatcher: Dispatcher) -> None: if dispatcher.latest_bot_messages: for m in dispatcher.latest_bot_messages: bot_utterance = BotUttered(text=m.text, data=m.data) logger.debug("Bot utterance '{}'".format(bot_utterance)) tracker.update(bot_utterance) dispatcher.latest_bot_messages = []
def test_traveling_back_in_time(default_domain): tracker = DialogueStateTracker("default", default_domain.slots) # the retrieved tracker should be empty assert len(tracker.events) == 0 intent = {"name": "greet", "confidence": 1.0} tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) tracker.update(UserUttered("/greet", intent, [])) import time time.sleep(1) time_for_timemachine = time.time() time.sleep(1) tracker.update(ActionExecuted("my_action")) tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) # Expecting count of 4: # +3 executed actions # +1 final state assert tracker.latest_action_name == ACTION_LISTEN_NAME assert len(tracker.events) == 4 assert len(list(tracker.generate_all_prior_trackers())) == 4 tracker = tracker.travel_back_in_time(time_for_timemachine) # Expecting count of 2: # +1 executed actions # +1 final state assert tracker.latest_action_name == ACTION_LISTEN_NAME assert len(tracker.events) == 2 assert len(list(tracker.generate_all_prior_trackers())) == 2
def test_traveling_back_in_time(default_domain): tracker = DialogueStateTracker("default", default_domain.slots) # the retrieved tracker should be empty assert len(tracker.events) == 0 intent = {"name": "greet", "confidence": 1.0} tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) tracker.update(UserUttered("/greet", intent, [])) import time time.sleep(1) time_for_timemachine = time.time() time.sleep(1) tracker.update(ActionExecuted("my_action")) tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) # Expecting count of 4: # +3 executed actions # +1 final state assert tracker.latest_action_name == ACTION_LISTEN_NAME assert len(tracker.events) == 4 assert len(list(tracker.generate_all_prior_trackers())) == 4 tracker = tracker.travel_back_in_time(time_for_timemachine) # Expecting count of 2: # +1 executed actions # +1 final state assert tracker.latest_action_name == ACTION_LISTEN_NAME assert len(tracker.events) == 2 assert len(list(tracker.generate_all_prior_trackers())) == 2
def test_tracker_entity_retrieval(default_domain): tracker = DialogueStateTracker("default", default_domain.slots) # the retrieved tracker should be empty assert len(tracker.events) == 0 assert list(tracker.get_latest_entity_values("entity_name")) == [] intent = {"name": "greet", "confidence": 1.0} tracker.update(UserUttered("/greet", intent, [{ "start": 1, "end": 5, "value": "greet", "entity": "entity_name", "extractor": "manual" }])) assert list(tracker.get_latest_entity_values("entity_name")) == ["greet"] assert list(tracker.get_latest_entity_values("unknown")) == []
def test_tracker_entity_retrieval(default_domain): tracker = DialogueStateTracker("default", default_domain.slots) # the retrieved tracker should be empty assert len(tracker.events) == 0 assert list(tracker.get_latest_entity_values("entity_name")) == [] intent = {"name": "greet", "confidence": 1.0} tracker.update(UserUttered("/greet", intent, [{ "start": 1, "end": 5, "value": "greet", "entity": "entity_name", "extractor": "manual" }])) assert list(tracker.get_latest_entity_values("entity_name")) == ["greet"] assert list(tracker.get_latest_entity_values("unknown")) == []
def _handle_message_with_tracker(self, message: UserMessage, tracker: DialogueStateTracker) -> None: if message.parse_data: parse_data = message.parse_data else: parse_data = self._parse_message(message) # don't ever directly mutate the tracker # - instead pass its events to log tracker.update(UserUttered(message.text, parse_data["intent"], parse_data["entities"], parse_data, input_channel=message.input_channel)) # store all entities as slots for e in self.domain.slots_for_entities(parse_data["entities"]): tracker.update(e) logger.debug("Logged UserUtterance - " "tracker now has {} events".format(len(tracker.events)))
def predict_action_probabilities(self, tracker: DialogueStateTracker, domain: Domain) -> List[float]: """Predicts the corresponding form action if there is an active form""" result = [0.0] * domain.num_actions if tracker.active_form.get('name'): logger.debug("There is an active form '{}'" "".format(tracker.active_form['name'])) if tracker.latest_action_name == ACTION_LISTEN_NAME: # predict form action after user utterance if tracker.active_form.get('rejected'): # since it is assumed that training stories contain # only unhappy paths, notify the form that # it should not be validated if predicted by other policy tracker_as_states = self.featurizer.prediction_states( [tracker], domain) states = tracker_as_states[0] memorized_form = self.recall(states, tracker, domain) if memorized_form == tracker.active_form['name']: logger.debug("There is a memorized tracker state {}, " "added `FormValidation(False)` event" "".format(self._modified_states(states))) tracker.update(FormValidation(False)) return result idx = domain.index_for_action(tracker.active_form['name']) result[idx] = FORM_SCORE elif tracker.latest_action_name == tracker.active_form.get('name'): # predict action_listen after form action idx = domain.index_for_action(ACTION_LISTEN_NAME) result[idx] = FORM_SCORE else: logger.debug("There is no active form") return result
def predict_action_probabilities(self, tracker: DialogueStateTracker, domain: Domain) -> List[float]: """Predicts the next action the bot should take after seeing the tracker. Returns the list of probabilities for the next actions""" if not self.connected: # TODO: connection seems to expire, # if expired we should set connected to false again. self.connect_to_rabbit() result = [0.0] * domain.num_actions intent = tracker.latest_message.intent if tracker.latest_action_name == self.custom_response_action_name: result = [0.0] * domain.num_actions idx = domain.index_for_action(ACTION_LISTEN_NAME) result[idx] = 1.0 elif (intent.get("name") is None and intent.get("confidence") < self.nlu_threshold): text = tracker.latest_message.text or "" answer = self.call(text) logger.info("\n\n -- Answer Selected -- ") logger.info("Bot: " + answer["bot"]) logger.info("Confidence: " + str(answer["intent_confidence"])) logger.info("Confidence: " + str(answer["utter_confidence"])) logger.info("Total Confidence: " + str(answer["total_confidence"])) logger.info("Policy: " + str(answer["policy_name"])) logger.info("Intent Name: " + answer["intent_name"]) set_answer_slot_event = SlotSet("bot_answers", answer["messages"]) tracker.update(set_answer_slot_event) result = self.bottis_score(result, domain, self.core_threshold) return result
def test_revert_user_utterance_event(default_domain): tracker = DialogueStateTracker("default", default_domain.slots) # the retrieved tracker should be empty assert len(tracker.events) == 0 intent1 = {"name": "greet", "confidence": 1.0} tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) tracker.update(UserUttered("/greet", intent1, [])) tracker.update(ActionExecuted("my_action_1")) tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) intent2 = {"name": "goodbye", "confidence": 1.0} tracker.update(UserUttered("/goodbye", intent2, [])) tracker.update(ActionExecuted("my_action_2")) tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) # Expecting count of 6: # +5 executed actions # +1 final state assert tracker.latest_action_name == ACTION_LISTEN_NAME assert len(list(tracker.generate_all_prior_trackers())) == 6 tracker.update(UserUtteranceReverted()) # Expecting count of 3: # +5 executed actions # +1 final state # -2 rewound actions associated with the /goodbye # -1 rewound action from the listen right before /goodbye assert tracker.latest_action_name == "my_action_1" assert len(list(tracker.generate_all_prior_trackers())) == 3 dialogue = tracker.as_dialogue() recovered = DialogueStateTracker("default", default_domain.slots) recovered.recreate_from_dialogue(dialogue) assert recovered.current_state() == tracker.current_state() assert tracker.latest_action_name == "my_action_1" assert len(list(tracker.generate_all_prior_trackers())) == 3
def test_revert_user_utterance_event(default_domain): tracker = DialogueStateTracker("default", default_domain.slots) # the retrieved tracker should be empty assert len(tracker.events) == 0 intent1 = {"name": "greet", "confidence": 1.0} tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) tracker.update(UserUttered("/greet", intent1, [])) tracker.update(ActionExecuted("my_action_1")) tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) intent2 = {"name": "goodbye", "confidence": 1.0} tracker.update(UserUttered("/goodbye", intent2, [])) tracker.update(ActionExecuted("my_action_2")) tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) # Expecting count of 6: # +5 executed actions # +1 final state assert tracker.latest_action_name == ACTION_LISTEN_NAME assert len(list(tracker.generate_all_prior_trackers())) == 6 tracker.update(UserUtteranceReverted()) # Expecting count of 3: # +5 executed actions # +1 final state # -2 rewound actions associated with the /goodbye # -1 rewound action from the listen right before /goodbye assert tracker.latest_action_name == "my_action_1" assert len(list(tracker.generate_all_prior_trackers())) == 3 dialogue = tracker.as_dialogue() recovered = DialogueStateTracker("default", default_domain.slots) recovered.recreate_from_dialogue(dialogue) assert recovered.current_state() == tracker.current_state() assert tracker.latest_action_name == "my_action_1" assert len(list(tracker.generate_all_prior_trackers())) == 3