def test_revert_action_event(default_domain): tracker = DialogueStateTracker("nlu", default_domain.slots, default_domain.topics, default_domain.default_topic) # the retrieved tracker should be empty assert len(tracker.events) == 0 intent = {"name": "greet", "confidence": 1.0} tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) tracker.update(UserUttered("_greet", intent, [])) tracker.update(ActionExecuted("my_action")) tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) assert tracker.latest_action_name == ACTION_LISTEN_NAME assert len(list(tracker.generate_all_prior_states())) == 4 tracker.update(ActionReverted()) assert tracker.latest_action_name == "my_action" assert len(list(tracker.generate_all_prior_states())) == 3 dialogue = tracker.as_dialogue() recovered = DialogueStateTracker("nlu", default_domain.slots, default_domain.topics, default_domain.default_topic) recovered.recreate_from_dialogue(dialogue) assert recovered.current_state() == tracker.current_state() assert tracker.latest_action_name == "my_action" assert len(list(tracker.generate_all_prior_states())) == 3
def explicit_events(self, domain, interpreter, should_append_final_listen=True): # type: (Domain, NaturalLanguageInterpreter) -> List[Event] """Returns events contained in the story step including implicit events. Not all events are always listed in the story dsl. This includes listen actions as well as implicitly set slots. This functions makes these events explicit and returns them with the rest of the steps events.""" events = [] for e in self.events: if isinstance(e, UserUttered): parse_data = interpreter.parse(e.text) updated_utterance = UserUttered(e.text, parse_data["intent"], parse_data["entities"], parse_data) events.append(ActionExecuted(ActionListen().name())) events.append(updated_utterance) events.extend(domain.slots_for_entities( parse_data["entities"])) else: events.append(e) if self.end_checkpoint is None and should_append_final_listen: events.append(ActionExecuted(ActionListen().name())) return events
def _prepare_events(self, step): # type: (StoryStep) -> List[Event] """Returns events contained in the story step inserting implicit events. Not all events are always listed in the story dsl. This includes listen actions as well as implicitly set slots. This functions makes these events explicit and returns them with the rest of the steps events.""" events = [] for e in step.events: if isinstance(e, UserUttered): parse_data = self.interpreter.parse(e.text) updated_utterance = UserUttered(e.text, parse_data["intent"], parse_data["entities"], parse_data) events.append(ActionExecuted(ActionListen().name())) events.append(updated_utterance) events.extend( self.domain.slots_for_entities(parse_data["entities"])) else: events.append(e) if step.end_checkpoint is None: events.append(ActionExecuted(ActionListen().name())) return events
def test_revert_action_event(default_domain): tracker = DialogueStateTracker("default", default_domain.slots) # the retrieved tracker should be empty assert len(tracker.events) == 0 intent = {"name": "greet", "confidence": 1.0} tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) tracker.update(UserUttered("/greet", intent, [])) tracker.update(ActionExecuted("my_action")) tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) # Expecting count of 4: # +3 executed actions # +1 final state assert tracker.latest_action_name == ACTION_LISTEN_NAME assert len(list(tracker.generate_all_prior_trackers())) == 4 tracker.update(ActionReverted()) # Expecting count of 3: # +3 executed actions # +1 final state # -1 reverted action assert tracker.latest_action_name == "my_action" assert len(list(tracker.generate_all_prior_trackers())) == 3 dialogue = tracker.as_dialogue() recovered = DialogueStateTracker("default", default_domain.slots) recovered.recreate_from_dialogue(dialogue) assert recovered.current_state() == tracker.current_state() assert tracker.latest_action_name == "my_action" assert len(list(tracker.generate_all_prior_trackers())) == 3
def send_action( endpoint, # type: EndpointConfig sender_id, # type: Text action_name, # type: Text policy=None, # type: Optional[Text] confidence=None, # type: Optional[float] is_new_action=False # bool ): # type: (...) -> Dict[Text, Any] """Log an action to a conversation.""" payload = ActionExecuted(action_name, policy, confidence).as_dict() subpath = "/conversations/{}/execute".format(sender_id) try: r = endpoint.request(json=payload, method="post", subpath=subpath) return _response_as_json(r) except requests.exceptions.HTTPError: if is_new_action: logger.warning("You have created a new action: {} " "which was not successfully executed. \n" "If this action does not return any events, " "you do not need to do anything. \n" "If this is a custom action which returns events, " "you are recommended to implement this action " "in your action server and try again." "".format(action_name)) payload = ActionExecuted(action_name).as_dict() return send_event(endpoint, sender_id, payload) else: logger.error("failed to execute action!") raise
def test_can_read_test_story(default_domain): trackers = training.load_data( "data/test_stories/stories.md", default_domain, use_story_concatenation=False, tracker_limit=1000, remove_duplicates=False ) assert len(trackers) == 7 # this should be the story simple_story_with_only_end -> show_it_all # the generated stories are in a non stable order - therefore we need to # do some trickery to find the one we want to test tracker = [t for t in trackers if len(t.events) == 5][0] assert tracker.events[0] == ActionExecuted("action_listen") assert tracker.events[1] == UserUttered( "simple", intent={"name": "simple", "confidence": 1.0}, parse_data={'text': '/simple', 'intent_ranking': [{'confidence': 1.0, 'name': 'simple'}], 'intent': {'confidence': 1.0, 'name': 'simple'}, 'entities': []}) assert tracker.events[2] == ActionExecuted("utter_default") assert tracker.events[3] == ActionExecuted("utter_greet") assert tracker.events[4] == ActionExecuted("action_listen")
def test_restart_event(default_domain): tracker = DialogueStateTracker("default", default_domain.slots) # the retrieved tracker should be empty assert len(tracker.events) == 0 intent = {"name": "greet", "confidence": 1.0} tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) tracker.update(UserUttered("/greet", intent, [])) tracker.update(ActionExecuted("my_action")) tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) assert len(tracker.events) == 4 assert tracker.latest_message.text == "/greet" assert len(list(tracker.generate_all_prior_trackers())) == 4 tracker.update(Restarted()) assert len(tracker.events) == 5 assert tracker.followup_action is not None assert tracker.followup_action == ACTION_LISTEN_NAME assert tracker.latest_message.text is None assert len(list(tracker.generate_all_prior_trackers())) == 1 dialogue = tracker.as_dialogue() recovered = DialogueStateTracker("default", default_domain.slots) recovered.recreate_from_dialogue(dialogue) assert recovered.current_state() == tracker.current_state() assert len(recovered.events) == 5 assert recovered.latest_message.text is None assert len(list(recovered.generate_all_prior_trackers())) == 1
def test_traveling_back_in_time(default_domain): tracker = DialogueStateTracker("default", default_domain.slots) # the retrieved tracker should be empty assert len(tracker.events) == 0 intent = {"name": "greet", "confidence": 1.0} tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) tracker.update(UserUttered("/greet", intent, [])) import time time.sleep(1) time_for_timemachine = time.time() time.sleep(1) tracker.update(ActionExecuted("my_action")) tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) # Expecting count of 4: # +3 executed actions # +1 final state assert tracker.latest_action_name == ACTION_LISTEN_NAME assert len(tracker.events) == 4 assert len(list(tracker.generate_all_prior_trackers())) == 4 tracker = tracker.travel_back_in_time(time_for_timemachine) # Expecting count of 2: # +1 executed actions # +1 final state assert tracker.latest_action_name == ACTION_LISTEN_NAME assert len(tracker.events) == 2 assert len(list(tracker.generate_all_prior_trackers())) == 2
def send_action(endpoint: EndpointConfig, sender_id: Text, action_name: Text, policy: Optional[Text] = None, confidence: Optional[float] = None, is_new_action: bool = False) -> Dict[Text, Any]: """Log an action to a conversation.""" payload = ActionExecuted(action_name, policy, confidence).as_dict() subpath = "/conversations/{}/execute".format(sender_id) try: r = endpoint.request(json=payload, method="post", subpath=subpath) return _response_as_json(r) except requests.exceptions.HTTPError: if is_new_action: warning_questions = questionary.confirm( "WARNING: You have created a new action: '{}', " "which was not successfully executed. " "If this action does not return any events, " "you do not need to do anything. " "If this is a custom action which returns events, " "you are recommended to implement this action " "in your action server and try again." "".format(action_name)) _ask_or_abort(warning_questions, sender_id, endpoint) payload = ActionExecuted(action_name).as_dict() return send_event(endpoint, sender_id, payload) else: logger.error("failed to execute action!") raise
def test_reminder_scheduled(default_processor): out = CollectingOutputChannel() sender_id = uuid.uuid4().hex d = Dispatcher(sender_id, out, default_processor.nlg) r = ReminderScheduled("utter_greet", datetime.datetime.now()) t = default_processor.tracker_store.get_or_create_tracker(sender_id) t.update(UserUttered("test")) t.update(ActionExecuted("action_reminder_reminder")) t.update(r) default_processor.tracker_store.save(t) default_processor.handle_reminder(r, d) # retrieve the updated tracker t = default_processor.tracker_store.retrieve(sender_id) assert t.events[-4] == UserUttered(None) assert t.events[-3] == ActionExecuted("utter_greet") assert t.events[-2] == BotUttered("hey there None!", { 'elements': None, 'buttons': None, 'attachment': None }) assert t.events[-1] == ActionExecuted("action_listen")
def test_last_executed_has_not_name(): events = [ ActionExecuted('one'), user_uttered('two', 1), ActionExecuted(ACTION_LISTEN_NAME) ] tracker = get_tracker(events) assert tracker.last_executed_action_has('another') is False
def test_get_last_event_for_with_skip(): events = [ ActionExecuted('one'), user_uttered('two', 1), ActionExecuted('three') ] tracker = get_tracker(events) assert (tracker.get_last_event_for(ActionExecuted, skip=1).action_name == 'one')
def test_ask_rephrase(self, trained_policy, default_domain): events = [ActionExecuted(ACTION_LISTEN_NAME), user_uttered("greet", 0.2), ActionExecuted(ACTION_DEFAULT_ASK_AFFIRMATION_NAME), ActionExecuted(ACTION_LISTEN_NAME), user_uttered('deny', 1)] next_action = self._get_next_action(trained_policy, events, default_domain) assert next_action == ACTION_DEFAULT_ASK_REPHRASE_NAME
def test_get_last_event_for_with_exclude(): events = [ ActionExecuted('one'), user_uttered('two', 1), ActionExecuted('three') ] tracker = get_tracker(events) assert (tracker.get_last_event_for( ActionExecuted, action_names_to_exclude=['three']).action_name == 'one')
def test_unknown_instead_affirmation(self, trained_policy, default_domain): events = [ActionExecuted(ACTION_LISTEN_NAME), user_uttered("greet", 0.2), ActionExecuted(ACTION_DEFAULT_ASK_AFFIRMATION_NAME), ActionExecuted(ACTION_LISTEN_NAME), user_uttered("greet", 0.2), ] next_action = self._get_next_action(trained_policy, events, default_domain) assert next_action == ACTION_DEFAULT_FALLBACK_NAME
def test_common_action_prefix_unequal(): this = [ ActionExecuted("action_listen"), ActionExecuted("greet"), UserUttered("hey"), ] other = [ ActionExecuted("greet"), ActionExecuted("action_listen"), UserUttered("hey"), ] num_common = visualization._length_of_common_action_prefix(this, other) assert num_common == 0
def _log_action_on_tracker(self, tracker, action_name, events, policy, policy_confidence): # Ensures that the code still works even if a lazy programmer missed # to type `return []` at the end of an action or the run method # returns `None` for some other reason. if events is None: events = [] logger.debug("Action '{}' ended with events '{}'".format( action_name, ['{}'.format(e) for e in events])) self._warn_about_new_slots(tracker, action_name, events) if action_name is not None: # log the action and its produced events tracker.update( ActionExecuted(action_name, policy, policy_confidence)) for e in events: # this makes sure the events are ordered by timestamp - # since the event objects are created somewhere else, # the timestamp would indicate a time before the time # of the action executed e.timestamp = time.time() tracker.update(e)
def test_affirm_rephrased_intent(self, trained_policy, default_domain): events = [ ActionExecuted(ACTION_LISTEN_NAME), user_uttered("greet", 0.2), ActionExecuted(ACTION_DEFAULT_ASK_AFFIRMATION_NAME), ActionExecuted(ACTION_LISTEN_NAME), user_uttered(USER_INTENT_DENY, 1), ActionExecuted(ACTION_DEFAULT_ASK_REPHRASE_NAME), ActionExecuted(ACTION_LISTEN_NAME), user_uttered("greet", 0.2), ] next_action = self._get_next_action(trained_policy, events, default_domain) assert next_action == ACTION_DEFAULT_ASK_AFFIRMATION_NAME
async def _predict_till_next_listen(endpoint: EndpointConfig, sender_id: Text, finetune: bool, sender_ids: List[Text], plot_file: Optional[Text]) -> None: """Predict and validate actions until we need to wait for a user msg.""" listen = False while not listen: result = await request_prediction(endpoint, sender_id) predictions = result.get("scores") probabilities = [prediction["score"] for prediction in predictions] pred_out = int(np.argmax(probabilities)) action_name = predictions[pred_out].get("action") policy = result.get("policy") confidence = result.get("confidence") await _print_history(sender_id, endpoint) await _plot_trackers(sender_ids, plot_file, endpoint, unconfirmed=[ActionExecuted(action_name)]) listen = await _validate_action(action_name, policy, confidence, predictions, endpoint, sender_id, finetune=finetune) await _plot_trackers(sender_ids, plot_file, endpoint)
def test_listen_after_hand_off(self, trained_policy, default_domain): events = [ActionExecuted(ACTION_DEFAULT_FALLBACK_NAME)] next_action = self._get_next_action(trained_policy, events, default_domain) assert next_action == ACTION_LISTEN_NAME
def test_ask_affirmation(self, trained_policy, default_domain): events = [ActionExecuted(ACTION_LISTEN_NAME), user_uttered("Hi", 0.2)] next_action = self._get_next_action(trained_policy, events, default_domain) assert next_action == ACTION_DEFAULT_ASK_AFFIRMATION_NAME
def test_missing_classes_filled_correctly(self, default_domain, trackers, tracker, featurizer): # Pretend that a couple of classes are missing and check that # those classes are predicted as 0, while the other class # probabilities are predicted normally. policy = self.create_policy(featurizer=featurizer, cv=None) classes = [1, 3] new_trackers = [] for tr in trackers: new_tracker = DialogueStateTracker(UserMessage.DEFAULT_SENDER_ID, default_domain.slots, default_domain.topics, default_domain.default_topic) for e in tr.applied_events(): if isinstance(e, ActionExecuted): new_action = default_domain.action_for_index( np.random.choice(classes)).name() new_tracker.update(ActionExecuted(new_action)) else: new_tracker.update(e) new_trackers.append(new_tracker) policy.train(new_trackers, domain=default_domain) predicted_probabilities = policy.predict_action_probabilities( tracker, default_domain) assert len(predicted_probabilities) == default_domain.num_actions assert np.allclose(sum(predicted_probabilities), 1.0) for i, prob in enumerate(predicted_probabilities): if i in classes: assert prob >= 0.0 else: assert prob == 0.0
def _predict_till_next_listen( endpoint, # type: EndpointConfig sender_id, # type: Text finetune, # type: bool sender_ids, # type: List[Text] plot_file # type: Optional[Text] ): # type: (...) -> None """Predict and validate actions until we need to wait for a user msg.""" listen = False while not listen: response = request_prediction(endpoint, sender_id) predictions = response.get("scores") probabilities = [prediction["score"] for prediction in predictions] pred_out = int(np.argmax(probabilities)) action_name = predictions[pred_out].get("action") _print_history(sender_id, endpoint) _plot_trackers(sender_ids, plot_file, endpoint, unconfirmed=[ActionExecuted(action_name)]) listen = _validate_action(action_name, predictions, endpoint, sender_id, finetune=finetune) _plot_trackers(sender_ids, plot_file, endpoint)
def _revert_single_affirmation_events() -> List[Event]: return [ UserUtteranceReverted(), # revert affirmation and request # revert original intent (has to be re-added later) UserUtteranceReverted(), # add action listen intent ActionExecuted(action_name=ACTION_LISTEN_NAME) ]
def _revert_rephrasing_events() -> List[Event]: return [UserUtteranceReverted(), # remove rephrasing # remove feedback and rephrase request UserUtteranceReverted(), # remove affirmation request and false intent UserUtteranceReverted(), # replace action with action listen ActionExecuted(action_name=ACTION_LISTEN_NAME)]
def as_dialogue(self, sender_id, domain): events = [] for step in self.story_steps: events.extend( step.explicit_events(domain, should_append_final_listen=False)) events.append(ActionExecuted(ActionListen().name())) return Dialogue(sender_id, events)
def test_json_parse_action(): # DOCS MARKER ActionExecuted evt = \ { 'event': 'action', 'name': 'my_action' } # DOCS END assert Event.from_parameters(evt) == ActionExecuted("my_action")
def test_revert_user_utterance_event(default_domain): tracker = DialogueStateTracker("default", default_domain.slots, default_domain.topics, default_domain.default_topic) # the retrieved tracker should be empty assert len(tracker.events) == 0 intent1 = {"name": "greet", "confidence": 1.0} tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) tracker.update(UserUttered("/greet", intent1, [])) tracker.update(ActionExecuted("my_action_1")) tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) intent2 = {"name": "goodbye", "confidence": 1.0} tracker.update(UserUttered("/goodbye", intent2, [])) tracker.update(ActionExecuted("my_action_2")) tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) # Expecting count of 6: # +5 executed actions # +1 final state assert tracker.latest_action_name == ACTION_LISTEN_NAME assert len(list(tracker.generate_all_prior_states())) == 6 tracker.update(UserUtteranceReverted()) # Expecting count of 3: # +5 executed actions # +1 final state # -2 rewound actions associated with the /goodbye # -1 rewound action from the listen right before /goodbye assert tracker.latest_action_name == "my_action_1" assert len(list(tracker.generate_all_prior_states())) == 3 dialogue = tracker.as_dialogue() recovered = DialogueStateTracker("default", default_domain.slots, default_domain.topics, default_domain.default_topic) recovered.recreate_from_dialogue(dialogue) assert recovered.current_state() == tracker.current_state() assert tracker.latest_action_name == "my_action_1" assert len(list(tracker.generate_all_prior_states())) == 3
def test_successful_rephrasing(self, trained_policy, default_dispatcher_collecting, default_domain): events = [ ActionExecuted(ACTION_LISTEN_NAME), user_uttered("greet", 0.2), ActionExecuted(ACTION_DEFAULT_ASK_AFFIRMATION_NAME), ActionExecuted(ACTION_LISTEN_NAME), user_uttered('deny', 1), ActionExecuted(ACTION_DEFAULT_ASK_REPHRASE_NAME), ActionExecuted(ACTION_LISTEN_NAME), user_uttered("bye", 1), ] tracker = self._get_tracker_after_reverts( events, default_dispatcher_collecting, default_domain) assert 'bye' == tracker.latest_message.parse_data['intent']['name'] assert tracker.export_stories() == "## sender\n* bye\n"
def test_affirmation(self, default_dispatcher_collecting, default_domain): events = [ActionExecuted(ACTION_LISTEN_NAME), user_uttered('greet', 1), ActionExecuted('utter_hello'), ActionExecuted(ACTION_LISTEN_NAME), user_uttered('greet', 0.2), ActionExecuted(ACTION_DEFAULT_ASK_AFFIRMATION_NAME), ActionExecuted(ACTION_LISTEN_NAME), user_uttered('greet', 1)] tracker = self._get_tracker_after_reverts(events, default_dispatcher_collecting, default_domain) assert 'greet' == tracker.latest_message.parse_data['intent']['name'] assert tracker.export_stories() == ("## sender\n" "* greet\n" " - utter_hello\n" "* greet\n")