def test_session_start_is_not_serialised(domain: Domain): tracker = DialogueStateTracker("default", domain.slots) # the retrieved tracker should be empty assert len(tracker.events) == 0 # add SlotSet event tracker.update(SlotSet("slot", "value")) # add the two SessionStarted events and a user event tracker.update(ActionExecuted(ACTION_SESSION_START_NAME)) tracker.update(SessionStarted()) tracker.update( UserUttered("say something", intent={INTENT_NAME_KEY: "some_intent"})) tracker.update(DefinePrevUserUtteredFeaturization(False)) YAMLStoryWriter().dumps( Story.from_events(tracker.events, "some-story01").story_steps) expected = """version: "2.0" stories: - story: some-story01 steps: - slot_was_set: - slot: value - intent: some_intent """ actual = YAMLStoryWriter().dumps( Story.from_events(tracker.events, "some-story01").story_steps) assert actual == expected
def _log_action_on_tracker( self, tracker: DialogueStateTracker, action_name: Text, events: Optional[List[Event]], prediction: PolicyPrediction, ) -> None: # Ensures that the code still works even if a lazy programmer missed # to type `return []` at the end of an action or the run method # returns `None` for some other reason. if events is None: events = [] self._warn_about_new_slots(tracker, action_name, events) action_was_rejected_manually = any( isinstance(event, ActionExecutionRejected) for event in events) if action_name is not None and not action_was_rejected_manually: logger.debug( f"Policy prediction ended with events '{prediction.events}'.") tracker.update_with_events(prediction.events, self.domain) # log the action and its produced events tracker.update( ActionExecuted(action_name, prediction.policy_name, prediction.max_confidence)) logger.debug(f"Action '{action_name}' ended with events '{events}'.") tracker.update_with_events(events, self.domain)
def _log_action_on_tracker( self, tracker: DialogueStateTracker, action_name: Text, events: Optional[List[Event]], policy: Optional[Text], confidence: Optional[float], ) -> None: # Ensures that the code still works even if a lazy programmer missed # to type `return []` at the end of an action or the run method # returns `None` for some other reason. if events is None: events = [] logger.debug( f"Action '{action_name}' ended with events '{[e for e in events]}'." ) self._warn_about_new_slots(tracker, action_name, events) action_was_rejected_manually = any( isinstance(event, ActionExecutionRejected) for event in events) if action_name is not None and not action_was_rejected_manually: # log the action and its produced events tracker.update(ActionExecuted(action_name, policy, confidence)) for e in events: # this makes sure the events are ordered by timestamp - # since the event objects are created somewhere else, # the timestamp would indicate a time before the time # of the action executed e.timestamp = time.time() tracker.update(e, self.domain)
def test_revert_action_event(default_domain: Domain): tracker = DialogueStateTracker("default", default_domain.slots) # the retrieved tracker should be empty assert len(tracker.events) == 0 intent = {"name": "greet", "confidence": 1.0} tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) tracker.update(UserUttered("/greet", intent, [])) tracker.update(ActionExecuted("my_action")) tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) # Expecting count of 4: # +3 executed actions # +1 final state assert tracker.latest_action.get(ACTION_NAME) == ACTION_LISTEN_NAME assert len(list(tracker.generate_all_prior_trackers())) == 4 tracker.update(ActionReverted()) # Expecting count of 3: # +3 executed actions # +1 final state # -1 reverted action assert tracker.latest_action.get(ACTION_NAME) == "my_action" assert len(list(tracker.generate_all_prior_trackers())) == 3 dialogue = tracker.as_dialogue() recovered = DialogueStateTracker("default", default_domain.slots) recovered.recreate_from_dialogue(dialogue) assert recovered.current_state() == tracker.current_state() assert tracker.latest_action.get(ACTION_NAME) == "my_action" assert len(list(tracker.generate_all_prior_trackers())) == 3
def test_restart_event(default_domain: Domain): tracker = DialogueStateTracker("default", default_domain.slots) # the retrieved tracker should be empty assert len(tracker.events) == 0 intent = {"name": "greet", "confidence": 1.0} tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) tracker.update(UserUttered("/greet", intent, [])) tracker.update(ActionExecuted("my_action")) tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) assert len(tracker.events) == 4 assert tracker.latest_message.text == "/greet" assert len(list(tracker.generate_all_prior_trackers())) == 4 tracker.update(Restarted()) assert len(tracker.events) == 5 assert tracker.followup_action is not None assert tracker.followup_action == ACTION_LISTEN_NAME assert tracker.latest_message.text is None assert len(list(tracker.generate_all_prior_trackers())) == 1 dialogue = tracker.as_dialogue() recovered = DialogueStateTracker("default", default_domain.slots) recovered.recreate_from_dialogue(dialogue) assert recovered.current_state() == tracker.current_state() assert len(recovered.events) == 5 assert recovered.latest_message.text is None assert len(list(recovered.generate_all_prior_trackers())) == 1
async def _handle_message_with_tracker( self, message: UserMessage, tracker: DialogueStateTracker) -> None: if message.parse_data: parse_data = message.parse_data else: parse_data = await self.parse_message(message) # don't ever directly mutate the tracker # - instead pass its events to log tracker.update( UserUttered( message.text, parse_data["intent"], parse_data["entities"], parse_data, input_channel=message.input_channel, message_id=message.message_id, metadata=message.metadata, ), self.domain, ) if parse_data["entities"]: self._log_slots(tracker) logger.debug( f"Logged UserUtterance - tracker now has {len(tracker.events)} events." )
def _log_action_on_tracker( self, tracker: DialogueStateTracker, action: Action, events: Optional[List[Event]], prediction: PolicyPrediction, ) -> None: # Ensures that the code still works even if a lazy programmer missed # to type `return []` at the end of an action or the run method # returns `None` for some other reason. if events is None: events = [] action_was_rejected_manually = any( isinstance(event, ActionExecutionRejected) for event in events) if not action_was_rejected_manually: logger.debug( f"Policy prediction ended with events '{prediction.events}'.") tracker.update_with_events(prediction.events, self.domain) # log the action and its produced events tracker.update(action.event_for_successful_execution(prediction)) logger.debug(f"Action '{action.name()}' ended with events '{events}'.") tracker.update_with_events(events, self.domain)
def test_tracker_update_slots_with_entity(default_domain: Domain): tracker = DialogueStateTracker("default", default_domain.slots) test_entity = default_domain.entities[0] expected_slot_value = "test user" intent = {"name": "greet", PREDICTED_CONFIDENCE_KEY: 1.0} tracker.update( UserUttered( "/greet", intent, [ { "start": 1, "end": 5, "value": expected_slot_value, "entity": test_entity, "extractor": "manual", } ], ), default_domain, ) assert tracker.get_slot(test_entity) == expected_slot_value
def predict_action_probabilities( self, tracker: DialogueStateTracker, domain: Domain, interpreter: NaturalLanguageInterpreter, **kwargs: Any, ) -> List[float]: """Predicts the corresponding form action if there is an active form""" result = self._default_predictions(domain) if tracker.active_loop_name: logger.debug( "There is an active form '{}'".format(tracker.active_loop_name) ) if tracker.latest_action_name == ACTION_LISTEN_NAME: # predict form action after user utterance if tracker.active_loop.get(LOOP_REJECTED): if self.state_is_unhappy(tracker, domain): tracker.update(LoopInterrupted(True)) return result result = self._prediction_result( tracker.active_loop_name, tracker, domain ) elif tracker.latest_action_name == tracker.active_loop_name: # predict action_listen after form action result = self._prediction_result(ACTION_LISTEN_NAME, tracker, domain) else: logger.debug("There is no active form") return result
def _collect_action_executed_predictions( processor: "MessageProcessor", partial_tracker: DialogueStateTracker, event: ActionExecuted, fail_on_prediction_errors: bool, circuit_breaker_tripped: bool, ) -> Tuple[EvaluationStore, Optional[Text], Optional[float]]: from rasa.core.policies.form_policy import FormPolicy action_executed_eval_store = EvaluationStore() gold = event.action_name or event.action_text if circuit_breaker_tripped: predicted = "circuit breaker tripped" policy = None confidence = None else: action, policy, confidence = processor.predict_next_action( partial_tracker) predicted = action.name() if (policy and predicted != gold and _form_might_have_been_rejected( processor.domain, partial_tracker, predicted)): # Wrong action was predicted, # but it might be Ok if form action is rejected. emulate_loop_rejection(partial_tracker) # try again action, policy, confidence = processor.predict_next_action( partial_tracker) # Even if the prediction is also wrong, we don't have to undo the emulation # of the action rejection as we know that the user explicitly specified # that something else than the form was supposed to run. predicted = action.name() action_executed_eval_store.add_to_store(action_predictions=[predicted], action_targets=[gold]) if action_executed_eval_store.has_prediction_target_mismatch(): partial_tracker.update( WronglyPredictedAction(gold, predicted, policy, confidence, event.timestamp)) if fail_on_prediction_errors: story_dump = YAMLStoryWriter().dumps( partial_tracker.as_story().story_steps) error_msg = (f"Model predicted a wrong action. Failed Story: " f"\n\n{story_dump}") if FormPolicy.__name__ in policy: error_msg += ("FormAction is not run during " "evaluation therefore it is impossible to know " "if validation failed or this story is wrong. " "If the story is correct, add it to the " "training stories and retrain.") raise WrongPredictionException(error_msg) else: partial_tracker.update( ActionExecuted(predicted, policy, confidence, event.timestamp)) return action_executed_eval_store, policy, confidence
def _collect_user_uttered_predictions( event: UserUttered, predicted: Dict[Text, Any], partial_tracker: DialogueStateTracker, fail_on_prediction_errors: bool, ) -> EvaluationStore: user_uttered_eval_store = EvaluationStore() # intent from the test story, may either be base intent or full retrieval intent base_intent = event.intent.get(INTENT_NAME_KEY) full_retrieval_intent = event.intent.get(FULL_RETRIEVAL_INTENT_NAME_KEY) intent_gold = full_retrieval_intent if full_retrieval_intent else base_intent # predicted intent: note that this is only the base intent at this point predicted_base_intent = predicted.get(INTENT, {}).get(INTENT_NAME_KEY) # if the test story only provides the base intent AND the prediction was correct, # we are not interested in full retrieval intents and skip this section. # In any other case we are interested in the full retrieval intent (e.g. for report) if intent_gold != predicted_base_intent: predicted_base_intent = _get_full_retrieval_intent(predicted) user_uttered_eval_store.add_to_store( intent_targets=[intent_gold], intent_predictions=[predicted_base_intent]) entity_gold = event.entities predicted_entities = predicted.get(ENTITIES) if entity_gold or predicted_entities: user_uttered_eval_store.add_to_store( entity_targets=_clean_entity_results(event.text, entity_gold), entity_predictions=_clean_entity_results(event.text, predicted_entities), ) if user_uttered_eval_store.check_prediction_target_mismatch(): partial_tracker.update( WronglyClassifiedUserUtterance(event, user_uttered_eval_store)) if fail_on_prediction_errors: story_dump = YAMLStoryWriter().dumps( partial_tracker.as_story().story_steps) raise WrongPredictionException( f"NLU model predicted a wrong intent or entities. Failed Story:" f" \n\n{story_dump}") else: response_selector_info = ({ RESPONSE_SELECTOR_PROPERTY_NAME: predicted[RESPONSE_SELECTOR_PROPERTY_NAME] } if RESPONSE_SELECTOR_PROPERTY_NAME in predicted else None) end_to_end_user_utterance = EndToEndUserUtterance( text=event.text, intent=event.intent, entities=event.entities, parse_data=response_selector_info, ) partial_tracker.update(end_to_end_user_utterance) return user_uttered_eval_store
def test_session_start(default_domain: Domain): tracker = DialogueStateTracker("default", default_domain.slots) # the retrieved tracker should be empty assert len(tracker.events) == 0 # add a SessionStarted event tracker.update(SessionStarted()) # tracker has one event assert len(tracker.events) == 1
def test_slot_mapping_entity_is_desired(slot_name: Text, expected: bool): domain = Domain.from_file("data/test_domains/travel_form.yml") tracker = DialogueStateTracker("test_id", slots=domain.slots) event = UserUttered( text="I'm travelling to Vancouver.", intent={"name": "inform", "confidence": 0.9604260921478271}, entities=[{"entity": "GPE", "value": "Vancouver", "role": "destination"}], ) tracker.update(event, domain) slot_mappings = domain.as_dict().get("slots").get(slot_name).get("mappings") assert SlotMapping.entity_is_desired(slot_mappings[0], tracker) is expected
def emulate_loop_rejection(partial_tracker: DialogueStateTracker) -> None: """Add `ActionExecutionRejected` event to the tracker. During evaluation, we don't run action server, therefore in order to correctly test unhappy paths of the loops, we need to emulate loop rejection. Args: partial_tracker: a :class:`rasa.core.trackers.DialogueStateTracker` """ from rasa.shared.core.events import ActionExecutionRejected rejected_action_name: Text = partial_tracker.active_loop_name partial_tracker.update(ActionExecutionRejected(rejected_action_name))
async def trigger_external_user_uttered( self, intent_name: Text, entities: Optional[Union[List[Dict[Text, Any]], Dict[Text, Text]]], tracker: DialogueStateTracker, output_channel: OutputChannel, ) -> None: """Triggers an external message. Triggers an external message (like a user message, but invisible; used, e.g., by a reminder or the trigger_intent endpoint). Args: intent_name: Name of the intent to be triggered. entities: Entities to be passed on. tracker: The tracker to which the event should be added. output_channel: The output channel. """ if isinstance(entities, list): entity_list = entities elif isinstance(entities, dict): # Allow for a short-hand notation {"ent1": "val1", "ent2": "val2", ...}. # Useful if properties like 'start', 'end', or 'extractor' are not given, # e.g. for external events. entity_list = [{ "entity": ent, "value": val } for ent, val in entities.items()] elif not entities: entity_list = [] else: rasa.shared.utils.io.raise_warning( f"Invalid entity specification: {entities}. Assuming no entities." ) entity_list = [] # Set the new event's input channel to the latest input channel, so # that we don't lose this property. input_channel = tracker.get_latest_input_channel() tracker.update( UserUttered.create_external(intent_name, entity_list, input_channel), self.domain, ) tracker = await self.run_action_extract_slots(output_channel, tracker) await self._run_prediction_loop(output_channel, tracker) # save tracker state to continue conversation from this state self.save_tracker(tracker)
def test_traveling_back_in_time(default_domain: Domain): tracker = DialogueStateTracker("default", default_domain.slots) # the retrieved tracker should be empty assert len(tracker.events) == 0 intent = {"name": "greet", "confidence": 1.0} tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) tracker.update(UserUttered("/greet", intent, [])) import time time.sleep(1) time_for_timemachine = time.time() time.sleep(1) tracker.update(ActionExecuted("my_action")) tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) # Expecting count of 4: # +3 executed actions # +1 final state assert tracker.latest_action.get(ACTION_NAME) == ACTION_LISTEN_NAME assert len(tracker.events) == 4 assert len(list(tracker.generate_all_prior_trackers())) == 4 tracker = tracker.travel_back_in_time(time_for_timemachine) # Expecting count of 2: # +1 executed actions # +1 final state assert tracker.latest_action.get(ACTION_NAME) == ACTION_LISTEN_NAME assert len(tracker.events) == 2 assert len(list(tracker.generate_all_prior_trackers())) == 2
def marker_sqlite_tracker(tmp_path: Path) -> Tuple[SQLTrackerStore, Text]: domain = Domain.empty() db_path = str(tmp_path / "rasa.db") tracker_store = SQLTrackerStore(dialect="sqlite", db=db_path) for i in range(5): tracker = DialogueStateTracker(str(i), None) tracker.update_with_events([SlotSet(str(j), "slot") for j in range(5)], domain) tracker.update(ActionExecuted(ACTION_SESSION_START_NAME)) tracker.update(UserUttered("hello")) tracker.update_with_events( [SlotSet(str(5 + j), "slot") for j in range(5)], domain) tracker_store.save(tracker) return tracker_store, db_path
async def _run_action( self, action: rasa.core.actions.action.Action, tracker: DialogueStateTracker, output_channel: OutputChannel, nlg: NaturalLanguageGenerator, prediction: PolicyPrediction, metadata: Optional[Dict[Text, Any]] = None, ) -> bool: # events and return values are used to update # the tracker state after an action has been taken try: # Here we set optional metadata to the ActionSessionStart, which will then # be passed to the SessionStart event. Otherwise the metadata will be lost. if action.name() == ACTION_SESSION_START_NAME: action.metadata = metadata # Use temporary tracker as we might need to discard the policy events in # case of a rejection. temporary_tracker = tracker.copy() temporary_tracker.update_with_events(prediction.events, self.domain) events = await action.run(output_channel, nlg, temporary_tracker, self.domain) except rasa.core.actions.action.ActionExecutionRejection: events = [ ActionExecutionRejected(action.name(), prediction.policy_name, prediction.max_confidence) ] tracker.update(events[0]) return self.should_predict_another_action(action.name()) except Exception: logger.exception( f"Encountered an exception while running action '{action.name()}'." "Bot will continue, but the actions events are lost. " "Please check the logs of your action server for " "more information.") events = [] self._log_action_on_tracker(tracker, action.name(), events, prediction) if action.name() != ACTION_LISTEN_NAME and not action.name( ).startswith(UTTER_PREFIX): self._log_slots(tracker) await self.execute_side_effects(events, tracker, output_channel) return self.should_predict_another_action(action.name())
async def _update_tracker_session( self, tracker: DialogueStateTracker, output_channel: OutputChannel, metadata: Optional[Dict] = None, ) -> None: """Check the current session in `tracker` and update it if expired. An 'action_session_start' is run if the latest tracker session has expired, or if the tracker does not yet contain any events (only those after the last restart are considered). Args: metadata: Data sent from client associated with the incoming user message. tracker: Tracker to inspect. output_channel: Output channel for potential utterances in a custom `ActionSessionStart`. """ if not tracker.applied_events() or self._has_session_expired(tracker): logger.debug( f"Starting a new session for conversation ID '{tracker.sender_id}'." ) action_session_start = self._get_action(ACTION_SESSION_START_NAME) # TODO: Remove in 3.0.0 and describe migration to `session_start_metadata` # slot in migration guide. if isinstance( action_session_start, rasa.core.actions.action.ActionSessionStart ): # Here we set optional metadata to the ActionSessionStart, which will # then be passed to the SessionStart event. action_session_start.metadata = metadata if metadata: tracker.update( SlotSet(SESSION_START_METADATA_SLOT, metadata), self.domain ) await self._run_action( action=action_session_start, tracker=tracker, output_channel=output_channel, nlg=self.nlg, prediction=PolicyPrediction.for_action_name( self.domain, ACTION_SESSION_START_NAME ), )
async def cancel_reminder_and_check( tracker: DialogueStateTracker, default_processor: MessageProcessor, reminder_canceled_event: ReminderCancelled, num_jobs_before: int, num_jobs_after: int, ) -> None: # cancel the sixth reminder tracker.update(reminder_canceled_event) # check that the jobs were added assert len((await jobs.scheduler()).get_jobs()) == num_jobs_before await default_processor._cancel_reminders(tracker.events, tracker) # check that only one job was removed assert len((await jobs.scheduler()).get_jobs()) == num_jobs_after
def test_missing_classes_filled_correctly( self, default_domain: Domain, trackers: List[TrackerWithCachedStates], tracker: DialogueStateTracker, featurizer: Optional[TrackerFeaturizer], priority: int, ): # Pretend that a couple of classes are missing and check that # those classes are predicted as 0, while the other class # probabilities are predicted normally. policy = self.create_policy(featurizer=featurizer, priority=priority, cv=None) classes = [1, 3] new_trackers = [] for tr in trackers: new_tracker = DialogueStateTracker(DEFAULT_SENDER_ID, default_domain.slots) for e in tr.applied_events(): if isinstance(e, ActionExecuted): new_action = rasa.core.actions.action.action_for_index( np.random.choice(classes), default_domain, action_endpoint=None).name() new_tracker.update(ActionExecuted(new_action)) else: new_tracker.update(e) new_trackers.append(new_tracker) policy.train(new_trackers, domain=default_domain, interpreter=RegexInterpreter()) prediction = policy.predict_action_probabilities( tracker, default_domain, RegexInterpreter()) assert not prediction.is_end_to_end_prediction assert len(prediction.probabilities) == default_domain.num_actions assert np.allclose(sum(prediction.probabilities), 1.0) for i, prob in enumerate(prediction.probabilities): if i in classes: assert prob >= 0.0 else: assert prob == 0.0
def test_get_latest_entity_values(entities: List[Dict[Text, Any]], expected_values: List[Text], domain: Domain): entity_type = entities[0].get("entity") entity_role = entities[0].get("role") entity_group = entities[0].get("group") tracker = DialogueStateTracker("default", domain.slots) # the retrieved tracker should be empty assert len(tracker.events) == 0 assert list(tracker.get_latest_entity_values(entity_type)) == [] intent = {"name": "greet", PREDICTED_CONFIDENCE_KEY: 1.0} tracker.update(UserUttered("/greet", intent, entities)) assert (list( tracker.get_latest_entity_values( entity_type, entity_role=entity_role, entity_group=entity_group)) == expected_values) assert list(tracker.get_latest_entity_values("unknown")) == []
def _collect_user_uttered_predictions( event: UserUttered, predicted: Dict[Text, Any], partial_tracker: DialogueStateTracker, fail_on_prediction_errors: bool, ) -> EvaluationStore: user_uttered_eval_store = EvaluationStore() intent_gold = event.intent.get("name") predicted_intent = predicted.get(INTENT, {}).get("name") user_uttered_eval_store.add_to_store( intent_predictions=[predicted_intent], intent_targets=[intent_gold] ) entity_gold = event.entities predicted_entities = predicted.get(ENTITIES) if entity_gold or predicted_entities: user_uttered_eval_store.add_to_store( entity_targets=_clean_entity_results(event.text, entity_gold), entity_predictions=_clean_entity_results(event.text, predicted_entities), ) if user_uttered_eval_store.has_prediction_target_mismatch(): partial_tracker.update( WronglyClassifiedUserUtterance(event, user_uttered_eval_store) ) if fail_on_prediction_errors: raise ValueError( "NLU model predicted a wrong intent. Failed Story:" " \n\n{}".format( YAMLStoryWriter().dumps(partial_tracker.as_story().story_steps) ) ) else: end_to_end_user_utterance = EndToEndUserUtterance( event.text, event.intent, event.entities ) partial_tracker.update(end_to_end_user_utterance) return user_uttered_eval_store
def add( self, predictions: List[Message], tracker: DialogueStateTracker, domain: Domain, original_message: UserMessage, ) -> DialogueStateTracker: """Adds NLU predictions to the tracker. Args: predictions: A list of NLU predictions wrapped as Messages tracker: The tracker the predictions should be attached to domain: The domain of the model. original_message: An original message from the user with extra metadata to annotate the predictions (e.g. channel) Returns: The original tracker updated with events created from the predictions """ for message in predictions: user_event = UserUttered( message.data.get(TEXT), message.data.get(INTENT), message.data.get(ENTITIES), input_channel=original_message.input_channel, message_id=message.data.get("message_id"), metadata=original_message.metadata, ) tracker.update(user_event, domain) if user_event.entities: # Log currently set slots slot_values = "\n".join( [f"\t{s.name}: {s.value}" for s in tracker.slots.values()]) if slot_values.strip(): logger.debug(f"Current slot values: \n{slot_values}") logger.debug(f"Logged {len(predictions)} UserUtterance(s) - \ tracker now has {len(tracker.events)} events.") return tracker
async def test_markers_cli_results_save_correctly(tmp_path: Path): domain = Domain.empty() store = InMemoryTrackerStore(domain) for i in range(5): tracker = DialogueStateTracker(str(i), None) tracker.update_with_events([SlotSet(str(j), "slot") for j in range(5)], domain) tracker.update(ActionExecuted(ACTION_SESSION_START_NAME)) tracker.update(UserUttered("hello")) tracker.update_with_events( [SlotSet(str(5 + j), "slot") for j in range(5)], domain) await store.save(tracker) tracker_loader = MarkerTrackerLoader(store, "all") results_path = tmp_path / "results.csv" markers = OrMarker(markers=[ SlotSetMarker("2", name="marker1"), SlotSetMarker("7", name="marker2") ]) await markers.evaluate_trackers(tracker_loader.load(), results_path) with open(results_path, "r") as results: result_reader = csv.DictReader(results) senders = set() for row in result_reader: senders.add(row["sender_id"]) if row["marker"] == "marker1": assert row["session_idx"] == "0" assert int(row["event_idx"]) >= 2 assert row["num_preceding_user_turns"] == "0" if row["marker"] == "marker2": assert row["session_idx"] == "1" assert int(row["event_idx"]) >= 3 assert row["num_preceding_user_turns"] == "1" assert len(senders) == 5
async def test_action_session_start_with_slots( default_channel: CollectingOutputChannel, template_nlg: TemplatedNaturalLanguageGenerator, template_sender_tracker: DialogueStateTracker, domain: Domain, session_config: SessionConfig, expected_events: List[Event], ): # set a few slots on tracker slot_set_event_1 = SlotSet("my_slot", "value") slot_set_event_2 = SlotSet("another-slot", "value2") for event in [slot_set_event_1, slot_set_event_2]: template_sender_tracker.update(event) domain.session_config = session_config events = await ActionSessionStart().run(default_channel, template_nlg, template_sender_tracker, domain) assert events == expected_events # make sure that the list of events has ascending timestamps assert sorted(events, key=lambda x: x.timestamp) == events
def test_session_start_is_not_serialised(default_domain: Domain): tracker = DialogueStateTracker("default", default_domain.slots) # the retrieved tracker should be empty assert len(tracker.events) == 0 # add SlotSet event tracker.update(SlotSet("slot", "value")) # add the two SessionStarted events and a user event tracker.update(ActionExecuted(ACTION_SESSION_START_NAME)) tracker.update(SessionStarted()) tracker.update(UserUttered("say something")) # make sure session start is not serialised story = Story.from_events(tracker.events, "some-story01") expected = """## some-story01 - slot{"slot": "value"} * say something """ assert story.as_story_string(flat=True) == expected
def test_slot_mapping_intent_is_desired(domain: Domain): domain = Domain.from_file("examples/formbot/domain.yml") tracker = DialogueStateTracker("sender_id_test", slots=domain.slots) event1 = UserUttered( text="I'd like to book a restaurant for 2 people.", intent={ "name": "request_restaurant", "confidence": 0.9604260921478271 }, entities=[{ "entity": "number", "value": 2 }], ) tracker.update(event1, domain) mappings_for_num_people = ( domain.as_dict().get("slots").get("num_people").get("mappings")) assert SlotMapping.intent_is_desired(mappings_for_num_people[0], tracker, domain) event2 = UserUttered( text="Yes, 2 please", intent={ "name": "affirm", "confidence": 0.9604260921478271 }, entities=[{ "entity": "number", "value": 2 }], ) tracker.update(event2, domain) assert (SlotMapping.intent_is_desired(mappings_for_num_people[0], tracker, domain) is False) event3 = UserUttered( text="Yes, please", intent={ "name": "affirm", "confidence": 0.9604260921478271 }, entities=[], ) tracker.update(event3, domain) mappings_for_preferences = ( domain.as_dict().get("slots").get("preferences").get("mappings")) assert (SlotMapping.intent_is_desired(mappings_for_preferences[0], tracker, domain) is False)
def _collect_action_executed_predictions( processor: "MessageProcessor", partial_tracker: DialogueStateTracker, event: ActionExecuted, fail_on_prediction_errors: bool, ) -> Tuple[EvaluationStore, PolicyPrediction, Optional[EntityEvaluationResult]]: action_executed_eval_store = EvaluationStore() expected_action_name = event.action_name expected_action_text = event.action_text expected_action = expected_action_name or expected_action_text policy_entity_result = None prev_action_unlikely_intent = False try: predicted_action, prediction, policy_entity_result = _run_action_prediction( processor, partial_tracker, expected_action) except ActionLimitReached: prediction = PolicyPrediction([], policy_name=None) predicted_action = "circuit breaker tripped" predicted_action_unlikely_intent = predicted_action == ACTION_UNLIKELY_INTENT_NAME if predicted_action_unlikely_intent and predicted_action != expected_action: partial_tracker.update( WronglyPredictedAction( predicted_action, expected_action_text, predicted_action, prediction.policy_name, prediction.max_confidence, event.timestamp, metadata=prediction.action_metadata, )) prev_action_unlikely_intent = True try: predicted_action, prediction, policy_entity_result = _run_action_prediction( processor, partial_tracker, expected_action) except ActionLimitReached: prediction = PolicyPrediction([], policy_name=None) predicted_action = "circuit breaker tripped" action_executed_eval_store.add_to_store( action_predictions=[predicted_action], action_targets=[expected_action]) if action_executed_eval_store.has_prediction_target_mismatch(): partial_tracker.update( WronglyPredictedAction( expected_action_name, expected_action_text, predicted_action, prediction.policy_name, prediction.max_confidence, event.timestamp, metadata=prediction.action_metadata, predicted_action_unlikely_intent=prev_action_unlikely_intent, )) if (fail_on_prediction_errors and predicted_action != ACTION_UNLIKELY_INTENT_NAME and predicted_action != expected_action): story_dump = YAMLStoryWriter().dumps( partial_tracker.as_story().story_steps) error_msg = (f"Model predicted a wrong action. Failed Story: " f"\n\n{story_dump}") raise WrongPredictionException(error_msg) elif prev_action_unlikely_intent: partial_tracker.update( WarningPredictedAction( ACTION_UNLIKELY_INTENT_NAME, predicted_action, prediction.policy_name, prediction.max_confidence, event.timestamp, prediction.action_metadata, )) else: partial_tracker.update( ActionExecuted( predicted_action, prediction.policy_name, prediction.max_confidence, event.timestamp, metadata=prediction.action_metadata, )) return action_executed_eval_store, prediction, policy_entity_result
def test_revert_user_utterance_event(default_domain: Domain): tracker = DialogueStateTracker("default", default_domain.slots) # the retrieved tracker should be empty assert len(tracker.events) == 0 intent1 = {"name": "greet", "confidence": 1.0} tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) tracker.update(UserUttered("/greet", intent1, [])) tracker.update(ActionExecuted("my_action_1")) tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) intent2 = {"name": "goodbye", "confidence": 1.0} tracker.update(UserUttered("/goodbye", intent2, [])) tracker.update(ActionExecuted("my_action_2")) tracker.update(ActionExecuted(ACTION_LISTEN_NAME)) # Expecting count of 6: # +5 executed actions # +1 final state assert tracker.latest_action.get(ACTION_NAME) == ACTION_LISTEN_NAME assert len(list(tracker.generate_all_prior_trackers())) == 6 tracker.update(UserUtteranceReverted()) # Expecting count of 3: # +5 executed actions # +1 final state # -2 rewound actions associated with the /goodbye # -1 rewound action from the listen right before /goodbye assert tracker.latest_action.get(ACTION_NAME) == "my_action_1" assert len(list(tracker.generate_all_prior_trackers())) == 3 dialogue = tracker.as_dialogue() recovered = DialogueStateTracker("default", default_domain.slots) recovered.recreate_from_dialogue(dialogue) assert recovered.current_state() == tracker.current_state() assert tracker.latest_action.get(ACTION_NAME) == "my_action_1" assert len(list(tracker.generate_all_prior_trackers())) == 3