[ SlotSet("num_people", None), ActiveLoop(None), SlotSet("num_tables", 5), SlotSet(REQUESTED_SLOT, None), ActiveLoop(None), ], ), # User rejected manually ( [{ "event": "action_execution_rejected", "name": "my form" }], [ ActionExecutionRejected("my form"), SlotSet("num_tables", 5), SlotSet("num_people", "hi"), SlotSet(REQUESTED_SLOT, None), ], ), ], ) async def test_validate_slots(validate_return_events: List[Dict], expected_events: List[Event]): form_name = "my form" slot_name = "num_people" slot_value = "hi" events = [ ActiveLoop(form_name), SlotSet(REQUESTED_SLOT, slot_name),
@pytest.mark.parametrize( "events, expected_applied_events", [ ( [ # Form is triggered and requests slot. ActionExecuted(ACTION_LISTEN_NAME), user_uttered("greet"), ActionExecuted("loop"), ActiveLoop("loop"), SlotSet(REQUESTED_SLOT, "bla"), # User sends chitchat instead of answering form. ActionExecuted(ACTION_LISTEN_NAME), user_uttered("chitchat"), # Form rejected execution. ActionExecutionRejected("loop"), # Action which deals with unhappy path. ActionExecuted("handling chitchat"), # We immediately return to form after executing an action to handle it. ActionExecuted("loop"), # Form happy path continues until all slots are filled. SlotSet(REQUESTED_SLOT, "bla"), ActionExecuted(ACTION_LISTEN_NAME), user_uttered("fill slots"), ActionExecuted("loop"), SlotSet("slot", "value"), SlotSet(REQUESTED_SLOT, None), ActiveLoop(None), ], [ ActionExecuted(ACTION_LISTEN_NAME),
async def test_persist_legacy_form_story(): domain = Domain.load("data/test_domains/form.yml") tracker = DialogueStateTracker("", domain.slots) story = ("* greet\n" " - utter_greet\n" "* start_form\n" " - some_form\n" ' - form{"name": "some_form"}\n' "* default\n" " - utter_default\n" " - some_form\n" "* stop\n" " - utter_ask_continue\n" "* affirm\n" " - some_form\n" "* stop\n" " - utter_ask_continue\n" "* inform\n" " - some_form\n" ' - form{"name": null}\n' "* goodbye\n" " - utter_goodbye\n") # simulate talking to the form events = [ UserUttered(intent={"name": "greet"}), ActionExecuted("utter_greet"), ActionExecuted("action_listen"), # start the form UserUttered(intent={"name": "start_form"}), ActionExecuted("some_form"), ActiveLoop("some_form"), ActionExecuted("action_listen"), # out of form input UserUttered(intent={"name": "default"}), ActionExecutionRejected("some_form"), ActionExecuted("utter_default"), ActionExecuted("some_form"), ActionExecuted("action_listen"), # out of form input UserUttered(intent={"name": "stop"}), ActionExecutionRejected("some_form"), ActionExecuted("utter_ask_continue"), ActionExecuted("action_listen"), # out of form input but continue with the form UserUttered(intent={"name": "affirm"}), LoopInterrupted(True), ActionExecuted("some_form"), ActionExecuted("action_listen"), # out of form input UserUttered(intent={"name": "stop"}), ActionExecutionRejected("some_form"), ActionExecuted("utter_ask_continue"), ActionExecuted("action_listen"), # form input UserUttered(intent={"name": "inform"}), LoopInterrupted(False), ActionExecuted("some_form"), ActionExecuted("action_listen"), ActiveLoop(None), UserUttered(intent={"name": "goodbye"}), ActionExecuted("utter_goodbye"), ActionExecuted("action_listen"), ] [tracker.update(e) for e in events] story = story.replace(f"- {LegacyForm.type_name}", f"- {ActiveLoop.type_name}") assert story in tracker.export_stories()
async def test_form_unhappy_path_no_validation_from_story(): form_name = "some_form" handle_rejection_action_name = "utter_handle_rejection" domain = Domain.from_yaml(f""" intents: - {GREET_INTENT_NAME} actions: - {UTTER_GREET_ACTION} - {handle_rejection_action_name} - some-action slots: {REQUESTED_SLOT}: type: unfeaturized forms: - {form_name} """) unhappy_story = TrackerWithCachedStates.from_events( "bla", domain=domain, slots=domain.slots, evts=[ # We are in an active form ActionExecuted(form_name), ActiveLoop(form_name), ActionExecuted(ACTION_LISTEN_NAME), # When a user says "hi", and the form is unhappy, # we want to run a specific action UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(handle_rejection_action_name), ActionExecuted(ACTION_LISTEN_NAME), # Next user utterance is an answer to the previous question # and shouldn't be validated by the form UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(form_name), ActionExecuted(ACTION_LISTEN_NAME), ], ) policy = RulePolicy() policy.train([unhappy_story], domain, RegexInterpreter()) # Check that RulePolicy predicts no validation to handle unhappy path conversation_events = [ ActionExecuted(form_name), ActiveLoop(form_name), SlotSet(REQUESTED_SLOT, "some value"), ActionExecuted(ACTION_LISTEN_NAME), UserUttered("haha", {"name": GREET_INTENT_NAME}), ActionExecutionRejected(form_name), ActionExecuted(handle_rejection_action_name), ActionExecuted(ACTION_LISTEN_NAME), UserUttered("haha", {"name": GREET_INTENT_NAME}), ] tracker = DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots) action_probabilities = policy.predict_action_probabilities( tracker, domain, RegexInterpreter()) # there is no rule for next action assert max(action_probabilities) == policy._core_fallback_threshold # check that RulePolicy entered unhappy path based on the training story assert tracker.events[-1] == LoopInterrupted(True)
assert action_received_events tracker = default_processor.get_tracker(conversation_id) # The action was logged on the tracker as well expected_events.append(ActionExecuted(ACTION_LISTEN_NAME)) for event, expected in zip(tracker.events, expected_events): assert event == expected # noinspection PyTypeChecker @pytest.mark.parametrize( "reject_fn", [ lambda: [ActionExecutionRejected(ACTION_LISTEN_NAME)], lambda: (_ for _ in ()).throw(ActionExecutionRejection(ACTION_LISTEN_NAME)), ], ) async def test_policy_events_not_applied_if_rejected( default_processor: MessageProcessor, monkeypatch: MonkeyPatch, reject_fn: Callable[[], List[Event]], ): expected_action = ACTION_LISTEN_NAME expected_events = [LoopInterrupted(True)] conversation_id = "test_policy_events_are_applied_to_tracker" user_message = "/greet" class ConstantEnsemble(PolicyEnsemble): def probabilities_using_best_policy(
async def test_form_unhappy_path_from_story(): form_name = "some_form" handle_rejection_action_name = "utter_handle_rejection" domain = Domain.from_yaml(f""" intents: - {GREET_INTENT_NAME} actions: - {UTTER_GREET_ACTION} - {handle_rejection_action_name} - some-action slots: {REQUESTED_SLOT}: type: unfeaturized forms: - {form_name} """) unhappy_story = TrackerWithCachedStates.from_events( "bla", domain=domain, slots=domain.slots, evts=[ # We are in an active form ActionExecuted(form_name), ActiveLoop(form_name), ActionExecuted(ACTION_LISTEN_NAME), # in training stories there is either intent or text, never both UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(UTTER_GREET_ACTION), # After our bot says "hi", we want to run a specific action ActionExecuted(handle_rejection_action_name), ActionExecuted(form_name), ActionExecuted(ACTION_LISTEN_NAME), ], ) policy = RulePolicy() policy.train([GREET_RULE, unhappy_story], domain, RegexInterpreter()) # Check that RulePolicy predicts action to handle unhappy path conversation_events = [ ActionExecuted(form_name), ActiveLoop(form_name), SlotSet(REQUESTED_SLOT, "some value"), ActionExecuted(ACTION_LISTEN_NAME), UserUttered("haha", {"name": GREET_INTENT_NAME}), ActionExecutionRejected(form_name), ] action_probabilities = policy.predict_action_probabilities( DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots), domain, RegexInterpreter(), ) assert_predicted_action(action_probabilities, domain, UTTER_GREET_ACTION) # Check that RulePolicy doesn't trigger form or action_listen # after handling unhappy path conversation_events.append(ActionExecuted(handle_rejection_action_name)) action_probabilities = policy.predict_action_probabilities( DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots), domain, RegexInterpreter(), ) assert max(action_probabilities) == policy._core_fallback_threshold
async def test_form_unhappy_path_no_validation_from_rule(): form_name = "some_form" handle_rejection_action_name = "utter_handle_rejection" domain = Domain.from_yaml(f""" intents: - {GREET_INTENT_NAME} actions: - {UTTER_GREET_ACTION} - {handle_rejection_action_name} - some-action slots: {REQUESTED_SLOT}: type: unfeaturized forms: - {form_name} """) unhappy_rule = TrackerWithCachedStates.from_events( "bla", domain=domain, slots=domain.slots, evts=[ # We are in an active form ActiveLoop(form_name), SlotSet(REQUESTED_SLOT, "bla"), ActionExecuted(RULE_SNIPPET_ACTION_NAME), ActionExecuted(ACTION_LISTEN_NAME), # When a user says "hi", and the form is unhappy, # we want to run a specific action UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(handle_rejection_action_name), # Next user utterance is an answer to the previous question # and shouldn't be validated by the form ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(form_name), ActionExecuted(ACTION_LISTEN_NAME), ], is_rule_tracker=True, ) # unhappy rule is multi user turn rule, therefore remove restriction for policy policy = RulePolicy(restrict_rules=False) # RulePolicy should memorize that unhappy_rule overrides GREET_RULE policy.train([GREET_RULE, unhappy_rule], domain, RegexInterpreter()) # Check that RulePolicy predicts action to handle unhappy path conversation_events = [ ActionExecuted(form_name), ActiveLoop(form_name), SlotSet(REQUESTED_SLOT, "some value"), ActionExecuted(ACTION_LISTEN_NAME), UserUttered("haha", {"name": GREET_INTENT_NAME}), ActionExecutionRejected(form_name), ] action_probabilities = policy.predict_action_probabilities( DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots), domain, RegexInterpreter(), ) assert_predicted_action(action_probabilities, domain, handle_rejection_action_name) # Check that RulePolicy predicts action_listen conversation_events.append(ActionExecuted(handle_rejection_action_name)) action_probabilities = policy.predict_action_probabilities( DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots), domain, RegexInterpreter(), ) assert_predicted_action(action_probabilities, domain, ACTION_LISTEN_NAME) # Check that RulePolicy triggers form again after handling unhappy path conversation_events.append(ActionExecuted(ACTION_LISTEN_NAME)) tracker = DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots) action_probabilities = policy.predict_action_probabilities( tracker, domain, RegexInterpreter()) assert_predicted_action(action_probabilities, domain, form_name) # check that RulePolicy entered unhappy path based on the training story assert tracker.events[-1] == LoopInterrupted(True)
}, { "entity": "count", "value": 1 }, ], timestamp=None, ), DefinePrevUserUtteredFeaturization(use_text_for_featurization=False, timestamp=None, metadata=None), ReminderCancelled(timestamp=1621590172.3872123), ReminderScheduled(timestamp=None, trigger_date_time=datetime.now(), intent="greet"), ActionExecutionRejected(action_name="my_action"), LegacyFormValidation(validate=True, timestamp=None), LoopInterrupted(timestamp=None, is_interrupted=False), ActiveLoop(name="loop"), LegacyForm(name="my_form"), AllSlotsReset(), SlotSet(key="my_slot", value={}), SlotSet(key="my slot", value=[]), SlotSet(key="test", value=1), SlotSet(key="test", value="text"), ConversationResumed(), ConversationPaused(), FollowupAction(name="test"), StoryExported(), Restarted(), ActionReverted(),
async def test_form_unhappy_path_from_in_form_rule(): form_name = "some_form" handle_rejection_action_name = "utter_handle_rejection" domain = Domain.from_yaml(f""" intents: - {GREET_INTENT_NAME} actions: - {UTTER_GREET_ACTION} - {handle_rejection_action_name} - some-action slots: {REQUESTED_SLOT}: type: unfeaturized forms: - {form_name} """) unhappy_rule = TrackerWithCachedStates.from_events( "bla", domain=domain, slots=domain.slots, evts=[ # We are in an active form ActiveLoop(form_name), SlotSet(REQUESTED_SLOT, "bla"), ActionExecuted(RULE_SNIPPET_ACTION_NAME), ActionExecuted(ACTION_LISTEN_NAME), # When a user says "hi", and the form is unhappy, # we want to run a specific action UserUttered(intent={"name": GREET_INTENT_NAME}), ActionExecuted(handle_rejection_action_name), ActionExecuted(form_name), ActionExecuted(ACTION_LISTEN_NAME), ], is_rule_tracker=True, ) policy = RulePolicy() # RulePolicy should memorize that unhappy_rule overrides GREET_RULE policy.train([GREET_RULE, unhappy_rule], domain, RegexInterpreter()) # Check that RulePolicy predicts action to handle unhappy path conversation_events = [ ActionExecuted(form_name), ActiveLoop(form_name), SlotSet(REQUESTED_SLOT, "some value"), ActionExecuted(ACTION_LISTEN_NAME), UserUttered("haha", {"name": GREET_INTENT_NAME}), ActionExecutionRejected(form_name), ] action_probabilities = policy.predict_action_probabilities( DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots), domain, RegexInterpreter(), ) assert_predicted_action(action_probabilities, domain, handle_rejection_action_name) # Check that RulePolicy triggers form again after handling unhappy path conversation_events.append(ActionExecuted(handle_rejection_action_name)) action_probabilities = policy.predict_action_probabilities( DialogueStateTracker.from_events("casd", evts=conversation_events, slots=domain.slots), domain, RegexInterpreter(), ) assert_predicted_action(action_probabilities, domain, form_name)
class TestUnexpecTEDIntentPolicy(TestTEDPolicy): @staticmethod def _policy_class_to_test() -> Type[UnexpecTEDIntentPolicy]: return UnexpecTEDIntentPolicy @pytest.fixture(scope="class") def featurizer(self) -> TrackerFeaturizer: featurizer = IntentMaxHistoryTrackerFeaturizer( IntentTokenizerSingleStateFeaturizer(), max_history=self.max_history ) return featurizer @staticmethod def persist_and_load_policy( trained_policy: UnexpecTEDIntentPolicy, model_storage: ModelStorage, resource: Resource, execution_context: ExecutionContext, ): return trained_policy.__class__.load( trained_policy.config, model_storage, resource, execution_context ) def test_ranking_length(self, trained_policy: UnexpecTEDIntentPolicy): assert trained_policy.config[RANKING_LENGTH] == LABEL_RANKING_LENGTH def test_ranking_length_and_renormalization( self, trained_policy: UnexpecTEDIntentPolicy, tracker: DialogueStateTracker, default_domain: Domain, ): precomputations = None prediction_metadata = trained_policy.predict_action_probabilities( tracker, default_domain, precomputations, ).action_metadata assert ( prediction_metadata is None or len(prediction_metadata[RANKING_KEY]) == trained_policy.config[RANKING_LENGTH] ) def test_label_data_assembly( self, trained_policy: UnexpecTEDIntentPolicy, default_domain: Domain ): # Construct input data state_featurizer = trained_policy.featurizer.state_featurizer encoded_all_labels = state_featurizer.encode_all_labels( default_domain, precomputations=None ) attribute_data, _ = model_data_utils.convert_to_data_format(encoded_all_labels) assembled_label_data = trained_policy._assemble_label_data( attribute_data, default_domain ) assembled_label_data_signature = assembled_label_data.get_signature() assert list(assembled_label_data_signature.keys()) == [ f"{LABEL}_{INTENT}", LABEL, ] assert assembled_label_data.num_examples == len(default_domain.intents) assert list(assembled_label_data_signature[f"{LABEL}_{INTENT}"].keys()) == [ MASK, SENTENCE, ] assert list(assembled_label_data_signature[LABEL].keys()) == [IDS] assert assembled_label_data_signature[f"{LABEL}_{INTENT}"][SENTENCE][ 0 ].units == len(default_domain.intents) def test_training_with_no_intent( self, featurizer: Optional[TrackerFeaturizer], default_domain: Domain, tmp_path: Path, caplog: LogCaptureFixture, model_storage: ModelStorage, resource: Resource, execution_context: ExecutionContext, ): stories = tmp_path / "stories.yml" stories.write_text( """ version: "3.0" stories: - story: test path steps: - action: utter_greet """ ) policy = self.create_policy( featurizer=featurizer, model_storage=model_storage, resource=resource, execution_context=execution_context, ) import tests.core.test_policies training_trackers = tests.core.test_policies.train_trackers( default_domain, str(stories), augmentation_factor=20 ) with pytest.warns(UserWarning): policy.train(training_trackers, default_domain, precomputations=None) def test_prepared_data_for_threshold_prediction( self, trained_policy: UnexpecTEDIntentPolicy, default_domain: Domain, stories_path: Path, ): training_trackers = train_trackers( default_domain, stories_path, augmentation_factor=0 ) training_model_data, _ = trained_policy._prepare_for_training( training_trackers, default_domain, precomputations=None, ) data_for_prediction = trained_policy._prepare_data_for_prediction( training_model_data ) assert set(data_for_prediction.data.keys()).issubset(PREDICTION_FEATURES) def test_similarities_collection_for_label_ids(self): label_ids = np.array([[0, 1], [1, -1], [2, -1]]) outputs = { "similarities": np.array( [[[1.2, 0.3, 0.2]], [[0.5, 0.2, 1.6]], [[0.01, 0.1, 1.7]],] ) } label_id_similarities = UnexpecTEDIntentPolicy._collect_label_id_grouped_scores( outputs, label_ids ) # Should contain similarities for all label ids except padding token. assert sorted(list(label_id_similarities.keys())) == [0, 1, 2] # Cross-check that the collected similarities are correct for each label id. assert label_id_similarities[0] == { POSITIVE_SCORES_KEY: [1.2], NEGATIVE_SCORES_KEY: [0.5, 0.01], } assert label_id_similarities[1] == { POSITIVE_SCORES_KEY: [0.3, 0.2], NEGATIVE_SCORES_KEY: [0.1], } assert label_id_similarities[2] == { POSITIVE_SCORES_KEY: [1.7], NEGATIVE_SCORES_KEY: [0.2, 1.6], } def test_label_quantiles_computation(self): label_id_scores = { 0: { POSITIVE_SCORES_KEY: [1.3, 0.2], NEGATIVE_SCORES_KEY: [ -0.1, -1.2, -2.3, -4.1, -0.5, 0.2, 0.8, 0.9, -3.2, -2.7, ], }, 3: {POSITIVE_SCORES_KEY: [1.3, 0.2], NEGATIVE_SCORES_KEY: [-0.1]}, 6: {POSITIVE_SCORES_KEY: [1.3, 0.2], NEGATIVE_SCORES_KEY: []}, } expected_thresholds = { 0: [ 0.2, 0.2, 0.2, 0.2, 0.2, -0.1, -0.1, -0.5, -0.5, -1.2, -1.2, -1.2, -2.3, -2.3, -2.7, -2.7, -3.2, -3.2, -4.1, -4.1, ], 3: [ -0.1, -0.1, -0.1, -0.1, -0.1, -0.1, -0.1, -0.1, -0.1, -0.1, -0.1, -0.1, -0.1, -0.1, -0.1, -0.1, -0.1, -0.1, -0.1, -0.1, ], 6: [ 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, ], } thresholds = UnexpecTEDIntentPolicy._compute_label_quantiles(label_id_scores) assert sorted(list(thresholds.keys())) == sorted( list(expected_thresholds.keys()) ) for label_id, tolerance_thresholds in thresholds.items(): assert expected_thresholds[label_id] == tolerance_thresholds def test_post_training_threshold_computation( self, trained_policy: UnexpecTEDIntentPolicy, default_domain: Domain, stories_path: Path, ): training_trackers = train_trackers( default_domain, stories_path, augmentation_factor=0 ) training_model_data, label_ids = trained_policy._prepare_for_training( training_trackers, default_domain, precomputations=None, ) trained_policy.compute_label_quantiles_post_training( training_model_data, label_ids ) computed_thresholds = trained_policy.label_quantiles # -1 is used for padding and hence is not expected in the keys expected_keys = list(np.unique(label_ids)) expected_keys.remove(-1) assert sorted(list(computed_thresholds.keys())) == sorted(expected_keys) @pytest.mark.parametrize( "tolerance, expected_thresholds", [ (0.0, [0.2, -0.1, 0.2]), (0.75, [-2.9, -0.1, -4.3]), (0.72, [-2.7, -0.1, -4.0]), (0.78, [-2.9, -0.1, -4.3]), (1.0, [-4.1, -0.1, -5.5]), ], ) def test_pick_thresholds_for_labels( self, tolerance: float, expected_thresholds: List[float] ): label_id_tolerance_thresholds = { 0: [ 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, -0.1, -0.1, -0.5, -0.5, -1.2, -1.2, -2.3, -2.3, -2.7, -2.9, -3.2, -3.2, -4.1, -4.1, ], 3: [ -0.1, -0.1, -0.1, -0.1, -0.1, -0.1, -0.1, -0.1, -0.1, -0.1, -0.1, -0.1, -0.1, -0.1, -0.1, -0.1, -0.1, -0.1, -0.1, -0.1, ], 4: [0.2 - (index * 0.3) for index in range(20)], } thresholds = UnexpecTEDIntentPolicy._pick_thresholds( label_id_tolerance_thresholds, tolerance ) assert sorted(list(thresholds.keys())) == sorted( list(label_id_tolerance_thresholds.keys()) ) computed_values = list(thresholds.values()) assert expected_thresholds == computed_values @pytest.mark.parametrize( "predicted_similarity, threshold_value, is_unlikely", [(1.2, 0.2, False), (0.3, -0.1, False), (-1.5, 0.03, True)], ) def test_unlikely_intent_check( self, trained_policy: UnexpecTEDIntentPolicy, model_storage: ModelStorage, resource: Resource, execution_context: ExecutionContext, default_domain: Domain, predicted_similarity: float, threshold_value: float, is_unlikely: bool, tmp_path: Path, ): loaded_policy = self.persist_and_load_policy( trained_policy, model_storage, resource, execution_context ) # Construct dummy similarities similarities = np.array([[0.0] * len(default_domain.intents)]) dummy_intent_index = 4 similarities[0, dummy_intent_index] = predicted_similarity loaded_policy.label_thresholds[dummy_intent_index] = threshold_value query_intent = default_domain.intents[dummy_intent_index] unlikely_intent_prediction = loaded_policy._check_unlikely_intent( default_domain, similarities, query_intent ) assert is_unlikely == unlikely_intent_prediction def test_should_check_for_intent( self, trained_policy: UnexpecTEDIntentPolicy, model_storage: ModelStorage, resource: Resource, execution_context: ExecutionContext, default_domain: Domain, tmp_path: Path, ): loaded_policy = self.persist_and_load_policy( trained_policy, model_storage, resource, execution_context ) intent_index = 0 assert ( loaded_policy._should_check_for_intent( default_domain.intents[intent_index], default_domain ) is False ) intent_index = 4 assert loaded_policy._should_check_for_intent( default_domain.intents[intent_index], default_domain ) loaded_policy.config[IGNORE_INTENTS_LIST] = [ default_domain.intents[intent_index] ] assert ( loaded_policy._should_check_for_intent( default_domain.intents[intent_index], default_domain ) is False ) def test_no_action_unlikely_intent_prediction( self, trained_policy: UnexpecTEDIntentPolicy, model_storage: ModelStorage, resource: Resource, execution_context: ExecutionContext, default_domain: Domain, tmp_path: Path, ): loaded_policy = self.persist_and_load_policy( trained_policy, model_storage, resource, execution_context ) expected_probabilities = [0] * default_domain.num_actions precomputations = None tracker = DialogueStateTracker(sender_id="init", slots=default_domain.slots) prediction = loaded_policy.predict_action_probabilities( tracker, default_domain, precomputations ) assert prediction.probabilities == expected_probabilities tracker.update_with_events( [ UserUttered(text="hello", intent={"name": "greet"}), ActionExecuted(action_name="utter_greet"), ], default_domain, ) prediction = loaded_policy.predict_action_probabilities( tracker, default_domain, precomputations ) assert prediction.probabilities == expected_probabilities loaded_policy.model = None prediction = loaded_policy.predict_action_probabilities( tracker, default_domain, precomputations ) assert prediction.probabilities == expected_probabilities @pytest.mark.parametrize( "predicted_similarity, threshold_value, is_unlikely", [(1.2, 0.2, False), (0.3, -0.1, False), (-1.5, 0.03, True)], ) def test_action_unlikely_intent_prediction( self, trained_policy: UnexpecTEDIntentPolicy, model_storage: ModelStorage, resource: Resource, execution_context: ExecutionContext, default_domain: Domain, predicted_similarity: float, threshold_value: float, is_unlikely: bool, monkeypatch: MonkeyPatch, tmp_path: Path, ): loaded_policy = self.persist_and_load_policy( trained_policy, model_storage, resource, execution_context ) similarities = np.array([[[0.0] * len(default_domain.intents)]]) dummy_intent_index = 4 similarities[0, 0, dummy_intent_index] = predicted_similarity query_intent = default_domain.intents[dummy_intent_index] loaded_policy.label_thresholds[dummy_intent_index] = threshold_value precomputations = None tracker = DialogueStateTracker(sender_id="init", slots=default_domain.slots) tracker.update_with_events( [UserUttered(text="hello", intent={"name": query_intent})], default_domain, ) # Preset the model predictions to the similarity values # so that we don't need to hardcode for particular model predictions. monkeypatch.setattr( loaded_policy.model, "run_inference", lambda data: {"similarities": similarities}, ) prediction = loaded_policy.predict_action_probabilities( tracker, default_domain, precomputations ) if not is_unlikely: assert prediction.probabilities == [0.0] * default_domain.num_actions else: assert ( prediction.probabilities[ default_domain.index_for_action(ACTION_UNLIKELY_INTENT_NAME) ] == 1.0 ) # Make sure metadata is set. The exact structure # of the metadata is tested separately and # not as part of this test. assert prediction.action_metadata is not None # Assert metadata is serializable assert json.dumps(prediction.action_metadata) @pytest.mark.parametrize( "tracker_events, should_skip", [ ([], True), ([ActionExecuted("action_listen")], True), ( [ ActionExecuted("action_listen"), UserUttered("hi", intent={"name": "greet"}), ], False, ), ( [ ActionExecuted("action_listen"), UserUttered("hi", intent={"name": "greet"}), EntitiesAdded([{"name": "dummy"}]), ], False, ), ( [ ActionExecuted("action_listen"), UserUttered("hi", intent={"name": "greet"}), SlotSet("name"), ], False, ), ( [ ActiveLoop("loop"), ActionExecuted("action_listen"), UserUttered("hi", intent={"name": "greet"}), ActionExecutionRejected("loop"), ], False, ), ( [ ActionExecuted("action_listen"), UserUttered("hi", intent={"name": "greet"}), ActionExecuted("utter_greet"), ], True, ), ], ) def test_skip_predictions_to_prevent_loop( self, trained_policy: UnexpecTEDIntentPolicy, model_storage: ModelStorage, resource: Resource, execution_context: ExecutionContext, default_domain: Domain, caplog: LogCaptureFixture, tracker_events: List[Event], should_skip: bool, tmp_path: Path, ): """Skips predictions to prevent loop.""" loaded_policy = self.persist_and_load_policy( trained_policy, model_storage, resource, execution_context ) precomputations = None tracker = DialogueStateTracker(sender_id="init", slots=default_domain.slots) tracker.update_with_events(tracker_events, default_domain) with caplog.at_level(logging.DEBUG): prediction = loaded_policy.predict_action_probabilities( tracker, default_domain, precomputations ) assert ( "Skipping predictions for UnexpecTEDIntentPolicy" in caplog.text ) == should_skip if should_skip: assert prediction.probabilities == loaded_policy._default_predictions( default_domain ) @pytest.mark.parametrize( "tracker_events", [ [ ActionExecuted("action_listen"), UserUttered("hi", intent={"name": "inexistent_intent"}), ], [ ActionExecuted("action_listen"), UserUttered("hi", intent={"name": "inexistent_intent"}), EntitiesAdded([{"name": "dummy"}]), ], [ ActionExecuted("action_listen"), UserUttered("hi", intent={"name": "inexistent_intent"}), SlotSet("name"), ], [ ActiveLoop("loop"), ActionExecuted("action_listen"), UserUttered("hi", intent={"name": "inexistent_intent"}), ActionExecutionRejected("loop"), ], ], ) def test_skip_predictions_if_new_intent( self, trained_policy: UnexpecTEDIntentPolicy, model_storage: ModelStorage, resource: Resource, execution_context: ExecutionContext, default_domain: Domain, caplog: LogCaptureFixture, tracker_events: List[Event], ): """Skips predictions if there's a new intent created.""" loaded_policy = self.persist_and_load_policy( trained_policy, model_storage, resource, execution_context ) tracker = DialogueStateTracker(sender_id="init", slots=default_domain.slots) tracker.update_with_events(tracker_events, default_domain) with caplog.at_level(logging.DEBUG): prediction = loaded_policy.predict_action_probabilities( tracker, default_domain, precomputations=None, ) assert "Skipping predictions for UnexpecTEDIntentPolicy" in caplog.text assert prediction.probabilities == loaded_policy._default_predictions( default_domain ) @pytest.mark.parametrize( "tracker_events_with_action, tracker_events_without_action", [ ( [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(text="hello", intent={"name": "greet"}), ActionExecuted(ACTION_UNLIKELY_INTENT_NAME), ActionExecuted("utter_greet"), UserUttered(text="sad", intent={"name": "thank_you"}), ], [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(text="hello", intent={"name": "greet"}), ActionExecuted("utter_greet"), UserUttered(text="sad", intent={"name": "thank_you"}), ], ), ( [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(text="hello", intent={"name": "greet"}), EntitiesAdded(entities=[{"entity": "name", "value": "Peter"},]), ActionExecuted(ACTION_UNLIKELY_INTENT_NAME), ActionExecuted("utter_greet"), UserUttered(text="sad", intent={"name": "thank_you"}), ], [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(text="hello", intent={"name": "greet"}), EntitiesAdded(entities=[{"entity": "name", "value": "Peter"},]), ActionExecuted("utter_greet"), UserUttered(text="sad", intent={"name": "thank_you"}), ], ), ( [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(text="hello", intent={"name": "greet"}), ActionExecuted(ACTION_UNLIKELY_INTENT_NAME), ActionExecuted("some_form"), ActiveLoop("some_form"), ActionExecuted(ACTION_LISTEN_NAME), UserUttered(text="default", intent={"name": "default"}), ActionExecuted(ACTION_UNLIKELY_INTENT_NAME), UserUttered(text="sad", intent={"name": "thank_you"}), ], [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(text="hello", intent={"name": "greet"}), ActionExecuted(ACTION_UNLIKELY_INTENT_NAME), ActionExecuted("some_form"), ActiveLoop("some_form"), ActionExecuted(ACTION_LISTEN_NAME), UserUttered(text="default", intent={"name": "default"}), UserUttered(text="sad", intent={"name": "thank_you"}), ], ), ], ) def test_ignore_action_unlikely_intent( self, trained_policy: UnexpecTEDIntentPolicy, model_storage: ModelStorage, resource: Resource, execution_context: ExecutionContext, default_domain: Domain, tracker_events_with_action: List[Event], tracker_events_without_action: List[Event], tmp_path: Path, ): loaded_policy = self.persist_and_load_policy( trained_policy, model_storage, resource, execution_context ) precomputations = None tracker_with_action = DialogueStateTracker.from_events( "test 1", evts=tracker_events_with_action ) tracker_without_action = DialogueStateTracker.from_events( "test 2", evts=tracker_events_without_action ) prediction_with_action = loaded_policy.predict_action_probabilities( tracker_with_action, default_domain, precomputations ) prediction_without_action = loaded_policy.predict_action_probabilities( tracker_without_action, default_domain, precomputations ) # If the weights didn't change then both trackers # should result in same prediction. For `UnexpecTEDIntentPolicy`, the real # prediction is inside action metadata. assert ( prediction_with_action.action_metadata == prediction_without_action.action_metadata ) def test_label_embedding_collection(self, trained_policy: UnexpecTEDIntentPolicy): label_ids = tf.constant([[[2], [-1]], [[1], [2]], [[0], [-1]]], dtype=tf.int32) all_label_embeddings = np.random.random((10, 20)) # `-1` is used as padding label id. The embedding for it # will be the same as `label_id=0` expected_extracted_label_embeddings = tf.constant( np.concatenate( [ all_label_embeddings[2], all_label_embeddings[0], all_label_embeddings[1], all_label_embeddings[2], all_label_embeddings[0], all_label_embeddings[0], ] ).reshape((3, 2, 20)), dtype=tf.float32, ) actual_extracted_label_embeddings = trained_policy.model._get_labels_embed( label_ids, tf.constant(all_label_embeddings, dtype=tf.float32) ) assert np.all( expected_extracted_label_embeddings == actual_extracted_label_embeddings ) @pytest.mark.parametrize( "query_intent_index, ranking_length", [(0, 0), (1, 3), (2, 1), (5, 0)] ) def test_collect_action_metadata( self, trained_policy: UnexpecTEDIntentPolicy, model_storage: ModelStorage, resource: Resource, execution_context: ExecutionContext, default_domain: Domain, tmp_path: Path, query_intent_index: int, ranking_length: int, ): loaded_policy = self.persist_and_load_policy( trained_policy, model_storage, resource, execution_context ) def test_individual_label_metadata( label_metadata: Dict[Text, Optional[float]], all_thresholds: Dict[int, float], all_similarities: np.array, label_index: int, ): expected_score = all_similarities[0][label_index] expected_threshold = ( all_thresholds[label_index] if label_index in all_thresholds else None ) expected_severity = ( expected_threshold - expected_score if expected_threshold else None ) assert label_metadata.get(SCORE_KEY) == expected_score assert label_metadata.get(THRESHOLD_KEY) == expected_threshold assert label_metadata.get(SEVERITY_KEY) == expected_severity # Monkey-patch certain attributes of the policy to make the testing easier. label_thresholds = {0: 1.2, 1: -0.3, 4: -2.3, 5: 0.2} loaded_policy.label_thresholds = label_thresholds loaded_policy.config[RANKING_LENGTH] = ranking_length # Some dummy similarities similarities = np.array([[3.2, 0.2, -1.2, -4.3, -5.1, 2.3]]) query_intent = default_domain.intents[query_intent_index] metadata = loaded_policy._collect_action_metadata( default_domain, similarities, query_intent=query_intent ) # Expected outer-most keys assert sorted(list(metadata.keys())) == sorted([QUERY_INTENT_KEY, RANKING_KEY]) # Schema validation for query intent key assert sorted(list(metadata[QUERY_INTENT_KEY].keys())) == sorted( [NAME, SCORE_KEY, THRESHOLD_KEY, SEVERITY_KEY] ) # Test all elements of metadata for query intent assert metadata[QUERY_INTENT_KEY].get(NAME) == query_intent test_individual_label_metadata( metadata.get(QUERY_INTENT_KEY), label_thresholds, similarities, query_intent_index, ) # Check if ranking is sorted correctly and truncated to `ranking_length` sorted_label_similarities = sorted( [(index, score) for index, score in enumerate(similarities[0])], key=lambda x: -x[1], ) sorted_label_similarities = ( sorted_label_similarities[:ranking_length] if ranking_length else sorted_label_similarities ) expected_label_rankings = [ default_domain.intents[index] for index, _ in sorted_label_similarities ] collected_label_rankings = [ label_metadata.get(NAME) for label_metadata in metadata.get(RANKING_KEY) ] assert collected_label_rankings == expected_label_rankings # Test all elements of metadata for all labels in ranking for label_metadata in metadata.get(RANKING_KEY): label_index = default_domain.intents.index(label_metadata.get(NAME)) test_individual_label_metadata( label_metadata, label_thresholds, similarities, label_index ) @pytest.mark.parametrize( "tracker_events_for_training, expected_trackers_with_events", [ # Filter because of no intent and action name ( [ [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(text="hello", intent={"name": "greet"}), ActionExecuted("utter_greet"), ActionExecuted(ACTION_LISTEN_NAME), UserUttered( text="happy to make it work", intent={"name": "goodbye"} ), ActionExecuted("utter_goodbye"), ActionExecuted(ACTION_LISTEN_NAME), ], [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(text="hello"), ActionExecuted("utter_greet"), ActionExecuted(ACTION_LISTEN_NAME), UserUttered(text="happy to make it work"), ActionExecuted(action_text="Great!"), ActionExecuted(ACTION_LISTEN_NAME), ], ], [ [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(text="hello", intent={"name": "greet"}), ActionExecuted("utter_greet"), ActionExecuted(ACTION_LISTEN_NAME), UserUttered( text="happy to make it work", intent={"name": "goodbye"} ), ActionExecuted("utter_goodbye"), ActionExecuted(ACTION_LISTEN_NAME), ], ], ), # Filter because of no action name ( [ [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(text="hello", intent={"name": "greet"}), ActionExecuted("utter_greet"), ActionExecuted(ACTION_LISTEN_NAME), UserUttered( text="happy to make it work", intent={"name": "goodbye"} ), ActionExecuted("utter_goodbye"), ActionExecuted(ACTION_LISTEN_NAME), ], [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(text="hello"), ActionExecuted("utter_greet"), ActionExecuted(ACTION_LISTEN_NAME), UserUttered( text="happy to make it work", intent={"name": "goodbye"} ), ActionExecuted(action_text="Great!"), ActionExecuted(ACTION_LISTEN_NAME), ], ], [ [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(text="hello", intent={"name": "greet"}), ActionExecuted("utter_greet"), ActionExecuted(ACTION_LISTEN_NAME), UserUttered( text="happy to make it work", intent={"name": "goodbye"} ), ActionExecuted("utter_goodbye"), ActionExecuted(ACTION_LISTEN_NAME), ], ], ), # Filter because of no intent ( [ [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(text="hello", intent={"name": "greet"}), ActionExecuted("utter_greet"), ActionExecuted(ACTION_LISTEN_NAME), UserUttered( text="happy to make it work", intent={"name": "goodbye"} ), ActionExecuted("utter_goodbye"), ActionExecuted(ACTION_LISTEN_NAME), ], [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(text="hello"), ActionExecuted("utter_greet"), ActionExecuted(ACTION_LISTEN_NAME), UserUttered(text="happy to make it work"), ActionExecuted("utter_goodbye"), ActionExecuted(ACTION_LISTEN_NAME), ], ], [ [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(text="hello", intent={"name": "greet"}), ActionExecuted("utter_greet"), ActionExecuted(ACTION_LISTEN_NAME), UserUttered( text="happy to make it work", intent={"name": "goodbye"} ), ActionExecuted("utter_goodbye"), ActionExecuted(ACTION_LISTEN_NAME), ], ], ), # No filter needed ( [ [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(text="hello", intent={"name": "greet"}), ActionExecuted("utter_greet"), ActionExecuted(ACTION_LISTEN_NAME), UserUttered( text="happy to make it work", intent={"name": "goodbye"} ), ActionExecuted("utter_goodbye"), ActionExecuted(ACTION_LISTEN_NAME), ], ], [ [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(text="hello", intent={"name": "greet"}), ActionExecuted("utter_greet"), ActionExecuted(ACTION_LISTEN_NAME), UserUttered( text="happy to make it work", intent={"name": "goodbye"} ), ActionExecuted("utter_goodbye"), ActionExecuted(ACTION_LISTEN_NAME), ], ], ), # Filter to return empty list of trackers ( [ [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered(text="hello", intent={"name": "greet"}), ActionExecuted("utter_greet"), ActionExecuted(ACTION_LISTEN_NAME), UserUttered( text="happy to make it work", intent={"name": "goodbye"} ), ActionExecuted(action_text="Great!"), ActionExecuted(ACTION_LISTEN_NAME), ], ], [], ), ], ) def test_filter_training_trackers( self, tracker_events_for_training: List[List[Event]], expected_trackers_with_events: List[List[Event]], domain: Domain, ): trackers_for_training = [ TrackerWithCachedStates.from_events( sender_id=f"{tracker_index}", evts=events, domain=domain ) for tracker_index, events in enumerate(tracker_events_for_training) ] filtered_trackers = UnexpecTEDIntentPolicy._get_trackers_for_training( trackers_for_training ) assert len(filtered_trackers) == len(expected_trackers_with_events) for collected_tracker, expected_tracker_events in zip( filtered_trackers, expected_trackers_with_events ): collected_tracker_events = list(collected_tracker.events) assert collected_tracker_events == expected_tracker_events
async def test_request_correct_slots_after_unhappy_path_with_custom_required_slots( ): form_name = "some_form" slot_name_1 = "slot_1" slot_name_2 = "slot_2" domain = f""" slots: {slot_name_1}: type: any {slot_name_2}: type: any forms: {form_name}: {slot_name_1}: - type: from_intent intent: some_intent value: some_value {slot_name_2}: - type: from_intent intent: some_intent value: some_value actions: - validate_{form_name} """ domain = Domain.from_yaml(domain) tracker = DialogueStateTracker.from_events( "default", [ ActiveLoop(form_name), SlotSet(REQUESTED_SLOT, "slot_2"), ActionExecuted(ACTION_LISTEN_NAME), UserUttered( "hello", intent={ "name": "greet", "confidence": 1.0 }, ), ActionExecutionRejected(form_name), ActionExecuted("utter_greet"), ], ) action_server_url = "http://my-action-server:5055/webhook" # Custom form validation action changes the order of the requested slots validate_return_events = [ { "event": "slot", "name": REQUESTED_SLOT, "value": slot_name_2 }, ] # The form should ask the same slot again when coming back after unhappy path expected_events = [SlotSet(REQUESTED_SLOT, slot_name_2)] with aioresponses() as mocked: mocked.post(action_server_url, payload={"events": validate_return_events}) action_server = EndpointConfig(action_server_url) action = FormAction(form_name, action_server) events = await action.run( CollectingOutputChannel(), TemplatedNaturalLanguageGenerator(domain.responses), tracker, domain, ) assert events == expected_events
async def test_switch_forms_with_same_slot(default_agent: Agent): """Tests switching of forms, where the first slot is the same in both forms. Tests the fix for issue 7710""" # Define two forms in the domain, with same first slot slot_a = "my_slot_a" form_1 = "my_form_1" utter_ask_form_1 = f"Please provide the value for {slot_a} of form 1" form_2 = "my_form_2" utter_ask_form_2 = f"Please provide the value for {slot_a} of form 2" domain = f""" version: "2.0" nlu: - intent: order_status examples: | - check status of my order - when are my shoes coming in - intent: return examples: | - start a return - I don't want my shoes anymore forms: {form_1}: {slot_a}: - type: from_entity entity: number {form_2}: {slot_a}: - type: from_entity entity: number responses: utter_ask_{form_1}_{slot_a}: - text: {utter_ask_form_1} utter_ask_{form_2}_{slot_a}: - text: {utter_ask_form_2} """ domain = Domain.from_yaml(domain) # Driving it like rasa/core/processor processor = MessageProcessor( default_agent.interpreter, default_agent.policy_ensemble, domain, InMemoryTrackerStore(domain), InMemoryLockStore(), TemplatedNaturalLanguageGenerator(domain.responses), ) # activate the first form tracker = DialogueStateTracker.from_events( "some-sender", evts=[ ActionExecuted(ACTION_LISTEN_NAME), UserUttered("order status", { "name": "form_1", "confidence": 1.0 }), DefinePrevUserUtteredFeaturization(False), ], ) # rasa/core/processor.predict_next_action prediction = PolicyPrediction([], "some_policy") action_1 = FormAction(form_1, None) await processor._run_action( action_1, tracker, CollectingOutputChannel(), TemplatedNaturalLanguageGenerator(domain.responses), prediction, ) events_expected = [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered("order status", { "name": "form_1", "confidence": 1.0 }), DefinePrevUserUtteredFeaturization(False), ActionExecuted(form_1), ActiveLoop(form_1), SlotSet(REQUESTED_SLOT, slot_a), BotUttered( text=utter_ask_form_1, metadata={"utter_action": f"utter_ask_{form_1}_{slot_a}"}, ), ] assert tracker.applied_events() == events_expected next_events = [ ActionExecuted(ACTION_LISTEN_NAME), UserUttered("return my shoes", { "name": "form_2", "confidence": 1.0 }), DefinePrevUserUtteredFeaturization(False), ] tracker.update_with_events( next_events, domain, ) events_expected.extend(next_events) # form_1 is still active, and bot will first validate if the user utterance # provides valid data for the requested slot, which is rejected await processor._run_action( action_1, tracker, CollectingOutputChannel(), TemplatedNaturalLanguageGenerator(domain.responses), prediction, ) events_expected.extend([ActionExecutionRejected(action_name=form_1)]) assert tracker.applied_events() == events_expected # Next, bot predicts form_2 action_2 = FormAction(form_2, None) await processor._run_action( action_2, tracker, CollectingOutputChannel(), TemplatedNaturalLanguageGenerator(domain.responses), prediction, ) events_expected.extend([ ActionExecuted(form_2), ActiveLoop(form_2), SlotSet(REQUESTED_SLOT, slot_a), BotUttered( text=utter_ask_form_2, metadata={"utter_action": f"utter_ask_{form_2}_{slot_a}"}, ), ]) assert tracker.applied_events() == events_expected
async def test_whole_loop(): expected_activation_events = [ ActionExecutionRejected("tada"), ActionExecuted("test"), ] expected_do_events = [ActionExecuted("do")] expected_deactivation_events = [SlotSet("deactivated")] form_name = "my form" class MyLoop(LoopAction): def name(self) -> Text: return form_name async def activate(self, *args: Any) -> List[Event]: return expected_activation_events async def do(self, *args: Any) -> List[Event]: events_so_far = args[-1] assert events_so_far == [ ActiveLoop(form_name), *expected_activation_events ] return expected_do_events async def deactivate(self, *args) -> List[Event]: events_so_far = args[-1] assert events_so_far == [ ActiveLoop(form_name), *expected_activation_events, *expected_do_events, ActiveLoop(None), ] return expected_deactivation_events async def is_done(self, *args) -> bool: events_so_far = args[-1] return events_so_far == [ ActiveLoop(form_name), *expected_activation_events, *expected_do_events, ] tracker = DialogueStateTracker.from_events("some sender", []) domain = Domain.empty() action = MyLoop() actual = await action.run( CollectingOutputChannel(), TemplatedNaturalLanguageGenerator(domain.templates), tracker, domain, ) assert actual == [ ActiveLoop(form_name), *expected_activation_events, *expected_do_events, ActiveLoop(None), *expected_deactivation_events, ]
def _emulate_form_rejection(partial_tracker: DialogueStateTracker) -> None: from rasa.shared.core.events import ActionExecutionRejected rejected_action_name: Text = partial_tracker.active_loop_name partial_tracker.update(ActionExecutionRejected(rejected_action_name))