def test_test_does_not_use_rules(tmp_path: Path, default_agent: Agent): from rasa.core.test import _create_data_generator test_file = tmp_path / "test.yml" test_name = "my test story" tests = f""" stories: - story: {test_name} steps: - intent: greet - action: utter_greet rules: - rule: rule which is ignored steps: - intent: greet - action: utter_greet """ test_file.write_text(tests) generator = _create_data_generator(str(test_file), default_agent) test_trackers = generator.generate_story_trackers() assert len(test_trackers) == 1 assert test_trackers[0].sender_id == test_name
async def test_end_to_evaluation_trips_circuit_breaker( e2e_story_file_trips_circuit_breaker_path: Text, trained_async: Callable, tmp_path: Path, ): config = textwrap.dedent( f""" version: '{LATEST_TRAINING_DATA_FORMAT_VERSION}' policies: - name: MemoizationPolicy max_history: 11 pipeline: [] """ ) config_path = tmp_path / "config.yml" rasa.shared.utils.io.write_text_file(config, config_path) model_path = await trained_async( "data/test_domains/default.yml", str(config_path), e2e_story_file_trips_circuit_breaker_path, ) agent = await load_agent(model_path) generator = _create_data_generator( e2e_story_file_trips_circuit_breaker_path, agent, use_conversation_test_files=True, ) test_stories = generator.generate_story_trackers() story_evaluation, num_stories, _ = await _collect_story_predictions( test_stories, agent ) circuit_trip_predicted = [ "utter_greet", "utter_greet", "utter_greet", "utter_greet", "utter_greet", "utter_greet", "utter_greet", "utter_greet", "utter_greet", "utter_greet", "circuit breaker tripped", "circuit breaker tripped", ] assert ( story_evaluation.evaluation_store.action_predictions == circuit_trip_predicted ) assert num_stories == 1
async def test_retrieval_intent(response_selector_agent: Agent, test_file: Text): generator = _create_data_generator( test_file, response_selector_agent, use_conversation_test_files=True, ) test_stories = generator.generate_story_trackers() story_evaluation, num_stories, _ = await _collect_story_predictions( test_stories, response_selector_agent ) # check that test story can either specify base intent or full retrieval intent assert not story_evaluation.evaluation_store.check_prediction_target_mismatch()
async def test_end_to_evaluation_with_forms(form_bot_agent: Agent): generator = _create_data_generator( "data/test_evaluations/test_form_end_to_end_stories.yml", form_bot_agent, use_conversation_test_files=True, ) test_stories = generator.generate_story_trackers() story_evaluation, num_stories, _ = await _collect_story_predictions( test_stories, form_bot_agent) assert not story_evaluation.evaluation_store.check_prediction_target_mismatch( )
async def test_end_to_end_evaluation_script_unknown_entity( default_agent: Agent, e2e_story_file_unknown_entity_path: Text): generator = _create_data_generator( e2e_story_file_unknown_entity_path, default_agent, use_conversation_test_files=True, ) completed_trackers = generator.generate_story_trackers() story_evaluation, num_stories, _ = await _collect_story_predictions( completed_trackers, default_agent) assert story_evaluation.evaluation_store.check_prediction_target_mismatch() assert len(story_evaluation.failed_stories) == 1 assert num_stories == 1
async def test_end_to_end_evaluation_script( default_agent: Agent, end_to_end_story_path: Text ): generator = _create_data_generator( end_to_end_story_path, default_agent, use_conversation_test_files=True ) completed_trackers = generator.generate_story_trackers() story_evaluation, num_stories, _ = await _collect_story_predictions( completed_trackers, default_agent, use_e2e=True ) serialised_store = [ "utter_greet", "action_listen", "utter_greet", "action_listen", "utter_default", "action_listen", "utter_goodbye", "action_listen", "utter_greet", "action_listen", "utter_default", "action_listen", "greet", "greet", "default", "goodbye", "greet", "default", '[{"name": "Max"}]{"entity": "name", "value": "Max"}', ] assert story_evaluation.evaluation_store.serialise()[0] == serialised_store assert not story_evaluation.evaluation_store.check_prediction_target_mismatch() assert len(story_evaluation.failed_stories) == 0 assert num_stories == 3
async def test_end_to_evaluation_trips_circuit_breaker( e2e_story_file_trips_circuit_breaker_path: Text, ): agent = Agent( domain="data/test_domains/default.yml", policies=[MemoizationPolicy(max_history=11)], ) training_data = agent.load_data(e2e_story_file_trips_circuit_breaker_path) agent.train(training_data) generator = _create_data_generator( e2e_story_file_trips_circuit_breaker_path, agent, use_conversation_test_files=True, ) test_stories = generator.generate_story_trackers() story_evaluation, num_stories, _ = await _collect_story_predictions( test_stories, agent) circuit_trip_predicted = [ "utter_greet", "utter_greet", "utter_greet", "utter_greet", "utter_greet", "utter_greet", "utter_greet", "utter_greet", "utter_greet", "utter_greet", "circuit breaker tripped", "circuit breaker tripped", ] assert (story_evaluation.evaluation_store.action_predictions == circuit_trip_predicted) assert num_stories == 1