async def test_end_to_end_evaluation_script(default_agent: Agent): completed_trackers = await _generate_trackers(END_TO_END_STORY_FILE, default_agent, use_e2e=True) story_evaluation, num_stories = collect_story_predictions( completed_trackers, default_agent, use_e2e=True) serialised_store = [ "utter_greet", "action_listen", "utter_greet", "action_listen", "utter_default", "action_listen", "utter_goodbye", "action_listen", "utter_greet", "action_listen", "utter_default", "action_listen", "greet", "greet", "default", "goodbye", "greet", "default", '[{"name": "Max"}](name:Max)', ] assert story_evaluation.evaluation_store.serialise()[0] == serialised_store assert not story_evaluation.evaluation_store.has_prediction_target_mismatch( ) assert len(story_evaluation.failed_stories) == 0 assert num_stories == 3
async def test_end_to_evaluation_trips_circuit_breaker(): agent = Agent( domain="data/test_domains/default.yml", policies=[MemoizationPolicy(max_history=11)], ) training_data = await agent.load_data(STORY_FILE_TRIPS_CIRCUIT_BREAKER) agent.train(training_data) test_stories = await _generate_trackers( E2E_STORY_FILE_TRIPS_CIRCUIT_BREAKER, agent, use_e2e=True) story_evaluation, num_stories = collect_story_predictions(test_stories, agent, use_e2e=True) circuit_trip_predicted = [ "utter_greet", "utter_greet", "utter_greet", "utter_greet", "utter_greet", "utter_greet", "utter_greet", "utter_greet", "utter_greet", "utter_greet", "circuit breaker tripped", "circuit breaker tripped", ] assert (story_evaluation.evaluation_store.action_predictions == circuit_trip_predicted) assert num_stories == 1
async def test_end_to_end_evaluation_script(restaurantbot: Text): restaurantbot = Agent.load(restaurantbot) completed_trackers = await _generate_trackers( END_TO_END_STORY_FILE, restaurantbot, use_e2e=True ) story_evaluation, num_stories = collect_story_predictions( completed_trackers, restaurantbot, use_e2e=True ) serialised_store = [ "utter_ask_howcanhelp", "action_listen", "utter_ask_howcanhelp", "action_listen", "utter_on_it", "utter_ask_cuisine", "action_listen", "utter_ask_numpeople", "action_listen", "utter_ask_howcanhelp", "action_listen", "utter_on_it", "utter_ask_numpeople", "action_listen", "utter_ask_moreupdates", "action_listen", "utter_ask_moreupdates", "action_listen", "utter_ack_dosearch", "action_search_restaurants", "action_suggest", "action_listen", "utter_ask_howcanhelp", "action_listen", "greet", "greet", "inform", "inform", "greet", "inform", "inform", "inform", "deny", "greet", "[moderately](price:moderate)", "[east](location)", "[french](cuisine)", "[cheap](price:lo)", "[french](cuisine)", "[bombay](location)", "[six](people:6)", "[moderately](price:moderate)", ] assert story_evaluation.evaluation_store.serialise()[0] == serialised_store assert not story_evaluation.evaluation_store.has_prediction_target_mismatch() assert len(story_evaluation.failed_stories) == 0 assert num_stories == 4
async def test_end_to_evaluation_with_forms(form_bot: Agent): test_stories = await _generate_trackers( "examples/formbot/tests/end-to-end-stories.md", form_bot, use_e2e=True ) story_evaluation, num_stories = collect_story_predictions( test_stories, form_bot, use_e2e=True ) assert not story_evaluation.evaluation_store.has_prediction_target_mismatch()
async def test_end_to_end_evaluation_script_unknown_entity( default_agent: Agent): completed_trackers = await _generate_trackers( E2E_STORY_FILE_UNKNOWN_ENTITY, default_agent, use_e2e=True) story_evaluation, num_stories = collect_story_predictions( completed_trackers, default_agent, use_e2e=True) assert story_evaluation.evaluation_store.has_prediction_target_mismatch() assert len(story_evaluation.failed_stories) == 1 assert num_stories == 1
async def test_action_evaluation_script(tmpdir, default_agent): completed_trackers = await _generate_trackers(DEFAULT_STORIES_FILE, default_agent, use_e2e=False) story_evaluation, num_stories = collect_story_predictions( completed_trackers, default_agent, use_e2e=False) assert not story_evaluation.evaluation_store.has_prediction_target_mismatch( ) assert len(story_evaluation.failed_stories) == 0 assert num_stories == 3
async def test_end_to_end_evaluation_script(tmpdir, default_agent): completed_trackers = await _generate_trackers(END_TO_END_STORY_FILE, default_agent, use_e2e=True) story_evaluation, num_stories = collect_story_predictions( completed_trackers, default_agent, use_e2e=True) assert not story_evaluation.evaluation_store.has_prediction_target_mismatch( ) assert len(story_evaluation.failed_stories) == 0 assert num_stories == 2