async def test_end_to_end_evaluation_script(default_agent: Agent):
completed_trackers = await _generate_trackers(END_TO_END_STORY_FILE,
default_agent,
use_e2e=True)
story_evaluation, num_stories = collect_story_predictions(
completed_trackers, default_agent, use_e2e=True)
serialised_store = [
"utter_greet",
"action_listen",
"utter_greet",
"action_listen",
"utter_default",
"action_listen",
"utter_goodbye",
"action_listen",
"utter_greet",
"action_listen",
"utter_default",
"action_listen",
"greet",
"greet",
"default",
"goodbye",
"greet",
"default",
'[{"name": "Max"}](name:Max)',
]
assert story_evaluation.evaluation_store.serialise()[0] == serialised_store
assert not story_evaluation.evaluation_store.has_prediction_target_mismatch(
)
assert len(story_evaluation.failed_stories) == 0
assert num_stories == 3
async def test_end_to_evaluation_trips_circuit_breaker():
agent = Agent(
domain="data/test_domains/default.yml",
policies=[MemoizationPolicy(max_history=11)],
)
training_data = await agent.load_data(STORY_FILE_TRIPS_CIRCUIT_BREAKER)
agent.train(training_data)
test_stories = await _generate_trackers(
E2E_STORY_FILE_TRIPS_CIRCUIT_BREAKER, agent, use_e2e=True)
story_evaluation, num_stories = collect_story_predictions(test_stories,
agent,
use_e2e=True)
circuit_trip_predicted = [
"utter_greet",
"utter_greet",
"utter_greet",
"utter_greet",
"utter_greet",
"utter_greet",
"utter_greet",
"utter_greet",
"utter_greet",
"utter_greet",
"circuit breaker tripped",
"circuit breaker tripped",
]
assert (story_evaluation.evaluation_store.action_predictions ==
circuit_trip_predicted)
assert num_stories == 1
async def test_end_to_end_evaluation_script(restaurantbot: Text):
restaurantbot = Agent.load(restaurantbot)
completed_trackers = await _generate_trackers(
END_TO_END_STORY_FILE, restaurantbot, use_e2e=True
)
story_evaluation, num_stories = collect_story_predictions(
completed_trackers, restaurantbot, use_e2e=True
)
serialised_store = [
"utter_ask_howcanhelp",
"action_listen",
"utter_ask_howcanhelp",
"action_listen",
"utter_on_it",
"utter_ask_cuisine",
"action_listen",
"utter_ask_numpeople",
"action_listen",
"utter_ask_howcanhelp",
"action_listen",
"utter_on_it",
"utter_ask_numpeople",
"action_listen",
"utter_ask_moreupdates",
"action_listen",
"utter_ask_moreupdates",
"action_listen",
"utter_ack_dosearch",
"action_search_restaurants",
"action_suggest",
"action_listen",
"utter_ask_howcanhelp",
"action_listen",
"greet",
"greet",
"inform",
"inform",
"greet",
"inform",
"inform",
"inform",
"deny",
"greet",
"[moderately](price:moderate)",
"[east](location)",
"[french](cuisine)",
"[cheap](price:lo)",
"[french](cuisine)",
"[bombay](location)",
"[six](people:6)",
"[moderately](price:moderate)",
]
assert story_evaluation.evaluation_store.serialise()[0] == serialised_store
assert not story_evaluation.evaluation_store.has_prediction_target_mismatch()
assert len(story_evaluation.failed_stories) == 0
assert num_stories == 4
async def test_end_to_evaluation_with_forms(form_bot: Agent):
test_stories = await _generate_trackers(
"examples/formbot/tests/end-to-end-stories.md", form_bot, use_e2e=True
)
story_evaluation, num_stories = collect_story_predictions(
test_stories, form_bot, use_e2e=True
)
assert not story_evaluation.evaluation_store.has_prediction_target_mismatch()
async def test_end_to_end_evaluation_script_unknown_entity(
default_agent: Agent):
completed_trackers = await _generate_trackers(
E2E_STORY_FILE_UNKNOWN_ENTITY, default_agent, use_e2e=True)
story_evaluation, num_stories = collect_story_predictions(
completed_trackers, default_agent, use_e2e=True)
assert story_evaluation.evaluation_store.has_prediction_target_mismatch()
assert len(story_evaluation.failed_stories) == 1
assert num_stories == 1
async def test_action_evaluation_script(tmpdir, default_agent):
completed_trackers = await _generate_trackers(DEFAULT_STORIES_FILE,
default_agent,
use_e2e=False)
story_evaluation, num_stories = collect_story_predictions(
completed_trackers, default_agent, use_e2e=False)
assert not story_evaluation.evaluation_store.has_prediction_target_mismatch(
)
assert len(story_evaluation.failed_stories) == 0
assert num_stories == 3
async def test_end_to_end_evaluation_script(tmpdir, default_agent):
completed_trackers = await _generate_trackers(END_TO_END_STORY_FILE,
default_agent,
use_e2e=True)
story_evaluation, num_stories = collect_story_predictions(
completed_trackers, default_agent, use_e2e=True)
assert not story_evaluation.evaluation_store.has_prediction_target_mismatch(
)
assert len(story_evaluation.failed_stories) == 0
assert num_stories == 2
