async def test_end_to_evaluation_trips_circuit_breaker():
agent = Agent(
domain="data/test_domains/default.yml",
policies=[MemoizationPolicy(max_history=11)],
)
training_data = await agent.load_data(STORY_FILE_TRIPS_CIRCUIT_BREAKER)
agent.train(training_data)
test_stories = await _generate_trackers(
E2E_STORY_FILE_TRIPS_CIRCUIT_BREAKER, agent, use_e2e=True)
story_evaluation, num_stories = _collect_story_predictions(test_stories,
agent,
use_e2e=True)
circuit_trip_predicted = [
"utter_greet",
"utter_greet",
"utter_greet",
"utter_greet",
"utter_greet",
"utter_greet",
"utter_greet",
"utter_greet",
"utter_greet",
"utter_greet",
"circuit breaker tripped",
"circuit breaker tripped",
]
assert (story_evaluation.evaluation_store.action_predictions ==
circuit_trip_predicted)
assert num_stories == 1
async def test_end_to_end_evaluation_script(default_agent: Agent):
completed_trackers = await _generate_trackers(END_TO_END_STORY_FILE,
default_agent,
use_e2e=True)
story_evaluation, num_stories = _collect_story_predictions(
completed_trackers, default_agent, use_e2e=True)
serialised_store = [
"utter_greet",
"action_listen",
"utter_greet",
"action_listen",
"utter_default",
"action_listen",
"utter_goodbye",
"action_listen",
"utter_greet",
"action_listen",
"utter_default",
"action_listen",
"greet",
"greet",
"default",
"goodbye",
"greet",
"default",
'[{"name": "Max"}]{"entity": "name", "value": "Max"}',
]
assert story_evaluation.evaluation_store.serialise()[0] == serialised_store
assert not story_evaluation.evaluation_store.has_prediction_target_mismatch(
)
assert len(story_evaluation.failed_stories) == 0
assert num_stories == 3
async def test_end_to_end_evaluation_script_unknown_entity(
default_agent: Agent):
completed_trackers = await _generate_trackers(
E2E_STORY_FILE_UNKNOWN_ENTITY, default_agent, use_e2e=True)
story_evaluation, num_stories = _collect_story_predictions(
completed_trackers, default_agent, use_e2e=True)
assert story_evaluation.evaluation_store.has_prediction_target_mismatch()
assert len(story_evaluation.failed_stories) == 1
assert num_stories == 1
async def test_end_to_evaluation_with_forms(form_bot_agent: Agent):
test_stories = await _generate_trackers(
"data/test_evaluations/form-end-to-end-stories.md",
form_bot_agent,
use_e2e=True)
story_evaluation, num_stories = _collect_story_predictions(test_stories,
form_bot_agent,
use_e2e=True)
assert not story_evaluation.evaluation_store.has_prediction_target_mismatch(
)
