def test_test_does_not_use_rules(tmp_path: Path, default_agent: Agent):
from rasa.core.test import _create_data_generator
test_file = tmp_path / "test.yml"
test_name = "my test story"
tests = f"""
stories:
- story: {test_name}
steps:
- intent: greet
- action: utter_greet
rules:
- rule: rule which is ignored
steps:
- intent: greet
- action: utter_greet
"""
test_file.write_text(tests)
generator = _create_data_generator(str(test_file), default_agent)
test_trackers = generator.generate_story_trackers()
assert len(test_trackers) == 1
assert test_trackers[0].sender_id == test_name
async def test_end_to_evaluation_trips_circuit_breaker(
e2e_story_file_trips_circuit_breaker_path: Text,
trained_async: Callable,
tmp_path: Path,
):
config = textwrap.dedent(
f"""
version: '{LATEST_TRAINING_DATA_FORMAT_VERSION}'
policies:
- name: MemoizationPolicy
max_history: 11
pipeline: []
"""
)
config_path = tmp_path / "config.yml"
rasa.shared.utils.io.write_text_file(config, config_path)
model_path = await trained_async(
"data/test_domains/default.yml",
str(config_path),
e2e_story_file_trips_circuit_breaker_path,
)
agent = await load_agent(model_path)
generator = _create_data_generator(
e2e_story_file_trips_circuit_breaker_path,
agent,
use_conversation_test_files=True,
)
test_stories = generator.generate_story_trackers()
story_evaluation, num_stories, _ = await _collect_story_predictions(
test_stories, agent
)
circuit_trip_predicted = [
"utter_greet",
"utter_greet",
"utter_greet",
"utter_greet",
"utter_greet",
"utter_greet",
"utter_greet",
"utter_greet",
"utter_greet",
"utter_greet",
"circuit breaker tripped",
"circuit breaker tripped",
]
assert (
story_evaluation.evaluation_store.action_predictions == circuit_trip_predicted
)
assert num_stories == 1
async def test_retrieval_intent(response_selector_agent: Agent, test_file: Text):
generator = _create_data_generator(
test_file, response_selector_agent, use_conversation_test_files=True,
)
test_stories = generator.generate_story_trackers()
story_evaluation, num_stories, _ = await _collect_story_predictions(
test_stories, response_selector_agent
)
# check that test story can either specify base intent or full retrieval intent
assert not story_evaluation.evaluation_store.check_prediction_target_mismatch()
async def test_end_to_evaluation_with_forms(form_bot_agent: Agent):
generator = _create_data_generator(
"data/test_evaluations/test_form_end_to_end_stories.yml",
form_bot_agent,
use_conversation_test_files=True,
)
test_stories = generator.generate_story_trackers()
story_evaluation, num_stories, _ = await _collect_story_predictions(
test_stories, form_bot_agent)
assert not story_evaluation.evaluation_store.check_prediction_target_mismatch(
)
async def test_end_to_end_evaluation_script_unknown_entity(
default_agent: Agent, e2e_story_file_unknown_entity_path: Text):
generator = _create_data_generator(
e2e_story_file_unknown_entity_path,
default_agent,
use_conversation_test_files=True,
)
completed_trackers = generator.generate_story_trackers()
story_evaluation, num_stories, _ = await _collect_story_predictions(
completed_trackers, default_agent)
assert story_evaluation.evaluation_store.check_prediction_target_mismatch()
assert len(story_evaluation.failed_stories) == 1
assert num_stories == 1
async def test_end_to_end_evaluation_script(
default_agent: Agent, end_to_end_story_path: Text
):
generator = _create_data_generator(
end_to_end_story_path, default_agent, use_conversation_test_files=True
)
completed_trackers = generator.generate_story_trackers()
story_evaluation, num_stories, _ = await _collect_story_predictions(
completed_trackers, default_agent, use_e2e=True
)
serialised_store = [
"utter_greet",
"action_listen",
"utter_greet",
"action_listen",
"utter_default",
"action_listen",
"utter_goodbye",
"action_listen",
"utter_greet",
"action_listen",
"utter_default",
"action_listen",
"greet",
"greet",
"default",
"goodbye",
"greet",
"default",
'[{"name": "Max"}]{"entity": "name", "value": "Max"}',
]
assert story_evaluation.evaluation_store.serialise()[0] == serialised_store
assert not story_evaluation.evaluation_store.check_prediction_target_mismatch()
assert len(story_evaluation.failed_stories) == 0
assert num_stories == 3
async def test_end_to_evaluation_trips_circuit_breaker(
e2e_story_file_trips_circuit_breaker_path: Text, ):
agent = Agent(
domain="data/test_domains/default.yml",
policies=[MemoizationPolicy(max_history=11)],
)
training_data = agent.load_data(e2e_story_file_trips_circuit_breaker_path)
agent.train(training_data)
generator = _create_data_generator(
e2e_story_file_trips_circuit_breaker_path,
agent,
use_conversation_test_files=True,
)
test_stories = generator.generate_story_trackers()
story_evaluation, num_stories, _ = await _collect_story_predictions(
test_stories, agent)
circuit_trip_predicted = [
"utter_greet",
"utter_greet",
"utter_greet",
"utter_greet",
"utter_greet",
"utter_greet",
"utter_greet",
"utter_greet",
"utter_greet",
"utter_greet",
"circuit breaker tripped",
"circuit breaker tripped",
]
assert (story_evaluation.evaluation_store.action_predictions ==
circuit_trip_predicted)
assert num_stories == 1
