def test_verify_actions_in_stories_not_in_domain(tmp_path: Path, domain_path: Text):
story_file_name = tmp_path / "stories.yml"
story_file_name.write_text(
"""
version: "3.0"
stories:
- story: story path 1
steps:
- intent: greet
- action: action_test_1
"""
)
importer = RasaFileImporter(
domain_path=domain_path, training_data_paths=[story_file_name]
)
validator = Validator.from_importer(importer)
with pytest.warns(UserWarning) as warning:
validity = validator.verify_actions_in_stories_rules()
assert validity is False
assert (
"The action 'action_test_1' is used in the 'story path 1' block, "
"but it is not listed in the domain file." in warning[0].message.args[0]
)
def validate_files(args: argparse.Namespace, stories_only: bool = False) -> None:
"""Validates either the story structure or the entire project.
Args:
args: Commandline arguments
stories_only: If `True`, only the story structure is validated.
"""
loop = asyncio.get_event_loop()
file_importer = RasaFileImporter(
domain_path=args.domain, training_data_paths=args.data
)
validator = loop.run_until_complete(Validator.from_importer(file_importer))
if stories_only:
all_good = _validate_story_structure(validator, args)
else:
all_good = (
_validate_domain(validator)
and _validate_nlu(validator, args)
and _validate_story_structure(validator, args)
)
if not all_good:
rasa.shared.utils.cli.print_error_and_exit(
"Project validation completed with errors."
)
def test_verify_bad_e2e_story_structure_when_text_identical(tmp_path: Path):
story_file_name = tmp_path / "stories.yml"
story_file_name.write_text("""
version: "3.0"
stories:
- story: path 1
steps:
- user: |
amazing!
- action: utter_happy
- story: path 2 (should always conflict path 1)
steps:
- user: |
amazing!
- action: utter_cheer_up
""")
# The two stories with identical user texts
importer = RasaFileImporter(
config_file="data/test_config/config_defaults.yml",
domain_path="data/test_domains/default.yml",
training_data_paths=[story_file_name],
training_type=TrainingType.NLU,
)
validator = Validator.from_importer(importer)
assert not validator.verify_story_structure(ignore_warnings=False)
def test_early_exit_on_invalid_domain():
domain_path = "data/test_domains/duplicate_intents.yml"
importer = RasaFileImporter(domain_path=domain_path)
with pytest.warns(UserWarning) as record:
validator = Validator.from_importer(importer)
validator.verify_domain_validity()
# two for non-unique domains, 2 for auto-fill removal
assert len(record) == 4
non_unique_warnings = list(
filter(
lambda warning: f"Loading domain from '{domain_path}' failed. "
f"Using empty domain. Error: 'Intents are not unique! "
f"Found multiple intents with name(s) ['default', 'goodbye']. "
f"Either rename or remove the duplicate ones.'" in warning.message.
args[0],
record,
))
assert len(non_unique_warnings) == 2
auto_fill_warnings = list(
filter(
lambda warning: "Slot auto-fill has been removed in 3.0" in warning
.message.args[0],
record,
))
assert len(auto_fill_warnings) == 2
def validate_files(args: argparse.Namespace, stories_only: bool = False) -> None:
"""Validates either the story structure or the entire project.
Args:
args: Commandline arguments
stories_only: If `True`, only the story structure is validated.
"""
from rasa.validator import Validator
config = rasa.cli.utils.get_validated_path(
args.config, "config", DEFAULT_CONFIG_PATH, none_is_valid=True
)
file_importer = RasaFileImporter(
domain_path=args.domain, training_data_paths=args.data, config_file=config,
)
validator = Validator.from_importer(file_importer)
if stories_only:
all_good = _validate_story_structure(validator, args)
else:
all_good = (
_validate_domain(validator)
and _validate_nlu(validator, args)
and _validate_story_structure(validator, args)
)
telemetry.track_validate_files(all_good)
if not all_good:
rasa.shared.utils.cli.print_error_and_exit(
"Project validation completed with errors."
)
def test_verify_intents_does_fail_on_invalid_data(nlu_data_path: Text):
# domain and nlu data are from different domain and should produce warnings
importer = RasaFileImporter(
domain_path="data/test_domains/default.yml", training_data_paths=[nlu_data_path]
)
validator = Validator.from_importer(importer)
assert not validator.verify_intents()
def test_verify_form_slots_invalid_domain(tmp_path: Path):
domain = tmp_path / "domain.yml"
domain.write_text("""
version: "3.0"
forms:
name_form:
required_slots:
- first_name
- last_nam
slots:
first_name:
type: text
mappings:
- type: from_text
last_name:
type: text
mappings:
- type: from_text
""")
importer = RasaFileImporter(domain_path=domain)
validator = Validator.from_importer(importer)
with pytest.warns(UserWarning) as w:
validity = validator.verify_form_slots()
assert validity is False
assert (
w[0].message.args[0] == "The form slot 'last_nam' in form 'name_form' "
"is not present in the domain slots."
"Please add the correct slot or check for typos.")
def test_verify_nlu_with_e2e_story(tmp_path: Path, nlu_data_path: Path):
story_file_name = tmp_path / "stories.yml"
with open(story_file_name, "w") as file:
file.write(
"""
stories:
- story: path 1
steps:
- user: |
hello assistant! Can you help me today?
- action: utter_greet
- story: path 2
steps:
- intent: greet
- action: utter_greet
"""
)
importer = RasaFileImporter(
config_file="data/test_moodbot/config.yml",
domain_path="data/test_moodbot/domain.yml",
training_data_paths=[story_file_name, nlu_data_path],
)
validator = Validator.from_importer(importer)
assert validator.verify_nlu()
def test_valid_stories_rules_default_actions(
file_name: Text, data_type: Text, tmp_path: Path
):
domain = tmp_path / "domain.yml"
domain.write_text(
"""
version: "3.0"
intents:
- greet
"""
)
file_name = tmp_path / f"{file_name}.yml"
file_name.write_text(
f"""
version: "3.0"
{file_name}:
- {data_type}: test path
steps:
- intent: greet
- action: action_restart
"""
)
importer = RasaFileImporter(domain_path=domain, training_data_paths=[file_name])
validator = Validator.from_importer(importer)
assert validator.verify_actions_in_stories_rules()
def test_verify_there_is_example_repetition_in_intents(nlu_data_path: Text):
# moodbot nlu data already has duplicated example 'good afternoon'
# for intents greet and goodbye
importer = RasaFileImporter(domain_path="data/test_moodbot/domain.yml",
training_data_paths=[nlu_data_path])
validator = Validator.from_importer(importer)
assert not validator.verify_example_repetition_in_intents(False)
def test_verify_story_structure(stories_path: Text):
importer = RasaFileImporter(
domain_path="data/test_domains/default.yml",
training_data_paths=[stories_path],
)
validator = Validator.from_importer(importer)
assert validator.verify_story_structure(ignore_warnings=False)
def test_verify_bad_story_structure_ignore_warnings():
importer = RasaFileImporter(
domain_path="data/test_domains/default.yml",
training_data_paths=["data/test_yaml_stories/stories_conflicting_2.yml"],
)
validator = Validator.from_importer(importer)
assert validator.verify_story_structure(ignore_warnings=True)
def test_verify_slot_mappings_mapping_active_loop_not_in_forms(tmp_path: Path):
domain = tmp_path / "domain.yml"
slot_name = "some_slot"
domain.write_text(f"""
version: "3.0"
entities:
- some_entity
slots:
{slot_name}:
type: text
influence_conversation: false
mappings:
- type: from_entity
entity: some_entity
conditions:
- active_loop: som_form
forms:
some_form:
required_slots:
- {slot_name}
""")
importer = RasaFileImporter(domain_path=domain)
validator = Validator.from_importer(importer)
with pytest.warns(
UserWarning,
match=r"Slot 'some_slot' has a mapping condition "
r"for form 'som_form' which is not listed "
r"in domain forms.*",
):
assert not validator.verify_slot_mappings()
def test_verify_intents_does_not_fail_on_valid_data(nlu_data_path: Text):
importer = RasaFileImporter(
domain_path="data/test_moodbot/domain.yml",
training_data_paths=[nlu_data_path],
)
validator = Validator.from_importer(importer)
assert validator.verify_intents()
def test_verify_slot_mappings_valid(tmp_path: Path):
domain = tmp_path / "domain.yml"
domain.write_text("""
version: "3.0"
intents:
- activate_booking
entities:
- city
slots:
location:
type: text
influence_conversation: false
mappings:
- type: from_entity
entity: city
conditions:
- active_loop: booking_form
started_booking_form:
type: bool
influence_conversation: false
mappings:
- type: from_trigger_intent
intent: activate_booking
value: true
forms:
booking_form:
required_slots:
- started_booking_form
- location
""")
importer = RasaFileImporter(domain_path=domain)
validator = Validator.from_importer(importer)
assert validator.verify_slot_mappings()
def test_verify_correct_e2e_story_structure(tmp_path: Path):
story_file_name = tmp_path / "stories.yml"
with open(story_file_name, "w") as file:
file.write("""
stories:
- story: path 1
steps:
- user: |
hello assistant! Can you help me today?
- action: utter_greet
- story: path 2 - state is similar but different from the one in path 1
steps:
- user: |
hello assistant! you Can help me today?
- action: utter_goodbye
- story: path 3
steps:
- user: |
That's it for today. Chat again tomorrow!
- action: utter_goodbye
""")
importer = RasaFileImporter(
config_file="data/test_config/config_defaults.yml",
domain_path="data/test_domains/default.yml",
training_data_paths=[story_file_name],
training_type=TrainingType.NLU,
)
validator = Validator.from_importer(importer)
assert validator.verify_story_structure(ignore_warnings=False)
def test_verify_there_is_not_example_repetition_in_intents():
importer = RasaFileImporter(
domain_path="data/test_moodbot/domain.yml",
training_data_paths=["examples/knowledgebasebot/data/nlu.yml"],
)
validator = Validator.from_importer(importer)
assert validator.verify_example_repetition_in_intents(False)
def test_verify_valid_responses():
importer = RasaFileImporter(
domain_path="data/test_domains/selectors.yml",
training_data_paths=[
"data/test_selectors/nlu.yml",
"data/test_selectors/stories.yml",
],
)
validator = Validator.from_importer(importer)
assert validator.verify_utterances_in_stories()
def validator_under_test() -> Validator:
importer = RasaFileImporter(
domain_path="data/test_validation/domain.yml",
training_data_paths=[
"data/test_validation/data/nlu.yml",
"data/test_validation/data/stories.yml",
],
)
validator = Validator.from_importer(importer)
return validator
def test_verify_valid_responses_in_rules(nlu_data_path: Text):
importer = RasaFileImporter(
domain_path="data/test_domains/default.yml",
training_data_paths=[
nlu_data_path,
"data/test_yaml_stories/rules_without_stories_and_wrong_names.yml",
],
)
validator = Validator.from_importer(importer)
assert not validator.verify_utterances_in_stories()
def test_response_selector_responses_in_domain_no_errors():
importer = RasaFileImporter(
config_file="data/test_config/config_defaults.yml",
domain_path="data/test_domains/response_selector_responses_in_domain.yml",
training_data_paths=[
"data/test_yaml_stories/test_base_retrieval_intent_story.yml"
],
)
validator = Validator.from_importer(importer)
assert validator.verify_utterances_in_stories(ignore_warnings=True)
def test_verify_logging_message_for_repetition_in_intents(
caplog, nlu_data_path: Text):
# moodbot nlu data already has duplicated example 'good afternoon'
# for intents greet and goodbye
importer = RasaFileImporter(domain_path="data/test_moodbot/domain.yml",
training_data_paths=[nlu_data_path])
validator = Validator.from_importer(importer)
caplog.clear() # clear caplog to avoid counting earlier debug messages
with pytest.warns(UserWarning) as record:
validator.verify_example_repetition_in_intents(False)
assert len(record) == 1
assert "You should fix that conflict " in record[0].message.args[0]
def _setup_trackers_for_testing(
domain_path: Text, training_data_file: Text
) -> Tuple[List[TrackerWithCachedStates], Domain]:
importer = RasaFileImporter(domain_path=domain_path,
training_data_paths=[training_data_file])
validator = Validator.from_importer(importer)
trackers = TrainingDataGenerator(
validator.story_graph,
domain=validator.domain,
remove_duplicates=False,
augmentation_factor=0,
).generate()
return trackers, validator.domain
def test_early_exit_on_invalid_domain():
domain_path = "data/test_domains/duplicate_intents.yml"
importer = RasaFileImporter(domain_path=domain_path)
with pytest.warns(UserWarning) as record:
validator = Validator.from_importer(importer)
validator.verify_domain_validity()
# two for non-unique domains
assert len(record) == 2
assert (
f"Loading domain from '{domain_path}' failed. Using empty domain. "
"Error: 'Intents are not unique! Found multiple intents with name(s) "
"['default', 'goodbye']. Either rename or remove the duplicate ones.'"
in record[0].message.args[0])
assert record[0].message.args[0] == record[1].message.args[0]
def test_valid_form_slots_in_domain(tmp_path: Path):
domain = tmp_path / "domain.yml"
domain.write_text("""
version: "2.0"
forms:
name_form:
required_slots:
first_name:
- type: from_text
last_name:
- type: from_text
slots:
first_name:
type: text
last_name:
type: text
""")
importer = RasaFileImporter(domain_path=domain)
validator = Validator.from_importer(importer)
assert validator.verify_form_slots()
def test_verify_correct_e2e_story_structure_with_intents(tmp_path: Path):
story_file_name = tmp_path / "stories.yml"
with open(story_file_name, "w") as file:
file.write("""
stories:
- story: path 1
steps:
- intent: greet
- action: utter_greet
- story: path 2
steps:
- intent: goodbye
- action: utter_goodbye
""")
importer = RasaFileImporter(
config_file="data/test_config/config_defaults.yml",
domain_path="data/test_domains/default.yml",
training_data_paths=[story_file_name],
)
validator = Validator.from_importer(importer)
assert validator.verify_story_structure(ignore_warnings=False)
def test_verify_actions_in_rules_not_in_domain(tmp_path: Path,
domain_path: Text):
rules_file_name = tmp_path / "rules.yml"
rules_file_name.write_text(f"""
version: "{LATEST_TRAINING_DATA_FORMAT_VERSION}"
rules:
- rule: rule path 1
steps:
- intent: goodbye
- action: action_test_2
""")
importer = RasaFileImporter(domain_path=domain_path,
training_data_paths=[rules_file_name])
validator = Validator.from_importer(importer)
with pytest.warns(UserWarning) as warning:
validity = validator.verify_actions_in_stories_rules()
assert validity is False
assert ("The action 'action_test_2' is used in the 'rule path 1' block, "
"but it is not listed in the domain file."
in warning[0].message.args[0])
def test_valid_form_slots_in_domain(tmp_path: Path):
domain = tmp_path / "domain.yml"
domain.write_text(f"""
version: "{LATEST_TRAINING_DATA_FORMAT_VERSION}"
forms:
name_form:
required_slots:
- first_name
- last_name
slots:
first_name:
type: text
mappings:
- type: from_text
last_name:
type: text
mappings:
- type: from_text
""")
importer = RasaFileImporter(domain_path=domain)
validator = Validator.from_importer(importer)
assert validator.verify_form_slots()
def test_verify_slot_mappings_slot_with_mapping_conditions_not_in_form(
tmp_path: Path):
domain = tmp_path / "domain.yml"
domain.write_text("""
version: "3.0"
intents:
- activate_booking
entities:
- city
slots:
location:
type: text
influence_conversation: false
mappings:
- type: from_entity
entity: city
conditions:
- active_loop: booking_form
started_booking_form:
type: bool
influence_conversation: false
mappings:
- type: from_trigger_intent
intent: activate_booking
value: true
forms:
booking_form:
required_slots:
- started_booking_form
""")
importer = RasaFileImporter(domain_path=domain)
validator = Validator.from_importer(importer)
with pytest.warns(
UserWarning,
match=
r"Slot 'location' has a mapping condition for form 'booking_form', "
r"but it's not present in 'booking_form' form's 'required_slots'.*",
):
assert not validator.verify_slot_mappings()
def test_verify_slot_mappings_from_trigger_intent_mapping_slot_not_in_forms(
tmp_path: Path,
):
domain = tmp_path / "domain.yml"
slot_name = "started_booking_form"
domain.write_text(
f"""
version: "3.0"
intents:
- activate_booking
entities:
- city
slots:
{slot_name}:
type: bool
influence_conversation: false
mappings:
- type: from_trigger_intent
intent: activate_booking
value: true
location:
type: text
mappings:
- type: from_entity
entity: city
forms:
booking_form:
required_slots:
- location
"""
)
importer = RasaFileImporter(domain_path=domain)
validator = Validator.from_importer(importer)
with pytest.warns(
UserWarning,
match=f"Slot '{slot_name}' has a 'from_trigger_intent' mapping, "
f"but it's not listed in any form 'required_slots'.",
):
assert not validator.verify_slot_mappings()
