def test_verify_logging_message_for_intent_not_used_in_story(
caplog: LogCaptureFixture, validator_under_test: Validator):
caplog.clear()
with pytest.warns(UserWarning) as record:
validator_under_test.verify_intents_in_stories(False)
assert "The intent 'goodbye' is not used in any story or rule." in (
m.message.args[0] for m in record)
def test_verify_logging_message_for_unused_utterance(
caplog: LogCaptureFixture, validator_under_test: Validator):
caplog.clear()
with pytest.warns(UserWarning) as record:
validator_under_test.verify_utterances_in_stories(False)
assert "The utterance 'utter_chatter' is not used in any story or rule." in (
m.message.args[0] for m in record)
def test_verify_logging_message_for_intent_not_used_in_nlu(
caplog: LogCaptureFixture, validator_under_test: Validator):
caplog.clear()
with pytest.warns(UserWarning) as record:
validator_under_test.verify_intents(False)
assert ("The intent 'goodbye' is listed in the domain file, "
"but is not found in the NLU training data."
in (m.message.args[0] for m in record))
def test_verify_domain_with_duplicates(
duplicates: Optional[Dict[Text, List[Text]]],
is_valid: bool,
warning_type: Any,
messages: List[Text],
):
domain = Domain([], [], [], {}, [], {}, duplicates=duplicates)
validator = Validator(domain, None, None, None)
with pytest.warns(warning_type) as warning:
assert validator.verify_domain_duplicates() is is_valid
assert len(warning) == len(messages)
for i in range(len(messages)):
assert messages[i] in warning[i].message.args[0]
def test_verify_correct_e2e_story_structure(tmp_path: Path):
story_file_name = tmp_path / "stories.yml"
with open(story_file_name, "w") as file:
file.write("""
stories:
- story: path 1
steps:
- user: |
hello assistant! Can you help me today?
- action: utter_greet
- story: path 2 - state is similar but different from the one in path 1
steps:
- user: |
hello assistant! you Can help me today?
- action: utter_goodbye
- story: path 3
steps:
- user: |
That's it for today. Chat again tomorrow!
- action: utter_goodbye
""")
importer = RasaFileImporter(
config_file="data/test_config/config_defaults.yml",
domain_path="data/test_domains/default.yml",
training_data_paths=[story_file_name],
training_type=TrainingType.NLU,
)
validator = Validator.from_importer(importer)
assert validator.verify_story_structure(ignore_warnings=False)
def test_verify_intents_does_not_fail_on_valid_data(nlu_data_path: Text):
importer = RasaFileImporter(
domain_path="data/test_moodbot/domain.yml",
training_data_paths=[nlu_data_path],
)
validator = Validator.from_importer(importer)
assert validator.verify_intents()
def test_verify_there_is_example_repetition_in_intents(nlu_data_path: Text):
# moodbot nlu data already has duplicated example 'good afternoon'
# for intents greet and goodbye
importer = RasaFileImporter(domain_path="data/test_moodbot/domain.yml",
training_data_paths=[nlu_data_path])
validator = Validator.from_importer(importer)
assert not validator.verify_example_repetition_in_intents(False)
def test_verify_bad_e2e_story_structure_when_text_identical(tmp_path: Path):
story_file_name = tmp_path / "stories.yml"
story_file_name.write_text("""
version: "3.0"
stories:
- story: path 1
steps:
- user: |
amazing!
- action: utter_happy
- story: path 2 (should always conflict path 1)
steps:
- user: |
amazing!
- action: utter_cheer_up
""")
# The two stories with identical user texts
importer = RasaFileImporter(
config_file="data/test_config/config_defaults.yml",
domain_path="data/test_domains/default.yml",
training_data_paths=[story_file_name],
training_type=TrainingType.NLU,
)
validator = Validator.from_importer(importer)
assert not validator.verify_story_structure(ignore_warnings=False)
def validate_files(args: argparse.Namespace, stories_only: bool = False) -> None:
"""Validates either the story structure or the entire project.
Args:
args: Commandline arguments
stories_only: If `True`, only the story structure is validated.
"""
loop = asyncio.get_event_loop()
file_importer = RasaFileImporter(
domain_path=args.domain, training_data_paths=args.data
)
validator = loop.run_until_complete(Validator.from_importer(file_importer))
if stories_only:
all_good = _validate_story_structure(validator, args)
else:
all_good = (
_validate_domain(validator)
and _validate_nlu(validator, args)
and _validate_story_structure(validator, args)
)
if not all_good:
rasa.shared.utils.cli.print_error_and_exit(
"Project validation completed with errors."
)
def reads(self, string: Text, **kwargs: Any) -> "TrainingData":
"""Reads TrainingData in YAML format from a string.
Args:
string: String with YAML training data.
**kwargs: Keyword arguments.
Returns:
New `TrainingData` object with parsed training data.
"""
from rasa.nlu.training_data import TrainingData
from rasa.validator import Validator
self.validate(string)
yaml_content = io_utils.read_yaml(string)
if not Validator.validate_training_data_format_version(
yaml_content, self.filename):
return TrainingData()
for key, value in yaml_content.items(): # pytype: disable=attribute-error
if key == KEY_NLU:
self._parse_nlu(value)
elif key == KEY_RESPONSES:
self.responses = value
return TrainingData(
self.training_examples,
self.entity_synonyms,
self.regex_features,
self.lookup_tables,
self.responses,
)
def validate_files(args: argparse.Namespace, stories_only: bool = False) -> None:
"""Validates either the story structure or the entire project.
Args:
args: Commandline arguments
stories_only: If `True`, only the story structure is validated.
"""
from rasa.validator import Validator
config = rasa.cli.utils.get_validated_path(
args.config, "config", DEFAULT_CONFIG_PATH, none_is_valid=True
)
file_importer = RasaFileImporter(
domain_path=args.domain, training_data_paths=args.data, config_file=config,
)
validator = Validator.from_importer(file_importer)
if stories_only:
all_good = _validate_story_structure(validator, args)
else:
all_good = (
_validate_domain(validator)
and _validate_nlu(validator, args)
and _validate_story_structure(validator, args)
)
telemetry.track_validate_files(all_good)
if not all_good:
rasa.shared.utils.cli.print_error_and_exit(
"Project validation completed with errors."
)
def test_verify_bad_story_structure_ignore_warnings():
importer = RasaFileImporter(
domain_path="data/test_domains/default.yml",
training_data_paths=["data/test_yaml_stories/stories_conflicting_2.yml"],
)
validator = Validator.from_importer(importer)
assert validator.verify_story_structure(ignore_warnings=True)
def test_verify_story_structure(stories_path: Text):
importer = RasaFileImporter(
domain_path="data/test_domains/default.yml",
training_data_paths=[stories_path],
)
validator = Validator.from_importer(importer)
assert validator.verify_story_structure(ignore_warnings=False)
def read_from_parsed_yaml(
self, parsed_content: Dict[Text, Union[Dict,
List]]) -> List[StoryStep]:
"""Read stories from parsed YAML.
Args:
parsed_content: The parsed YAML as a dictionary.
Returns:
The parsed stories or rules.
"""
from rasa.validator import Validator
if not Validator.validate_training_data_format_version(
parsed_content, self.source_name):
return []
stories = parsed_content.get(KEY_STORIES, [])
self._parse_data(stories, is_rule_data=False)
rules = parsed_content.get(KEY_RULES, [])
self._parse_data(rules, is_rule_data=True)
self._add_current_stories_to_result()
return self.story_steps
def test_verify_intents_does_fail_on_invalid_data(nlu_data_path: Text):
# domain and nlu data are from different domain and should produce warnings
importer = RasaFileImporter(
domain_path="data/test_domains/default.yml", training_data_paths=[nlu_data_path]
)
validator = Validator.from_importer(importer)
assert not validator.verify_intents()
def test_valid_stories_rules_default_actions(
file_name: Text, data_type: Text, tmp_path: Path
):
domain = tmp_path / "domain.yml"
domain.write_text(
"""
version: "3.0"
intents:
- greet
"""
)
file_name = tmp_path / f"{file_name}.yml"
file_name.write_text(
f"""
version: "3.0"
{file_name}:
- {data_type}: test path
steps:
- intent: greet
- action: action_restart
"""
)
importer = RasaFileImporter(domain_path=domain, training_data_paths=[file_name])
validator = Validator.from_importer(importer)
assert validator.verify_actions_in_stories_rules()
def test_verify_actions_in_stories_not_in_domain(tmp_path: Path, domain_path: Text):
story_file_name = tmp_path / "stories.yml"
story_file_name.write_text(
"""
version: "3.0"
stories:
- story: story path 1
steps:
- intent: greet
- action: action_test_1
"""
)
importer = RasaFileImporter(
domain_path=domain_path, training_data_paths=[story_file_name]
)
validator = Validator.from_importer(importer)
with pytest.warns(UserWarning) as warning:
validity = validator.verify_actions_in_stories_rules()
assert validity is False
assert (
"The action 'action_test_1' is used in the 'story path 1' block, "
"but it is not listed in the domain file." in warning[0].message.args[0]
)
def test_verify_nlu_with_e2e_story(tmp_path: Path, nlu_data_path: Path):
story_file_name = tmp_path / "stories.yml"
with open(story_file_name, "w") as file:
file.write(
"""
stories:
- story: path 1
steps:
- user: |
hello assistant! Can you help me today?
- action: utter_greet
- story: path 2
steps:
- intent: greet
- action: utter_greet
"""
)
importer = RasaFileImporter(
config_file="data/test_moodbot/config.yml",
domain_path="data/test_moodbot/domain.yml",
training_data_paths=[story_file_name, nlu_data_path],
)
validator = Validator.from_importer(importer)
assert validator.verify_nlu()
def read_from_parsed_yaml(
self, parsed_content: Dict[Text, Union[Dict,
List]]) -> List[StoryStep]:
"""Read stories from parsed YAML.
Args:
parsed_content: The parsed YAML as a dictionary.
Returns:
The parsed stories or rules.
"""
from rasa.validator import Validator
if not Validator.validate_training_data_format_version(
parsed_content, self.source_name):
return []
for key, parser_class in {
KEY_STORIES: StoryParser,
KEY_RULES: RuleParser,
}.items():
data = parsed_content.get(key, [])
parser = parser_class.from_reader(self)
parser.parse_data(data)
self.story_steps.extend(parser.get_steps())
return self.story_steps
def test_verify_slot_mappings_valid(tmp_path: Path):
domain = tmp_path / "domain.yml"
domain.write_text("""
version: "3.0"
intents:
- activate_booking
entities:
- city
slots:
location:
type: text
influence_conversation: false
mappings:
- type: from_entity
entity: city
conditions:
- active_loop: booking_form
started_booking_form:
type: bool
influence_conversation: false
mappings:
- type: from_trigger_intent
intent: activate_booking
value: true
forms:
booking_form:
required_slots:
- started_booking_form
- location
""")
importer = RasaFileImporter(domain_path=domain)
validator = Validator.from_importer(importer)
assert validator.verify_slot_mappings()
def test_early_exit_on_invalid_domain():
domain_path = "data/test_domains/duplicate_intents.yml"
importer = RasaFileImporter(domain_path=domain_path)
with pytest.warns(UserWarning) as record:
validator = Validator.from_importer(importer)
validator.verify_domain_validity()
# two for non-unique domains, 2 for auto-fill removal
assert len(record) == 4
non_unique_warnings = list(
filter(
lambda warning: f"Loading domain from '{domain_path}' failed. "
f"Using empty domain. Error: 'Intents are not unique! "
f"Found multiple intents with name(s) ['default', 'goodbye']. "
f"Either rename or remove the duplicate ones.'" in warning.message.
args[0],
record,
))
assert len(non_unique_warnings) == 2
auto_fill_warnings = list(
filter(
lambda warning: "Slot auto-fill has been removed in 3.0" in warning
.message.args[0],
record,
))
assert len(auto_fill_warnings) == 2
def test_verify_slot_mappings_mapping_active_loop_not_in_forms(tmp_path: Path):
domain = tmp_path / "domain.yml"
slot_name = "some_slot"
domain.write_text(f"""
version: "3.0"
entities:
- some_entity
slots:
{slot_name}:
type: text
influence_conversation: false
mappings:
- type: from_entity
entity: some_entity
conditions:
- active_loop: som_form
forms:
some_form:
required_slots:
- {slot_name}
""")
importer = RasaFileImporter(domain_path=domain)
validator = Validator.from_importer(importer)
with pytest.warns(
UserWarning,
match=r"Slot 'some_slot' has a mapping condition "
r"for form 'som_form' which is not listed "
r"in domain forms.*",
):
assert not validator.verify_slot_mappings()
def test_verify_form_slots_invalid_domain(tmp_path: Path):
domain = tmp_path / "domain.yml"
domain.write_text("""
version: "3.0"
forms:
name_form:
required_slots:
- first_name
- last_nam
slots:
first_name:
type: text
mappings:
- type: from_text
last_name:
type: text
mappings:
- type: from_text
""")
importer = RasaFileImporter(domain_path=domain)
validator = Validator.from_importer(importer)
with pytest.warns(UserWarning) as w:
validity = validator.verify_form_slots()
assert validity is False
assert (
w[0].message.args[0] == "The form slot 'last_nam' in form 'name_form' "
"is not present in the domain slots."
"Please add the correct slot or check for typos.")
def test_verify_there_is_not_example_repetition_in_intents():
importer = RasaFileImporter(
domain_path="data/test_moodbot/domain.yml",
training_data_paths=["examples/knowledgebasebot/data/nlu.yml"],
)
validator = Validator.from_importer(importer)
assert validator.verify_example_repetition_in_intents(False)
async def test_invalid_training_data_format_version_warns():
invalid_version_1 = {KEY_TRAINING_DATA_FORMAT_VERSION: 2.0}
invalid_version_2 = {KEY_TRAINING_DATA_FORMAT_VERSION: "Rasa"}
for version in [invalid_version_1, invalid_version_2]:
with pytest.warns(UserWarning):
assert Validator.validate_training_data_format_version(version, "")
async def test_future_training_data_format_version_not_compatible():
next_minor = str(Version(LATEST_TRAINING_DATA_FORMAT_VERSION).next_minor())
incompatible_version = {KEY_TRAINING_DATA_FORMAT_VERSION: next_minor}
with pytest.warns(UserWarning):
assert not Validator.validate_training_data_format_version(
incompatible_version, "")
def test_verify_valid_responses_in_rules(nlu_data_path: Text):
importer = RasaFileImporter(
domain_path="data/test_domains/default.yml",
training_data_paths=[
nlu_data_path,
"data/test_yaml_stories/rules_without_stories_and_wrong_names.yml",
],
)
validator = Validator.from_importer(importer)
assert not validator.verify_utterances_in_stories()
def test_verify_valid_responses():
importer = RasaFileImporter(
domain_path="data/test_domains/selectors.yml",
training_data_paths=[
"data/test_selectors/nlu.yml",
"data/test_selectors/stories.yml",
],
)
validator = Validator.from_importer(importer)
assert validator.verify_utterances_in_stories()
def _validate_story_structure(validator: Validator, args: argparse.Namespace) -> bool:
# Check if a valid setting for `max_history` was given
if isinstance(args.max_history, int) and args.max_history < 1:
raise argparse.ArgumentTypeError(
f"The value of `--max-history {args.max_history}` is not a positive integer."
)
return validator.verify_story_structure(
not args.fail_on_warnings, max_history=args.max_history
)
def validator_under_test() -> Validator:
importer = RasaFileImporter(
domain_path="data/test_validation/domain.yml",
training_data_paths=[
"data/test_validation/data/nlu.yml",
"data/test_validation/data/stories.yml",
],
)
validator = Validator.from_importer(importer)
return validator
