def test_verify_logging_message_for_intent_not_used_in_story( caplog: LogCaptureFixture, validator_under_test: Validator): caplog.clear() with pytest.warns(UserWarning) as record: validator_under_test.verify_intents_in_stories(False) assert "The intent 'goodbye' is not used in any story or rule." in ( m.message.args[0] for m in record)
def test_verify_logging_message_for_unused_utterance( caplog: LogCaptureFixture, validator_under_test: Validator): caplog.clear() with pytest.warns(UserWarning) as record: validator_under_test.verify_utterances_in_stories(False) assert "The utterance 'utter_chatter' is not used in any story or rule." in ( m.message.args[0] for m in record)
def test_verify_logging_message_for_intent_not_used_in_nlu( caplog: LogCaptureFixture, validator_under_test: Validator): caplog.clear() with pytest.warns(UserWarning) as record: validator_under_test.verify_intents(False) assert ("The intent 'goodbye' is listed in the domain file, " "but is not found in the NLU training data." in (m.message.args[0] for m in record))
def test_verify_domain_with_duplicates( duplicates: Optional[Dict[Text, List[Text]]], is_valid: bool, warning_type: Any, messages: List[Text], ): domain = Domain([], [], [], {}, [], {}, duplicates=duplicates) validator = Validator(domain, None, None, None) with pytest.warns(warning_type) as warning: assert validator.verify_domain_duplicates() is is_valid assert len(warning) == len(messages) for i in range(len(messages)): assert messages[i] in warning[i].message.args[0]
def test_verify_correct_e2e_story_structure(tmp_path: Path): story_file_name = tmp_path / "stories.yml" with open(story_file_name, "w") as file: file.write(""" stories: - story: path 1 steps: - user: | hello assistant! Can you help me today? - action: utter_greet - story: path 2 - state is similar but different from the one in path 1 steps: - user: | hello assistant! you Can help me today? - action: utter_goodbye - story: path 3 steps: - user: | That's it for today. Chat again tomorrow! - action: utter_goodbye """) importer = RasaFileImporter( config_file="data/test_config/config_defaults.yml", domain_path="data/test_domains/default.yml", training_data_paths=[story_file_name], training_type=TrainingType.NLU, ) validator = Validator.from_importer(importer) assert validator.verify_story_structure(ignore_warnings=False)
def test_verify_intents_does_not_fail_on_valid_data(nlu_data_path: Text): importer = RasaFileImporter( domain_path="data/test_moodbot/domain.yml", training_data_paths=[nlu_data_path], ) validator = Validator.from_importer(importer) assert validator.verify_intents()
def test_verify_there_is_example_repetition_in_intents(nlu_data_path: Text): # moodbot nlu data already has duplicated example 'good afternoon' # for intents greet and goodbye importer = RasaFileImporter(domain_path="data/test_moodbot/domain.yml", training_data_paths=[nlu_data_path]) validator = Validator.from_importer(importer) assert not validator.verify_example_repetition_in_intents(False)
def test_verify_bad_e2e_story_structure_when_text_identical(tmp_path: Path): story_file_name = tmp_path / "stories.yml" story_file_name.write_text(""" version: "3.0" stories: - story: path 1 steps: - user: | amazing! - action: utter_happy - story: path 2 (should always conflict path 1) steps: - user: | amazing! - action: utter_cheer_up """) # The two stories with identical user texts importer = RasaFileImporter( config_file="data/test_config/config_defaults.yml", domain_path="data/test_domains/default.yml", training_data_paths=[story_file_name], training_type=TrainingType.NLU, ) validator = Validator.from_importer(importer) assert not validator.verify_story_structure(ignore_warnings=False)
def validate_files(args: argparse.Namespace, stories_only: bool = False) -> None: """Validates either the story structure or the entire project. Args: args: Commandline arguments stories_only: If `True`, only the story structure is validated. """ loop = asyncio.get_event_loop() file_importer = RasaFileImporter( domain_path=args.domain, training_data_paths=args.data ) validator = loop.run_until_complete(Validator.from_importer(file_importer)) if stories_only: all_good = _validate_story_structure(validator, args) else: all_good = ( _validate_domain(validator) and _validate_nlu(validator, args) and _validate_story_structure(validator, args) ) if not all_good: rasa.shared.utils.cli.print_error_and_exit( "Project validation completed with errors." )
def reads(self, string: Text, **kwargs: Any) -> "TrainingData": """Reads TrainingData in YAML format from a string. Args: string: String with YAML training data. **kwargs: Keyword arguments. Returns: New `TrainingData` object with parsed training data. """ from rasa.nlu.training_data import TrainingData from rasa.validator import Validator self.validate(string) yaml_content = io_utils.read_yaml(string) if not Validator.validate_training_data_format_version( yaml_content, self.filename): return TrainingData() for key, value in yaml_content.items(): # pytype: disable=attribute-error if key == KEY_NLU: self._parse_nlu(value) elif key == KEY_RESPONSES: self.responses = value return TrainingData( self.training_examples, self.entity_synonyms, self.regex_features, self.lookup_tables, self.responses, )
def validate_files(args: argparse.Namespace, stories_only: bool = False) -> None: """Validates either the story structure or the entire project. Args: args: Commandline arguments stories_only: If `True`, only the story structure is validated. """ from rasa.validator import Validator config = rasa.cli.utils.get_validated_path( args.config, "config", DEFAULT_CONFIG_PATH, none_is_valid=True ) file_importer = RasaFileImporter( domain_path=args.domain, training_data_paths=args.data, config_file=config, ) validator = Validator.from_importer(file_importer) if stories_only: all_good = _validate_story_structure(validator, args) else: all_good = ( _validate_domain(validator) and _validate_nlu(validator, args) and _validate_story_structure(validator, args) ) telemetry.track_validate_files(all_good) if not all_good: rasa.shared.utils.cli.print_error_and_exit( "Project validation completed with errors." )
def test_verify_bad_story_structure_ignore_warnings(): importer = RasaFileImporter( domain_path="data/test_domains/default.yml", training_data_paths=["data/test_yaml_stories/stories_conflicting_2.yml"], ) validator = Validator.from_importer(importer) assert validator.verify_story_structure(ignore_warnings=True)
def test_verify_story_structure(stories_path: Text): importer = RasaFileImporter( domain_path="data/test_domains/default.yml", training_data_paths=[stories_path], ) validator = Validator.from_importer(importer) assert validator.verify_story_structure(ignore_warnings=False)
def read_from_parsed_yaml( self, parsed_content: Dict[Text, Union[Dict, List]]) -> List[StoryStep]: """Read stories from parsed YAML. Args: parsed_content: The parsed YAML as a dictionary. Returns: The parsed stories or rules. """ from rasa.validator import Validator if not Validator.validate_training_data_format_version( parsed_content, self.source_name): return [] stories = parsed_content.get(KEY_STORIES, []) self._parse_data(stories, is_rule_data=False) rules = parsed_content.get(KEY_RULES, []) self._parse_data(rules, is_rule_data=True) self._add_current_stories_to_result() return self.story_steps
def test_verify_intents_does_fail_on_invalid_data(nlu_data_path: Text): # domain and nlu data are from different domain and should produce warnings importer = RasaFileImporter( domain_path="data/test_domains/default.yml", training_data_paths=[nlu_data_path] ) validator = Validator.from_importer(importer) assert not validator.verify_intents()
def test_valid_stories_rules_default_actions( file_name: Text, data_type: Text, tmp_path: Path ): domain = tmp_path / "domain.yml" domain.write_text( """ version: "3.0" intents: - greet """ ) file_name = tmp_path / f"{file_name}.yml" file_name.write_text( f""" version: "3.0" {file_name}: - {data_type}: test path steps: - intent: greet - action: action_restart """ ) importer = RasaFileImporter(domain_path=domain, training_data_paths=[file_name]) validator = Validator.from_importer(importer) assert validator.verify_actions_in_stories_rules()
def test_verify_actions_in_stories_not_in_domain(tmp_path: Path, domain_path: Text): story_file_name = tmp_path / "stories.yml" story_file_name.write_text( """ version: "3.0" stories: - story: story path 1 steps: - intent: greet - action: action_test_1 """ ) importer = RasaFileImporter( domain_path=domain_path, training_data_paths=[story_file_name] ) validator = Validator.from_importer(importer) with pytest.warns(UserWarning) as warning: validity = validator.verify_actions_in_stories_rules() assert validity is False assert ( "The action 'action_test_1' is used in the 'story path 1' block, " "but it is not listed in the domain file." in warning[0].message.args[0] )
def test_verify_nlu_with_e2e_story(tmp_path: Path, nlu_data_path: Path): story_file_name = tmp_path / "stories.yml" with open(story_file_name, "w") as file: file.write( """ stories: - story: path 1 steps: - user: | hello assistant! Can you help me today? - action: utter_greet - story: path 2 steps: - intent: greet - action: utter_greet """ ) importer = RasaFileImporter( config_file="data/test_moodbot/config.yml", domain_path="data/test_moodbot/domain.yml", training_data_paths=[story_file_name, nlu_data_path], ) validator = Validator.from_importer(importer) assert validator.verify_nlu()
def read_from_parsed_yaml( self, parsed_content: Dict[Text, Union[Dict, List]]) -> List[StoryStep]: """Read stories from parsed YAML. Args: parsed_content: The parsed YAML as a dictionary. Returns: The parsed stories or rules. """ from rasa.validator import Validator if not Validator.validate_training_data_format_version( parsed_content, self.source_name): return [] for key, parser_class in { KEY_STORIES: StoryParser, KEY_RULES: RuleParser, }.items(): data = parsed_content.get(key, []) parser = parser_class.from_reader(self) parser.parse_data(data) self.story_steps.extend(parser.get_steps()) return self.story_steps
def test_verify_slot_mappings_valid(tmp_path: Path): domain = tmp_path / "domain.yml" domain.write_text(""" version: "3.0" intents: - activate_booking entities: - city slots: location: type: text influence_conversation: false mappings: - type: from_entity entity: city conditions: - active_loop: booking_form started_booking_form: type: bool influence_conversation: false mappings: - type: from_trigger_intent intent: activate_booking value: true forms: booking_form: required_slots: - started_booking_form - location """) importer = RasaFileImporter(domain_path=domain) validator = Validator.from_importer(importer) assert validator.verify_slot_mappings()
def test_early_exit_on_invalid_domain(): domain_path = "data/test_domains/duplicate_intents.yml" importer = RasaFileImporter(domain_path=domain_path) with pytest.warns(UserWarning) as record: validator = Validator.from_importer(importer) validator.verify_domain_validity() # two for non-unique domains, 2 for auto-fill removal assert len(record) == 4 non_unique_warnings = list( filter( lambda warning: f"Loading domain from '{domain_path}' failed. " f"Using empty domain. Error: 'Intents are not unique! " f"Found multiple intents with name(s) ['default', 'goodbye']. " f"Either rename or remove the duplicate ones.'" in warning.message. args[0], record, )) assert len(non_unique_warnings) == 2 auto_fill_warnings = list( filter( lambda warning: "Slot auto-fill has been removed in 3.0" in warning .message.args[0], record, )) assert len(auto_fill_warnings) == 2
def test_verify_slot_mappings_mapping_active_loop_not_in_forms(tmp_path: Path): domain = tmp_path / "domain.yml" slot_name = "some_slot" domain.write_text(f""" version: "3.0" entities: - some_entity slots: {slot_name}: type: text influence_conversation: false mappings: - type: from_entity entity: some_entity conditions: - active_loop: som_form forms: some_form: required_slots: - {slot_name} """) importer = RasaFileImporter(domain_path=domain) validator = Validator.from_importer(importer) with pytest.warns( UserWarning, match=r"Slot 'some_slot' has a mapping condition " r"for form 'som_form' which is not listed " r"in domain forms.*", ): assert not validator.verify_slot_mappings()
def test_verify_form_slots_invalid_domain(tmp_path: Path): domain = tmp_path / "domain.yml" domain.write_text(""" version: "3.0" forms: name_form: required_slots: - first_name - last_nam slots: first_name: type: text mappings: - type: from_text last_name: type: text mappings: - type: from_text """) importer = RasaFileImporter(domain_path=domain) validator = Validator.from_importer(importer) with pytest.warns(UserWarning) as w: validity = validator.verify_form_slots() assert validity is False assert ( w[0].message.args[0] == "The form slot 'last_nam' in form 'name_form' " "is not present in the domain slots." "Please add the correct slot or check for typos.")
def test_verify_there_is_not_example_repetition_in_intents(): importer = RasaFileImporter( domain_path="data/test_moodbot/domain.yml", training_data_paths=["examples/knowledgebasebot/data/nlu.yml"], ) validator = Validator.from_importer(importer) assert validator.verify_example_repetition_in_intents(False)
async def test_invalid_training_data_format_version_warns(): invalid_version_1 = {KEY_TRAINING_DATA_FORMAT_VERSION: 2.0} invalid_version_2 = {KEY_TRAINING_DATA_FORMAT_VERSION: "Rasa"} for version in [invalid_version_1, invalid_version_2]: with pytest.warns(UserWarning): assert Validator.validate_training_data_format_version(version, "")
async def test_future_training_data_format_version_not_compatible(): next_minor = str(Version(LATEST_TRAINING_DATA_FORMAT_VERSION).next_minor()) incompatible_version = {KEY_TRAINING_DATA_FORMAT_VERSION: next_minor} with pytest.warns(UserWarning): assert not Validator.validate_training_data_format_version( incompatible_version, "")
def test_verify_valid_responses_in_rules(nlu_data_path: Text): importer = RasaFileImporter( domain_path="data/test_domains/default.yml", training_data_paths=[ nlu_data_path, "data/test_yaml_stories/rules_without_stories_and_wrong_names.yml", ], ) validator = Validator.from_importer(importer) assert not validator.verify_utterances_in_stories()
def test_verify_valid_responses(): importer = RasaFileImporter( domain_path="data/test_domains/selectors.yml", training_data_paths=[ "data/test_selectors/nlu.yml", "data/test_selectors/stories.yml", ], ) validator = Validator.from_importer(importer) assert validator.verify_utterances_in_stories()
def _validate_story_structure(validator: Validator, args: argparse.Namespace) -> bool: # Check if a valid setting for `max_history` was given if isinstance(args.max_history, int) and args.max_history < 1: raise argparse.ArgumentTypeError( f"The value of `--max-history {args.max_history}` is not a positive integer." ) return validator.verify_story_structure( not args.fail_on_warnings, max_history=args.max_history )
def validator_under_test() -> Validator: importer = RasaFileImporter( domain_path="data/test_validation/domain.yml", training_data_paths=[ "data/test_validation/data/nlu.yml", "data/test_validation/data/stories.yml", ], ) validator = Validator.from_importer(importer) return validator