def test_responses_text_multiline_is_preserved(): responses_yml = textwrap.dedent(""" responses: utter_confirm: - text: |- First line Second line Third line - text: One more response utter_cancel: - text: First line - text: Second line """) reader = RasaYAMLReader() result = reader.reads(responses_yml) dumped = RasaYAMLWriter().dumps(result) validation_reader = RasaYAMLReader() dumped_result = validation_reader.reads(dumped) assert dumped_result.responses == result.responses # dumping again should also not change the format assert dumped == RasaYAMLWriter().dumps(dumped_result)
def test_nlg_multimedia_load_dump_roundtrip(): responses_yml = textwrap.dedent( """ responses: utter_chitchat/ask_weather: - text: Where do you want to check the weather? image: https://example.com/weather.jpg utter_chitchat/ask_name: - text: My name is Sara. """ ) reader = RasaYAMLReader() result = reader.reads(responses_yml) dumped = RasaYAMLWriter().dumps(result) validation_reader = RasaYAMLReader() dumped_result = validation_reader.reads(dumped) assert dumped_result.responses == result.responses # dumping again should also not change the format assert dumped == RasaYAMLWriter().dumps(dumped_result)
async def convert_and_write(cls, source_path: Path, output_path: Path) -> None: """Converts the given training data file and saves it to the output directory. Args: source_path: Path to the training data file. output_path: Path to the output directory. """ reader = NLGMarkdownReader() writer = RasaYAMLWriter() output_nlg_path = cls.generate_path_for_converted_training_data_file( source_path, output_path ) training_data = reader.read(source_path) converted_responses = {} for response_name, examples in training_data.responses.items(): new_response_name = cls._normalize_response_name(response_name) converted_responses[new_response_name] = examples converted_training_data = TrainingData(responses=converted_responses) writer.dump(output_nlg_path, converted_training_data) print_success(f"Converted NLG file: '{source_path}' >> '{output_nlg_path}'.")
def convert_training_data(data_file: Union[list, Text], out_file: Text, output_format: Text, language: Text) -> None: """Convert training data. Args: data_file (Union[list, Text]): Path to the file or directory containing Rasa data. out_file (Text): File or existing path where to save training data in Rasa format. output_format (Text): Output format the training data should be converted into. language (Text): Language of the data. """ if isinstance(data_file, list): data_file = data_file[0] if not os.path.exists(str(data_file)): print_error( "Data file '{}' does not exist. Provide a valid NLU data file using " "the '--data' argument.".format(data_file)) return td = rasa.shared.nlu.training_data.loading.load_data(data_file, language) if output_format == "json": output = td.nlu_as_json(indent=2) else: output = RasaYAMLWriter().dumps(td) write_to_file(out_file, output)
def test_responses_are_converted_from_markdown(): responses_md = textwrap.dedent(""" ## ask name * chitchat/ask_name - my name is Sara, Rasa's documentation bot! """) result = NLGMarkdownReader().reads(responses_md) dumped = RasaYAMLWriter().dumps(result) validation_reader = RasaYAMLReader() dumped_result = validation_reader.reads(dumped) assert dumped_result.responses == result.responses # dumping again should also not change the format assert dumped == RasaYAMLWriter().dumps(dumped_result)
def test_write_metadata_stripped(): reader = RasaYAMLReader() result = reader.reads(INTENT_EXAMPLES_WITH_METADATA) # Add strippable characters to first example text result.training_examples[0].data["text"] += " \r\n " dumped = RasaYAMLWriter().dumps(result) assert dumped == INTENT_EXAMPLES_WITH_METADATA
def convert_and_write(cls, source_path: Path, output_path: Path) -> None: """Converts the given training data file and saves it to the output directory. Args: source_path: Path to the training data file. output_path: Path to the output directory. """ reader = NLGMarkdownReader() writer = RasaYAMLWriter() output_nlg_path = cls.generate_path_for_converted_training_data_file( source_path, output_path) yaml_training_data = reader.read(source_path) writer.dump(output_nlg_path, yaml_training_data) print_success( f"Converted NLG file: '{source_path}' >> '{output_nlg_path}'.")
def nlu_as_yaml(self) -> Text: from rasa.shared.nlu.training_data.formats.rasa_yaml import RasaYAMLWriter # avoid dumping NLG data (responses). this is a workaround until we # can remove the distinction between nlu & nlg when converting to a string # (so until after we remove markdown support) no_responses_training_data = copy.copy(self) no_responses_training_data.responses = {} return RasaYAMLWriter().dumps(no_responses_training_data)
def test_training_data_as_yaml_dict(): from collections import OrderedDict parser = RasaYAMLReader() writer = RasaYAMLWriter() training_data = parser.reads(""" nlu: - intent: some_intent examples: | - an example responses: utter_something: - text: hello world """) structure = writer.training_data_to_dict(training_data) assert isinstance(structure, OrderedDict) assert "nlu" in structure assert "responses" in structure
def test_metadata_roundtrip(): reader = RasaYAMLReader() result = reader.reads(INTENT_EXAMPLES_WITH_METADATA) dumped = RasaYAMLWriter().dumps(result) assert dumped == INTENT_EXAMPLES_WITH_METADATA validation_reader = RasaYAMLReader() dumped_result = validation_reader.reads(dumped) assert dumped_result.training_examples == result.training_examples
def nlg_as_yaml(self) -> Text: """Generates yaml representation of the response phrases (NLG) of TrainingData. Returns: responses in yaml format as a string """ from rasa.shared.nlu.training_data.formats.rasa_yaml import RasaYAMLWriter # only dump responses. at some point it might make sense to remove the # differentiation between dumping NLU and dumping responses. but we # can't do that until after we remove markdown support. return RasaYAMLWriter().dumps(TrainingData(responses=self.responses))
def prepare_nlu_text(example: Text, entities: List[Dict]): """ combines plain text and entities into training example format :param example: training example plain text :param entities: list of entities :return: trianing example combine with enities """ if not Utility.check_empty_string(example): if entities: from rasa.shared.nlu.training_data.formats.rasa_yaml import RasaYAMLWriter example = RasaYAMLWriter.generate_message({'text': example, "entities": entities}) return example
def test_intent_examples_multiline_consistency(tmp_path: pathlib.Path): """Test that multiline examples are written back as multiline examples.""" training_data_file = (pathlib.Path("data") / "test_multiline_intent_examples_yaml" / "nlu.yml") training_data_from_disc = RasaYAMLReader().read( filename=training_data_file) tmp_file = tmp_path / "nlu.yml" RasaYAMLWriter().dump(tmp_file, training_data_from_disc) rewritten_file_content = tmp_file.read_text(encoding="utf-8") original_file_content = training_data_file.read_text(encoding="utf-8") assert original_file_content == rewritten_file_content
async def _convert_nlu_training_data( in_path: Text, out_path: Text, language: Text, ): if rasa.shared.data.is_likely_yaml_file(out_path): from rasa.shared.nlu.training_data.loading import load_data from rasa.shared.nlu.training_data.formats.rasa_yaml import RasaYAMLWriter training_data = load_data(in_path, language) RasaYAMLWriter().dump(out_path, training_data) else: from rasa.nlu.convert import convert_training_data convert_training_data( in_path, out_path, Path(out_path).suffix.replace('.', ''), language, )
def convert_and_write(cls, source_path: Path, output_path: Path) -> None: """Converts the given training data file and saves it to the output directory. Args: source_path: Path to the training data file. output_path: Path to the output directory. """ output_nlu_path = cls.generate_path_for_converted_training_data_file( source_path, output_path) yaml_training_data = MarkdownReader().read(source_path) RasaYAMLWriter().dump(output_nlu_path, yaml_training_data) for lookup_table in yaml_training_data.lookup_tables: cls._write_nlu_lookup_table_yaml(lookup_table, output_path) print_success( f"Converted NLU file: '{source_path}' >> '{output_nlu_path}'.")
def _write_nlu_lookup_table_yaml(cls, lookup_table: Dict[Text, Any], output_dir_path: Path) -> None: """Converts and writes lookup tables examples from `txt` to `YAML` format. Args: lookup_table: Lookup tables items. output_dir_path: Path to the target output directory. """ lookup_table_file = lookup_table.get("elements") if not lookup_table_file or not isinstance(lookup_table_file, str): return examples_from_file = read_lookup_table_file(lookup_table_file) target_filename = cls.generate_path_for_converted_training_data_file( Path(lookup_table_file), output_dir_path) entity_name = Path(lookup_table_file).stem RasaYAMLWriter().dump( target_filename, TrainingData(lookup_tables=[{ "name": entity_name, "elements": examples_from_file }]), )
def test_yaml_examples_are_written(example: Text): parser = RasaYAMLReader() writer = RasaYAMLWriter() training_data = parser.reads(example) assert example.strip() == writer.dumps(training_data).strip()