def test_responses_text_multiline_is_preserved():
responses_yml = textwrap.dedent("""
responses:
utter_confirm:
- text: |-
First line
Second line
Third line
- text: One more response
utter_cancel:
- text: First line
- text: Second line
""")
reader = RasaYAMLReader()
result = reader.reads(responses_yml)
dumped = RasaYAMLWriter().dumps(result)
validation_reader = RasaYAMLReader()
dumped_result = validation_reader.reads(dumped)
assert dumped_result.responses == result.responses
# dumping again should also not change the format
assert dumped == RasaYAMLWriter().dumps(dumped_result)
def test_nlg_multimedia_load_dump_roundtrip():
responses_yml = textwrap.dedent(
"""
responses:
utter_chitchat/ask_weather:
- text: Where do you want to check the weather?
image: https://example.com/weather.jpg
utter_chitchat/ask_name:
- text: My name is Sara.
"""
)
reader = RasaYAMLReader()
result = reader.reads(responses_yml)
dumped = RasaYAMLWriter().dumps(result)
validation_reader = RasaYAMLReader()
dumped_result = validation_reader.reads(dumped)
assert dumped_result.responses == result.responses
# dumping again should also not change the format
assert dumped == RasaYAMLWriter().dumps(dumped_result)
async def convert_and_write(cls, source_path: Path, output_path: Path) -> None:
"""Converts the given training data file and saves it to the output directory.
Args:
source_path: Path to the training data file.
output_path: Path to the output directory.
"""
reader = NLGMarkdownReader()
writer = RasaYAMLWriter()
output_nlg_path = cls.generate_path_for_converted_training_data_file(
source_path, output_path
)
training_data = reader.read(source_path)
converted_responses = {}
for response_name, examples in training_data.responses.items():
new_response_name = cls._normalize_response_name(response_name)
converted_responses[new_response_name] = examples
converted_training_data = TrainingData(responses=converted_responses)
writer.dump(output_nlg_path, converted_training_data)
print_success(f"Converted NLG file: '{source_path}' >> '{output_nlg_path}'.")
def convert_training_data(data_file: Union[list, Text], out_file: Text,
output_format: Text, language: Text) -> None:
"""Convert training data.
Args:
data_file (Union[list, Text]): Path to the file or directory
containing Rasa data.
out_file (Text): File or existing path where to save
training data in Rasa format.
output_format (Text): Output format the training data
should be converted into.
language (Text): Language of the data.
"""
if isinstance(data_file, list):
data_file = data_file[0]
if not os.path.exists(str(data_file)):
print_error(
"Data file '{}' does not exist. Provide a valid NLU data file using "
"the '--data' argument.".format(data_file))
return
td = rasa.shared.nlu.training_data.loading.load_data(data_file, language)
if output_format == "json":
output = td.nlu_as_json(indent=2)
else:
output = RasaYAMLWriter().dumps(td)
write_to_file(out_file, output)
def test_responses_are_converted_from_markdown():
responses_md = textwrap.dedent("""
## ask name
* chitchat/ask_name
- my name is Sara, Rasa's documentation bot!
""")
result = NLGMarkdownReader().reads(responses_md)
dumped = RasaYAMLWriter().dumps(result)
validation_reader = RasaYAMLReader()
dumped_result = validation_reader.reads(dumped)
assert dumped_result.responses == result.responses
# dumping again should also not change the format
assert dumped == RasaYAMLWriter().dumps(dumped_result)
def test_write_metadata_stripped():
reader = RasaYAMLReader()
result = reader.reads(INTENT_EXAMPLES_WITH_METADATA)
# Add strippable characters to first example text
result.training_examples[0].data["text"] += " \r\n "
dumped = RasaYAMLWriter().dumps(result)
assert dumped == INTENT_EXAMPLES_WITH_METADATA
def convert_and_write(cls, source_path: Path, output_path: Path) -> None:
"""Converts the given training data file and saves it to the output directory.
Args:
source_path: Path to the training data file.
output_path: Path to the output directory.
"""
reader = NLGMarkdownReader()
writer = RasaYAMLWriter()
output_nlg_path = cls.generate_path_for_converted_training_data_file(
source_path, output_path)
yaml_training_data = reader.read(source_path)
writer.dump(output_nlg_path, yaml_training_data)
print_success(
f"Converted NLG file: '{source_path}' >> '{output_nlg_path}'.")
def nlu_as_yaml(self) -> Text:
from rasa.shared.nlu.training_data.formats.rasa_yaml import RasaYAMLWriter
# avoid dumping NLG data (responses). this is a workaround until we
# can remove the distinction between nlu & nlg when converting to a string
# (so until after we remove markdown support)
no_responses_training_data = copy.copy(self)
no_responses_training_data.responses = {}
return RasaYAMLWriter().dumps(no_responses_training_data)
def test_training_data_as_yaml_dict():
from collections import OrderedDict
parser = RasaYAMLReader()
writer = RasaYAMLWriter()
training_data = parser.reads("""
nlu:
- intent: some_intent
examples: |
- an example
responses:
utter_something:
- text: hello world
""")
structure = writer.training_data_to_dict(training_data)
assert isinstance(structure, OrderedDict)
assert "nlu" in structure
assert "responses" in structure
def test_metadata_roundtrip():
reader = RasaYAMLReader()
result = reader.reads(INTENT_EXAMPLES_WITH_METADATA)
dumped = RasaYAMLWriter().dumps(result)
assert dumped == INTENT_EXAMPLES_WITH_METADATA
validation_reader = RasaYAMLReader()
dumped_result = validation_reader.reads(dumped)
assert dumped_result.training_examples == result.training_examples
def nlg_as_yaml(self) -> Text:
"""Generates yaml representation of the response phrases (NLG) of TrainingData.
Returns:
responses in yaml format as a string
"""
from rasa.shared.nlu.training_data.formats.rasa_yaml import RasaYAMLWriter
# only dump responses. at some point it might make sense to remove the
# differentiation between dumping NLU and dumping responses. but we
# can't do that until after we remove markdown support.
return RasaYAMLWriter().dumps(TrainingData(responses=self.responses))
def prepare_nlu_text(example: Text, entities: List[Dict]):
"""
combines plain text and entities into training example format
:param example: training example plain text
:param entities: list of entities
:return: trianing example combine with enities
"""
if not Utility.check_empty_string(example):
if entities:
from rasa.shared.nlu.training_data.formats.rasa_yaml import RasaYAMLWriter
example = RasaYAMLWriter.generate_message({'text': example, "entities": entities})
return example
def test_intent_examples_multiline_consistency(tmp_path: pathlib.Path):
"""Test that multiline examples are written back as multiline examples."""
training_data_file = (pathlib.Path("data") /
"test_multiline_intent_examples_yaml" / "nlu.yml")
training_data_from_disc = RasaYAMLReader().read(
filename=training_data_file)
tmp_file = tmp_path / "nlu.yml"
RasaYAMLWriter().dump(tmp_file, training_data_from_disc)
rewritten_file_content = tmp_file.read_text(encoding="utf-8")
original_file_content = training_data_file.read_text(encoding="utf-8")
assert original_file_content == rewritten_file_content
async def _convert_nlu_training_data(
in_path: Text, out_path: Text, language: Text,
):
if rasa.shared.data.is_likely_yaml_file(out_path):
from rasa.shared.nlu.training_data.loading import load_data
from rasa.shared.nlu.training_data.formats.rasa_yaml import RasaYAMLWriter
training_data = load_data(in_path, language)
RasaYAMLWriter().dump(out_path, training_data)
else:
from rasa.nlu.convert import convert_training_data
convert_training_data(
in_path, out_path, Path(out_path).suffix.replace('.', ''), language,
)
def convert_and_write(cls, source_path: Path, output_path: Path) -> None:
"""Converts the given training data file and saves it to the output directory.
Args:
source_path: Path to the training data file.
output_path: Path to the output directory.
"""
output_nlu_path = cls.generate_path_for_converted_training_data_file(
source_path, output_path)
yaml_training_data = MarkdownReader().read(source_path)
RasaYAMLWriter().dump(output_nlu_path, yaml_training_data)
for lookup_table in yaml_training_data.lookup_tables:
cls._write_nlu_lookup_table_yaml(lookup_table, output_path)
print_success(
f"Converted NLU file: '{source_path}' >> '{output_nlu_path}'.")
def _write_nlu_lookup_table_yaml(cls, lookup_table: Dict[Text, Any],
output_dir_path: Path) -> None:
"""Converts and writes lookup tables examples from `txt` to `YAML` format.
Args:
lookup_table: Lookup tables items.
output_dir_path: Path to the target output directory.
"""
lookup_table_file = lookup_table.get("elements")
if not lookup_table_file or not isinstance(lookup_table_file, str):
return
examples_from_file = read_lookup_table_file(lookup_table_file)
target_filename = cls.generate_path_for_converted_training_data_file(
Path(lookup_table_file), output_dir_path)
entity_name = Path(lookup_table_file).stem
RasaYAMLWriter().dump(
target_filename,
TrainingData(lookup_tables=[{
"name": entity_name,
"elements": examples_from_file
}]),
)
def test_yaml_examples_are_written(example: Text):
parser = RasaYAMLReader()
writer = RasaYAMLWriter()
training_data = parser.reads(example)
assert example.strip() == writer.dumps(training_data).strip()
