async def test_simple_story(tmpdir: Path, default_domain: Domain,
input_md_file: Text, input_yaml_file: Text):
original_md_reader = MarkdownStoryReader(default_domain,
None,
False,
input_md_file,
is_used_for_training=False)
original_md_story_steps = original_md_reader.read_from_file(input_md_file)
assert not YAMLStoryWriter.stories_contain_loops(original_md_story_steps)
original_yaml_reader = YAMLStoryReader(default_domain, None, False)
original_yaml_story_steps = original_yaml_reader.read_from_file(
input_yaml_file)
target_story_filename = tmpdir / "test.yml"
writer = YAMLStoryWriter()
writer.dump(target_story_filename, original_md_story_steps)
processed_yaml_reader = YAMLStoryReader(default_domain, None, False)
processed_yaml_story_steps = processed_yaml_reader.read_from_file(
target_story_filename)
assert len(processed_yaml_story_steps) == len(original_yaml_story_steps)
for processed_step, original_step in zip(processed_yaml_story_steps,
original_yaml_story_steps):
assert len(processed_step.events) == len(original_step.events)
def convert_and_write(cls, source_path: Path, output_path: Path) -> None:
"""Converts the given training data file and saves it to the output directory.
Args:
source_path: Path to the training data file.
output_path: Path to the output directory.
"""
from rasa.shared.core.training_data.story_reader.yaml_story_reader import (
KEY_ACTIVE_LOOP, )
output_core_path = cls.generate_path_for_converted_training_data_file(
source_path, output_path)
reader = MarkdownStoryReader(unfold_or_utterances=False)
writer = YAMLStoryWriter()
loop = asyncio.get_event_loop()
steps = loop.run_until_complete(reader.read_from_file(source_path))
if YAMLStoryWriter.stories_contain_loops(steps):
print_warning(
f"Training data file '{source_path}' contains forms. "
f"Any 'form' events will be converted to '{KEY_ACTIVE_LOOP}' events. "
f"Please note that in order for these stories to work you still "
f"need the 'FormPolicy' to be active. However the 'FormPolicy' is "
f"deprecated, please consider switching to the new 'RulePolicy', "
f"for which you can find the documentation here: {DOCS_URL_RULES}."
)
writer.dump(output_core_path, steps)
print_success(
f"Converted Core file: '{source_path}' >> '{output_core_path}'.")
async def test_story_start_checkpoint_is_skipped(default_domain: Domain):
input_md_file = "data/test_stories/stories.md"
original_md_reader = MarkdownStoryReader(default_domain,
None,
False,
input_md_file,
is_used_for_training=False)
original_md_story_steps = original_md_reader.read_from_file(input_md_file)
yaml_text = YAMLStoryWriter().dumps(original_md_story_steps)
assert STORY_START not in yaml_text
def filter(cls, source_path: Path) -> bool:
"""Checks if the given training data file contains Core data in `Markdown`
format and can be converted to `YAML`.
Args:
source_path: Path to the training data file.
Returns:
`True` if the given file can be converted, `False` otherwise
"""
return MarkdownStoryReader.is_stories_file(
source_path) or MarkdownStoryReader.is_test_stories_file(
source_path)
def _guess_reader(
filename: Text,
domain: Domain,
template_variables: Optional[Dict] = None,
use_e2e: bool = False,
) -> StoryReader:
if YAMLStoryReader.is_stories_file(filename):
return YAMLStoryReader(domain, template_variables, use_e2e, filename)
elif MarkdownStoryReader.is_stories_file(filename):
return MarkdownStoryReader(domain, template_variables, use_e2e, filename)
raise ValueError(
f"Failed to find a reader class for the story file `{filename}`. "
f"Supported formats are "
f"{', '.join(MARKDOWN_FILE_EXTENSIONS + YAML_FILE_EXTENSIONS)}."
)
async def test_forms_are_converted(default_domain: Domain):
original_md_reader = MarkdownStoryReader(default_domain,
None,
False,
is_used_for_training=False)
original_md_story_steps = original_md_reader.read_from_file(
"data/test_stories/stories_form.md")
assert YAMLStoryWriter.stories_contain_loops(original_md_story_steps)
writer = YAMLStoryWriter()
with pytest.warns(None) as record:
writer.dumps(original_md_story_steps)
assert len(record) == 0
async def test_read_stories_with_multiline_comments(tmpdir,
default_domain: Domain):
reader = MarkdownStoryReader(default_domain)
story_steps = reader.read_from_file(
"data/test_stories/stories_with_multiline_comments.md")
assert len(story_steps) == 4
assert story_steps[0].block_name == "happy path"
assert len(story_steps[0].events) == 4
assert story_steps[1].block_name == "sad path 1"
assert len(story_steps[1].events) == 7
assert story_steps[2].block_name == "sad path 2"
assert len(story_steps[2].events) == 7
assert story_steps[3].block_name == "say goodbye"
assert len(story_steps[3].events) == 2
async def _convert_core_training_data(
in_path: Text,
out_path: Text,
):
from rasa.shared.core.training_data.story_reader.markdown_story_reader import (
MarkdownStoryReader, )
from rasa.shared.core.training_data.story_reader.yaml_story_reader import (
YAMLStoryReader, )
reader = (MarkdownStoryReader()
if rasa.shared.data.is_likely_markdown_file(in_path) else
YAMLStoryReader())
steps = reader.read_from_file(in_path)
YAMLStoryWriter().dump(out_path, steps)
def _get_reader(
filename: Text,
domain: Domain,
template_variables: Optional[Dict] = None,
use_e2e: bool = False,
) -> StoryReader:
if rasa.shared.data.is_likely_markdown_file(filename):
return MarkdownStoryReader(domain, template_variables, use_e2e, filename)
elif rasa.shared.data.is_likely_yaml_file(filename):
return YAMLStoryReader(domain, template_variables, use_e2e, filename)
else:
# This is a use case for uploading the story over REST API.
# The source file has a random name.
return _guess_reader(filename, domain, template_variables, use_e2e)
def is_story_file(file_path: Text) -> bool:
"""Checks if a file is a Rasa story file.
Args:
file_path: Path of the file which should be checked.
Returns:
`True` if it's a story file, otherwise `False`.
"""
from rasa.shared.core.training_data.story_reader.yaml_story_reader import (
YAMLStoryReader, )
from rasa.shared.core.training_data.story_reader.markdown_story_reader import (
MarkdownStoryReader, )
return YAMLStoryReader.is_yaml_story_file(
file_path) or MarkdownStoryReader.is_markdown_story_file(file_path)
async def convert_and_write(cls, source_path: Path,
output_path: Path) -> None:
"""Converts the given training data file and saves it to the output directory.
Args:
source_path: Path to the training data file.
output_path: Path to the output directory.
"""
from rasa.shared.core.training_data.story_reader.yaml_story_reader import (
KEY_ACTIVE_LOOP, )
# check if source file is test stories file
if MarkdownStoryReader.is_test_stories_file(source_path):
reader = MarkdownStoryReader(
is_used_for_training=False,
use_e2e=True,
ignore_deprecation_warning=True,
)
output_core_path = cls._generate_path_for_converted_test_data_file(
source_path, output_path)
else:
reader = MarkdownStoryReader(is_used_for_training=False,
ignore_deprecation_warning=True)
output_core_path = cls.generate_path_for_converted_training_data_file(
source_path, output_path)
steps = reader.read_from_file(source_path)
if YAMLStoryWriter.stories_contain_loops(steps):
print_warning(
f"Training data file '{source_path}' contains forms. "
f"Any 'form' events will be converted to '{KEY_ACTIVE_LOOP}' events. "
f"Please note that in order for these stories to work you still "
f"need the 'FormPolicy' to be active. However the 'FormPolicy' is "
f"deprecated, please consider switching to the new 'RulePolicy', "
f"for which you can find the documentation here: "
f"{rasa.shared.constants.DOCS_URL_RULES}.")
writer = YAMLStoryWriter()
writer.dump(
output_core_path,
steps,
is_test_story=MarkdownStoryReader.is_test_stories_file(
source_path),
)
print_success(
f"Converted Core file: '{source_path}' >> '{output_core_path}'.")
def test_invalid_end_to_end_format(line: Text):
reader = MarkdownStoryReader()
with pytest.raises(ValueError):
# noinspection PyProtectedMember
_ = reader.parse_e2e_message(line)
def test_e2e_parsing(line: Text, expected: Dict):
actual = MarkdownStoryReader.parse_e2e_message(line)
assert actual.as_dict() == expected
def test_skip_markdown_reading_deprecation():
with pytest.warns(None) as warnings:
MarkdownStoryReader(ignore_deprecation_warning=True)
assert not warnings
def test_markdown_reading_deprecation():
with pytest.warns(FutureWarning):
MarkdownStoryReader()
