async def convert_and_write(cls, source_path: Path, output_path: Path) -> None: """Converts the given training data file and saves it to the output directory. Args: source_path: Path to the training data file. output_path: Path to the output directory. """ from rasa.shared.core.training_data.story_reader.yaml_story_reader import ( KEY_ACTIVE_LOOP, ) # check if source file is test stories file if MarkdownStoryReader.is_test_stories_file(source_path): reader = MarkdownStoryReader( is_used_for_training=False, use_e2e=True, ignore_deprecation_warning=True, ) output_core_path = cls._generate_path_for_converted_test_data_file( source_path, output_path) else: reader = MarkdownStoryReader(is_used_for_training=False, ignore_deprecation_warning=True) output_core_path = cls.generate_path_for_converted_training_data_file( source_path, output_path) steps = reader.read_from_file(source_path) if YAMLStoryWriter.stories_contain_loops(steps): print_warning( f"Training data file '{source_path}' contains forms. " f"Any 'form' events will be converted to '{KEY_ACTIVE_LOOP}' events. " f"Please note that in order for these stories to work you still " f"need the 'FormPolicy' to be active. However the 'FormPolicy' is " f"deprecated, please consider switching to the new 'RulePolicy', " f"for which you can find the documentation here: " f"{rasa.shared.constants.DOCS_URL_RULES}.") writer = YAMLStoryWriter() writer.dump( output_core_path, steps, is_test_story=MarkdownStoryReader.is_test_stories_file( source_path), ) print_success( f"Converted Core file: '{source_path}' >> '{output_core_path}'.")
def convert_and_write(cls, source_path: Path, output_path: Path) -> None: """Converts the given training data file and saves it to the output directory. Args: source_path: Path to the training data file. output_path: Path to the output directory. """ from rasa.shared.core.training_data.story_reader.yaml_story_reader import ( KEY_ACTIVE_LOOP, ) output_core_path = cls.generate_path_for_converted_training_data_file( source_path, output_path) reader = MarkdownStoryReader(unfold_or_utterances=False) writer = YAMLStoryWriter() loop = asyncio.get_event_loop() steps = loop.run_until_complete(reader.read_from_file(source_path)) if YAMLStoryWriter.stories_contain_loops(steps): print_warning( f"Training data file '{source_path}' contains forms. " f"Any 'form' events will be converted to '{KEY_ACTIVE_LOOP}' events. " f"Please note that in order for these stories to work you still " f"need the 'FormPolicy' to be active. However the 'FormPolicy' is " f"deprecated, please consider switching to the new 'RulePolicy', " f"for which you can find the documentation here: {DOCS_URL_RULES}." ) writer.dump(output_core_path, steps) print_success( f"Converted Core file: '{source_path}' >> '{output_core_path}'.")
async def test_simple_story(tmpdir: Path, default_domain: Domain, input_md_file: Text, input_yaml_file: Text): original_md_reader = MarkdownStoryReader(default_domain, None, False, input_md_file, is_used_for_training=False) original_md_story_steps = original_md_reader.read_from_file(input_md_file) assert not YAMLStoryWriter.stories_contain_loops(original_md_story_steps) original_yaml_reader = YAMLStoryReader(default_domain, None, False) original_yaml_story_steps = original_yaml_reader.read_from_file( input_yaml_file) target_story_filename = tmpdir / "test.yml" writer = YAMLStoryWriter() writer.dump(target_story_filename, original_md_story_steps) processed_yaml_reader = YAMLStoryReader(default_domain, None, False) processed_yaml_story_steps = processed_yaml_reader.read_from_file( target_story_filename) assert len(processed_yaml_story_steps) == len(original_yaml_story_steps) for processed_step, original_step in zip(processed_yaml_story_steps, original_yaml_story_steps): assert len(processed_step.events) == len(original_step.events)
async def test_story_start_checkpoint_is_skipped(default_domain: Domain): input_md_file = "data/test_stories/stories.md" original_md_reader = MarkdownStoryReader(default_domain, None, False, input_md_file, is_used_for_training=False) original_md_story_steps = original_md_reader.read_from_file(input_md_file) yaml_text = YAMLStoryWriter().dumps(original_md_story_steps) assert STORY_START not in yaml_text
async def _convert_core_training_data( in_path: Text, out_path: Text, ): from rasa.shared.core.training_data.story_reader.markdown_story_reader import ( MarkdownStoryReader, ) from rasa.shared.core.training_data.story_reader.yaml_story_reader import ( YAMLStoryReader, ) reader = (MarkdownStoryReader() if rasa.shared.data.is_likely_markdown_file(in_path) else YAMLStoryReader()) steps = reader.read_from_file(in_path) YAMLStoryWriter().dump(out_path, steps)
def _guess_reader( filename: Text, domain: Domain, template_variables: Optional[Dict] = None, use_e2e: bool = False, ) -> StoryReader: if YAMLStoryReader.is_stories_file(filename): return YAMLStoryReader(domain, template_variables, use_e2e, filename) elif MarkdownStoryReader.is_stories_file(filename): return MarkdownStoryReader(domain, template_variables, use_e2e, filename) raise ValueError( f"Failed to find a reader class for the story file `{filename}`. " f"Supported formats are " f"{', '.join(MARKDOWN_FILE_EXTENSIONS + YAML_FILE_EXTENSIONS)}." )
def _get_reader( filename: Text, domain: Domain, template_variables: Optional[Dict] = None, use_e2e: bool = False, ) -> StoryReader: if rasa.shared.data.is_likely_markdown_file(filename): return MarkdownStoryReader(domain, template_variables, use_e2e, filename) elif rasa.shared.data.is_likely_yaml_file(filename): return YAMLStoryReader(domain, template_variables, use_e2e, filename) else: # This is a use case for uploading the story over REST API. # The source file has a random name. return _guess_reader(filename, domain, template_variables, use_e2e)
async def test_forms_are_converted(default_domain: Domain): original_md_reader = MarkdownStoryReader(default_domain, None, False, is_used_for_training=False) original_md_story_steps = original_md_reader.read_from_file( "data/test_stories/stories_form.md") assert YAMLStoryWriter.stories_contain_loops(original_md_story_steps) writer = YAMLStoryWriter() with pytest.warns(None) as record: writer.dumps(original_md_story_steps) assert len(record) == 0
async def test_read_stories_with_multiline_comments(tmpdir, default_domain: Domain): reader = MarkdownStoryReader(default_domain) story_steps = reader.read_from_file( "data/test_stories/stories_with_multiline_comments.md") assert len(story_steps) == 4 assert story_steps[0].block_name == "happy path" assert len(story_steps[0].events) == 4 assert story_steps[1].block_name == "sad path 1" assert len(story_steps[1].events) == 7 assert story_steps[2].block_name == "sad path 2" assert len(story_steps[2].events) == 7 assert story_steps[3].block_name == "say goodbye" assert len(story_steps[3].events) == 2
def test_invalid_end_to_end_format(line: Text): reader = MarkdownStoryReader() with pytest.raises(ValueError): # noinspection PyProtectedMember _ = reader.parse_e2e_message(line)
def test_skip_markdown_reading_deprecation(): with pytest.warns(None) as warnings: MarkdownStoryReader(ignore_deprecation_warning=True) assert not warnings
def test_markdown_reading_deprecation(): with pytest.warns(FutureWarning): MarkdownStoryReader()
def test_e2e_parsing(line: Text, expected: Dict): actual = MarkdownStoryReader().parse_e2e_message(line) assert actual.as_dict() == expected