def dump( self, target: Union[Text, Path, ruamel_yaml.StringIO], story_steps: List[StoryStep], ) -> None: """Writes Story steps into a target file/stream. Args: target: name of the target file/stream to write the YAML to. story_steps: Original story steps to be converted to the YAML. """ from rasa.validator import KEY_TRAINING_DATA_FORMAT_VERSION self.target = target stories = [] for story_step in story_steps: processed_story_step = self.process_story_step(story_step) if processed_story_step: stories.append(processed_story_step) result = OrderedDict() result[KEY_TRAINING_DATA_FORMAT_VERSION] = DoubleQuotedScalarString( LATEST_TRAINING_DATA_FORMAT_VERSION) result[KEY_STORIES] = stories io_utils.write_yaml(result, self.target, True)
def test_get_valid_config(parameters): config_path = None if parameters["config_data"] is not None: config_path = os.path.join(tempfile.mkdtemp(), "config.yml") io_utils.write_yaml(parameters["config_data"], config_path) default_config_path = None if parameters["default_config"] is not None: default_config_path = os.path.join(tempfile.mkdtemp(), "default-config.yml") io_utils.write_yaml(parameters["default_config"], default_config_path) if parameters["error"]: with pytest.raises(SystemExit): _get_valid_config(config_path, parameters["mandatory_keys"]) else: config_path = _get_valid_config(config_path, parameters["mandatory_keys"], default_config_path) config_data = io_utils.read_yaml_file(config_path) for k in parameters["mandatory_keys"]: assert k in config_data
def create_simple_project(path: Path): scaffold.create_initial_project(str(path)) # create a config file # for the cli test the resulting model is not important, use components that are # fast to train write_yaml( { "language": "en", "pipeline": [{ "name": "KeywordIntentClassifier" }], "policies": [ { "name": "MappingPolicy" }, { "name": "MemoizationPolicy", "max_history": 3 }, ], }, path / "config.yml", ) return path
def dump(self, target: Union[Text, Path, StringIO], training_data: "TrainingData") -> None: """Writes training data into a file in a YAML format. Args: target: Name of the target object to write the YAML to. training_data: TrainingData object. """ from rasa.validator import KEY_TRAINING_DATA_FORMAT_VERSION from ruamel.yaml.scalarstring import DoubleQuotedScalarString nlu_items = [] nlu_items.extend(self.process_intents(training_data)) nlu_items.extend(self.process_synonyms(training_data)) nlu_items.extend(self.process_regexes(training_data)) nlu_items.extend(self.process_lookup_tables(training_data)) result = OrderedDict() result[KEY_TRAINING_DATA_FORMAT_VERSION] = DoubleQuotedScalarString( LATEST_TRAINING_DATA_FORMAT_VERSION) if nlu_items: result[KEY_NLU] = nlu_items if training_data.responses: result[KEY_RESPONSES] = training_data.responses io_utils.write_yaml(result, target, True)
def write_endpoint_config_to_yaml( path: Path, data: Dict[Text, Any], endpoints_filename: Text = "endpoints.yml") -> Path: endpoints_path = path / endpoints_filename # write endpoints config to file io_utils.write_yaml(data, endpoints_path) return endpoints_path
def test_write_utf_8_yaml_file(tmp_path: Path): """This test makes sure that dumping a yaml doesn't result in Uxxxx sequences but rather directly dumps the unicode character.""" file_path = str(tmp_path / "test.yml") data = {"data": "amazing 🌈"} io_utils.write_yaml(data, file_path) assert io_utils.read_file(file_path) == "data: amazing 🌈\n"
def test_load_from_config(tmpdir: Path): import rasa.utils.io as io_utils config_path = str(tmpdir / "config.yml") io_utils.write_yaml({"importers": [{ "name": "MultiProjectImporter" }]}, config_path) importer = TrainingDataImporter.load_from_config(config_path) assert isinstance(importer, CombinedDataImporter) assert isinstance(importer._importers[0], MultiProjectImporter)
def test_prepare_credentials_for_rasa_x_if_rasa_channel_not_given( tmpdir: Path): credentials_path = str(tmpdir / "credentials.yml") io_utils.write_yaml({}, credentials_path) tmp_credentials = x._prepare_credentials_for_rasa_x( credentials_path, "http://localhost:5002") actual = io_utils.read_config_file(tmp_credentials) assert actual["rasa"]["url"] == "http://localhost:5002"
def test_test_core_comparison_after_train( run_in_simple_project: Callable[..., RunResult]): write_yaml({ "language": "en", "policies": [{ "name": "MemoizationPolicy" }] }, "config_1.yml") write_yaml({ "language": "en", "policies": [{ "name": "MemoizationPolicy" }] }, "config_2.yml") run_in_simple_project( "train", "core", "-c", "config_1.yml", "config_2.yml", "--stories", "data/stories.yml", "--runs", "2", "--percentages", "25", "75", "--augmentation", "5", "--out", "comparison_models", ) assert os.path.exists("comparison_models") assert os.path.exists("comparison_models/run_1") assert os.path.exists("comparison_models/run_2") run_in_simple_project( "test", "core", "-m", "comparison_models", "--stories", "data/stories", "--evaluate-model-directory", ) assert os.path.exists(os.path.join(DEFAULT_RESULTS_PATH, RESULTS_FILE)) assert os.path.exists( os.path.join(DEFAULT_RESULTS_PATH, "core_model_comparison_graph.pdf"))
def test_train_core_compare(run_in_simple_project: Callable[..., RunResult]): temp_dir = os.getcwd() io_utils.write_yaml( { "language": "en", "pipeline": "supervised_embeddings", "policies": [{ "name": "MemoizationPolicy" }], }, "config_1.yml", ) io_utils.write_yaml( { "language": "en", "pipeline": "supervised_embeddings", "policies": [{ "name": "MemoizationPolicy" }], }, "config_2.yml", ) run_in_simple_project( "train", "core", "-c", "config_1.yml", "config_2.yml", "--stories", "data/stories.yml", "--out", "core_comparison_results", "--runs", "2", "--percentages", "25", "75", "--augmentation", "5", ) assert os.path.exists(os.path.join(temp_dir, "core_comparison_results")) run_directories = io_utils.list_subdirectories( os.path.join(temp_dir, "core_comparison_results")) assert len(run_directories) == 2 model_files = io_utils.list_files( os.path.join(temp_dir, "core_comparison_results", run_directories[0])) assert len(model_files) == 4 assert model_files[0].endswith("tar.gz")
def dump_obj_as_yaml_to_file(filename: Union[Text, Path], obj: Any, should_preserve_key_order: bool = False) -> None: """Writes `obj` to the filename in YAML repr. Args: filename: Target filename. obj: Object to dump. should_preserve_key_order: Whether to preserve key order in `obj`. """ io_utils.write_yaml(obj, filename, should_preserve_key_order=should_preserve_key_order)
def dump( self, target: Union[Text, Path, yaml.StringIO], story_steps: List[StoryStep], ) -> None: """Writes Story steps into a target file/stream. Args: target: name of the target file/stream to write the YAML to. story_steps: Original story steps to be converted to the YAML. """ result = self.stories_to_yaml(story_steps) io_utils.write_yaml(result, target, True)
def update_number_of_epochs(config_path: Text, output_file: Text): config = io_utils.read_yaml_file(config_path) if "pipeline" not in config.keys(): raise ValueError(f"Invalid config provided! File: '{config_path}'.") for component in config["pipeline"]: # do not update epochs for pipeline templates if not isinstance(component, dict): continue if component["name"] in [DIETClassifier.name, ResponseSelector.name]: component[EPOCHS] = 1 io_utils.write_yaml(config, output_file)
async def test_fail_on_invalid_utterances(tmpdir): # domain and stories are from different domain and should produce warnings invalid_domain = str(tmpdir / "invalid_domain.yml") io_utils.write_yaml( { "responses": {"utter_greet": {"text": "hello"}}, "actions": [ "utter_greet", "utter_non_existent", # error: utter template odes not exist ], }, invalid_domain, ) importer = RasaFileImporter(domain_path=invalid_domain) validator = await Validator.from_importer(importer) assert not validator.verify_utterances()
def test_test(run_in_simple_project_with_model: Callable[..., RunResult]): write_yaml( { "pipeline": "KeywordIntentClassifier", "policies": [{ "name": "MemoizationPolicy" }], }, "config2.yml", ) run_in_simple_project_with_model("test") assert os.path.exists("results") assert os.path.exists("results/intent_histogram.png") assert os.path.exists("results/intent_confusion_matrix.png")
def test_prepare_credentials_if_already_valid(tmpdir: Path): credentials_path = str(tmpdir / "credentials.yml") credentials = { "rasa": { "url": "my-custom-url" }, "another-channel": { "url": "some-url" }, } io_utils.write_yaml(credentials, credentials_path) x._prepare_credentials_for_rasa_x(credentials_path) actual = io_utils.read_config_file(credentials_path) assert actual == credentials
def test_test_nlu_comparison(run_in_simple_project: Callable[..., RunResult]): write_yaml({"pipeline": "KeywordIntentClassifier"}, "config.yml") write_yaml({"pipeline": "KeywordIntentClassifier"}, "config2.yml") run_in_simple_project( "test", "nlu", "--config", "config.yml", "config2.yml", "--run", "2", "--percentages", "75", "25", ) assert os.path.exists("results/run_1") assert os.path.exists("results/run_2")
def test_dump_yaml_key_order(tmp_path: Path, should_preserve_key_order: bool, expected_keys: List[Text]): file = tmp_path / "test.yml" # create YAML file with keys in reverse-alphabetical order content = "" for i in reversed(string.ascii_lowercase): content += f"{i}: {uuid.uuid4().hex}\n" file.write_text(content) # load this file and ensure keys are in correct reverse-alphabetical order data = io_utils.read_yaml_file(file) assert list(data.keys()) == list(reversed(string.ascii_lowercase)) # dumping `data` will result in alphabetical or reverse-alphabetical list of keys, # depending on the value of `should_preserve_key_order` io_utils.write_yaml(data, file, should_preserve_key_order=should_preserve_key_order) with file.open() as f: keys = [line.split(":")[0] for line in f.readlines()] assert keys == expected_keys
def test_minimal_yaml_nlu_file(tmp_path): target_file = tmp_path / "test_nlu_file.yaml" io_utils.write_yaml(MINIMAL_VALID_EXAMPLE, target_file, True) assert RasaYAMLReader.is_yaml_nlu_file(target_file)
def _write_endpoint_config_to_yaml(path: Path, data: Dict[Text, Any]) -> Path: endpoints_path = path / "endpoints.yml" # write endpoints config to file io_utils.write_yaml(data, endpoints_path) return endpoints_path