def dump(
self,
target: Union[Text, Path, ruamel_yaml.StringIO],
story_steps: List[StoryStep],
) -> None:
"""Writes Story steps into a target file/stream.
Args:
target: name of the target file/stream to write the YAML to.
story_steps: Original story steps to be converted to the YAML.
"""
from rasa.validator import KEY_TRAINING_DATA_FORMAT_VERSION
self.target = target
stories = []
for story_step in story_steps:
processed_story_step = self.process_story_step(story_step)
if processed_story_step:
stories.append(processed_story_step)
result = OrderedDict()
result[KEY_TRAINING_DATA_FORMAT_VERSION] = DoubleQuotedScalarString(
LATEST_TRAINING_DATA_FORMAT_VERSION)
result[KEY_STORIES] = stories
io_utils.write_yaml(result, self.target, True)
def test_get_valid_config(parameters):
config_path = None
if parameters["config_data"] is not None:
config_path = os.path.join(tempfile.mkdtemp(), "config.yml")
io_utils.write_yaml(parameters["config_data"], config_path)
default_config_path = None
if parameters["default_config"] is not None:
default_config_path = os.path.join(tempfile.mkdtemp(),
"default-config.yml")
io_utils.write_yaml(parameters["default_config"], default_config_path)
if parameters["error"]:
with pytest.raises(SystemExit):
_get_valid_config(config_path, parameters["mandatory_keys"])
else:
config_path = _get_valid_config(config_path,
parameters["mandatory_keys"],
default_config_path)
config_data = io_utils.read_yaml_file(config_path)
for k in parameters["mandatory_keys"]:
assert k in config_data
def create_simple_project(path: Path):
scaffold.create_initial_project(str(path))
# create a config file
# for the cli test the resulting model is not important, use components that are
# fast to train
write_yaml(
{
"language":
"en",
"pipeline": [{
"name": "KeywordIntentClassifier"
}],
"policies": [
{
"name": "MappingPolicy"
},
{
"name": "MemoizationPolicy",
"max_history": 3
},
],
},
path / "config.yml",
)
return path
def dump(self, target: Union[Text, Path, StringIO],
training_data: "TrainingData") -> None:
"""Writes training data into a file in a YAML format.
Args:
target: Name of the target object to write the YAML to.
training_data: TrainingData object.
"""
from rasa.validator import KEY_TRAINING_DATA_FORMAT_VERSION
from ruamel.yaml.scalarstring import DoubleQuotedScalarString
nlu_items = []
nlu_items.extend(self.process_intents(training_data))
nlu_items.extend(self.process_synonyms(training_data))
nlu_items.extend(self.process_regexes(training_data))
nlu_items.extend(self.process_lookup_tables(training_data))
result = OrderedDict()
result[KEY_TRAINING_DATA_FORMAT_VERSION] = DoubleQuotedScalarString(
LATEST_TRAINING_DATA_FORMAT_VERSION)
if nlu_items:
result[KEY_NLU] = nlu_items
if training_data.responses:
result[KEY_RESPONSES] = training_data.responses
io_utils.write_yaml(result, target, True)
def write_endpoint_config_to_yaml(
path: Path,
data: Dict[Text, Any],
endpoints_filename: Text = "endpoints.yml") -> Path:
endpoints_path = path / endpoints_filename
# write endpoints config to file
io_utils.write_yaml(data, endpoints_path)
return endpoints_path
def test_write_utf_8_yaml_file(tmp_path: Path):
"""This test makes sure that dumping a yaml doesn't result in Uxxxx sequences
but rather directly dumps the unicode character."""
file_path = str(tmp_path / "test.yml")
data = {"data": "amazing 🌈"}
io_utils.write_yaml(data, file_path)
assert io_utils.read_file(file_path) == "data: amazing 🌈\n"
def test_load_from_config(tmpdir: Path):
import rasa.utils.io as io_utils
config_path = str(tmpdir / "config.yml")
io_utils.write_yaml({"importers": [{
"name": "MultiProjectImporter"
}]}, config_path)
importer = TrainingDataImporter.load_from_config(config_path)
assert isinstance(importer, CombinedDataImporter)
assert isinstance(importer._importers[0], MultiProjectImporter)
def test_prepare_credentials_for_rasa_x_if_rasa_channel_not_given(
tmpdir: Path):
credentials_path = str(tmpdir / "credentials.yml")
io_utils.write_yaml({}, credentials_path)
tmp_credentials = x._prepare_credentials_for_rasa_x(
credentials_path, "http://localhost:5002")
actual = io_utils.read_config_file(tmp_credentials)
assert actual["rasa"]["url"] == "http://localhost:5002"
def test_test_core_comparison_after_train(
run_in_simple_project: Callable[..., RunResult]):
write_yaml({
"language": "en",
"policies": [{
"name": "MemoizationPolicy"
}]
}, "config_1.yml")
write_yaml({
"language": "en",
"policies": [{
"name": "MemoizationPolicy"
}]
}, "config_2.yml")
run_in_simple_project(
"train",
"core",
"-c",
"config_1.yml",
"config_2.yml",
"--stories",
"data/stories.yml",
"--runs",
"2",
"--percentages",
"25",
"75",
"--augmentation",
"5",
"--out",
"comparison_models",
)
assert os.path.exists("comparison_models")
assert os.path.exists("comparison_models/run_1")
assert os.path.exists("comparison_models/run_2")
run_in_simple_project(
"test",
"core",
"-m",
"comparison_models",
"--stories",
"data/stories",
"--evaluate-model-directory",
)
assert os.path.exists(os.path.join(DEFAULT_RESULTS_PATH, RESULTS_FILE))
assert os.path.exists(
os.path.join(DEFAULT_RESULTS_PATH, "core_model_comparison_graph.pdf"))
def test_train_core_compare(run_in_simple_project: Callable[..., RunResult]):
temp_dir = os.getcwd()
io_utils.write_yaml(
{
"language": "en",
"pipeline": "supervised_embeddings",
"policies": [{
"name": "MemoizationPolicy"
}],
},
"config_1.yml",
)
io_utils.write_yaml(
{
"language": "en",
"pipeline": "supervised_embeddings",
"policies": [{
"name": "MemoizationPolicy"
}],
},
"config_2.yml",
)
run_in_simple_project(
"train",
"core",
"-c",
"config_1.yml",
"config_2.yml",
"--stories",
"data/stories.yml",
"--out",
"core_comparison_results",
"--runs",
"2",
"--percentages",
"25",
"75",
"--augmentation",
"5",
)
assert os.path.exists(os.path.join(temp_dir, "core_comparison_results"))
run_directories = io_utils.list_subdirectories(
os.path.join(temp_dir, "core_comparison_results"))
assert len(run_directories) == 2
model_files = io_utils.list_files(
os.path.join(temp_dir, "core_comparison_results", run_directories[0]))
assert len(model_files) == 4
assert model_files[0].endswith("tar.gz")
def dump_obj_as_yaml_to_file(filename: Union[Text, Path],
obj: Any,
should_preserve_key_order: bool = False) -> None:
"""Writes `obj` to the filename in YAML repr.
Args:
filename: Target filename.
obj: Object to dump.
should_preserve_key_order: Whether to preserve key order in `obj`.
"""
io_utils.write_yaml(obj,
filename,
should_preserve_key_order=should_preserve_key_order)
def dump(
self,
target: Union[Text, Path, yaml.StringIO],
story_steps: List[StoryStep],
) -> None:
"""Writes Story steps into a target file/stream.
Args:
target: name of the target file/stream to write the YAML to.
story_steps: Original story steps to be converted to the YAML.
"""
result = self.stories_to_yaml(story_steps)
io_utils.write_yaml(result, target, True)
def update_number_of_epochs(config_path: Text, output_file: Text):
config = io_utils.read_yaml_file(config_path)
if "pipeline" not in config.keys():
raise ValueError(f"Invalid config provided! File: '{config_path}'.")
for component in config["pipeline"]:
# do not update epochs for pipeline templates
if not isinstance(component, dict):
continue
if component["name"] in [DIETClassifier.name, ResponseSelector.name]:
component[EPOCHS] = 1
io_utils.write_yaml(config, output_file)
async def test_fail_on_invalid_utterances(tmpdir):
# domain and stories are from different domain and should produce warnings
invalid_domain = str(tmpdir / "invalid_domain.yml")
io_utils.write_yaml(
{
"responses": {"utter_greet": {"text": "hello"}},
"actions": [
"utter_greet",
"utter_non_existent", # error: utter template odes not exist
],
},
invalid_domain,
)
importer = RasaFileImporter(domain_path=invalid_domain)
validator = await Validator.from_importer(importer)
assert not validator.verify_utterances()
def test_test(run_in_simple_project_with_model: Callable[..., RunResult]):
write_yaml(
{
"pipeline": "KeywordIntentClassifier",
"policies": [{
"name": "MemoizationPolicy"
}],
},
"config2.yml",
)
run_in_simple_project_with_model("test")
assert os.path.exists("results")
assert os.path.exists("results/intent_histogram.png")
assert os.path.exists("results/intent_confusion_matrix.png")
def test_prepare_credentials_if_already_valid(tmpdir: Path):
credentials_path = str(tmpdir / "credentials.yml")
credentials = {
"rasa": {
"url": "my-custom-url"
},
"another-channel": {
"url": "some-url"
},
}
io_utils.write_yaml(credentials, credentials_path)
x._prepare_credentials_for_rasa_x(credentials_path)
actual = io_utils.read_config_file(credentials_path)
assert actual == credentials
def test_test_nlu_comparison(run_in_simple_project: Callable[..., RunResult]):
write_yaml({"pipeline": "KeywordIntentClassifier"}, "config.yml")
write_yaml({"pipeline": "KeywordIntentClassifier"}, "config2.yml")
run_in_simple_project(
"test",
"nlu",
"--config",
"config.yml",
"config2.yml",
"--run",
"2",
"--percentages",
"75",
"25",
)
assert os.path.exists("results/run_1")
assert os.path.exists("results/run_2")
def test_dump_yaml_key_order(tmp_path: Path, should_preserve_key_order: bool,
expected_keys: List[Text]):
file = tmp_path / "test.yml"
# create YAML file with keys in reverse-alphabetical order
content = ""
for i in reversed(string.ascii_lowercase):
content += f"{i}: {uuid.uuid4().hex}\n"
file.write_text(content)
# load this file and ensure keys are in correct reverse-alphabetical order
data = io_utils.read_yaml_file(file)
assert list(data.keys()) == list(reversed(string.ascii_lowercase))
# dumping `data` will result in alphabetical or reverse-alphabetical list of keys,
# depending on the value of `should_preserve_key_order`
io_utils.write_yaml(data,
file,
should_preserve_key_order=should_preserve_key_order)
with file.open() as f:
keys = [line.split(":")[0] for line in f.readlines()]
assert keys == expected_keys
def test_minimal_yaml_nlu_file(tmp_path):
target_file = tmp_path / "test_nlu_file.yaml"
io_utils.write_yaml(MINIMAL_VALID_EXAMPLE, target_file, True)
assert RasaYAMLReader.is_yaml_nlu_file(target_file)
def _write_endpoint_config_to_yaml(path: Path, data: Dict[Text, Any]) -> Path:
endpoints_path = path / "endpoints.yml"
# write endpoints config to file
io_utils.write_yaml(data, endpoints_path)
return endpoints_path
