def test_dump_config(
tmp_path: Path,
input_file: Text,
expected_file: Text,
capsys: CaptureFixture,
autoconfig_keys: Set[Text],
):
config_file = str(tmp_path / "config.yml")
shutil.copyfile(str(CONFIG_FOLDER / input_file), config_file)
autoconfig.get_configuration(config_file)
actual = io_utils.read_file(config_file)
expected = io_utils.read_file(str(CONFIG_FOLDER / expected_file))
assert actual == expected
captured = capsys.readouterr()
assert "does not exist or is empty" not in captured.out
for k in CONFIG_AUTOCONFIGURABLE_KEYS:
if k in autoconfig_keys:
assert k in captured.out
else:
assert k not in captured.out
def guess_format(filename: Text) -> Text:
"""Applies heuristics to guess the data format of a file.
Args:
filename: file whose type should be guessed
Returns:
Guessed file format.
"""
from rasa.nlu.training_data.formats import RasaYAMLReader, markdown
guess = UNK
content = ""
try:
content = io_utils.read_file(filename)
js = json.loads(content)
except ValueError:
if any(marker in content for marker in markdown.MARKDOWN_SECTION_MARKERS):
guess = MARKDOWN
elif _is_nlg_story_format(content):
guess = MARKDOWN_NLG
elif RasaYAMLReader.is_yaml_nlu_file(filename):
guess = RASA_YAML
else:
for fformat, format_heuristic in _json_format_heuristics.items():
if format_heuristic(js, filename):
guess = fformat
break
logger.debug(f"Training data format of '{filename}' is '{guess}'.")
return guess
async def read_from_file(self, filename: Union[Text,
Path]) -> List[StoryStep]:
"""Read stories or rules from file.
Args:
filename: Path to the story/rule file.
Returns:
`StoryStep`s read from `filename`.
"""
self.source_name = filename
try:
file_content = io_utils.read_file(filename,
io_utils.DEFAULT_ENCODING)
validate_yaml_schema(file_content, CORE_SCHEMA_FILE)
yaml_content = io_utils.read_yaml(file_content)
except (ValueError, ParserError) as e:
common_utils.raise_warning(
f"Failed to read YAML from '{filename}', it will be skipped. Error: {e}"
)
return []
except InvalidYamlFileError as e:
raise ValueError from e
return self.read_from_parsed_yaml(yaml_content)
def guess_format(filename: Text) -> Text:
"""Applies heuristics to guess the data format of a file.
Args:
filename: file whose type should be guessed
Returns:
Guessed file format.
"""
guess = UNK
content = ""
try:
content = io_utils.read_file(filename)
js = json.loads(content)
except ValueError:
if any([marker in content for marker in _markdown_section_markers]):
guess = MARKDOWN
elif _is_nlg_story_format(content):
guess = MARKDOWN_NLG
else:
for fformat, format_heuristic in _json_format_heuristics.items():
if format_heuristic(js, filename):
guess = fformat
break
logger.debug("Training data format of '{}' is '{}'.".format(
filename, guess))
return guess
def inject_domain(
domain_path: Text,
domain_service: "DomainService",
project_id: Text = config.project_name,
username: Text = config.default_username,
) -> Dict[Text, Any]:
"""Load Rasa Core domain at `path` and save it to database.
Quits the application if domain cannot be loaded.
"""
if not os.path.exists(domain_path):
rasa_cli_utils.print_error_and_exit(
f"domain.yml could not be found at '{os.path.abspath(domain_path)}'. "
f"Rasa X requires a domain in the project root directory.")
try:
domain_service.validate_and_store_domain_yaml(
domain_yaml=read_file(domain_path),
project_id=project_id,
path=domain_path,
store_responses=True,
username=username,
should_dump_domain=False,
)
except InvalidDomain as e:
rasa_cli_utils.print_error_and_exit(
f"Could not inject domain. Details:\n{e}")
return domain_service.get_or_create_domain(project_id)
def test_nlu(args: argparse.Namespace) -> None:
from rasa import data
from rasa.test import compare_nlu_models, perform_nlu_cross_validation, test_nlu
nlu_data = cli_utils.get_validated_path(args.nlu, "nlu", DEFAULT_DATA_PATH)
nlu_data = data.get_nlu_directory(nlu_data)
output = args.out or DEFAULT_RESULTS_PATH
io_utils.create_directory(output)
if args.config is not None and len(args.config) == 1:
args.config = os.path.abspath(args.config[0])
if os.path.isdir(args.config):
config_dir = args.config
config_files = os.listdir(config_dir)
args.config = [
os.path.join(config_dir, os.path.abspath(config))
for config in config_files
]
if isinstance(args.config, list):
logger.info(
"Multiple configuration files specified, running nlu comparison mode."
)
config_files = []
for file in args.config:
try:
validation_utils.validate_yaml_schema(
io_utils.read_file(file),
CONFIG_SCHEMA_FILE,
show_validation_errors=False,
)
config_files.append(file)
except validation_utils.InvalidYamlFileError:
logger.debug(
"Ignoring file '{}' as it is not a valid config file.".
format(file))
continue
compare_nlu_models(
configs=config_files,
nlu=nlu_data,
output=output,
runs=args.runs,
exclusion_percentages=args.percentages,
)
elif args.cross_validation:
logger.info("Test model using cross validation.")
config = cli_utils.get_validated_path(args.config, "config",
DEFAULT_CONFIG_PATH)
perform_nlu_cross_validation(config, nlu_data, output, vars(args))
else:
model_path = cli_utils.get_validated_path(args.model, "model",
DEFAULT_MODELS_PATH)
test_nlu(model_path, nlu_data, output, vars(args))
def test_write_utf_8_yaml_file(tmp_path: Path):
"""This test makes sure that dumping a yaml doesn't result in Uxxxx sequences
but rather directly dumps the unicode character."""
file_path = str(tmp_path / "test.yml")
data = {"data": "amazing 🌈"}
io_utils.write_yaml_file(data, file_path)
assert io_utils.read_file(file_path) == "data: amazing 🌈\n"
def json_unpickle(file_name: Text) -> Any:
"""Unpickle an object from file using json."""
import jsonpickle.ext.numpy as jsonpickle_numpy
import jsonpickle
jsonpickle_numpy.register_handlers()
file_content = io_utils.read_file(file_name)
return jsonpickle.loads(file_content)
def get_private_ssh_key() -> Text:
"""Get the private key which Rasa X can use to authenticate with a Git server.
Returns: A private SSH key which can be used together with the public key which
the user has to store at their Git Server.
"""
if not Path(GIT_GENERATED_PRIVATE_KEY_PATH).exists():
raise ValueError("Please request the public key first.")
return io_utils.read_file(GIT_GENERATED_PRIVATE_KEY_PATH)
def get_public_ssh_key() -> Text:
"""Get a public SSH key which can be used to setup Integrated Version Control.
Returns: A public SSH key which has to be provided to the Git server so that
Rasa X can authenticate with its private key.
"""
if not Path(GIT_GENERATED_PRIVATE_KEY_PATH).exists():
GitSSHKeyProvider.generate_new_ssh_keys()
return io_utils.read_file(GIT_GENERATED_PUBLIC_KEY_PATH)
def test_get_configuration_for_different_training_types(
tmp_path: Path,
input_file: Text,
expected_file: Text,
expected_file_windows: Text,
training_type: autoconfig.TrainingType,
):
config_file = str(tmp_path / "config.yml")
shutil.copyfile(str(CONFIG_FOLDER / input_file), config_file)
autoconfig.get_configuration(config_file, training_type)
actual = io_utils.read_file(config_file)
if sys.platform == "win32":
expected = io_utils.read_file(
str(CONFIG_FOLDER / expected_file_windows))
else:
expected = io_utils.read_file(str(CONFIG_FOLDER / expected_file))
assert actual == expected
async def test_pull_runtime_config_from_server():
config_url = "http://example.com/api/config?token=token"
credentials = "rasa: http://example.com:5002/api"
endpoint_config = """
event_broker:
url: http://example.com/event_broker
username: some_username
password: PASSWORD
queue: broker_queue
"""
with aioresponses() as mocked:
mocked.get(
config_url,
payload={
"credentials": credentials,
"endpoints": endpoint_config
},
)
endpoints_path, credentials_path = await x._pull_runtime_config_from_server(
config_url, 1, 0)
assert io_utils.read_file(endpoints_path) == endpoint_config
assert io_utils.read_file(credentials_path) == credentials
def load(path: Text) -> Optional["TrackerFeaturizer"]:
"""Load the featurizer from file.
Args:
path: The path to load the tracker featurizer from.
Returns:
The loaded tracker featurizer.
"""
featurizer_file = os.path.join(path, "featurizer.json")
if os.path.isfile(featurizer_file):
return jsonpickle.decode(io_utils.read_file(featurizer_file))
logger.error(f"Couldn't load featurizer for policy. "
f"File '{featurizer_file}' doesn't exist.")
return None
def persist_graph(graph: "networkx.Graph", output_file: Text) -> None:
"""Plots the graph and persists it into a html file."""
import networkx as nx
import rasa.utils.io as io_utils
expg = nx.nx_pydot.to_pydot(graph)
template = io_utils.read_file(visualization_html_path())
# Insert graph into template
template = template.replace("// { is-client }", "isClient = true", 1)
graph_as_text = expg.to_string()
# escape backslashes
graph_as_text = graph_as_text.replace("\\", "\\\\")
template = template.replace("// { graph-content }",
f"graph = `{graph_as_text}`", 1)
io_utils.write_text_file(template, output_file)
def get_tests_from_file(filename: Optional[Text] = None) -> List[Text]:
"""Returns an list of tests from a `filename`.
Args:
filename: Path to a test file.
"""
if not filename:
filename = utils.get_project_directory() / DEFAULT_FILENAME
try:
content = io_utils.read_file(filename)
return _split_tests(content)
except ValueError as e:
rasa_utils.raise_warning(
f"Unable to get tests from {filename}:\n{e} "
f"Please, make sure you have end-to-end tests added to your assistant. "
f"See https://rasa.com/docs/rasa/user-guide/testing-your-assistant/ "
f"for more information.",
UserWarning,
)
return []
def extract_partial_endpoint_config(
endpoint_config_path: Text, key: Text
) -> Optional["EndpointConfig"]:
"""Extracts partial endpoint config at `key`.
Args:
endpoint_config_path: Path to endpoint config file to read.
key: Endpoint config key (section) to extract.
Returns:
Endpoint config initialised only from `key`.
"""
from rasa.utils import endpoints
# read endpoint config file and create dictionary containing only one
# key-value pair
content = rasa_io_utils.read_file(endpoint_config_path)
endpoint_dict = {key: load_yaml(content)[key]}
# dump this sub-dictionary to a temporary file and load endpoint config from it
temp_path = dump_as_yaml_to_temporary_file(endpoint_dict)
return endpoints.read_endpoint_config(temp_path, key)
def test_validate_yaml_schema(file, schema):
# should raise no exception
validation_utils.validate_yaml_schema(io_utils.read_file(file), schema)
def read(self, filename: Text, **kwargs: Any) -> "TrainingData":
"""Reads TrainingData from a file."""
self.filename = filename
return self.reads(io_utils.read_file(filename), **kwargs)
def test_read_file_with_not_existing_path():
with pytest.raises(ValueError):
io_utils.read_file("some path")
def is_markdown_nlu_file(filename: Union[Text, Path]) -> bool:
content = io_utils.read_file(filename)
return any(marker in content for marker in MARKDOWN_SECTION_MARKERS)
def test_validate_yaml_schema_raise_exception(file, schema):
with pytest.raises(validation_utils.InvalidYamlFileError):
validation_utils.validate_yaml_schema(io_utils.read_file(file), schema)
def _read_data(paths: List[Text]) -> Generator[Tuple[Text, Text], None, None]:
for filename in paths:
try:
yield read_file(filename), filename
except ValueError:
rasa_cli_utils.print_warning(f"Cannot read file {filename}")
