def test_get_valid_config(parameters):
config_path = None
if parameters["config_data"] is not None:
config_path = os.path.join(tempfile.mkdtemp(), "config.yml")
io_utils.write_yaml_file(parameters["config_data"], config_path)
default_config_path = None
if parameters["default_config"] is not None:
default_config_path = os.path.join(tempfile.mkdtemp(),
"default-config.yml")
io_utils.write_yaml_file(parameters["default_config"],
default_config_path)
if parameters["error"]:
with pytest.raises(SystemExit):
_get_valid_config(config_path, parameters["mandatory_keys"])
else:
config_path = _get_valid_config(config_path,
parameters["mandatory_keys"],
default_config_path)
config_data = io_utils.read_yaml_file(config_path)
for k in parameters["mandatory_keys"]:
assert k in config_data
def is_yaml_nlu_file(filename: Text) -> bool:
"""Checks if the specified file possibly contains NLU training data in YAML.
Args:
filename: name of the file to check.
Returns:
`True` if the `filename` is possibly a valid YAML NLU file,
`False` otherwise.
"""
if not Path(filename).suffix in YAML_FILE_EXTENSIONS:
return False
try:
content = io_utils.read_yaml_file(filename)
if KEY_NLU in content:
return True
except (YAMLError, Warning) as e:
logger.error(
f"Tried to check if '{filename}' is an NLU file, but failed to "
f"read it. If this file contains NLU data, you should "
f"investigate this error, otherwise it is probably best to "
f"move the file to a different location. "
f"Error: {e}"
)
return False
async def _inject_domain_from_model(self, project: Text,
domain_path: Text) -> None:
from rasax.community.services.domain_service import DomainService
domain_service = DomainService(self.session)
# do not inject if domain_service already contains a domain
if not domain_service.has_empty_or_no_domain(project):
return
from rasax.community.services.nlg_service import NlgService
from rasa.utils.io import read_yaml_file
data = read_yaml_file(domain_path)
# store responses if no responses found in NLG service
_, number_of_responses = NlgService(self.session).fetch_responses()
should_store_responses = number_of_responses == 0
domain_service.store_domain(
data,
project,
path=None,
store_responses=should_store_responses,
# responses injected by a model were already included in a training
have_responses_been_edited=False,
username=config.SYSTEM_USER,
)
def test_train_docker_and_docs_configs(config_file: Text):
content = io_utils.read_yaml_file(config_file)
loaded_config = config.load(config_file)
assert len(loaded_config.component_names) > 1
assert loaded_config.language == content["language"]
def is_yaml_story_file(file_path: Text) -> bool:
"""Check if file contains Core training data or rule data in YAML format.
Args:
file_path: Path of the file to check.
Returns:
`True` in case the file is a Core YAML training data or rule data file,
`False` otherwise.
"""
suffix = Path(file_path).suffix
if suffix and suffix not in YAML_FILE_EXTENSIONS:
return False
try:
content = io_utils.read_yaml_file(file_path)
return any(key in content for key in [KEY_STORIES, KEY_RULES])
except Exception as e:
# Using broad `Exception` because yaml library is not exposing all Errors
common_utils.raise_warning(
f"Tried to check if '{file_path}' is a story or rule file, but failed "
f"to read it. If this file contains story or rule data, you should "
f"investigate this error, otherwise it is probably best to "
f"move the file to a different location. Error: {e}")
return False
def inject_config(
config_path: Text,
settings_service: "SettingsService") -> Optional[Dict[Text, Any]]:
"""Load a configuration file from `path` and save it to the database.
Quits the application if config cannot be loaded.
"""
if not os.path.exists(config_path):
rasa_cli_utils.print_error_and_exit(
f"Failed to inject Rasa configuration. The file "
f"'{os.path.abspath(config_path)}' does not exist.")
_config = read_yaml_file(config_path)
if not _config:
rasa_cli_utils.print_error_and_exit(
f"Failed to inject Rasa configuration:\n"
f"Reading of yaml '{os.path.abspath(config_path)}' file failed. Most "
f"likely the file was not found or uses the wrong syntax.")
settings_service.save_config(config.team_name,
"default",
_config,
config_path,
should_dump=False)
logger.debug("Loaded local configuration from '{}' into database".format(
os.path.abspath(config_path)))
return _config
def generate_domain_file(conf: BotsConf):
cnt = io_utils.read_yaml_file(f'{conf.templates_dir}/domain.yml')
intents = cnt['intents']
actions = cnt['actions']
for f in glob.glob(conf.ruleset_files):
rules = io_utils.read_json_file(f)
for rule in rules:
intents.append({rule['intent']: {'triggers': rule['action']}})
actions.append(rule['action'])
return cnt
def _from_file(cls, path: Text, skill_selector: "SkillSelector") -> "SkillSelector":
path = os.path.abspath(path)
if os.path.exists(path) and data.is_config_file(path):
config = io_utils.read_yaml_file(path)
if isinstance(config, dict):
parent_directory = os.path.dirname(path)
return cls._from_dict(config, parent_directory, skill_selector)
return cls.all_skills()
def test_emojis_in_tmp_file():
test_data = """
data:
- one 😁💯 👩🏿💻👨🏿💻
- two £ (?u)\\b\\w+\\b f\u00fcr
"""
test_file = io_utils.create_temporary_file(test_data)
content = io_utils.read_yaml_file(test_file)
assert content["data"][0] == "one 😁💯 👩🏿💻👨🏿💻"
assert content["data"][1] == "two £ (?u)\\b\\w+\\b für"
def test_prepare_credentials_for_rasa_x_if_rasa_channel_not_given(
tmpdir_factory):
directory = tmpdir_factory.mktemp("directory")
credentials_path = str(directory / "credentials.yml")
io_utils.write_yaml_file({}, credentials_path)
x._prepare_credentials_for_rasa_x(credentials_path)
actual = io_utils.read_yaml_file(credentials_path)
assert actual["rasa"]["url"] == "http://localhost:5002"
def extract_domain_from_model(
model_path: Text) -> Optional[Dict[Text, Any]]:
from rasa.utils.io import read_yaml_file
from rasa.constants import DEFAULT_DOMAIN_PATH
temp_model_dir = rasa.model.unpack_model(model_path)
domain_path = os.path.join(temp_model_dir, "core", DEFAULT_DOMAIN_PATH)
try:
return read_yaml_file(domain_path)
except (FileNotFoundError, UnicodeDecodeError) as e:
logger.error("Could not read domain file for model at "
"'{}'. Details:\n{}".format(model_path, e))
finally:
shutil.rmtree(temp_model_dir, ignore_errors=True)
def update_number_of_epochs(config_path: Text, output_file: Text):
config = io_utils.read_yaml_file(config_path)
if "pipeline" not in config.keys():
raise ValueError(f"Invalid config provided! File: '{config_path}'.")
for component in config["pipeline"]:
# do not update epochs for pipeline templates
if not isinstance(component, dict):
continue
if component["name"] in [DIETClassifier.name, ResponseSelector.name]:
component[EPOCHS] = 1
io_utils.write_yaml(config, output_file)
def _prepare_credentials_for_rasa_x(credentials_path: Optional[Text],
rasa_x_url=None) -> Text:
credentials_path = get_validated_path(credentials_path, "credentials",
DEFAULT_CREDENTIALS_PATH, True)
if credentials_path:
credentials = io_utils.read_yaml_file(credentials_path)
else:
credentials = {}
# this makes sure the Rasa X is properly configured no matter what
if rasa_x_url:
credentials["rasa"] = {"url": rasa_x_url}
dumped_credentials = yaml.dump(credentials, default_flow_style=False)
tmp_credentials = io_utils.create_temporary_file(dumped_credentials, "yml")
return tmp_credentials
def _prepare_credentials_for_rasa_x(credentials_path: Optional[Text]) -> Text:
credentials_path = get_validated_path(credentials_path, "credentials",
DEFAULT_CREDENTIALS_PATH, True)
if credentials_path:
credentials = io_utils.read_yaml_file(credentials_path)
else:
credentials = {}
# If no credentials are given, create a new credentials file.
credentials_path = DEFAULT_CREDENTIALS_PATH
if not credentials.get("rasa"):
credentials["rasa"] = {"url": "http://localhost:5002/api"}
io_utils.write_yaml_file(credentials, credentials_path)
logging.debug("No Rasa credentials given. Creating one in '{}'"
"".format(credentials_path))
return credentials_path
def test_prepare_credentials_if_already_valid(tmpdir_factory):
directory = tmpdir_factory.mktemp("directory")
credentials_path = str(directory / "credentials.yml")
credentials = {
"rasa": {
"url": "my-custom-url"
},
"another-channel": {
"url": "some-url"
},
}
io_utils.write_yaml_file(credentials, credentials_path)
x._prepare_credentials_for_rasa_x(credentials_path)
actual = io_utils.read_yaml_file(credentials_path)
assert actual == credentials
def is_key_in_yaml(cls, file_path: Text, *keys: Text) -> bool:
"""Check if all keys are contained in the parsed dictionary from a yaml file.
Arguments:
file_path: path to the yaml file
keys: keys to look for
Returns:
`True` if all the keys are contained in the file, `False` otherwise.
"""
try:
content = io_utils.read_yaml_file(file_path)
return any(key in content for key in keys)
except Exception as e:
# Using broad `Exception` because yaml library is not exposing all Errors
rasa.shared.utils.io.raise_warning(
f"Tried to open '{file_path}' and load its data, but failed "
f"to read it. There seems to be an error with the yaml syntax: {e}"
)
return False
def inject_environments_config_from_file(self, project_id: Text,
filename: Text):
"""Inject a deployment environments configuration file at `filename`
into the db.
"""
try:
_config = io_utils.read_yaml_file(filename)
self.inspect_environments_config(_config)
self.save_environments_config(project_id, _config)
logger.debug("Successfully injected deployment environments "
"configuration from file '{}'.".format(filename))
except (ValueError, FileNotFoundError) as e:
current_config = self.get_environments_config(project_id)
logger.warning(
"Could not inject deployment environments "
"configuration from file '{}'. Details:\n{}\nYou may "
"still use Rasa X if you currently have a "
"working configuration. Your current configuration "
"is:\n{}".format(filename, e,
json.dumps(current_config, indent=2)))
def test_dump_yaml_key_order(tmp_path: Path, should_preserve_key_order: bool,
expected_keys: List[Text]):
file = tmp_path / "test.yml"
# create YAML file with keys in reverse-alphabetical order
content = ""
for i in reversed(string.ascii_lowercase):
content += f"{i}: {uuid.uuid4().hex}\n"
file.write_text(content)
# load this file and ensure keys are in correct reverse-alphabetical order
data = io_utils.read_yaml_file(file)
assert list(data.keys()) == list(reversed(string.ascii_lowercase))
# dumping `data` will result in alphabetical or reverse-alphabetical list of keys,
# depending on the value of `should_preserve_key_order`
io_utils.write_yaml_file(
data, file, should_preserve_key_order=should_preserve_key_order)
with file.open() as f:
keys = [line.split(":")[0] for line in f.readlines()]
assert keys == expected_keys
async def get_config(self) -> Dict:
config_as_yaml = io_utils.read_yaml_file("config.yml")
return config_as_yaml