def test_read_yaml_string_with_env_var_not_exist():
config_with_env_var_not_exist = """
user: ${USER_NAME}
password: ${PASSWORD}
"""
with pytest.raises(ValueError):
io_utils.read_yaml(config_with_env_var_not_exist)
def test_read_yaml_string():
config_without_env_var = """
user: user
password: pass
"""
content = io_utils.read_yaml(config_without_env_var)
assert content["user"] == "user" and content["password"] == "pass"
def test_read_yaml_string_with_env_var():
config_with_env_var = """
user: ${USER_NAME}
password: ${PASS}
"""
content = io_utils.read_yaml(config_with_env_var)
assert content["user"] == "user" and content["password"] == "pass"
def test_read_yaml_string_with_multiple_env_vars_per_line():
config_with_env_var = """
user: ${USER_NAME} ${PASS}
password: ${PASS}
"""
content = io_utils.read_yaml(config_with_env_var)
assert content["user"] == "user pass" and content["password"] == "pass"
def test_read_yaml_string_with_env_var_prefix():
config_with_env_var_prefix = """
user: db_${USER_NAME}
password: db_${PASS}
"""
content = io_utils.read_yaml(config_with_env_var_prefix)
assert content["user"] == "db_user" and content["password"] == "db_pass"
def reads(self, string: Text, **kwargs: Any) -> "TrainingData":
"""Reads TrainingData in YAML format from a string.
Args:
string: String with YAML training data.
**kwargs: Keyword arguments.
Returns:
New `TrainingData` object with parsed training data.
"""
from rasa.nlu.training_data import TrainingData
from rasa.validator import Validator
self.validate(string)
yaml_content = io_utils.read_yaml(string)
if not Validator.validate_training_data_format_version(
yaml_content, self.filename):
return TrainingData()
for key, value in yaml_content.items(): # pytype: disable=attribute-error
if key == KEY_NLU:
self._parse_nlu(value)
elif key == KEY_RESPONSES:
self.responses = value
return TrainingData(
self.training_examples,
self.entity_synonyms,
self.regex_features,
self.lookup_tables,
self.responses,
)
def test_read_yaml_string_with_env_var_postfix():
config_with_env_var_postfix = """
user: ${USER_NAME}_admin
password: ${PASS}_admin
"""
content = io_utils.read_yaml(config_with_env_var_postfix)
assert content["user"] == "user_admin" and content["password"] == "pass_admin"
def test_environment_variable_dict_without_prefix_and_postfix():
os.environ["variable"] = "test"
content = "model: \n test: ${variable}"
content = io_utils.read_yaml(content)
assert content["model"]["test"] == "test"
def test_environment_variable_in_list():
os.environ["variable"] = "test"
content = "model: \n - value\n - ${variable}"
content = io_utils.read_yaml(content)
assert content["model"][1] == "test"
async def read_from_file(self, filename: Union[Text,
Path]) -> List[StoryStep]:
"""Read stories or rules from file.
Args:
filename: Path to the story/rule file.
Returns:
`StoryStep`s read from `filename`.
"""
self.source_name = filename
try:
file_content = io_utils.read_file(filename,
io_utils.DEFAULT_ENCODING)
validate_yaml_schema(file_content, CORE_SCHEMA_FILE)
yaml_content = io_utils.read_yaml(file_content)
except (ValueError, ParserError) as e:
common_utils.raise_warning(
f"Failed to read YAML from '{filename}', it will be skipped. Error: {e}"
)
return []
except InvalidYamlFileError as e:
raise ValueError from e
return self.read_from_parsed_yaml(yaml_content)
async def start_training_process(
self,
team: Text = config.team_name,
project_id: Text = config.project_name) -> Any:
url = "/model/train"
nlu_training_data = self.data_service.get_nlu_training_data_object(
should_include_lookup_table_entries=True, )
responses: Optional[Dict] = None
if self.data_service.training_data_contains_retrieval_intents(
nlu_training_data):
try:
from rasa.utils import io as io_utils
responses = io_utils.read_yaml(self._get_responses_file_name())
except ValueError as e:
rasa_cli_utils.print_error(
"Could not complete training request as your training data contains "
"retrieval intents of the form 'intent/response_key' but there is no "
"responses file found.")
raise ValueError(
f"Unable to train on data containing retrieval intents. "
f"Details:\n{e}")
nlu_training_data = nlu_training_data.filter_training_examples(
lambda ex: ex.get(RESPONSE_KEY_ATTRIBUTE) is None)
md_formatted_data = nlu_training_data.nlu_as_markdown().strip()
stories = self.story_services.fetch_stories(None)
combined_stories = self.story_services.get_stories_markdown(stories)
domain = self.domain_service.get_or_create_domain()
domain_yaml = dump_obj_as_yaml_to_string(domain)
_config = self.settings_service.get_config(team, project_id)
config_yaml = dump_obj_as_yaml_to_string(_config)
payload = dict(
domain=domain_yaml,
config=config_yaml,
nlu=md_formatted_data,
stories=combined_stories,
responses=dump_obj_as_yaml_to_string(responses),
force=False,
save_to_default_model_directory=False,
)
async with self._session() as session:
response = await session.post(
self._request_url(url),
params=self._query_parameters(),
json=payload,
timeout=24 * 60 * 60, # 24 hours
)
return await response.read()
def test_emojis_in_yaml():
test_data = """
data:
- one 😁💯 👩🏿💻👨🏿💻
- two £ (?u)\\b\\w+\\b f\u00fcr
"""
content = io_utils.read_yaml(test_data)
assert content["data"][0] == "one 😁💯 👩🏿💻👨🏿💻"
assert content["data"][1] == "two £ (?u)\\b\\w+\\b für"
def test_read_emojis_from_json():
import json
d = {"text": "hey 😁💯 👩🏿💻👨🏿💻🧜♂️(?u)\\b\\w+\\b} f\u00fcr"}
json_string = json.dumps(d, indent=2)
content = io_utils.read_yaml(json_string)
expected = "hey 😁💯 👩🏿💻👨🏿💻🧜♂️(?u)\\b\\w+\\b} für"
assert content.get("text") == expected
def test_bool_str():
test_data = """
one: "yes"
two: "true"
three: "True"
"""
content = io_utils.read_yaml(test_data)
assert content["one"] == "yes"
assert content["two"] == "true"
assert content["three"] == "True"
def test_emojis_in_tmp_file():
test_data = """
data:
- one 😁💯 👩🏿💻👨🏿💻
- two £ (?u)\\b\\w+\\b f\u00fcr
"""
test_file = io_utils.create_temporary_file(test_data)
with open(test_file, mode="r", encoding=io_utils.DEFAULT_ENCODING) as f:
content = f.read()
content = io_utils.read_yaml(content)
assert content["data"][0] == "one 😁💯 👩🏿💻👨🏿💻"
assert content["data"][1] == "two £ (?u)\\b\\w+\\b für"
async def test_warning_if_intent_not_in_domain(default_domain: Domain):
stories = """
stories:
- story: I am gonna make you explode 💥
steps:
# Intent defined in user key.
- intent: definitely not in domain
"""
reader = YAMLStoryReader(RegexInterpreter(), default_domain)
yaml_content = io_utils.read_yaml(stories)
with pytest.warns(UserWarning):
reader.read_from_parsed_yaml(yaml_content)
async def test_no_warning_if_intent_in_domain(default_domain: Domain):
stories = (f'version: "{LATEST_TRAINING_DATA_FORMAT_VERSION}"\n'
f"stories:\n"
f"- story: I am fine 💥\n"
f" steps:\n"
f" - intent: greet")
reader = YAMLStoryReader(RegexInterpreter(), default_domain)
yaml_content = io_utils.read_yaml(stories)
with pytest.warns(None) as record:
reader.read_from_parsed_yaml(yaml_content)
assert not len(record)
async def test_no_warning_if_intent_in_domain(default_domain: Domain):
stories = """
stories:
- story: I am fine 💥
steps:
- intent: greet
"""
reader = YAMLStoryReader(RegexInterpreter(), default_domain)
yaml_content = io_utils.read_yaml(stories)
with pytest.warns(None) as record:
reader.read_from_parsed_yaml(yaml_content)
assert len(record) == 0
async def test_active_loop_is_parsed(default_domain: Domain):
stories = (f'version: "{LATEST_TRAINING_DATA_FORMAT_VERSION}"\n'
f"stories:\n"
f"- story: name\n"
f" steps:\n"
f" - intent: greet\n"
f" - active_loop: null")
reader = YAMLStoryReader(RegexInterpreter(), default_domain)
yaml_content = io_utils.read_yaml(stories)
with pytest.warns(None) as record:
reader.read_from_parsed_yaml(yaml_content)
assert not len(record)
def run_trial(space):
"""The objective function is pickled and transferred to the workers.
Hence, this function has to contain all the imports we need.
"""
data_dir = os.environ.get("INPUT_DATA_DIRECTORY", "./data")
model_dir = os.environ.get("INPUT_MODEL_DIRECTORY", "./models")
target_metric = os.environ.get("INPUT_TARGET_METRIC", "f1_score")
if target_metric not in AVAILABLE_METRICS:
logger.error("The metric '{}' is not in the available metrics. "
"Please use one of the available metrics: {}."
"".format(target_metric, AVAILABLE_METRICS))
return {"loss": 1, "status": STATUS_FAIL}
logger.debug("Search space: {}".format(space))
# The epoch has to be an int since `tqdm` otherwise will cause an exception.
if "epochs" in space:
space["epochs"] = int(space["epochs"])
with open(os.path.join(data_dir, "template_config.yml")) as f:
config_yml = f.read().format(**space)
config = read_yaml(config_yml)
config = rasa.nlu.config.load(config)
trainer = Trainer(config)
training_data = load_data(os.path.join(data_dir, "train.md"))
test_data_path = os.path.join(data_dir, "validation.md")
# wrap in train and eval in try/except in case
# nlu_hyperopt proposes invalid combination of params
try:
model = trainer.train(training_data)
model_path = trainer.persist(model_dir)
if target_metric is None or target_metric == "threshold_loss":
loss = _get_threshold_loss(model, test_data_path)
else:
loss = _get_nlu_evaluation_loss(model_path, target_metric,
test_data_path)
return {"loss": loss, "status": STATUS_OK}
except Exception as e:
logger.error(e)
return {"loss": 1, "status": STATUS_FAIL}
async def train(request):
body = await request.json()
if 'rasa_nlu_data' not in body:
raise HTTPException(422)
model_name = request.path_params["model_name"]
nlu_data = body["rasa_nlu_data"]
if 'config' in body:
config = read_yaml(body['config'])
else:
config = read_config_file('./config.yml')
async_trainer.train(nlu_data, model_name, config)
return JSONResponse({"message": "Training started"})
def reads(self, string: Text, **kwargs: Any) -> "TrainingData":
"""Reads TrainingData in YAML format from a string.
Args:
string: String with YAML training data.
**kwargs: Keyword arguments.
Returns:
New `TrainingData` object with parsed training data.
"""
from rasa.nlu.training_data import TrainingData
yaml_content = io_utils.read_yaml(string)
for key, value in yaml_content.items(): # pytype: disable=attribute-error
if key == KEY_NLU:
self._parse_nlu(value)
return TrainingData(
self.training_examples,
self.entity_synonyms,
self.regex_features,
self.lookup_tables,
)
def test_environment_variable_not_existing():
content = "model: \n test: ${variable}"
with pytest.raises(ValueError):
io_utils.read_yaml(content)
async def get_config(self) -> Dict:
config_as_yaml = self.get_content("config.yml")
return io_utils.read_yaml(config_as_yaml)