def test_data_convert_nlu_yml(
run: Callable[..., RunResult], tmp_path: Path, request: FixtureRequest
):
target_file = tmp_path / "out.yml"
# The request rootdir is required as the `testdir` fixture in `run` changes the
# working directory
test_data_dir = Path(request.config.rootdir, "data", "examples", "rasa")
source_file = (test_data_dir / "demo-rasa.json").absolute()
result = run(
"data",
"convert",
"nlu",
"--data",
str(source_file),
"--out",
str(target_file),
"-f",
"yaml",
)
assert result.ret == 0
assert target_file.exists()
actual_data = RasaYAMLReader().read(target_file)
expected = RasaYAMLReader().read(test_data_dir / "demo-rasa.yml")
assert len(actual_data.training_examples) == len(expected.training_examples)
assert len(actual_data.entity_synonyms) == len(expected.entity_synonyms)
assert len(actual_data.regex_features) == len(expected.regex_features)
assert len(actual_data.lookup_tables) == len(expected.lookup_tables)
assert actual_data.entities == expected.entities
def test_docs_training_data(mdx_file_path: Path):
with mdx_file_path.open("r") as handle:
mdx_content = handle.read()
matches = TRAINING_DATA_CODEBLOCK_RE.finditer(mdx_content)
lines_with_errors: List[Text] = []
for match in matches:
yaml_path = match.group("yaml_path")
if yaml_path:
with (DOCS_BASE_DIR / yaml_path).open("r") as handle:
codeblock = handle.read()
else:
codeblock = match.group("codeblock")
start_index = match.span()[0]
line_number = mdx_content.count("\n", 0, start_index) + 1
try:
RasaYAMLReader.validate(codeblock)
except ValueError:
lines_with_errors.append(str(line_number))
if lines_with_errors:
raise AssertionError(
f"({mdx_file_path}): Invalid training data found "
f"at line{'s' if len(lines_with_errors) > 1 else ''} {', '.join(lines_with_errors)}"
)
def _validate_yaml_training_payload(yaml_text: Text) -> None:
try:
RasaYAMLReader.validate(yaml_text)
except Exception as e:
raise ErrorResponse(
400,
"BadRequest",
f"The request body does not contain valid YAML. Error: {e}",
help_url=DOCS_URL_TRAINING_DATA_NLU,
)
def test_train_model_training_data_persisted(
tmp_path: Path, nlu_as_json_path: Text, tmp_path_factory: TempPathFactory
):
config_file = tmp_path / "config.yml"
rasa.shared.utils.io.dump_obj_as_json_to_file(
config_file,
{"pipeline": [{"name": "KeywordIntentClassifier"}], "language": "en"},
)
persisted_path = rasa.model_training.train_nlu(
str(config_file),
nlu_as_json_path,
output=str(tmp_path),
persist_nlu_training_data=True,
)
assert Path(persisted_path).is_file()
model_dir = tmp_path_factory.mktemp("loaded")
storage, _ = LocalModelStorage.from_model_archive(model_dir, Path(persisted_path))
nlu_data_dir = model_dir / "nlu_training_data_provider"
assert nlu_data_dir.is_dir()
assert not RasaYAMLReader().read(nlu_data_dir / "training_data.yml").is_empty()
def _reader_factory(fformat: Text) -> Optional["TrainingDataReader"]:
"""Generates the appropriate reader class based on the file format."""
from rasa.shared.nlu.training_data.formats import (
RasaYAMLReader,
MarkdownReader,
WitReader,
LuisReader,
RasaReader,
DialogflowReader,
NLGMarkdownReader,
)
reader = None
if fformat == LUIS:
reader = LuisReader()
elif fformat == WIT:
reader = WitReader()
elif fformat in DIALOGFLOW_RELEVANT:
reader = DialogflowReader()
elif fformat == RASA:
reader = RasaReader()
elif fformat == MARKDOWN:
reader = MarkdownReader()
elif fformat == MARKDOWN_NLG:
reader = NLGMarkdownReader()
elif fformat == RASA_YAML:
reader = RasaYAMLReader()
return reader
def guess_format(filename: Text) -> Text:
"""Applies heuristics to guess the data format of a file.
Args:
filename: file whose type should be guessed
Returns:
Guessed file format.
"""
from rasa.shared.nlu.training_data.formats import RasaYAMLReader
guess = UNK
if not os.path.isfile(filename):
return guess
try:
content = rasa.shared.utils.io.read_file(filename)
js = json.loads(content)
except ValueError:
if MarkdownReader.is_markdown_nlu_file(filename):
guess = MARKDOWN
elif NLGMarkdownReader.is_markdown_nlg_file(filename):
guess = MARKDOWN_NLG
elif RasaYAMLReader.is_yaml_nlu_file(filename):
guess = RASA_YAML
else:
for file_format, format_heuristic in _json_format_heuristics.items():
if format_heuristic(js, filename):
guess = file_format
break
logger.debug(f"Training data format of '{filename}' is '{guess}'.")
return guess
async def test_multi_project_training(trained_async, tmp_path_factory: TempPathFactory):
example_directory = "data/test_multi_domain"
config_file = os.path.join(example_directory, "config.yml")
domain_file = os.path.join(example_directory, "domain.yml")
files_of_root_project = os.path.join(example_directory, "data")
trained_stack_model_path = await trained_async(
config=config_file,
domain=domain_file,
training_files=files_of_root_project,
force_training=True,
persist_nlu_training_data=True,
)
storage_path = tmp_path_factory.mktemp("storage_path")
model_storage, model_metadata = LocalModelStorage.from_model_archive(
storage_path, trained_stack_model_path
)
domain = model_metadata.domain
expected_intents = {
"greet",
"goodbye",
"affirm",
"deny",
"mood_great",
"mood_unhappy",
}
assert all([i in domain.intents for i in expected_intents])
with model_storage.read_from(
Resource("nlu_training_data_provider")
) as resource_dir:
nlu_training_data_file = resource_dir / "training_data.yml"
nlu_training_data = RasaYAMLReader().read(nlu_training_data_file)
assert expected_intents == nlu_training_data.intents
expected_actions = [
"utter_greet",
"utter_cheer_up",
"utter_did_that_help",
"utter_happy",
"utter_goodbye",
]
assert all([a in domain.action_names_or_texts for a in expected_actions])
async def test_multi_project_training(trained_async):
example_directory = "data/test_multi_domain"
config_file = os.path.join(example_directory, "config.yml")
domain_file = os.path.join(example_directory, "domain.yml")
files_of_root_project = os.path.join(example_directory, "data")
trained_stack_model_path = await trained_async(
config=config_file,
domain=domain_file,
training_files=files_of_root_project,
force_training=True,
persist_nlu_training_data=True,
)
unpacked = model.unpack_model(trained_stack_model_path)
domain_file = os.path.join(
unpacked, DEFAULT_CORE_SUBDIRECTORY_NAME, DEFAULT_DOMAIN_PATH
)
domain = Domain.load(domain_file)
expected_intents = {
"greet",
"goodbye",
"affirm",
"deny",
"mood_great",
"mood_unhappy",
}
assert all([i in domain.intents for i in expected_intents])
nlu_training_data_file = os.path.join(unpacked, "nlu", "training_data.yml")
nlu_training_data = RasaYAMLReader().read(nlu_training_data_file)
assert expected_intents == nlu_training_data.intents
expected_actions = [
"utter_greet",
"utter_cheer_up",
"utter_did_that_help",
"utter_happy",
"utter_goodbye",
]
assert all([a in domain.action_names for a in expected_actions])
def persist(
state: StateMachineState,
is_initial_state: bool,
domain_folder: str,
nlu_folder: str,
):
domain, nlu_data = get_domain_nlu(
state=state, is_initial_state=is_initial_state
)
# Generate filename
filename = "".join(
e.lower()
for e in state.name
if e.isalnum() or e.isspace() or e in ["-", "_"]
)
filename = "_".join(filename.split(" ")) + ".yaml"
# Persist domain
domain_filename = os.path.join(domain_folder, filename)
Path(domain_filename).parent.mkdir(parents=True, exist_ok=True)
rasa.shared.utils.validation.validate_yaml_schema(
domain.as_yaml(), rasa.shared.constants.DOMAIN_SCHEMA_FILE
)
# Delete domain_filename
if os.path.exists(domain_filename):
os.remove(domain_filename)
domain.persist(domain_filename)
# Persist NLU
nlu_filename = os.path.join(nlu_folder, filename)
nlu_data_yaml = dump_obj_as_yaml_to_string(
nlu_data, should_preserve_key_order=True
)
RasaYAMLReader().validate(nlu_data_yaml)
Path(nlu_filename).parent.mkdir(parents=True, exist_ok=True)
if os.path.exists(nlu_filename):
os.remove(nlu_filename)
write_text_file(nlu_data_yaml, nlu_filename)
def persist(
stories: List[Story],
domain_filename: str,
nlu_filename: str,
additional_intents: List[Intent],
additional_utterances: List[Utterance],
slots: List[Slot],
use_rules: bool = False,
):
all_domain = Domain.empty()
all_intents: Set[Intent] = set(additional_intents)
all_stories: List[Story] = []
all_slot_was_sets: Set[SlotWasSet] = set()
for story in stories:
domain, sub_stories, intents, slot_was_sets = story.get_domain_nlu(
use_rules=use_rules)
all_domain = all_domain.merge(domain)
all_intents.update(intents)
all_stories.extend(sub_stories)
all_slot_was_sets.update(slot_was_sets)
# Append consolidated slots
domain_slots = Domain(
intents=set([intent.name for intent in all_intents]),
entities=[slot.name for slot in slots],
slots=slots,
responses={
utterance.name: [{
"text": utterance.text
}]
for utterance in additional_utterances
},
action_names=[],
forms={},
)
all_domain = all_domain.merge(domain_slots)
# Validate domain
rasa.shared.utils.validation.validate_yaml_schema(
all_domain.as_yaml(), rasa.shared.constants.DOMAIN_SCHEMA_FILE)
# Write domain
if os.path.exists(domain_filename):
os.remove(domain_filename)
Path(domain_filename).parent.mkdir(parents=True, exist_ok=True)
all_domain.persist(domain_filename)
# Write NLU
nlu_data = {
"version":
"2.0",
"nlu": [
intent.as_nlu_yaml() for intent in all_intents
if isinstance(intent, IntentWithExamples)
],
"rules" if use_rules else "stories":
all_stories,
}
nlu_data_yaml = dump_obj_as_yaml_to_string(nlu_data,
should_preserve_key_order=True)
RasaYAMLReader().validate(nlu_data_yaml)
# TODO: Create folders if not existent
if os.path.exists(nlu_filename):
os.remove(nlu_filename)
Path(nlu_filename).parent.mkdir(parents=True, exist_ok=True)
write_text_file(nlu_data_yaml, nlu_filename)