def validate(string: Text) -> None:
"""Check if the string adheres to the NLU yaml data schema.
If the string is not in the right format, an exception will be raised."""
try:
validation.validate_yaml_schema(string, NLU_SCHEMA_FILE)
except validation.InvalidYamlFileError as e:
raise ValueError from e
def validate(self, string: Text) -> None:
"""Check if the string adheres to the NLU yaml data schema.
If the string is not in the right format, an exception will be raised."""
try:
validation.validate_yaml_schema(string, NLU_SCHEMA_FILE)
except YamlException as e:
e.filename = self.filename
raise e
def run_nlu_test(args: argparse.Namespace) -> None:
"""Run NLU tests."""
from rasa import data
from rasa.test import compare_nlu_models, perform_nlu_cross_validation, test_nlu
nlu_data = cli_utils.get_validated_path(args.nlu, "nlu", DEFAULT_DATA_PATH)
nlu_data = data.get_nlu_directory(nlu_data)
output = args.out or DEFAULT_RESULTS_PATH
args.errors = not args.no_errors
io_utils.create_directory(output)
if args.config is not None and len(args.config) == 1:
args.config = os.path.abspath(args.config[0])
if os.path.isdir(args.config):
args.config = rasa.shared.utils.io.list_files(args.config)
if isinstance(args.config, list):
logger.info(
"Multiple configuration files specified, running nlu comparison mode."
)
config_files = []
for file in args.config:
try:
validation_utils.validate_yaml_schema(
rasa.shared.utils.io.read_file(file),
CONFIG_SCHEMA_FILE,
show_validation_errors=False,
)
config_files.append(file)
except validation_utils.InvalidYamlFileError:
logger.debug(
f"Ignoring file '{file}' as it is not a valid config file."
)
continue
compare_nlu_models(
configs=config_files,
nlu=nlu_data,
output=output,
runs=args.runs,
exclusion_percentages=args.percentages,
)
elif args.cross_validation:
logger.info("Test model using cross validation.")
config = cli_utils.get_validated_path(args.config, "config",
DEFAULT_CONFIG_PATH)
perform_nlu_cross_validation(config, nlu_data, output, vars(args))
else:
model_path = cli_utils.get_validated_path(args.model, "model",
DEFAULT_MODELS_PATH)
test_nlu(model_path, nlu_data, output, vars(args))
def from_yaml(cls, yaml: Text, original_filename: Text = "") -> "Domain":
try:
validate_yaml_schema(yaml, DOMAIN_SCHEMA_FILE)
except InvalidYamlFileError as e:
raise InvalidDomain(str(e))
data = rasa.shared.utils.io.read_yaml(yaml)
if not validate_training_data_format_version(data, original_filename):
return Domain.empty()
return cls.from_dict(data)
def test_validate_yaml_schema_raise_exception_extra_hyphen_for_image():
domain = """
version: "2.0"
responses:
utter_cheer_up:
- image: https://i.imgur.com/nGF1K8f.jpg
- text: Here is something to cheer you up
"""
with pytest.raises(validation_utils.YamlValidationException) as e:
validation_utils.validate_yaml_schema(domain, DOMAIN_SCHEMA_FILE)
assert (
"Missing 'text' or 'custom' key in response or null 'text' value in response."
in str(e.value))
def test_validate_yaml_schema_raise_exception_null_text():
domain = """
version: "2.0"
responses:
utter_ask_email:
- text: What is your email ID?
utter_ask_name:
- text: null
"""
with pytest.raises(validation_utils.YamlValidationException) as e:
validation_utils.validate_yaml_schema(domain, DOMAIN_SCHEMA_FILE)
assert (
"Missing 'text' or 'custom' key in response or null 'text' value in response."
in str(e.value))
async def run_nlu_test_async(
config: Optional[Union[Text, List[Text]]],
data_path: Text,
models_path: Text,
output_dir: Text,
cross_validation: bool,
percentages: List[int],
runs: int,
no_errors: bool,
all_args: Dict[Text, Any],
) -> None:
"""Runs NLU tests.
Args:
all_args: all arguments gathered in a Dict so we can pass it as one argument
to other functions.
config: it refers to the model configuration file. It can be a single file or
a list of multiple files or a folder with multiple config files inside.
data_path: path for the nlu data.
models_path: path to a trained Rasa model.
output_dir: output path for any files created during the evaluation.
cross_validation: indicates if it should test the model using cross validation
or not.
percentages: defines the exclusion percentage of the training data.
runs: number of comparison runs to make.
no_errors: indicates if incorrect predictions should be written to a file
or not.
"""
from rasa.model_testing import (
compare_nlu_models,
perform_nlu_cross_validation,
test_nlu,
)
nlu_data = rasa.cli.utils.get_validated_path(data_path, "nlu",
DEFAULT_DATA_PATH)
nlu_data = rasa.shared.data.get_nlu_directory(nlu_data)
output = output_dir or DEFAULT_RESULTS_PATH
all_args["errors"] = not no_errors
rasa.shared.utils.io.create_directory(output)
if config is not None and len(config) == 1:
config = os.path.abspath(config[0])
if os.path.isdir(config):
config = rasa.shared.utils.io.list_files(config)
if isinstance(config, list):
logger.info(
"Multiple configuration files specified, running nlu comparison mode."
)
config_files = []
for file in config:
try:
validation_utils.validate_yaml_schema(
rasa.shared.utils.io.read_file(file),
CONFIG_SCHEMA_FILE,
)
config_files.append(file)
except YamlException:
rasa.shared.utils.io.raise_warning(
f"Ignoring file '{file}' as it is not a valid config file."
)
continue
await compare_nlu_models(
configs=config_files,
nlu=nlu_data,
output=output,
runs=runs,
exclusion_percentages=percentages,
)
elif cross_validation:
logger.info("Test model using cross validation.")
config = rasa.cli.utils.get_validated_path(config, "config",
DEFAULT_CONFIG_PATH)
perform_nlu_cross_validation(config, nlu_data, output, all_args)
else:
model_path = rasa.cli.utils.get_validated_path(models_path, "model",
DEFAULT_MODELS_PATH)
await test_nlu(model_path, nlu_data, output, all_args)
def test_validate_yaml_schema_raise_exception(file: Text, schema: Text):
with pytest.raises(YamlException):
validation_utils.validate_yaml_schema(
rasa.shared.utils.io.read_file(file), schema)
def test_validate_yaml_schema(file, schema):
# should raise no exception
validation_utils.validate_yaml_schema(rasa.shared.utils.io.read_file(file),
schema)
def test_validate_yaml_schema_raise_exception(file, schema):
with pytest.raises(validation_utils.InvalidYamlFileError):
validation_utils.validate_yaml_schema(
rasa.shared.utils.io.read_file(file), schema
)
