def test_nlu(args: argparse.Namespace) -> None:
from rasa import data
from rasa.test import compare_nlu_models, perform_nlu_cross_validation, test_nlu
nlu_data = cli_utils.get_validated_path(args.nlu, "nlu", DEFAULT_DATA_PATH)
nlu_data = data.get_nlu_directory(nlu_data)
output = args.out or DEFAULT_RESULTS_PATH
io_utils.create_directory(output)
if args.config is not None and len(args.config) == 1:
args.config = os.path.abspath(args.config[0])
if os.path.isdir(args.config):
config_dir = args.config
config_files = os.listdir(config_dir)
args.config = [
os.path.join(config_dir, os.path.abspath(config))
for config in config_files
]
if isinstance(args.config, list):
logger.info(
"Multiple configuration files specified, running nlu comparison mode."
)
config_files = []
for file in args.config:
try:
validation_utils.validate_yaml_schema(
io_utils.read_file(file),
CONFIG_SCHEMA_FILE,
show_validation_errors=False,
)
config_files.append(file)
except validation_utils.InvalidYamlFileError:
logger.debug(
"Ignoring file '{}' as it is not a valid config file.".
format(file))
continue
compare_nlu_models(
configs=config_files,
nlu=nlu_data,
output=output,
runs=args.runs,
exclusion_percentages=args.percentages,
)
elif args.cross_validation:
logger.info("Test model using cross validation.")
config = cli_utils.get_validated_path(args.config, "config",
DEFAULT_CONFIG_PATH)
perform_nlu_cross_validation(config, nlu_data, output, vars(args))
else:
model_path = cli_utils.get_validated_path(args.model, "model",
DEFAULT_MODELS_PATH)
test_nlu(model_path, nlu_data, output, vars(args))
def run_nlu_test(args: argparse.Namespace) -> None:
"""Run NLU tests."""
from rasa.test import compare_nlu_models, perform_nlu_cross_validation, test_nlu
nlu_data = rasa.cli.utils.get_validated_path(args.nlu, "nlu",
DEFAULT_DATA_PATH)
nlu_data = rasa.shared.data.get_nlu_directory(nlu_data)
output = args.out or DEFAULT_RESULTS_PATH
args.errors = not args.no_errors
rasa.shared.utils.io.create_directory(output)
if args.config is not None and len(args.config) == 1:
args.config = os.path.abspath(args.config[0])
if os.path.isdir(args.config):
args.config = rasa.shared.utils.io.list_files(args.config)
if isinstance(args.config, list):
logger.info(
"Multiple configuration files specified, running nlu comparison mode."
)
config_files = []
for file in args.config:
try:
validation_utils.validate_yaml_schema(
rasa.shared.utils.io.read_file(file),
CONFIG_SCHEMA_FILE,
show_validation_errors=False,
)
config_files.append(file)
except validation_utils.InvalidYamlFileError:
logger.debug(
f"Ignoring file '{file}' as it is not a valid config file."
)
continue
compare_nlu_models(
configs=config_files,
nlu=nlu_data,
output=output,
runs=args.runs,
exclusion_percentages=args.percentages,
)
elif args.cross_validation:
logger.info("Test model using cross validation.")
config = rasa.cli.utils.get_validated_path(args.config, "config",
DEFAULT_CONFIG_PATH)
perform_nlu_cross_validation(config, nlu_data, output, vars(args))
else:
model_path = rasa.cli.utils.get_validated_path(args.model, "model",
DEFAULT_MODELS_PATH)
test_nlu(model_path, nlu_data, output, vars(args))
def test_nlu_model_cross_validation() -> None:
nlu_data = DEFAULT_DATA_PATH + "/nlu"
perform_nlu_cross_validation('config.yml', nlu_data, {})
async def run_nlu_test_async(
config: Optional[Union[Text, List[Text]]],
data_path: Text,
models_path: Text,
output_dir: Text,
cross_validation: bool,
percentages: List[int],
runs: int,
no_errors: bool,
all_args: Dict[Text, Any],
) -> None:
"""Runs NLU tests.
Args:
all_args: all arguments gathered in a Dict so we can pass it as one argument
to other functions.
config: it refers to the model configuration file. It can be a single file or
a list of multiple files or a folder with multiple config files inside.
data_path: path for the nlu data.
models_path: path to a trained Rasa model.
output_dir: output path for any files created during the evaluation.
cross_validation: indicates if it should test the model using cross validation
or not.
percentages: defines the exclusion percentage of the training data.
runs: number of comparison runs to make.
no_errors: indicates if incorrect predictions should be written to a file
or not.
"""
from rasa.test import compare_nlu_models, perform_nlu_cross_validation, test_nlu
nlu_data = rasa.cli.utils.get_validated_path(data_path, "nlu",
DEFAULT_DATA_PATH)
nlu_data = rasa.shared.data.get_nlu_directory(nlu_data)
output = output_dir or DEFAULT_RESULTS_PATH
all_args["errors"] = not no_errors
rasa.shared.utils.io.create_directory(output)
if config is not None and len(config) == 1:
config = os.path.abspath(config[0])
if os.path.isdir(config):
config = rasa.shared.utils.io.list_files(config)
if isinstance(config, list):
logger.info(
"Multiple configuration files specified, running nlu comparison mode."
)
config_files = []
for file in config:
try:
validation_utils.validate_yaml_schema(
rasa.shared.utils.io.read_file(file),
CONFIG_SCHEMA_FILE,
)
config_files.append(file)
except YamlException:
rasa.shared.utils.io.raise_warning(
f"Ignoring file '{file}' as it is not a valid config file."
)
continue
await compare_nlu_models(
configs=config_files,
nlu=nlu_data,
output=output,
runs=runs,
exclusion_percentages=percentages,
)
elif cross_validation:
logger.info("Test model using cross validation.")
config = rasa.cli.utils.get_validated_path(config, "config",
DEFAULT_CONFIG_PATH)
perform_nlu_cross_validation(config, nlu_data, output, all_args)
else:
model_path = rasa.cli.utils.get_validated_path(models_path, "model",
DEFAULT_MODELS_PATH)
await test_nlu(model_path, nlu_data, output, all_args)
def test_nlu_model_cross_validation() -> None:
perform_nlu_cross_validation('config.yml', DEFAULT_DATA_PATH, {})