def test_nlu_comparison(tmpdir, config_path, config_path_duplicate):
# the configs need to be at a different path, otherwise the results are
# combined on the same dictionary key and cannot be plotted properly
configs = [config_path, config_path_duplicate]
output = tmpdir.strpath
compare_nlu_models(configs,
DEFAULT_DATA_PATH,
output,
runs=2,
exclusion_percentages=[50, 80])
assert set(os.listdir(output)) == {
"run_1",
"run_2",
"results.json",
"nlu_model_comparison_graph.pdf",
}
run_1_path = os.path.join(output, "run_1")
assert set(os.listdir(run_1_path)) == {
"50%_exclusion", "80%_exclusion", "test.md"
}
exclude_50_path = os.path.join(run_1_path, "50%_exclusion")
modelnames = [
os.path.splitext(os.path.basename(config))[0] for config in configs
]
modeloutputs = set(["train"] + [f"{m}_report" for m in modelnames] +
[f"{m}.tar.gz" for m in modelnames])
assert set(os.listdir(exclude_50_path)) == modeloutputs
def test_nlu_comparison(tmpdir, config_path):
configs = [config_path, config_path]
output = tmpdir.strpath
compare_nlu_models(configs,
DEFAULT_DATA_PATH,
output,
runs=2,
exclusion_percentages=[50, 80])
assert set(os.listdir(output)) == {
"run_1",
"run_2",
"results.json",
"nlu_model_comparison_graph.pdf",
}
run_1_path = os.path.join(output, "run_1")
assert set(os.listdir(run_1_path)) == {
"50%_exclusion", "80%_exclusion", "test.md"
}
exclude_50_path = os.path.join(run_1_path, "50%_exclusion")
modelnames = [
os.path.splitext(os.path.basename(config))[0] for config in configs
]
modeloutputs = set(["train"] + [f"{m}_report" for m in modelnames] +
[f"{m}.tar.gz" for m in modelnames])
assert set(os.listdir(exclude_50_path)) == modeloutputs
def test_nlu(args: argparse.Namespace) -> None:
from rasa import data
from rasa.test import compare_nlu_models, perform_nlu_cross_validation, test_nlu
nlu_data = cli_utils.get_validated_path(args.nlu, "nlu", DEFAULT_DATA_PATH)
nlu_data = data.get_nlu_directory(nlu_data)
output = args.out or DEFAULT_RESULTS_PATH
io_utils.create_directory(output)
if args.config is not None and len(args.config) == 1:
args.config = os.path.abspath(args.config[0])
if os.path.isdir(args.config):
config_dir = args.config
config_files = os.listdir(config_dir)
args.config = [
os.path.join(config_dir, os.path.abspath(config))
for config in config_files
]
if isinstance(args.config, list):
logger.info(
"Multiple configuration files specified, running nlu comparison mode."
)
config_files = []
for file in args.config:
try:
validation_utils.validate_yaml_schema(
io_utils.read_file(file),
CONFIG_SCHEMA_FILE,
show_validation_errors=False,
)
config_files.append(file)
except validation_utils.InvalidYamlFileError:
logger.debug(
"Ignoring file '{}' as it is not a valid config file.".
format(file))
continue
compare_nlu_models(
configs=config_files,
nlu=nlu_data,
output=output,
runs=args.runs,
exclusion_percentages=args.percentages,
)
elif args.cross_validation:
logger.info("Test model using cross validation.")
config = cli_utils.get_validated_path(args.config, "config",
DEFAULT_CONFIG_PATH)
perform_nlu_cross_validation(config, nlu_data, output, vars(args))
else:
model_path = cli_utils.get_validated_path(args.model, "model",
DEFAULT_MODELS_PATH)
test_nlu(model_path, nlu_data, output, vars(args))
def run_nlu_test(args: argparse.Namespace) -> None:
"""Run NLU tests."""
from rasa.test import compare_nlu_models, perform_nlu_cross_validation, test_nlu
nlu_data = rasa.cli.utils.get_validated_path(args.nlu, "nlu",
DEFAULT_DATA_PATH)
nlu_data = rasa.shared.data.get_nlu_directory(nlu_data)
output = args.out or DEFAULT_RESULTS_PATH
args.errors = not args.no_errors
rasa.shared.utils.io.create_directory(output)
if args.config is not None and len(args.config) == 1:
args.config = os.path.abspath(args.config[0])
if os.path.isdir(args.config):
args.config = rasa.shared.utils.io.list_files(args.config)
if isinstance(args.config, list):
logger.info(
"Multiple configuration files specified, running nlu comparison mode."
)
config_files = []
for file in args.config:
try:
validation_utils.validate_yaml_schema(
rasa.shared.utils.io.read_file(file),
CONFIG_SCHEMA_FILE,
show_validation_errors=False,
)
config_files.append(file)
except validation_utils.InvalidYamlFileError:
logger.debug(
f"Ignoring file '{file}' as it is not a valid config file."
)
continue
compare_nlu_models(
configs=config_files,
nlu=nlu_data,
output=output,
runs=args.runs,
exclusion_percentages=args.percentages,
)
elif args.cross_validation:
logger.info("Test model using cross validation.")
config = rasa.cli.utils.get_validated_path(args.config, "config",
DEFAULT_CONFIG_PATH)
perform_nlu_cross_validation(config, nlu_data, output, vars(args))
else:
model_path = rasa.cli.utils.get_validated_path(args.model, "model",
DEFAULT_MODELS_PATH)
test_nlu(model_path, nlu_data, output, vars(args))
def test_nlu_comparison(tmp_path: Path):
config = {
"language":
"en",
"pipeline": [
{
"name": "WhitespaceTokenizer"
},
{
"name": "KeywordIntentClassifier"
},
{
"name": "RegexEntityExtractor"
},
],
}
# the configs need to be at a different path, otherwise the results are
# combined on the same dictionary key and cannot be plotted properly
configs = [
write_file_config(config).name,
write_file_config(config).name,
]
output = str(tmp_path)
compare_nlu_models(configs,
DEFAULT_DATA_PATH,
output,
runs=2,
exclusion_percentages=[50, 80])
assert set(os.listdir(output)) == {
"run_1",
"run_2",
"results.json",
"nlu_model_comparison_graph.pdf",
}
run_1_path = os.path.join(output, "run_1")
assert set(os.listdir(run_1_path)) == {
"50%_exclusion", "80%_exclusion", "test.md"
}
exclude_50_path = os.path.join(run_1_path, "50%_exclusion")
modelnames = [
os.path.splitext(os.path.basename(config))[0] for config in configs
]
modeloutputs = set(["train"] + [f"{m}_report" for m in modelnames] +
[f"{m}.tar.gz" for m in modelnames])
assert set(os.listdir(exclude_50_path)) == modeloutputs
def test_nlu_comparison(tmpdir):
configs = [
NLU_DEFAULT_CONFIG_PATH,
"sample_configs/config_supervised_embeddings.yml",
]
output = tmpdir.strpath
compare_nlu_models(
configs, DEFAULT_DATA_PATH, output, runs=2, exclusion_percentages=[50, 80]
)
assert set(os.listdir(output)) == {
"run_1",
"run_2",
"results.json",
"nlu_model_comparison_graph.pdf",
}
run_1_path = os.path.join(output, "run_1")
assert set(os.listdir(run_1_path)) == {"50%_exclusion", "80%_exclusion", "test.md"}
