def test_nlu_comparison(tmpdir, config_path, config_path_duplicate): # the configs need to be at a different path, otherwise the results are # combined on the same dictionary key and cannot be plotted properly configs = [config_path, config_path_duplicate] output = tmpdir.strpath compare_nlu_models(configs, DEFAULT_DATA_PATH, output, runs=2, exclusion_percentages=[50, 80]) assert set(os.listdir(output)) == { "run_1", "run_2", "results.json", "nlu_model_comparison_graph.pdf", } run_1_path = os.path.join(output, "run_1") assert set(os.listdir(run_1_path)) == { "50%_exclusion", "80%_exclusion", "test.md" } exclude_50_path = os.path.join(run_1_path, "50%_exclusion") modelnames = [ os.path.splitext(os.path.basename(config))[0] for config in configs ] modeloutputs = set(["train"] + [f"{m}_report" for m in modelnames] + [f"{m}.tar.gz" for m in modelnames]) assert set(os.listdir(exclude_50_path)) == modeloutputs
def test_nlu_comparison(tmpdir, config_path): configs = [config_path, config_path] output = tmpdir.strpath compare_nlu_models(configs, DEFAULT_DATA_PATH, output, runs=2, exclusion_percentages=[50, 80]) assert set(os.listdir(output)) == { "run_1", "run_2", "results.json", "nlu_model_comparison_graph.pdf", } run_1_path = os.path.join(output, "run_1") assert set(os.listdir(run_1_path)) == { "50%_exclusion", "80%_exclusion", "test.md" } exclude_50_path = os.path.join(run_1_path, "50%_exclusion") modelnames = [ os.path.splitext(os.path.basename(config))[0] for config in configs ] modeloutputs = set(["train"] + [f"{m}_report" for m in modelnames] + [f"{m}.tar.gz" for m in modelnames]) assert set(os.listdir(exclude_50_path)) == modeloutputs
def test_nlu(args: argparse.Namespace) -> None: from rasa import data from rasa.test import compare_nlu_models, perform_nlu_cross_validation, test_nlu nlu_data = cli_utils.get_validated_path(args.nlu, "nlu", DEFAULT_DATA_PATH) nlu_data = data.get_nlu_directory(nlu_data) output = args.out or DEFAULT_RESULTS_PATH io_utils.create_directory(output) if args.config is not None and len(args.config) == 1: args.config = os.path.abspath(args.config[0]) if os.path.isdir(args.config): config_dir = args.config config_files = os.listdir(config_dir) args.config = [ os.path.join(config_dir, os.path.abspath(config)) for config in config_files ] if isinstance(args.config, list): logger.info( "Multiple configuration files specified, running nlu comparison mode." ) config_files = [] for file in args.config: try: validation_utils.validate_yaml_schema( io_utils.read_file(file), CONFIG_SCHEMA_FILE, show_validation_errors=False, ) config_files.append(file) except validation_utils.InvalidYamlFileError: logger.debug( "Ignoring file '{}' as it is not a valid config file.". format(file)) continue compare_nlu_models( configs=config_files, nlu=nlu_data, output=output, runs=args.runs, exclusion_percentages=args.percentages, ) elif args.cross_validation: logger.info("Test model using cross validation.") config = cli_utils.get_validated_path(args.config, "config", DEFAULT_CONFIG_PATH) perform_nlu_cross_validation(config, nlu_data, output, vars(args)) else: model_path = cli_utils.get_validated_path(args.model, "model", DEFAULT_MODELS_PATH) test_nlu(model_path, nlu_data, output, vars(args))
def run_nlu_test(args: argparse.Namespace) -> None: """Run NLU tests.""" from rasa.test import compare_nlu_models, perform_nlu_cross_validation, test_nlu nlu_data = rasa.cli.utils.get_validated_path(args.nlu, "nlu", DEFAULT_DATA_PATH) nlu_data = rasa.shared.data.get_nlu_directory(nlu_data) output = args.out or DEFAULT_RESULTS_PATH args.errors = not args.no_errors rasa.shared.utils.io.create_directory(output) if args.config is not None and len(args.config) == 1: args.config = os.path.abspath(args.config[0]) if os.path.isdir(args.config): args.config = rasa.shared.utils.io.list_files(args.config) if isinstance(args.config, list): logger.info( "Multiple configuration files specified, running nlu comparison mode." ) config_files = [] for file in args.config: try: validation_utils.validate_yaml_schema( rasa.shared.utils.io.read_file(file), CONFIG_SCHEMA_FILE, show_validation_errors=False, ) config_files.append(file) except validation_utils.InvalidYamlFileError: logger.debug( f"Ignoring file '{file}' as it is not a valid config file." ) continue compare_nlu_models( configs=config_files, nlu=nlu_data, output=output, runs=args.runs, exclusion_percentages=args.percentages, ) elif args.cross_validation: logger.info("Test model using cross validation.") config = rasa.cli.utils.get_validated_path(args.config, "config", DEFAULT_CONFIG_PATH) perform_nlu_cross_validation(config, nlu_data, output, vars(args)) else: model_path = rasa.cli.utils.get_validated_path(args.model, "model", DEFAULT_MODELS_PATH) test_nlu(model_path, nlu_data, output, vars(args))
def test_nlu_comparison(tmp_path: Path): config = { "language": "en", "pipeline": [ { "name": "WhitespaceTokenizer" }, { "name": "KeywordIntentClassifier" }, { "name": "RegexEntityExtractor" }, ], } # the configs need to be at a different path, otherwise the results are # combined on the same dictionary key and cannot be plotted properly configs = [ write_file_config(config).name, write_file_config(config).name, ] output = str(tmp_path) compare_nlu_models(configs, DEFAULT_DATA_PATH, output, runs=2, exclusion_percentages=[50, 80]) assert set(os.listdir(output)) == { "run_1", "run_2", "results.json", "nlu_model_comparison_graph.pdf", } run_1_path = os.path.join(output, "run_1") assert set(os.listdir(run_1_path)) == { "50%_exclusion", "80%_exclusion", "test.md" } exclude_50_path = os.path.join(run_1_path, "50%_exclusion") modelnames = [ os.path.splitext(os.path.basename(config))[0] for config in configs ] modeloutputs = set(["train"] + [f"{m}_report" for m in modelnames] + [f"{m}.tar.gz" for m in modelnames]) assert set(os.listdir(exclude_50_path)) == modeloutputs
def test_nlu_comparison(tmpdir): configs = [ NLU_DEFAULT_CONFIG_PATH, "sample_configs/config_supervised_embeddings.yml", ] output = tmpdir.strpath compare_nlu_models( configs, DEFAULT_DATA_PATH, output, runs=2, exclusion_percentages=[50, 80] ) assert set(os.listdir(output)) == { "run_1", "run_2", "results.json", "nlu_model_comparison_graph.pdf", } run_1_path = os.path.join(output, "run_1") assert set(os.listdir(run_1_path)) == {"50%_exclusion", "80%_exclusion", "test.md"}