def test_nlu(args: argparse.Namespace) -> None: from rasa import data from rasa.test import compare_nlu_models, perform_nlu_cross_validation, test_nlu nlu_data = cli_utils.get_validated_path(args.nlu, "nlu", DEFAULT_DATA_PATH) nlu_data = data.get_nlu_directory(nlu_data) output = args.out or DEFAULT_RESULTS_PATH io_utils.create_directory(output) if args.config is not None and len(args.config) == 1: args.config = os.path.abspath(args.config[0]) if os.path.isdir(args.config): config_dir = args.config config_files = os.listdir(config_dir) args.config = [ os.path.join(config_dir, os.path.abspath(config)) for config in config_files ] if isinstance(args.config, list): logger.info( "Multiple configuration files specified, running nlu comparison mode." ) config_files = [] for file in args.config: try: validation_utils.validate_yaml_schema( io_utils.read_file(file), CONFIG_SCHEMA_FILE, show_validation_errors=False, ) config_files.append(file) except validation_utils.InvalidYamlFileError: logger.debug( "Ignoring file '{}' as it is not a valid config file.". format(file)) continue compare_nlu_models( configs=config_files, nlu=nlu_data, output=output, runs=args.runs, exclusion_percentages=args.percentages, ) elif args.cross_validation: logger.info("Test model using cross validation.") config = cli_utils.get_validated_path(args.config, "config", DEFAULT_CONFIG_PATH) perform_nlu_cross_validation(config, nlu_data, output, vars(args)) else: model_path = cli_utils.get_validated_path(args.model, "model", DEFAULT_MODELS_PATH) test_nlu(model_path, nlu_data, output, vars(args))
def run_nlu_test(args: argparse.Namespace) -> None: """Run NLU tests.""" from rasa.test import compare_nlu_models, perform_nlu_cross_validation, test_nlu nlu_data = rasa.cli.utils.get_validated_path(args.nlu, "nlu", DEFAULT_DATA_PATH) nlu_data = rasa.shared.data.get_nlu_directory(nlu_data) output = args.out or DEFAULT_RESULTS_PATH args.errors = not args.no_errors rasa.shared.utils.io.create_directory(output) if args.config is not None and len(args.config) == 1: args.config = os.path.abspath(args.config[0]) if os.path.isdir(args.config): args.config = rasa.shared.utils.io.list_files(args.config) if isinstance(args.config, list): logger.info( "Multiple configuration files specified, running nlu comparison mode." ) config_files = [] for file in args.config: try: validation_utils.validate_yaml_schema( rasa.shared.utils.io.read_file(file), CONFIG_SCHEMA_FILE, show_validation_errors=False, ) config_files.append(file) except validation_utils.InvalidYamlFileError: logger.debug( f"Ignoring file '{file}' as it is not a valid config file." ) continue compare_nlu_models( configs=config_files, nlu=nlu_data, output=output, runs=args.runs, exclusion_percentages=args.percentages, ) elif args.cross_validation: logger.info("Test model using cross validation.") config = rasa.cli.utils.get_validated_path(args.config, "config", DEFAULT_CONFIG_PATH) perform_nlu_cross_validation(config, nlu_data, output, vars(args)) else: model_path = rasa.cli.utils.get_validated_path(args.model, "model", DEFAULT_MODELS_PATH) test_nlu(model_path, nlu_data, output, vars(args))
def test_nlu_model_cross_validation() -> None: nlu_data = DEFAULT_DATA_PATH + "/nlu" perform_nlu_cross_validation('config.yml', nlu_data, {})
async def run_nlu_test_async( config: Optional[Union[Text, List[Text]]], data_path: Text, models_path: Text, output_dir: Text, cross_validation: bool, percentages: List[int], runs: int, no_errors: bool, all_args: Dict[Text, Any], ) -> None: """Runs NLU tests. Args: all_args: all arguments gathered in a Dict so we can pass it as one argument to other functions. config: it refers to the model configuration file. It can be a single file or a list of multiple files or a folder with multiple config files inside. data_path: path for the nlu data. models_path: path to a trained Rasa model. output_dir: output path for any files created during the evaluation. cross_validation: indicates if it should test the model using cross validation or not. percentages: defines the exclusion percentage of the training data. runs: number of comparison runs to make. no_errors: indicates if incorrect predictions should be written to a file or not. """ from rasa.test import compare_nlu_models, perform_nlu_cross_validation, test_nlu nlu_data = rasa.cli.utils.get_validated_path(data_path, "nlu", DEFAULT_DATA_PATH) nlu_data = rasa.shared.data.get_nlu_directory(nlu_data) output = output_dir or DEFAULT_RESULTS_PATH all_args["errors"] = not no_errors rasa.shared.utils.io.create_directory(output) if config is not None and len(config) == 1: config = os.path.abspath(config[0]) if os.path.isdir(config): config = rasa.shared.utils.io.list_files(config) if isinstance(config, list): logger.info( "Multiple configuration files specified, running nlu comparison mode." ) config_files = [] for file in config: try: validation_utils.validate_yaml_schema( rasa.shared.utils.io.read_file(file), CONFIG_SCHEMA_FILE, ) config_files.append(file) except YamlException: rasa.shared.utils.io.raise_warning( f"Ignoring file '{file}' as it is not a valid config file." ) continue await compare_nlu_models( configs=config_files, nlu=nlu_data, output=output, runs=runs, exclusion_percentages=percentages, ) elif cross_validation: logger.info("Test model using cross validation.") config = rasa.cli.utils.get_validated_path(config, "config", DEFAULT_CONFIG_PATH) perform_nlu_cross_validation(config, nlu_data, output, all_args) else: model_path = rasa.cli.utils.get_validated_path(models_path, "model", DEFAULT_MODELS_PATH) await test_nlu(model_path, nlu_data, output, all_args)
def test_nlu_model_cross_validation() -> None: perform_nlu_cross_validation('config.yml', DEFAULT_DATA_PATH, {})