def test_nlu( model: Optional[Text], nlu_data: Optional[Text], output_directory: Text = DEFAULT_RESULTS_PATH, kwargs: Optional[Dict] = None, ): from rasa.nlu.test import run_evaluation from rasa.model import get_model try: unpacked_model = get_model(model) except ModelNotFound: print_error( "Could not find any model. Use 'rasa train nlu' to train a " "Rasa model and provide it via the '--model' argument.") return io_utils.create_directory(output_directory) nlu_model = os.path.join(unpacked_model, "nlu") if os.path.exists(nlu_model): kwargs = utils.minimal_kwargs(kwargs, run_evaluation, ["data_path", "model"]) run_evaluation(nlu_data, nlu_model, output_directory=output_directory, **kwargs) else: print_error( "Could not find any model. Use 'rasa train nlu' to train a " "Rasa model and provide it via the '--model' argument.")
def run_core_test(args: argparse.Namespace) -> None: """Run core tests.""" from rasa import data from rasa.test import test_core_models_in_directory, test_core, test_core_models endpoints = cli_utils.get_validated_path(args.endpoints, "endpoints", DEFAULT_ENDPOINTS_PATH, True) stories = cli_utils.get_validated_path(args.stories, "stories", DEFAULT_DATA_PATH) stories = data.get_core_directory(stories) output = args.out or DEFAULT_RESULTS_PATH io_utils.create_directory(output) if isinstance(args.model, list) and len(args.model) == 1: args.model = args.model[0] if isinstance(args.model, str): model_path = cli_utils.get_validated_path(args.model, "model", DEFAULT_MODELS_PATH) if args.evaluate_model_directory: test_core_models_in_directory(args.model, stories, output) else: test_core( model=model_path, stories=stories, endpoints=endpoints, output=output, additional_arguments=vars(args), ) else: test_core_models(args.model, stories, output)
def run_evaluation( data_path: Text, model_path: Text, out_directory: Optional[Text] = None, report: Optional[Text] = None, successes: Optional[Text] = None, errors: Optional[Text] = "errors.json", confmat: Optional[Text] = None, histogram: Optional[Text] = None, component_builder: Optional[ComponentBuilder] = None, ) -> Dict: # pragma: no cover """ Evaluate intent classification and entity extraction. :param data_path: path to the test data :param model_path: path to the model :param out_directory: path to folder where all output will be stored :param report: path to folder where reports are stored :param successes: path to file that will contain success cases :param errors: path to file that will contain error cases :param confmat: path to file that will show the confusion matrix :param histogram: path fo file that will show a histogram :param component_builder: component builder :return: dictionary containing evaluation results """ # get the metadata config from the package data interpreter = Interpreter.load(model_path, component_builder) interpreter.pipeline = remove_pretrained_extractors(interpreter.pipeline) test_data = training_data.load_data(data_path, interpreter.model_metadata.language) result = { "intent_evaluation": None, "entity_evaluation": None, } # type: Dict[Text, Optional[Dict]] if report: if out_directory: report = os.path.join(out_directory, report) io_utils.create_directory(report) intent_results, entity_results = get_eval_data(interpreter, test_data) if intent_results: logger.info("Intent evaluation results:") result["intent_evaluation"] = evaluate_intents(intent_results, report, successes, errors, confmat, histogram, out_directory) if entity_results: logger.info("Entity evaluation results:") extractors = get_entity_extractors(interpreter) result["entity_evaluation"] = evaluate_entities( entity_results, extractors, report, out_directory) return result
def test_core( model: Optional[Text] = None, stories: Optional[Text] = None, endpoints: Optional[Text] = None, output: Text = DEFAULT_RESULTS_PATH, kwargs: Optional[Dict] = None, ): import rasa.core.test import rasa.core.utils as core_utils import rasa.model from rasa.core.interpreter import RegexInterpreter, NaturalLanguageInterpreter from rasa.core.agent import Agent _endpoints = core_utils.AvailableEndpoints.read_endpoints(endpoints) if kwargs is None: kwargs = {} if output: io_utils.create_directory(output) try: unpacked_model = rasa.model.get_model(model) except ModelNotFound: print_error( "Unable to test: could not find a model. Use 'rasa train' to train a " "Rasa model and provide it via the '--model' argument." ) return core_path, nlu_path = rasa.model.get_model_subdirectories(unpacked_model) if not core_path: print_error( "Unable to test: could not find a Core model. Use 'rasa train' to train a " "Rasa model and provide it via the '--model' argument." ) use_e2e = kwargs["e2e"] if "e2e" in kwargs else False _interpreter = RegexInterpreter() if use_e2e: if nlu_path: _interpreter = NaturalLanguageInterpreter.create(_endpoints.nlu or nlu_path) else: print_warning( "No NLU model found. Using default 'RegexInterpreter' for end-to-end " "evaluation." ) _agent = Agent.load(unpacked_model, interpreter=_interpreter) kwargs = utils.minimal_kwargs(kwargs, rasa.core.test, ["stories", "agent"]) loop = asyncio.get_event_loop() loop.run_until_complete( rasa.core.test(stories, _agent, out_directory=output, **kwargs) )
def test_nlu(args: argparse.Namespace) -> None: from rasa import data from rasa.test import compare_nlu_models, perform_nlu_cross_validation, test_nlu nlu_data = cli_utils.get_validated_path(args.nlu, "nlu", DEFAULT_DATA_PATH) nlu_data = data.get_nlu_directory(nlu_data) output = args.out or DEFAULT_RESULTS_PATH io_utils.create_directory(output) if args.config is not None and len(args.config) == 1: args.config = os.path.abspath(args.config[0]) if os.path.isdir(args.config): config_dir = args.config config_files = os.listdir(config_dir) args.config = [ os.path.join(config_dir, os.path.abspath(config)) for config in config_files ] if isinstance(args.config, list): logger.info( "Multiple configuration files specified, running nlu comparison mode." ) config_files = [] for file in args.config: try: validation_utils.validate_yaml_schema( io_utils.read_file(file), CONFIG_SCHEMA_FILE, show_validation_errors=False, ) config_files.append(file) except validation_utils.InvalidYamlFileError: logger.debug( "Ignoring file '{}' as it is not a valid config file.". format(file)) continue compare_nlu_models( configs=config_files, nlu=nlu_data, output=output, runs=args.runs, exclusion_percentages=args.percentages, ) elif args.cross_validation: logger.info("Test model using cross validation.") config = cli_utils.get_validated_path(args.config, "config", DEFAULT_CONFIG_PATH) perform_nlu_cross_validation(config, nlu_data, output, vars(args)) else: model_path = cli_utils.get_validated_path(args.model, "model", DEFAULT_MODELS_PATH) test_nlu(model_path, nlu_data, output, vars(args))
def _add_ssh_credentials(self, ssh_key: Text, repository_id: int) -> Dict[Text, Text]: base_path = self._directory_for_git_clones / SSH_FILES_DIRECTORY io_utils.create_directory(base_path) path_to_key = base_path / f"{repository_id}.key" path_to_executable = self._ssh_executable_path(repository_id) return self._add_ssh_credentials_to_disk(ssh_key, path_to_key, path_to_executable)
def _create_directory_for_git_clones( directory_for_git_clones: Union[Text, Path]) -> Path: try: io_utils.create_directory(directory_for_git_clones) except OSError as e: # This should only happen when running tests locally logger.error(e) import tempfile directory_for_git_clones = tempfile.mkdtemp() return Path(directory_for_git_clones)
def test_core( model: Optional[Text] = None, stories: Optional[Text] = None, output: Text = DEFAULT_RESULTS_PATH, additional_arguments: Optional[Dict] = None, ) -> None: import rasa.model from rasa.core.interpreter import RegexInterpreter from rasa.core.agent import Agent if additional_arguments is None: additional_arguments = {} if output: io_utils.create_directory(output) try: unpacked_model = rasa.model.get_model(model) except ModelNotFound: cli_utils.print_error( "Unable to test: could not find a model. Use 'rasa train' to train a " "Rasa model and provide it via the '--model' argument." ) return _agent = Agent.load(unpacked_model) if _agent.policy_ensemble is None: cli_utils.print_error( "Unable to test: could not find a Core model. Use 'rasa train' to train a " "Rasa model and provide it via the '--model' argument." ) if isinstance(_agent.interpreter, RegexInterpreter): cli_utils.print_warning( "No NLU model found. Using default 'RegexInterpreter' for end-to-end " "evaluation. If you added actual user messages to your test stories " "this will likely lead to the tests failing. In that case, you need " "to train a NLU model first, e.g. using `rasa train`." ) from rasa.core.test import test kwargs = utils.minimal_kwargs(additional_arguments, test, ["stories", "agent"]) _test_core(stories, _agent, output, **kwargs)
def save_repository( self, repository_information: Dict[Text, Union[Text, int]] ) -> Dict[Text, Union[Text, int]]: """Saves the credentials for a repository. Args: repository_information: The information for the repository as `Dict`. Returns: The saved repository information including database ID. """ repository_credentials = self._get_repository_from_dict( repository_information) if not self.has_required_remote_permission( repository_credentials.repository_url, repository_credentials.ssh_key): raise ValueError( "Given repository credentials don't provide write " "permissions to the repository. Please make sure the ssh " "key is correct and the administrator of the remote " "repository gave you the required permissions.") self.add(repository_credentials) self.flush() # to assign database id self._add_ssh_credentials(repository_credentials.ssh_key, repository_credentials.id) io_utils.create_directory( self.repository_path(repository_credentials.id)) self.set_repository(repository_credentials.id) was_created_via_ui = bool( repository_information.get(REPOSITORY_USE_GENERATED_KEYS_KEY)) telemetry.track_repository_creation( repository_credentials.target_branch, was_created_via_ui) return repository_credentials.as_dict()
def test_create_directory_if_already_exists(tmp_path: Path): # This should not throw an exception io_utils.create_directory(str(tmp_path)) assert True
def test_create_directory_if_new(tmp_path: Path): directory = str(tmp_path / "a" / "b") io_utils.create_directory(directory) assert os.path.exists(directory)
def cross_validate( data: TrainingData, n_folds: int, nlu_config: Union[RasaNLUModelConfig, Text], output: Optional[Text] = None, successes: bool = False, errors: bool = False, confmat: Optional[Text] = None, histogram: Optional[Text] = None, ) -> Tuple[CVEvaluationResult, CVEvaluationResult]: """Stratified cross validation on data. Args: data: Training Data n_folds: integer, number of cv folds nlu_config: nlu config file report: path to folder where reports are stored successes: if true successful predictions are written to a file errors: if true incorrect predictions are written to a file confmat: path to file that will show the confusion matrix histogram: path fo file that will show a histogram Returns: dictionary with key, list structure, where each entry in list corresponds to the relevant result for one fold """ from collections import defaultdict if isinstance(nlu_config, str): nlu_config = config.load(nlu_config) if output: io_utils.create_directory(output) trainer = Trainer(nlu_config) trainer.pipeline = remove_pretrained_extractors(trainer.pipeline) intent_train_metrics = defaultdict(list) # type: IntentMetrics intent_test_metrics = defaultdict(list) # type: IntentMetrics entity_train_metrics = defaultdict( lambda: defaultdict(list)) # type: EntityMetrics entity_test_metrics = defaultdict( lambda: defaultdict(list)) # type: EntityMetrics intent_test_results = [] # type: List[IntentEvaluationResult] entity_test_results = [] # type: List[EntityEvaluationResult] intent_classifier_present = False extractors = set() # type: Set[Text] for train, test in generate_folds(n_folds, data): interpreter = trainer.train(train) # calculate train accuracy combine_result(intent_train_metrics, entity_train_metrics, interpreter, train) # calculate test accuracy combine_result( intent_test_metrics, entity_test_metrics, interpreter, test, intent_test_results, entity_test_results, ) if not extractors: extractors = get_entity_extractors(interpreter) if is_intent_classifier_present(interpreter): intent_classifier_present = True if intent_classifier_present: logger.info("Accumulated test folds intent evaluation results:") evaluate_intents(intent_test_results, output, successes, errors, confmat, histogram) if extractors: logger.info("Accumulated test folds entity evaluation results:") evaluate_entities(entity_test_results, extractors, output, successes, errors) return ( CVEvaluationResult(dict(intent_train_metrics), dict(intent_test_metrics)), CVEvaluationResult(dict(entity_train_metrics), dict(entity_test_metrics)), )
def run_evaluation( data_path: Text, model_path: Text, output_directory: Optional[Text] = None, successes: bool = False, errors: bool = False, confmat: Optional[Text] = None, histogram: Optional[Text] = None, component_builder: Optional[ComponentBuilder] = None, ) -> Dict: # pragma: no cover """ Evaluate intent classification, response selection and entity extraction. :param data_path: path to the test data :param model_path: path to the model :param output_directory: path to folder where all output will be stored :param successes: if true successful predictions are written to a file :param errors: if true incorrect predictions are written to a file :param confmat: path to file that will show the confusion matrix :param histogram: path fo file that will show a histogram :param component_builder: component builder :return: dictionary containing evaluation results """ if isinstance(model, Interpreter): interpreter = model # get the correct interpreter based on incremental flag elif incremental is False: interpreter = Interpreter.load(model, component_builder) else: interpreter = IncrementalInterpreter.load(model, component_builder) test_data = training_data.load_data(data_path, interpreter.model_metadata.language) # get the metadata config from the package data #interpreter = Interpreter.load(model_path, component_builder) interpreter.pipeline = remove_pretrained_extractors(interpreter.pipeline) test_data = training_data.load_data(data_path, interpreter.model_metadata.language) result = { "intent_evaluation": None, "entity_evaluation": None, "response_selection_evaluation": None, } # type: Dict[Text, Optional[Dict]] if output_directory: io_utils.create_directory(output_directory) intent_results, response_selection_results, entity_results, = get_eval_data( interpreter, test_data ) if intent_results: logger.info("Intent evaluation results:") result["intent_evaluation"] = evaluate_intents( intent_results, output_directory, successes, errors, confmat, histogram ) if response_selection_results: logger.info("Response selection evaluation results:") result["response_selection_evaluation"] = evaluate_response_selections( response_selection_results, output_directory ) if entity_results: logger.info("Entity evaluation results:") extractors = get_entity_extractors(interpreter) result["entity_evaluation"] = evaluate_entities( entity_results, extractors, output_directory, successes, errors ) return result
def test_creation_of_existing_dir(tmpdir): # makes sure there is no exception assert io_utils.create_directory(tmpdir.strpath) is None