async def test_without_additional_e2e_examples(tmp_path: Path): domain_path = tmp_path / "domain.yml" domain_path.write_text(Domain.empty().as_yaml()) config_path = tmp_path / "config.yml" config_path.touch() existing = TrainingDataImporter.load_from_dict( {}, str(config_path), str(domain_path), [] ) stories = StoryGraph( [ StoryStep( events=[ UserUttered(None, {"name": "greet_from_stories"}), ActionExecuted("utter_greet_from_stories"), ] ) ] ) # Patch to return our test stories existing.get_stories = asyncio.coroutine(lambda *args: stories) importer = E2EImporter(existing) training_data = await importer.get_nlu_data() assert training_data.training_examples assert not training_data.is_empty() assert len(training_data.nlu_examples) == 0
async def test_adding_e2e_actions_to_domain(project: Text): config_path = os.path.join(project, DEFAULT_CONFIG_PATH) domain_path = os.path.join(project, DEFAULT_DOMAIN_PATH) default_data_path = os.path.join(project, DEFAULT_DATA_PATH) existing = TrainingDataImporter.load_from_dict({}, config_path, domain_path, [default_data_path]) additional_actions = ["Hi Joey.", "it's sunny outside."] stories = StoryGraph([ StoryStep(events=[ UserUttered("greet_from_stories", {"name": "greet_from_stories"}), ActionExecuted("utter_greet_from_stories"), ]), StoryStep(events=[ UserUttered("how are you doing?", {"name": "greet_from_stories"}), ActionExecuted(additional_actions[0], action_text=additional_actions[0]), ActionExecuted(additional_actions[1], action_text=additional_actions[1]), ActionExecuted(additional_actions[1], action_text=additional_actions[1]), ]), ]) # Patch to return our test stories existing.get_stories = asyncio.coroutine(lambda *args: stories) importer = E2EImporter(existing) domain = await importer.get_domain() assert all(action_name in domain.action_names for action_name in additional_actions)
def _create_data_generator( resource_name: Text, agent: "Agent", max_stories: Optional[int] = None, use_conversation_test_files: bool = False, ) -> "TrainingDataGenerator": from rasa.shared.core.generator import TrainingDataGenerator from rasa.shared.constants import DEFAULT_DOMAIN_PATH from rasa.model import get_model_subdirectories core_model = None if agent.model_directory: core_model, _ = get_model_subdirectories(agent.model_directory) if core_model and os.path.exists( os.path.join(core_model, DEFAULT_DOMAIN_PATH)): domain_path = os.path.join(core_model, DEFAULT_DOMAIN_PATH) else: domain_path = None test_data_importer = TrainingDataImporter.load_from_dict( training_data_paths=[resource_name], domain_path=domain_path) if use_conversation_test_files: story_graph = test_data_importer.get_conversation_tests() else: story_graph = test_data_importer.get_stories() return TrainingDataGenerator( story_graph, agent.domain, use_story_concatenation=False, augmentation_factor=0, tracker_limit=max_stories, )
def _create_data_generator( resource_name: Text, agent: "Agent", max_stories: Optional[int] = None, use_conversation_test_files: bool = False, ) -> "TrainingDataGenerator": from rasa.shared.core.generator import TrainingDataGenerator tmp_domain_path = Path(tempfile.mkdtemp()) / "domain.yaml" agent.domain.persist(tmp_domain_path) test_data_importer = TrainingDataImporter.load_from_dict( training_data_paths=[resource_name], domain_path=str(tmp_domain_path) ) if use_conversation_test_files: story_graph = test_data_importer.get_conversation_tests() else: story_graph = test_data_importer.get_stories() return TrainingDataGenerator( story_graph, agent.domain, use_story_concatenation=False, augmentation_factor=0, tracker_limit=max_stories, )
def test_nlu_warn_of_competition_with_regex_extractor( monkeypatch: MonkeyPatch, component_types: List[Dict[Text, Text]], data_path: Text, should_warn: bool, ): importer = TrainingDataImporter.load_from_dict( training_data_paths=[data_path]) # there are no domain files for the above examples, so: monkeypatch.setattr(Domain, "check_missing_responses", lambda *args, **kwargs: None) graph_schema = GraphSchema({ f"{idx}": SchemaNode({}, component_type, "", "", {}) for idx, component_type in enumerate(component_types) }) validator = DefaultV1RecipeValidator(graph_schema) monkeypatch.setattr(validator, "_warn_if_some_training_data_is_unused", lambda *args, **kwargs: None) if should_warn: with pytest.warns( UserWarning, match=( f"You have an overlap between the " f"'{RegexEntityExtractor.__name__}' and the statistical"), ): validator.validate(importer) else: with pytest.warns(None) as records: validator.validate(importer) assert len(records) == 0
def default_importer(project: Text) -> TrainingDataImporter: config_path = os.path.join(project, DEFAULT_CONFIG_PATH) domain_path = os.path.join(project, DEFAULT_DOMAIN_PATH) default_data_path = os.path.join(project, DEFAULT_DATA_PATH) return TrainingDataImporter.load_from_dict({}, config_path, domain_path, [default_data_path])
def inner( train_schema: GraphSchema, cache: Optional[TrainingCache] = None, model_storage: Optional[ModelStorage] = None, path: Optional[Path] = None, force_retraining: bool = False, ) -> Path: if not path: path = tmp_path_factory.mktemp("model_storage_path") if not model_storage: model_storage = LocalModelStorage.create(path) if not cache: cache = local_cache_creator(path) graph_trainer = GraphTrainer( model_storage=model_storage, cache=cache, graph_runner_class=DaskGraphRunner ) output_filename = path / "model.tar.gz" graph_trainer.train( GraphModelConfiguration( train_schema=train_schema, predict_schema=GraphSchema({}), language=None, core_target=None, nlu_target="nlu", training_type=TrainingType.BOTH, ), importer=TrainingDataImporter.load_from_dict(domain_path=str(domain_path)), output_filename=output_filename, force_retraining=force_retraining, ) assert output_filename.is_file() return output_filename
def test_read_conversation_tests(project: Text): importer = TrainingDataImporter.load_from_dict(training_data_paths=[ str(Path(project) / DEFAULT_CONVERSATION_TEST_PATH) ]) test_stories = importer.get_conversation_tests() assert len(test_stories.story_steps) == 7
async def test_rasa_file_importer_with_invalid_domain(tmp_path: Path): config_file = tmp_path / "config.yml" config_file.write_text("") importer = TrainingDataImporter.load_from_dict({}, str(config_file), None, []) actual = await importer.get_domain() assert actual.as_dict() == Domain.empty().as_dict()
async def test_nlu_data_domain_sync_with_retrieval_intents(project: Text): config_path = os.path.join(project, DEFAULT_CONFIG_PATH) domain_path = "data/test_domains/default_retrieval_intents.yml" data_paths = [ "data/test_nlu/default_retrieval_intents.md", "data/test_responses/default.md", ] base_data_importer = TrainingDataImporter.load_from_dict( {}, config_path, domain_path, data_paths ) nlu_importer = NluDataImporter(base_data_importer) core_importer = CoreDataImporter(base_data_importer) importer = RetrievalModelsDataImporter( CombinedDataImporter([nlu_importer, core_importer]) ) domain = await importer.get_domain() nlu_data = await importer.get_nlu_data() assert domain.retrieval_intents == ["chitchat"] assert domain.intent_properties["chitchat"].get("is_retrieval_intent") assert domain.retrieval_intent_templates == nlu_data.responses assert domain.templates != nlu_data.responses assert "utter_chitchat" in domain.action_names
def test_importer_fingerprint(): importer = TrainingDataImporter.load_from_dict(training_data_paths=[ "./data/test_nlu_no_responses/nlu_with_unicode.yml" ]) fp1 = importer.fingerprint() fp2 = importer.fingerprint() assert fp1 != fp2
async def test_nlu_comparison( tmp_path: Path, monkeypatch: MonkeyPatch, nlu_as_json_path: Text ): config = { "language": "en", "pipeline": [ {"name": "WhitespaceTokenizer"}, {"name": "KeywordIntentClassifier"}, {"name": "RegexEntityExtractor"}, ], } # the configs need to be at a different path, otherwise the results are # combined on the same dictionary key and cannot be plotted properly configs = [write_file_config(config).name, write_file_config(config).name] # mock training monkeypatch.setattr(Interpreter, "load", Mock(spec=RasaNLUInterpreter)) monkeypatch.setattr(sys.modules["rasa.nlu"], "train", AsyncMock()) monkeypatch.setattr( sys.modules["rasa.nlu.test"], "get_eval_data", Mock(return_value=(1, None, (None,),)), ) monkeypatch.setattr( sys.modules["rasa.nlu.test"], "evaluate_intents", Mock(return_value={"f1_score": 1}), ) output = str(tmp_path) test_data_importer = TrainingDataImporter.load_from_dict( training_data_paths=[nlu_as_json_path] ) test_data = test_data_importer.get_nlu_data() await compare_nlu_models( configs, test_data, output, runs=2, exclusion_percentages=[50, 80] ) assert set(os.listdir(output)) == { "run_1", "run_2", "results.json", "nlu_model_comparison_graph.pdf", } run_1_path = os.path.join(output, "run_1") assert set(os.listdir(run_1_path)) == {"50%_exclusion", "80%_exclusion", "test.yml"} exclude_50_path = os.path.join(run_1_path, "50%_exclusion") modelnames = [os.path.splitext(os.path.basename(config))[0] for config in configs] modeloutputs = set( ["train"] + [f"{m}_report" for m in modelnames] + [f"{m}.tar.gz" for m in modelnames] ) assert set(os.listdir(exclude_50_path)) == modeloutputs
def test_importer_with_unicode_files(): importer = TrainingDataImporter.load_from_dict(training_data_paths=[ "./data/test_nlu_no_responses/nlu_with_unicode.yml" ]) # None of these should raise nlu_data = importer.get_nlu_data() assert not nlu_data.is_empty() importer.get_stories() importer.get_domain()
def test_nlu_data_domain_sync_responses(project: Text): config_path = os.path.join(project, DEFAULT_CONFIG_PATH) domain_path = "data/test_domains/default.yml" data_paths = ["data/test_responses/responses_utter_rasa.yml"] importer = TrainingDataImporter.load_from_dict({}, config_path, domain_path, data_paths) with pytest.warns(None): domain = importer.get_domain() # Responses were sync between "test_responses.yml" and the "domain.yml" assert "utter_rasa" in domain.responses.keys()
def test_subintent_response_matches_with_action(project: Text): """Tests retrieval intent responses are matched correctly to actions.""" config_path = os.path.join(project, DEFAULT_CONFIG_PATH) domain_path = "data/test_domains/simple_retrieval_intent.yml" data_path = "data/test/simple_retrieval_intent_nlu.yml" importer = TrainingDataImporter.load_from_dict({}, config_path, domain_path, data_path) domain = importer.get_domain() # Test retrieval intent response is matched correctly to actions # ie. utter_chitchat/faq response compatible with action utter_chitchat with pytest.warns(None) as record: domain.check_missing_responses() assert not record
def test_core_warn_if_data_but_no_policy(monkeypatch: MonkeyPatch, policy_type: Optional[Type[Policy]]): importer = TrainingDataImporter.load_from_dict( domain_path="data/test_e2ebot/domain.yml", training_data_paths=[ "data/test_e2ebot/data/nlu.yml", "data/test_e2ebot/data/stories.yml", ], ) nodes = { "tokenizer": SchemaNode({}, WhitespaceTokenizer, "", "", {}), "nlu-component": SchemaNode({}, DIETClassifier, "", "", {}), } if policy_type is not None: nodes["some-policy"] = SchemaNode({}, policy_type, "", "", {}) graph_schema = GraphSchema(nodes) validator = DefaultV1RecipeValidator(graph_schema) monkeypatch.setattr( validator, "_raise_if_a_rule_policy_is_incompatible_with_domain", lambda *args, **kwargs: None, ) monkeypatch.setattr(validator, "_warn_if_no_rule_policy_is_contained", lambda: None) monkeypatch.setattr( validator, "_warn_if_rule_based_data_is_unused_or_missing", lambda *args, **kwargs: None, ) if policy_type is None: with pytest.warns( UserWarning, match="Found data for training policies but no policy" ) as records: validator.validate(importer) assert len(records) == 1 else: with pytest.warns( UserWarning, match="Slot auto-fill has been removed in 3.0") as records: validator.validate(importer) assert all([ warn.message.args[0].startswith("Slot auto-fill has been removed") for warn in records.list ])
def test_load_from_dict( config: Dict, expected: List[Type["TrainingDataImporter"]], project: Text ): config_path = os.path.join(project, DEFAULT_CONFIG_PATH) domain_path = os.path.join(project, DEFAULT_DOMAIN_PATH) default_data_path = os.path.join(project, DEFAULT_DATA_PATH) actual = TrainingDataImporter.load_from_dict( config, config_path, domain_path, [default_data_path] ) assert isinstance(actual, E2EImporter) assert isinstance(actual.importer, ResponsesSyncImporter) actual_importers = [i.__class__ for i in actual.importer._importer._importers] assert actual_importers == expected
def test_spacy_preprocessor_process_training_data( spacy_nlp_component: SpacyNLP, spacy_model: SpacyModel): training_data = TrainingDataImporter.load_from_dict(training_data_paths=[ "data/test_e2ebot/data/nlu.yml", "data/test_e2ebot/data/stories.yml", ]).get_nlu_data() spacy_nlp_component.process_training_data(training_data, spacy_model) for message in training_data.training_examples: for attr in DENSE_FEATURIZABLE_ATTRIBUTES: attr_text = message.data.get(attr) if attr_text: doc = message.data[SPACY_DOCS[attr]] assert isinstance(doc, spacy.tokens.doc.Doc) assert doc.text == attr_text.lower()
def test_response_missing(project: Text): """Tests warning when response is missing.""" config_path = os.path.join(project, DEFAULT_CONFIG_PATH) domain_path = "data/test_domains/missing_chitchat_response.yml" data_path = "data/test/simple_retrieval_intent_nlu.yml" importer = TrainingDataImporter.load_from_dict({}, config_path, domain_path, data_path) domain = importer.get_domain() with pytest.warns(UserWarning) as record: domain.check_missing_responses() assert ( "Action 'utter_chitchat' is listed as a response action in the domain " "file, but there is no matching response defined. Please check your " "domain.") in record[0].message.args[0]
async def _create_data_generator( resource_name: Text, agent: "Agent", max_stories: Optional[int] = None, use_e2e: bool = False, ) -> "TrainingDataGenerator": from rasa.shared.core.generator import TrainingDataGenerator test_data_importer = TrainingDataImporter.load_from_dict( training_data_paths=[resource_name]) story_graph = await test_data_importer.get_stories(use_e2e=use_e2e) return TrainingDataGenerator( story_graph, agent.domain, use_story_concatenation=False, augmentation_factor=0, tracker_limit=max_stories, )
def test_nlu_data_domain_sync_with_retrieval_intents(project: Text): config_path = os.path.join(project, DEFAULT_CONFIG_PATH) domain_path = "data/test_domains/default_retrieval_intents.yml" data_paths = [ "data/test/stories_default_retrieval_intents.yml", "data/test_responses/default.yml", ] importer = TrainingDataImporter.load_from_dict({}, config_path, domain_path, data_paths) domain = importer.get_domain() nlu_data = importer.get_nlu_data() assert domain.retrieval_intents == ["chitchat"] assert domain.intent_properties["chitchat"].get("is_retrieval_intent") assert domain.retrieval_intent_responses == nlu_data.responses assert domain.responses != nlu_data.responses assert "utter_chitchat" in domain.action_names_or_texts
async def test_nlu_data_domain_sync_responses(project: Text): config_path = os.path.join(project, DEFAULT_CONFIG_PATH) domain_path = "data/test_domains/default.yml" data_paths = ["data/test_nlg/test_responses.yml"] base_data_importer = TrainingDataImporter.load_from_dict({}, config_path, domain_path, data_paths) nlu_importer = NluDataImporter(base_data_importer) core_importer = CoreDataImporter(base_data_importer) importer = ResponsesSyncImporter( CombinedDataImporter([nlu_importer, core_importer])) with pytest.warns(None): domain = await importer.get_domain() # Responses were sync between "test_responses.yml" and the "domain.yml" assert "utter_rasa" in domain.templates.keys()
def test_number_of_examples_per_intent_with_yaml(tmp_path: Path): domain_path = tmp_path / "domain.yml" domain_path.write_text(Domain.empty().as_yaml()) config_path = tmp_path / "config.yml" config_path.touch() importer = TrainingDataImporter.load_from_dict( {}, str(config_path), str(domain_path), [ "data/test_number_nlu_examples/nlu.yml", "data/test_number_nlu_examples/stories.yml", "data/test_number_nlu_examples/rules.yml", ], ) training_data = importer.get_nlu_data() assert training_data.intents == {"greet", "ask_weather"} assert training_data.number_of_examples_per_intent["greet"] == 2 assert training_data.number_of_examples_per_intent["ask_weather"] == 3
def test_core_warn_if_rule_data_unused( policy_type_not_consuming_rule_data: Type[Policy], ): importer = TrainingDataImporter.load_from_dict( domain_path="data/test_moodbot/domain.yml", training_data_paths=[ "data/test_moodbot/data/nlu.yml", "data/test_moodbot/data/rules.yml", ], ) graph_schema = GraphSchema({ "policy": SchemaNode({}, policy_type_not_consuming_rule_data, "", "", {}) }) validator = DefaultV1RecipeValidator(graph_schema) with pytest.warns( UserWarning, match=("Found rule-based training data but no policy " "supporting rule-based data."), ): validator.validate(importer)
def test_core_warn_if_rule_data_missing( policy_type_consuming_rule_data: Type[Policy]): importer = TrainingDataImporter.load_from_dict( domain_path="data/test_e2ebot/domain.yml", training_data_paths=[ "data/test_e2ebot/data/nlu.yml", "data/test_e2ebot/data/stories.yml", ], ) graph_schema = GraphSchema({ "policy": SchemaNode({}, policy_type_consuming_rule_data, "", "", {}) }) validator = DefaultV1RecipeValidator(graph_schema) with pytest.warns( UserWarning, match=("Found a rule-based policy in your configuration " "but no rule-based training data."), ): validator.validate(importer)
async def test_import_nlu_training_data_with_default_actions(project: Text): config_path = os.path.join(project, DEFAULT_CONFIG_PATH) domain_path = os.path.join(project, DEFAULT_DOMAIN_PATH) default_data_path = os.path.join(project, DEFAULT_DATA_PATH) importer = TrainingDataImporter.load_from_dict({}, config_path, domain_path, [default_data_path]) assert isinstance(importer, E2EImporter) importer_without_e2e = importer.importer # Check additional NLU training data from domain was added nlu_data = await importer.get_nlu_data() assert len(nlu_data.training_examples) > len( (await importer_without_e2e.get_nlu_data()).training_examples) extended_training_data = await importer.get_nlu_data() assert all( Message(data={ ACTION_NAME: action_name, ACTION_TEXT: "" }) in extended_training_data.training_examples for action_name in rasa.shared.core.constants.DEFAULT_ACTION_NAMES)
async def test_import_nlu_training_data_from_e2e_stories(project: Text): config_path = os.path.join(project, DEFAULT_CONFIG_PATH) domain_path = os.path.join(project, DEFAULT_DOMAIN_PATH) default_data_path = os.path.join(project, DEFAULT_DATA_PATH) importer = TrainingDataImporter.load_from_dict({}, config_path, domain_path, [default_data_path]) # The `E2EImporter` correctly wraps the underlying `CombinedDataImporter` assert isinstance(importer, E2EImporter) importer_without_e2e = importer.importer stories = StoryGraph([ StoryStep(events=[ SlotSet("some slot", "doesn't matter"), UserUttered("greet_from_stories", {"name": "greet_from_stories"}), ActionExecuted("utter_greet_from_stories"), ]), StoryStep(events=[ UserUttered("how are you doing?"), ActionExecuted("utter_greet_from_stories", action_text="Hi Joey."), ]), ]) # Patch to return our test stories importer_without_e2e.get_stories = asyncio.coroutine(lambda *args: stories) # The wrapping `E2EImporter` simply forwards these method calls assert (await importer_without_e2e.get_stories()).as_story_string() == ( await importer.get_stories()).as_story_string() assert (await importer_without_e2e.get_config()) == (await importer.get_config()) # Check additional NLU training data from stories was added nlu_data = await importer.get_nlu_data() # The `E2EImporter` adds NLU training data based on our training stories assert len(nlu_data.training_examples) > len( (await importer_without_e2e.get_nlu_data()).training_examples) # Check if the NLU training data was added correctly from the story training data expected_additional_messages = [ Message(data={ TEXT: "greet_from_stories", INTENT_NAME: "greet_from_stories" }), Message(data={ ACTION_NAME: "utter_greet_from_stories", ACTION_TEXT: "" }), Message(data={ TEXT: "how are you doing?", INTENT_NAME: None }), Message(data={ ACTION_NAME: "utter_greet_from_stories", ACTION_TEXT: "Hi Joey." }), ] assert all(m in nlu_data.training_examples for m in expected_additional_messages)
def test_loader_loads_graph_runner( default_model_storage: ModelStorage, temp_cache: TrainingCache, tmp_path: Path, tmp_path_factory: TempPathFactory, domain_path: Path, ): graph_trainer = GraphTrainer( model_storage=default_model_storage, cache=temp_cache, graph_runner_class=DaskGraphRunner, ) test_value = "test_value" train_schema = GraphSchema( { "train": SchemaNode( needs={}, uses=PersistableTestComponent, fn="train", constructor_name="create", config={"test_value": test_value}, is_target=True, ), "load": SchemaNode( needs={"resource": "train"}, uses=PersistableTestComponent, fn="run_inference", constructor_name="load", config={}, ), } ) predict_schema = GraphSchema( { "load": SchemaNode( needs={}, uses=PersistableTestComponent, fn="run_inference", constructor_name="load", config={}, is_target=True, resource=Resource("train"), ) } ) output_filename = tmp_path / "model.tar.gz" importer = TrainingDataImporter.load_from_dict( training_data_paths=[], domain_path=str(domain_path) ) trained_at = datetime.utcnow() with freezegun.freeze_time(trained_at): model_metadata = graph_trainer.train( GraphModelConfiguration( train_schema=train_schema, predict_schema=predict_schema, training_type=TrainingType.BOTH, language=None, core_target=None, nlu_target=None, ), importer=importer, output_filename=output_filename, ) assert isinstance(model_metadata, ModelMetadata) assert output_filename.is_file() loaded_model_storage_path = tmp_path_factory.mktemp("loaded model storage") model_metadata, loaded_predict_graph_runner = loader.load_predict_graph_runner( storage_path=loaded_model_storage_path, model_archive_path=output_filename, model_storage_class=LocalModelStorage, graph_runner_class=DaskGraphRunner, ) assert loaded_predict_graph_runner.run() == {"load": test_value} assert model_metadata.predict_schema == predict_schema assert model_metadata.train_schema == train_schema assert model_metadata.model_id assert model_metadata.domain.as_dict() == Domain.from_path(domain_path).as_dict() assert model_metadata.rasa_open_source_version == rasa.__version__ assert model_metadata.trained_at == trained_at
def test_graph_trainer_returns_model_metadata( default_model_storage: ModelStorage, temp_cache: TrainingCache, tmp_path: Path, domain_path: Path, ): graph_trainer = GraphTrainer( model_storage=default_model_storage, cache=temp_cache, graph_runner_class=DaskGraphRunner, ) test_value = "test_value" train_schema = GraphSchema( { "train": SchemaNode( needs={}, uses=PersistableTestComponent, fn="train", constructor_name="create", config={"test_value": test_value}, is_target=True, ), "load": SchemaNode( needs={"resource": "train"}, uses=PersistableTestComponent, fn="run_inference", constructor_name="load", config={}, ), } ) predict_schema = GraphSchema( { "load": SchemaNode( needs={}, uses=PersistableTestComponent, fn="run_inference", constructor_name="load", config={}, is_target=True, resource=Resource("train"), ) } ) output_filename = tmp_path / "model.tar.gz" model_metadata = graph_trainer.train( GraphModelConfiguration( train_schema=train_schema, predict_schema=predict_schema, language=None, core_target=None, nlu_target="nlu", training_type=TrainingType.BOTH, ), importer=TrainingDataImporter.load_from_dict(domain_path=str(domain_path)), output_filename=output_filename, ) assert model_metadata.model_id assert model_metadata.domain.as_dict() == Domain.from_path(domain_path).as_dict() assert model_metadata.train_schema == train_schema assert model_metadata.predict_schema == predict_schema
async def run_nlu_test_async( config: Optional[Union[Text, List[Text]]], data_path: Text, models_path: Text, output_dir: Text, cross_validation: bool, percentages: List[int], runs: int, no_errors: bool, all_args: Dict[Text, Any], ) -> None: """Runs NLU tests. Args: all_args: all arguments gathered in a Dict so we can pass it as one argument to other functions. config: it refers to the model configuration file. It can be a single file or a list of multiple files or a folder with multiple config files inside. data_path: path for the nlu data. models_path: path to a trained Rasa model. output_dir: output path for any files created during the evaluation. cross_validation: indicates if it should test the model using cross validation or not. percentages: defines the exclusion percentage of the training data. runs: number of comparison runs to make. no_errors: indicates if incorrect predictions should be written to a file or not. """ from rasa.model_testing import ( compare_nlu_models, perform_nlu_cross_validation, test_nlu, ) data_path = rasa.cli.utils.get_validated_path(data_path, "nlu", DEFAULT_DATA_PATH) test_data_importer = TrainingDataImporter.load_from_dict( training_data_paths=[data_path]) nlu_data = await test_data_importer.get_nlu_data() output = output_dir or DEFAULT_RESULTS_PATH all_args["errors"] = not no_errors rasa.shared.utils.io.create_directory(output) if config is not None and len(config) == 1: config = os.path.abspath(config[0]) if os.path.isdir(config): config = rasa.shared.utils.io.list_files(config) if isinstance(config, list): logger.info( "Multiple configuration files specified, running nlu comparison mode." ) config_files = [] for file in config: try: validation_utils.validate_yaml_schema( rasa.shared.utils.io.read_file(file), CONFIG_SCHEMA_FILE, ) config_files.append(file) except YamlException: rasa.shared.utils.io.raise_warning( f"Ignoring file '{file}' as it is not a valid config file." ) continue await compare_nlu_models( configs=config_files, test_data=nlu_data, output=output, runs=runs, exclusion_percentages=percentages, ) elif cross_validation: logger.info("Test model using cross validation.") config = rasa.cli.utils.get_validated_path(config, "config", DEFAULT_CONFIG_PATH) perform_nlu_cross_validation(config, nlu_data, output, all_args) else: model_path = rasa.cli.utils.get_validated_path(models_path, "model", DEFAULT_MODELS_PATH) await test_nlu(model_path, data_path, output, all_args)