def test_run_cv_evaluation_with_response_selector(monkeypatch: MonkeyPatch): training_data_obj = rasa.shared.nlu.training_data.loading.load_data( "data/examples/rasa/demo-rasa.yml") training_data_responses_obj = rasa.shared.nlu.training_data.loading.load_data( "data/examples/rasa/demo-rasa-responses.yml") training_data_obj = training_data_obj.merge(training_data_responses_obj) nlu_config = RasaNLUModelConfig({ "language": "en", "pipeline": [ { "name": "WhitespaceTokenizer" }, { "name": "CountVectorsFeaturizer" }, { "name": "DIETClassifier", EPOCHS: 2 }, { "name": "ResponseSelector", EPOCHS: 2 }, ], }) # mock training trainer = Trainer(nlu_config) trainer.pipeline = remove_pretrained_extractors(trainer.pipeline) mock = Mock(return_value=Interpreter(trainer.pipeline, None)) monkeypatch.setattr(Trainer, "train", mock) n_folds = 2 intent_results, entity_results, response_selection_results = cross_validate( training_data_obj, n_folds, nlu_config, successes=False, errors=False, disable_plotting=True, report_as_dict=True, ) assert len(intent_results.train["Accuracy"]) == n_folds assert len(intent_results.train["Precision"]) == n_folds assert len(intent_results.train["F1-score"]) == n_folds assert len(intent_results.test["Accuracy"]) == n_folds assert len(intent_results.test["Precision"]) == n_folds assert len(intent_results.test["F1-score"]) == n_folds assert all(key in intent_results.evaluation for key in ["errors", "report"]) assert any( isinstance(intent_report, dict) and intent_report.get("confused_with") is not None for intent_report in intent_results.evaluation["report"].values()) assert len(response_selection_results.train["Accuracy"]) == n_folds assert len(response_selection_results.train["Precision"]) == n_folds assert len(response_selection_results.train["F1-score"]) == n_folds assert len(response_selection_results.test["Accuracy"]) == n_folds assert len(response_selection_results.test["Precision"]) == n_folds assert len(response_selection_results.test["F1-score"]) == n_folds assert all(key in response_selection_results.evaluation for key in ["errors", "report"]) assert any( isinstance(intent_report, dict) and intent_report.get("confused_with") is not None for intent_report in response_selection_results.evaluation["report"].values()) assert len(entity_results.train["DIETClassifier"]["Accuracy"]) == n_folds assert len(entity_results.train["DIETClassifier"]["Precision"]) == n_folds assert len(entity_results.train["DIETClassifier"]["F1-score"]) == n_folds assert len(entity_results.test["DIETClassifier"]["Accuracy"]) == n_folds assert len(entity_results.test["DIETClassifier"]["Precision"]) == n_folds assert len(entity_results.test["DIETClassifier"]["F1-score"]) == n_folds for extractor_evaluation in entity_results.evaluation.values(): assert all(key in extractor_evaluation for key in ["errors", "report"])
def test_remove_pretrained_extractors(pretrained_interpreter): target_components_names = ["SpacyNLP"] filtered_pipeline = remove_pretrained_extractors( pretrained_interpreter.pipeline) filtered_components_names = [c.name for c in filtered_pipeline] assert filtered_components_names == target_components_names
def __init__(self, interpreter, intention_names): self.interpreter = interpreter self.interpreter.pipeline = remove_pretrained_extractors( self.interpreter.pipeline) self.intention_names = intention_names