def test_run_cv_evaluation_with_response_selector(monkeypatch: MonkeyPatch):
training_data_obj = rasa.shared.nlu.training_data.loading.load_data(
"data/examples/rasa/demo-rasa.yml")
training_data_responses_obj = rasa.shared.nlu.training_data.loading.load_data(
"data/examples/rasa/demo-rasa-responses.yml")
training_data_obj = training_data_obj.merge(training_data_responses_obj)
nlu_config = RasaNLUModelConfig({
"language":
"en",
"pipeline": [
{
"name": "WhitespaceTokenizer"
},
{
"name": "CountVectorsFeaturizer"
},
{
"name": "DIETClassifier",
EPOCHS: 2
},
{
"name": "ResponseSelector",
EPOCHS: 2
},
],
})
# mock training
trainer = Trainer(nlu_config)
trainer.pipeline = remove_pretrained_extractors(trainer.pipeline)
mock = Mock(return_value=Interpreter(trainer.pipeline, None))
monkeypatch.setattr(Trainer, "train", mock)
n_folds = 2
intent_results, entity_results, response_selection_results = cross_validate(
training_data_obj,
n_folds,
nlu_config,
successes=False,
errors=False,
disable_plotting=True,
report_as_dict=True,
)
assert len(intent_results.train["Accuracy"]) == n_folds
assert len(intent_results.train["Precision"]) == n_folds
assert len(intent_results.train["F1-score"]) == n_folds
assert len(intent_results.test["Accuracy"]) == n_folds
assert len(intent_results.test["Precision"]) == n_folds
assert len(intent_results.test["F1-score"]) == n_folds
assert all(key in intent_results.evaluation
for key in ["errors", "report"])
assert any(
isinstance(intent_report, dict)
and intent_report.get("confused_with") is not None
for intent_report in intent_results.evaluation["report"].values())
assert len(response_selection_results.train["Accuracy"]) == n_folds
assert len(response_selection_results.train["Precision"]) == n_folds
assert len(response_selection_results.train["F1-score"]) == n_folds
assert len(response_selection_results.test["Accuracy"]) == n_folds
assert len(response_selection_results.test["Precision"]) == n_folds
assert len(response_selection_results.test["F1-score"]) == n_folds
assert all(key in response_selection_results.evaluation
for key in ["errors", "report"])
assert any(
isinstance(intent_report, dict)
and intent_report.get("confused_with") is not None for intent_report in
response_selection_results.evaluation["report"].values())
assert len(entity_results.train["DIETClassifier"]["Accuracy"]) == n_folds
assert len(entity_results.train["DIETClassifier"]["Precision"]) == n_folds
assert len(entity_results.train["DIETClassifier"]["F1-score"]) == n_folds
assert len(entity_results.test["DIETClassifier"]["Accuracy"]) == n_folds
assert len(entity_results.test["DIETClassifier"]["Precision"]) == n_folds
assert len(entity_results.test["DIETClassifier"]["F1-score"]) == n_folds
for extractor_evaluation in entity_results.evaluation.values():
assert all(key in extractor_evaluation for key in ["errors", "report"])
def test_remove_pretrained_extractors(pretrained_interpreter):
target_components_names = ["SpacyNLP"]
filtered_pipeline = remove_pretrained_extractors(
pretrained_interpreter.pipeline)
filtered_components_names = [c.name for c in filtered_pipeline]
assert filtered_components_names == target_components_names
def __init__(self, interpreter, intention_names):
self.interpreter = interpreter
self.interpreter.pipeline = remove_pretrained_extractors(
self.interpreter.pipeline)
self.intention_names = intention_names