def test_response_selector_present():
response_selector_component = ResponseSelector()
interpreter_with_response_selector = Interpreter(
[response_selector_component], context=None)
interpreter_without_response_selector = Interpreter([], context=None)
assert is_response_selector_present(interpreter_with_response_selector)
assert not is_response_selector_present(
interpreter_without_response_selector)
def test_entity_evaluation_report(tmpdir_factory):
class EntityExtractorA(EntityExtractor):
provides = ["entities"]
def __init__(self, component_config=None) -> None:
super().__init__(component_config)
class EntityExtractorB(EntityExtractor):
provides = ["entities"]
def __init__(self, component_config=None) -> None:
super().__init__(component_config)
path = tmpdir_factory.mktemp("evaluation").strpath
report_folder = os.path.join(path, "reports")
report_filename_a = os.path.join(report_folder,
"EntityExtractorA_report.json")
report_filename_b = os.path.join(report_folder,
"EntityExtractorB_report.json")
rasa.utils.io.create_directory(report_folder)
mock_interpreter = Interpreter(
[
EntityExtractorA({"provides": ["entities"]}),
EntityExtractorB({"provides": ["entities"]}),
],
None,
)
extractors = get_entity_extractors(mock_interpreter)
result = evaluate_entities(
[EN_entity_result],
extractors,
report_folder,
errors=True,
successes=True,
disable_plotting=False,
)
report_a = json.loads(rasa.utils.io.read_file(report_filename_a))
report_b = json.loads(rasa.utils.io.read_file(report_filename_b))
assert len(report_a) == 6
assert report_a["datetime"]["support"] == 1.0
assert report_b["macro avg"]["recall"] == 0.0
assert report_a["macro avg"]["recall"] == 0.5
assert result["EntityExtractorA"]["accuracy"] == 0.75
assert os.path.exists(
os.path.join(report_folder, "EntityExtractorA_confusion_matrix.png"))
assert os.path.exists(
os.path.join(report_folder, "EntityExtractorA_errors.json"))
assert os.path.exists(
os.path.join(report_folder, "EntityExtractorA_successes.json"))
assert not os.path.exists(
os.path.join(report_folder, "EntityExtractorA_histogram.png"))
def test_get_entity_extractors(
components: List[Component], expected_extractors: Set[Text]
):
mock_interpreter = Interpreter(components, None)
extractors = get_entity_extractors(mock_interpreter)
assert extractors == expected_extractors
def get(self,
repository_version,
repository_authorization,
rasa_version,
use_cache=True):
update_request = backend().request_backend_parse_nlu(
repository_version, repository_authorization)
repository_name = (f"{update_request.get('version_id')}_"
f"{update_request.get('total_training_end')}_"
f"{update_request.get('language')}")
interpreter = self.interpreters.get(repository_name)
if interpreter and use_cache:
return interpreter
persistor = BothubPersistor(repository_version,
repository_authorization, rasa_version)
model_directory = mkdtemp()
persistor.retrieve(str(update_request.get("repository_uuid")),
model_directory)
self.interpreters[repository_name] = Interpreter(
None, {
"language": update_request.get("language")
}).load(model_directory,
components.ComponentBuilder(use_cache=False))
return self.get(repository_version, repository_authorization,
rasa_version)
def test_run_cv_evaluation(
pretrained_embeddings_spacy_config: RasaNLUModelConfig,
monkeypatch: MonkeyPatch):
td = rasa.shared.nlu.training_data.loading.load_data(
"data/examples/rasa/demo-rasa.json")
nlu_config = RasaNLUModelConfig({
"language":
"en",
"pipeline": [
{
"name": "WhitespaceTokenizer"
},
{
"name": "CountVectorsFeaturizer"
},
{
"name": "DIETClassifier",
EPOCHS: 2
},
],
})
# mock training
trainer = Trainer(nlu_config)
trainer.pipeline = remove_pretrained_extractors(trainer.pipeline)
mock = Mock(return_value=Interpreter(trainer.pipeline, None))
monkeypatch.setattr(Trainer, "train", mock)
n_folds = 2
intent_results, entity_results, response_selection_results = cross_validate(
td,
n_folds,
nlu_config,
successes=False,
errors=False,
disable_plotting=True,
report_as_dict=True,
)
assert len(intent_results.train["Accuracy"]) == n_folds
assert len(intent_results.train["Precision"]) == n_folds
assert len(intent_results.train["F1-score"]) == n_folds
assert len(intent_results.test["Accuracy"]) == n_folds
assert len(intent_results.test["Precision"]) == n_folds
assert len(intent_results.test["F1-score"]) == n_folds
assert all(key in intent_results.evaluation
for key in ["errors", "report"])
assert any(
isinstance(intent_report, dict)
and intent_report.get("confused_with") is not None
for intent_report in intent_results.evaluation["report"].values())
for extractor_evaluation in entity_results.evaluation.values():
assert all(key in extractor_evaluation for key in ["errors", "report"])
def test_entity_evaluation_report(tmpdir_factory):
class EntityExtractorA(EntityExtractor):
provides = ["entities"]
def __init__(self, component_config=None) -> None:
super(EntityExtractorA, self).__init__(component_config)
class EntityExtractorB(EntityExtractor):
provides = ["entities"]
def __init__(self, component_config=None) -> None:
super(EntityExtractorB, self).__init__(component_config)
path = tmpdir_factory.mktemp("evaluation").strpath
report_folder = os.path.join(path, "reports")
report_filename_a = os.path.join(report_folder,
"EntityExtractorA_report.json")
report_filename_b = os.path.join(report_folder,
"EntityExtractorB_report.json")
utils.create_dir(report_folder)
mock_interpreter = Interpreter(
[
EntityExtractorA({"provides": ["entities"]}),
EntityExtractorB({"provides": ["entities"]}),
],
None,
)
extractors = get_entity_extractors(mock_interpreter)
result = evaluate_entities([EN_entity_result], extractors, report_folder)
report_a = json.loads(rasa.utils.io.read_file(report_filename_a))
report_b = json.loads(rasa.utils.io.read_file(report_filename_b))
assert len(report_a) == 8
assert report_a["datetime"]["support"] == 1.0
assert report_b["macro avg"]["recall"] == 0.2
assert result["EntityExtractorA"]["accuracy"] == 0.75
def test_get_entity_extractors(components, expected_extractors):
mock_interpreter = Interpreter(components, None)
extractors = get_entity_extractors(mock_interpreter)
assert extractors == expected_extractors
def test_is_entity_extractor_present(components, expected_result):
interpreter = Interpreter(components, context=None)
assert is_entity_extractor_present(interpreter) == expected_result
