def evaluation_get_individual_report(self):
from sklearn import metrics
intent_report = metrics.classification_report(self.target_intents,
self.predicted_intents,
output_dict=True)
from rasa.nlu.test import align_all_entity_predictions, merge_labels, substitute_labels, get_entity_extractors
extractors = get_entity_extractors(self.interpreter)
aligned_predictions = align_all_entity_predictions(
self.entity_results, extractors)
merged_targets = merge_labels(aligned_predictions)
merged_targets = substitute_labels(merged_targets, "O", "no_entity")
entity_report = {}
for extractor in extractors:
merged_predictions = merge_labels(aligned_predictions, extractor)
merged_predictions = substitute_labels(merged_predictions, "O",
"no_entity")
entity_report = metrics.classification_report(merged_targets,
merged_predictions,
output_dict=True)
return [intent_report, entity_report]
def evaluate_entities(entity_results, extractors): # pragma: no cover
"""Creates summary statistics for each entity extractor.
Logs precision, recall, and F1 per entity type for each extractor."""
aligned_predictions = align_all_entity_predictions(entity_results, extractors)
merged_targets = merge_labels(aligned_predictions)
merged_targets = substitute_labels(merged_targets, "O", "no_entity")
result = {}
for extractor in extractors:
merged_predictions = merge_labels(aligned_predictions, extractor)
merged_predictions = substitute_labels(merged_predictions, "O", "no_entity")
report, precision, f1, accuracy = get_evaluation_metrics(
merged_targets,
merged_predictions,
output_dict=True,
exclude_label="no_entity",
)
result = {
"report": report,
"precision": precision,
"f1_score": f1,
"accuracy": accuracy,
}
return result
def test_label_replacement():
original_labels = ["O", "location"]
target_labels = ["no_entity", "location"]
assert substitute_labels(original_labels, "O",
"no_entity") == target_labels
def __evaluate_entities(entity_results, extractors: Set[Text]) -> Dict:
"""
Creates summary statistics for each entity extractor.
Logs precision, recall, and F1 per entity type for each extractor.
Args:
entity_results: entity evaluation results
extractors: entity extractors to consider
Returns: dictionary with evaluation results
"""
from rasa.model_testing import get_evaluation_metrics
from rasa.nlu.test import (
NO_ENTITY,
align_all_entity_predictions,
merge_labels,
substitute_labels,
collect_successful_entity_predictions,
collect_incorrect_entity_predictions
)
aligned_predictions = align_all_entity_predictions(entity_results, extractors)
merged_targets = merge_labels(aligned_predictions)
from rasa.shared.nlu.constants import NO_ENTITY_TAG
merged_targets = substitute_labels(merged_targets, NO_ENTITY_TAG, NO_ENTITY)
result = {}
for extractor in extractors:
merged_predictions = merge_labels(aligned_predictions, extractor)
merged_predictions = substitute_labels(
merged_predictions, NO_ENTITY_TAG, NO_ENTITY
)
report, precision, f1, accuracy = get_evaluation_metrics(
merged_targets,
merged_predictions,
output_dict=False,
exclude_label=NO_ENTITY,
)
successes = collect_successful_entity_predictions(
entity_results, merged_predictions, merged_targets
)
errors = collect_incorrect_entity_predictions(
entity_results, merged_predictions, merged_targets
)
result[extractor] = {
"total_count": len(successes) + len(errors),
"success_count": len(successes),
"failure_count": len(errors),
"precision": precision,
"f1_score": f1,
"accuracy": accuracy,
# 'successes': successes,
'errors': errors
}
return result