def collect_nlu_errors(
intent_results: List[IntentEvaluationResult], errors_filename: Text
) -> None:
"""Log messages which result in wrong predictions and save them to file"""
errors = [
{
"text": r.message,
"intent": r.intent_target,
"intent_prediction": {
"name": r.intent_prediction,
"confidence": r.confidence,
},
}
for r in intent_results
if r.intent_target != r.intent_prediction
]
if errors:
utils.write_json_to_file(errors_filename, errors)
logger.info("Incorrect intent predictions saved to {}.".format(errors_filename))
logger.debug(
"\n\nThese intent examples could not be classified "
"correctly: \n{}".format(errors)
)
else:
logger.info("Your model predicted all intents successfully.")
def collect_nlu_successes(
intent_results: List[IntentEvaluationResult], successes_filename: Text
) -> None:
"""Log messages which result in successful predictions
and save them to file"""
successes = [
{
"text": r.message,
"intent": r.intent_target,
"intent_prediction": {
"name": r.intent_prediction,
"confidence": r.confidence,
},
}
for r in intent_results
if r.intent_target == r.intent_prediction
]
if successes:
utils.write_json_to_file(successes_filename, successes)
logger.info(
"Successful intent predictions saved to {}.".format(successes_filename)
)
logger.debug(
"\n\nSuccessfully predicted the following intents: \n{}".format(successes)
)
else:
logger.info("No successful intent predictions found.")
def persist(self, file_name: Text,
model_dir: Text) -> Optional[Dict[Text, Any]]:
file_name = file_name + ".json"
utils.write_json_to_file(os.path.join(model_dir, file_name),
self.gazette,
indent=4)
return {"file": file_name}
def persist(self, file_name: Text, model_dir: Text) -> Optional[Dict[Text, Any]]:
"""Persist this model into the passed directory.
Return the metadata necessary to load the model again."""
file_name = file_name + ".pkl"
regex_file = os.path.join(model_dir, file_name)
utils.write_json_to_file(regex_file, self.known_patterns, indent=4)
return {"file": file_name}
def persist(self, file_name: Text,
model_dir: Text) -> Optional[Dict[Text, Any]]:
if self.regex_features:
file_name = file_name + ".json"
utils.write_json_to_file(os.path.join(model_dir, file_name),
self.regex_features)
return {"file": file_name}
return {"file": None}
def persist(self, model_dir):
# type: (Text) -> Optional[Dict[Text, Any]]
if self.composite_entities:
composite_entities_file = os.path \
.join(model_dir,
COMPOSITE_ENTITIES_FILE_NAME)
write_json_to_file(composite_entities_file,
self.composite_entities,
separators=(',', ': '))
def _persist(self) -> None:
if self.synonyms:
with self._model_storage.write_to(self._resource) as storage:
entity_synonyms_file = storage / EntitySynonymMapper.SYNONYM_FILENAME
write_json_to_file(entity_synonyms_file,
self.synonyms,
separators=(",", ": "))
def evaluate_response_selections(
response_selection_results: List[ResponseSelectionEvaluationResult],
report_folder: Optional[Text],
) -> Dict: # pragma: no cover
"""Creates summary statistics for response selection.
Only considers those examples with a set response.
Others are filtered out. Returns a dictionary of containing the
evaluation result.
"""
# remove empty intent targets
num_examples = len(response_selection_results)
response_selection_results = remove_empty_response_examples(
response_selection_results)
logger.info("Response Selection Evaluation: Only considering those "
"{} examples that have a defined response out "
"of {} examples".format(len(response_selection_results),
num_examples))
target_responses, predicted_responses = _targets_predictions_from(
response_selection_results, "response_target", "response_prediction")
if report_folder:
report, precision, f1, accuracy = get_evaluation_metrics(
target_responses, predicted_responses, output_dict=True)
report_filename = os.path.join(report_folder,
"response_selection_report.json")
utils.write_json_to_file(report_filename, report)
logger.info(
"Classification report saved to {}.".format(report_filename))
else:
report, precision, f1, accuracy = get_evaluation_metrics(
target_responses, predicted_responses)
if isinstance(report, str):
log_evaluation_table(report, precision, f1, accuracy)
predictions = [{
"text": res.message,
"intent_target": res.intent_target,
"response_target": res.response_target,
"response_predicted": res.response_prediction,
"confidence": res.confidence,
} for res in response_selection_results]
return {
"predictions": predictions,
"report": report,
"precision": precision,
"f1_score": f1,
"accuracy": accuracy,
}
def persist(self, file_name, dir_name):
if self.composite_entities:
composite_entities_file = os.path.join(
dir_name, COMPOSITE_ENTITIES_FILE_NAME)
write_json_to_file(
composite_entities_file,
self.composite_entities,
separators=(",", ": "),
)
def evaluate_entities(
entity_results: List[EntityEvaluationResult],
extractors: Set[Text],
report_folder: Optional[Text],
output_folder: Optional[Text] = None,
) -> Dict: # pragma: no cover
"""Creates summary statistics for each entity extractor.
Logs precision, recall, and F1 per entity type for each extractor."""
aligned_predictions = align_all_entity_predictions(entity_results,
extractors)
merged_targets = merge_labels(aligned_predictions)
merged_targets = substitute_labels(merged_targets, "O", NO_ENTITY)
result = {}
for extractor in extractors:
merged_predictions = merge_labels(aligned_predictions, extractor)
merged_predictions = substitute_labels(merged_predictions, "O",
NO_ENTITY)
logger.info("Evaluation for entity extractor: {} ".format(extractor))
if report_folder:
report_filename = extractor + "_report.json"
extractor_report_filename = os.path.join(report_folder,
report_filename)
report, precision, f1, accuracy = get_evaluation_metrics(
merged_targets,
merged_predictions,
output_dict=True,
exclude_label=NO_ENTITY,
)
utils.write_json_to_file(extractor_report_filename, report)
logger.info("Classification report for '{}' saved to '{}'."
"".format(extractor, extractor_report_filename))
else:
report, precision, f1, accuracy = get_evaluation_metrics(
merged_targets,
merged_predictions,
output_dict=False,
exclude_label=NO_ENTITY,
)
if isinstance(report, str):
log_evaluation_table(report, precision, f1, accuracy)
result[extractor] = {
"report": report,
"precision": precision,
"f1_score": f1,
"accuracy": accuracy,
}
return result
def persist(self, file_name: Text, model_dir: Text) -> Dict[Text, Any]:
"""Persist this model into the passed directory.
Return the metadata necessary to load the model again.
"""
file_name = file_name + ".json"
keyword_file = os.path.join(model_dir, file_name)
utils.write_json_to_file(keyword_file, self.intent_keyword_map)
return {"file": file_name}
def persist(self, file_name: Text, model_dir: Text) -> Optional[Dict[Text, Any]]:
"""Persist this component to disk for future loading."""
if self.regex_feature:
file_name = file_name + ".json"
regex_feature_file = os.path.join(model_dir, file_name)
write_json_to_file(
regex_feature_file,
self.regex_feature, separators=(",", ": "))
return {"file": file_name}
else:
return {"file": None}
def persist(self, file_name: Text,
model_dir: Text) -> Optional[Dict[Text, Any]]:
# type: (Text) -> Optional[Dict[Text, Any]]
if self.entity_customize:
entity_customize_file = os.path.join(model_dir, file_name)
write_json_to_file(entity_customize_file,
self.entity_customize,
separators=(',', ': '))
return {"customize_file": file_name}
else:
return {"customize_file": None}
def persist(self, model_dir: Text) -> Optional[Dict[Text, Any]]:
if self.synonyms:
entity_synonyms_file = os.path.join(model_dir,
ENTITY_SYNONYMS_FILE_NAME)
write_json_to_file(entity_synonyms_file,
self.synonyms,
separators=(',', ': '))
return {"synonyms_file": ENTITY_SYNONYMS_FILE_NAME}
else:
return {"synonyms_file": None}
def fake_model_dir(empty_model_dir):
metadata_file = "metadata.json"
metadata_content = {"pipeline": "pretrained_embeddings_spacy", "language": "en"}
metadata_path = os.path.join(empty_model_dir, metadata_file)
utils.write_json_to_file(metadata_path, metadata_content)
fake_obj = {"Fake", "model"}
fake_obj_path = os.path.join(empty_model_dir, "component.pkl")
with open(fake_obj_path, "wb") as f:
pickle.dump(fake_obj, f)
return empty_model_dir # not empty anymore ;)
def persist(self, file_name: Text, model_dir: Text) -> Optional[Dict[Text, Any]]:
if self.synonyms:
file_name = file_name + ".json"
entity_synonyms_file = os.path.join(model_dir, file_name)
write_json_to_file(
entity_synonyms_file, self.synonyms, separators=(",", ": ")
)
return {"file": file_name}
else:
return {"file": None}
def test_remove_model_with_files(empty_model_dir):
metadata_file = "metadata.json"
metadata_content = {"pipeline": "pretrained_embeddings_spacy", "language": "en"}
metadata_path = os.path.join(empty_model_dir, metadata_file)
write_json_to_file(metadata_path, metadata_content)
fake_obj = {"Fake", "model"}
fake_obj_path = os.path.join(empty_model_dir, "component.pkl")
with io.open(fake_obj_path, "wb") as f:
pickle.dump(fake_obj, f)
assert remove_model(empty_model_dir)
def persist(self, model_dir: Text):
"""Persists the metadata of a model to a given directory."""
metadata = self.metadata.copy()
metadata.update(
{
"trained_at": datetime.datetime.now().strftime("%Y%m%d-%H%M%S"),
"rasa_version": rasa.__version__,
}
)
filename = os.path.join(model_dir, "metadata.json")
write_json_to_file(filename, metadata, indent=4)
def persist(self, file_name: Text,
model_dir: Text) -> Optional[Dict[Text, Any]]:
"""Persist this model into the passed directory."""
file_name = file_name + ".json"
featurizer_file = os.path.join(model_dir, file_name)
data = {
"all_ngrams": self.all_ngrams,
"best_num_ngrams": self.best_num_ngrams
}
write_json_to_file(featurizer_file, data, separators=(",", ": "))
return {"file": file_name}
def persist(self, file_name: Text,
model_dir: Text) -> Optional[Dict[Text, Any]]:
"""Persist this model into the passed directory.
Args:
file_name: Prefix to add to all files stored as part of this component.
model_dir: Path where files should be stored.
Returns:
Metadata necessary to load the model again.
"""
patterns_file_name = file_name + ".patterns.pkl"
regex_file = Path(model_dir) / patterns_file_name
utils.write_json_to_file(regex_file, self.known_patterns, indent=4)
return {"file": file_name}
def write_incorrect_entity_predictions(
entity_results: List[EntityEvaluationResult],
merged_targets: List[Text],
merged_predictions: List[Text],
error_filename: Text,
):
errors = collect_incorrect_entity_predictions(entity_results,
merged_predictions,
merged_targets)
if errors:
utils.write_json_to_file(error_filename, errors)
logger.info(f"Incorrect entity predictions saved to {error_filename}.")
logger.debug("\n\nThese intent examples could not be classified "
"correctly: \n{}".format(errors))
else:
logger.info("Your model predicted all entities successfully.")
def evaluate_entities(entity_results, interpreter,
report_folder): # pragma: no cover
"""Creates summary statistics for each entity extractor.
Logs precision, recall, and F1 per entity type for each extractor."""
extractors = get_entity_extractors(interpreter)
aligned_predictions = align_all_entity_predictions(entity_results,
extractors)
merged_targets = merge_labels(aligned_predictions)
merged_targets = substitute_labels(merged_targets, "O", "no_entity")
result = {}
for extractor in extractors:
merged_predictions = merge_labels(aligned_predictions, extractor)
merged_predictions = substitute_labels(merged_predictions, "O",
"no_entity")
logger.info("Evaluation for entity extractor: {} ".format(extractor))
if report_folder:
report, precision, f1, accuracy = get_evaluation_metrics(
merged_targets, merged_predictions, output_dict=True)
report_filename = extractor + "_report.json"
extractor_report_filename = os.path.join(report_folder,
report_filename)
utils.write_json_to_file(extractor_report_filename, report)
logger.info("Classification report for '{}' saved to '{}'."
"".format(extractor, extractor_report_filename))
else:
report, precision, f1, accuracy = get_evaluation_metrics(
merged_targets, merged_predictions)
log_evaluation_table(report, precision, f1, accuracy)
result[extractor] = {
"report": report,
"precision": precision,
"f1_score": f1,
"accuracy": accuracy,
}
return result
def compare_nlu_models(
configs: List[Text],
nlu: Text,
output: Text,
runs: int,
exclusion_percentages: List[int],
):
"""Trains multiple models, compares them and saves the results."""
from rasa.nlu.test import drop_intents_below_freq
from rasa.nlu.training_data import load_data
from rasa.nlu.utils import write_json_to_file
from rasa.utils.io import create_path
from rasa.nlu.test import compare_nlu
from rasa.core.test import plot_nlu_results
data = load_data(nlu)
data = drop_intents_below_freq(data, cutoff=5)
create_path(output)
bases = [os.path.basename(nlu_config) for nlu_config in configs]
model_names = [os.path.splitext(base)[0] for base in bases]
f1_score_results = {
model_name: [[] for _ in range(runs)]
for model_name in model_names
}
training_examples_per_run = compare_nlu(
configs,
data,
exclusion_percentages,
f1_score_results,
model_names,
output,
runs,
)
f1_path = os.path.join(output, RESULTS_FILE)
write_json_to_file(f1_path, f1_score_results)
plot_nlu_results(output, training_examples_per_run)
def write_successful_entity_predictions(
entity_results: List[EntityEvaluationResult],
merged_targets: List[Text],
merged_predictions: List[Text],
successes_filename: Text,
):
successes = collect_successful_entity_predictions(entity_results,
merged_predictions,
merged_targets)
if successes:
utils.write_json_to_file(successes_filename, successes)
logger.info("Successful entity predictions saved to {}.".format(
successes_filename))
logger.debug(
"\n\nSuccessfully predicted the following entities: \n{}".format(
successes))
else:
logger.info("No successful entity prediction found.")
def collect_nlu_errors(intent_results, errors_filename):
"""Log messages which result in wrong predictions and save them to file"""
errors = [{
"text": r.message,
"intent": r.intent_target,
"intent_prediction": {
"name": r.intent_prediction,
"confidence": r.confidence,
},
} for r in intent_results if r.intent_target != r.intent_prediction]
if errors:
utils.write_json_to_file(errors_filename, errors)
logger.info(
"Model prediction errors saved to {}.".format(errors_filename))
logger.debug("\n\nThese intent examples could not be classified "
"correctly: \n{}".format(errors))
else:
logger.info("Your model made no errors")
def collect_nlu_successes(intent_results, successes_filename):
"""Log messages which result in successful predictions
and save them to file"""
successes = [{
"text": r.message,
"intent": r.intent_target,
"intent_prediction": {
"name": r.intent_prediction,
"confidence": r.confidence,
},
} for r in intent_results if r.intent_target == r.intent_prediction]
if successes:
utils.write_json_to_file(successes_filename, successes)
logger.info("Model prediction successes saved to {}.".format(
successes_filename))
logger.debug(
"\n\nSuccessfully predicted the following intents: \n{}".format(
successes))
else:
logger.info("Your model made no successful predictions")
def save_result_by_group(datasets_results, n_fold, out_config_directory,
datasets_names):
big_results = []
big_names = []
medium_results = []
medium_names = []
small_results = []
small_names = []
size = len(datasets_results)
for i in range(size):
size = int(datasets_results[i]['intent_evaluation']['report']
['weighted avg']['support']) * int(n_fold)
if size < 300:
small_results.append(datasets_results[i])
small_names.append(datasets_names[i])
elif size < 700:
medium_results.append(datasets_results[i])
medium_names.append(datasets_names[i])
else:
big_results.append(datasets_results[i])
big_names.append(datasets_names[i])
if len(small_results) > 0:
small_result = sum_results(small_results)
small_result['datasets'] = small_names
utils.write_json_to_file(
out_config_directory + 'Small_Datasets_Mean_Result', small_result)
if len(medium_results) > 0:
medium_result = sum_results(medium_results)
medium_result['datasets'] = medium_names
utils.write_json_to_file(
out_config_directory + 'Medium_Datasets_Mean_Result',
medium_result)
if len(big_results) > 0:
big_result = sum_results(big_results)
big_result['datasets'] = big_names
utils.write_json_to_file(
out_config_directory + 'Big_Datasets_Mean_Result', big_result)
def evaluate_entities(
entity_results: List[EntityEvaluationResult],
extractors: Set[Text],
output_directory: Optional[Text],
successes: bool = False,
errors: bool = False,
) -> Dict: # pragma: no cover
"""Creates summary statistics for each entity extractor.
Logs precision, recall, and F1 per entity type for each extractor."""
aligned_predictions = align_all_entity_predictions(entity_results,
extractors)
merged_targets = merge_labels(aligned_predictions)
merged_targets = substitute_labels(merged_targets, "O", NO_ENTITY)
result = {}
for extractor in extractors:
merged_predictions = merge_labels(aligned_predictions, extractor)
merged_predictions = substitute_labels(merged_predictions, "O",
NO_ENTITY)
logger.info("Evaluation for entity extractor: {} ".format(extractor))
if output_directory:
report_filename = "{}_report.json".format(extractor)
extractor_report_filename = os.path.join(output_directory,
report_filename)
report, precision, f1, accuracy = get_evaluation_metrics(
merged_targets,
merged_predictions,
output_dict=True,
exclude_label=NO_ENTITY,
)
utils.write_json_to_file(extractor_report_filename, report)
logger.info("Classification report for '{}' saved to '{}'."
"".format(extractor, extractor_report_filename))
else:
report, precision, f1, accuracy = get_evaluation_metrics(
merged_targets,
merged_predictions,
output_dict=False,
exclude_label=NO_ENTITY,
)
if isinstance(report, str):
log_evaluation_table(report, precision, f1, accuracy)
if successes:
successes_filename = "{}_successes.json".format(extractor)
if output_directory:
successes_filename = os.path.join(output_directory,
successes_filename)
# save classified samples to file for debugging
write_successful_entity_predictions(entity_results, merged_targets,
merged_predictions,
successes_filename)
if errors:
errors_filename = "{}_errors.json".format(extractor)
if output_directory:
errors_filename = os.path.join(output_directory,
errors_filename)
# log and save misclassified samples to file for debugging
write_incorrect_entity_predictions(entity_results, merged_targets,
merged_predictions,
errors_filename)
result[extractor] = {
"report": report,
"precision": precision,
"f1_score": f1,
"accuracy": accuracy,
}
return result
def evaluate_intents(
intent_results: List[IntentEvaluationResult],
report_folder: Optional[Text],
successes_filename: Optional[Text],
errors_filename: Optional[Text],
confmat_filename: Optional[Text],
intent_hist_filename: Optional[Text],
) -> Dict: # pragma: no cover
"""Creates a confusion matrix and summary statistics for intent predictions.
Log samples which could not be classified correctly and save them to file.
Creates a confidence histogram which is saved to file.
Wrong and correct prediction confidences will be
plotted in separate bars of the same histogram plot.
Only considers those examples with a set intent.
Others are filtered out. Returns a dictionary of containing the
evaluation result."""
# remove empty intent targets
num_examples = len(intent_results)
intent_results = remove_empty_intent_examples(intent_results)
logger.info("Intent Evaluation: Only considering those "
"{} examples that have a defined intent out "
"of {} examples".format(len(intent_results), num_examples))
target_intents, predicted_intents = _targets_predictions_from(
intent_results)
if report_folder:
report, precision, f1, accuracy = get_evaluation_metrics(
target_intents, predicted_intents, output_dict=True)
report_filename = os.path.join(report_folder, "intent_report.json")
utils.write_json_to_file(report_filename, report)
logger.info(
"Classification report saved to {}.".format(report_filename))
else:
report, precision, f1, accuracy = get_evaluation_metrics(
target_intents, predicted_intents)
if isinstance(report, str):
log_evaluation_table(report, precision, f1, accuracy)
if successes_filename:
# save classified samples to file for debugging
collect_nlu_successes(intent_results, successes_filename)
if errors_filename:
# log and save misclassified samples to file for debugging
collect_nlu_errors(intent_results, errors_filename)
if confmat_filename:
from sklearn.metrics import confusion_matrix
from sklearn.utils.multiclass import unique_labels
import matplotlib.pyplot as plt
cnf_matrix = confusion_matrix(target_intents, predicted_intents)
labels = unique_labels(target_intents, predicted_intents)
plot_confusion_matrix(
cnf_matrix,
classes=labels,
title="Intent Confusion matrix",
out=confmat_filename,
)
plt.show(block=False)
plot_intent_confidences(intent_results, intent_hist_filename)
plt.show(block=False)
predictions = [{
"text": res.message,
"intent": res.intent_target,
"predicted": res.intent_prediction,
"confidence": res.confidence,
} for res in intent_results]
return {
"predictions": predictions,
"report": report,
"precision": precision,
"f1_score": f1,
"accuracy": accuracy,
}
def persist(self) -> None:
"""Persist this model into the passed directory."""
with self._model_storage.write_to(self._resource) as model_dir:
file_name = f"{self.__class__.__name__}.json"
keyword_file = model_dir / file_name
utils.write_json_to_file(keyword_file.name, self.intent_keyword_map)
