def collect_nlu_errors( intent_results: List[IntentEvaluationResult], errors_filename: Text ) -> None: """Log messages which result in wrong predictions and save them to file""" errors = [ { "text": r.message, "intent": r.intent_target, "intent_prediction": { "name": r.intent_prediction, "confidence": r.confidence, }, } for r in intent_results if r.intent_target != r.intent_prediction ] if errors: utils.write_json_to_file(errors_filename, errors) logger.info("Incorrect intent predictions saved to {}.".format(errors_filename)) logger.debug( "\n\nThese intent examples could not be classified " "correctly: \n{}".format(errors) ) else: logger.info("Your model predicted all intents successfully.")
def collect_nlu_successes( intent_results: List[IntentEvaluationResult], successes_filename: Text ) -> None: """Log messages which result in successful predictions and save them to file""" successes = [ { "text": r.message, "intent": r.intent_target, "intent_prediction": { "name": r.intent_prediction, "confidence": r.confidence, }, } for r in intent_results if r.intent_target == r.intent_prediction ] if successes: utils.write_json_to_file(successes_filename, successes) logger.info( "Successful intent predictions saved to {}.".format(successes_filename) ) logger.debug( "\n\nSuccessfully predicted the following intents: \n{}".format(successes) ) else: logger.info("No successful intent predictions found.")
def persist(self, file_name: Text, model_dir: Text) -> Optional[Dict[Text, Any]]: file_name = file_name + ".json" utils.write_json_to_file(os.path.join(model_dir, file_name), self.gazette, indent=4) return {"file": file_name}
def persist(self, file_name: Text, model_dir: Text) -> Optional[Dict[Text, Any]]: """Persist this model into the passed directory. Return the metadata necessary to load the model again.""" file_name = file_name + ".pkl" regex_file = os.path.join(model_dir, file_name) utils.write_json_to_file(regex_file, self.known_patterns, indent=4) return {"file": file_name}
def persist(self, file_name: Text, model_dir: Text) -> Optional[Dict[Text, Any]]: if self.regex_features: file_name = file_name + ".json" utils.write_json_to_file(os.path.join(model_dir, file_name), self.regex_features) return {"file": file_name} return {"file": None}
def persist(self, model_dir): # type: (Text) -> Optional[Dict[Text, Any]] if self.composite_entities: composite_entities_file = os.path \ .join(model_dir, COMPOSITE_ENTITIES_FILE_NAME) write_json_to_file(composite_entities_file, self.composite_entities, separators=(',', ': '))
def _persist(self) -> None: if self.synonyms: with self._model_storage.write_to(self._resource) as storage: entity_synonyms_file = storage / EntitySynonymMapper.SYNONYM_FILENAME write_json_to_file(entity_synonyms_file, self.synonyms, separators=(",", ": "))
def evaluate_response_selections( response_selection_results: List[ResponseSelectionEvaluationResult], report_folder: Optional[Text], ) -> Dict: # pragma: no cover """Creates summary statistics for response selection. Only considers those examples with a set response. Others are filtered out. Returns a dictionary of containing the evaluation result. """ # remove empty intent targets num_examples = len(response_selection_results) response_selection_results = remove_empty_response_examples( response_selection_results) logger.info("Response Selection Evaluation: Only considering those " "{} examples that have a defined response out " "of {} examples".format(len(response_selection_results), num_examples)) target_responses, predicted_responses = _targets_predictions_from( response_selection_results, "response_target", "response_prediction") if report_folder: report, precision, f1, accuracy = get_evaluation_metrics( target_responses, predicted_responses, output_dict=True) report_filename = os.path.join(report_folder, "response_selection_report.json") utils.write_json_to_file(report_filename, report) logger.info( "Classification report saved to {}.".format(report_filename)) else: report, precision, f1, accuracy = get_evaluation_metrics( target_responses, predicted_responses) if isinstance(report, str): log_evaluation_table(report, precision, f1, accuracy) predictions = [{ "text": res.message, "intent_target": res.intent_target, "response_target": res.response_target, "response_predicted": res.response_prediction, "confidence": res.confidence, } for res in response_selection_results] return { "predictions": predictions, "report": report, "precision": precision, "f1_score": f1, "accuracy": accuracy, }
def persist(self, file_name, dir_name): if self.composite_entities: composite_entities_file = os.path.join( dir_name, COMPOSITE_ENTITIES_FILE_NAME) write_json_to_file( composite_entities_file, self.composite_entities, separators=(",", ": "), )
def evaluate_entities( entity_results: List[EntityEvaluationResult], extractors: Set[Text], report_folder: Optional[Text], output_folder: Optional[Text] = None, ) -> Dict: # pragma: no cover """Creates summary statistics for each entity extractor. Logs precision, recall, and F1 per entity type for each extractor.""" aligned_predictions = align_all_entity_predictions(entity_results, extractors) merged_targets = merge_labels(aligned_predictions) merged_targets = substitute_labels(merged_targets, "O", NO_ENTITY) result = {} for extractor in extractors: merged_predictions = merge_labels(aligned_predictions, extractor) merged_predictions = substitute_labels(merged_predictions, "O", NO_ENTITY) logger.info("Evaluation for entity extractor: {} ".format(extractor)) if report_folder: report_filename = extractor + "_report.json" extractor_report_filename = os.path.join(report_folder, report_filename) report, precision, f1, accuracy = get_evaluation_metrics( merged_targets, merged_predictions, output_dict=True, exclude_label=NO_ENTITY, ) utils.write_json_to_file(extractor_report_filename, report) logger.info("Classification report for '{}' saved to '{}'." "".format(extractor, extractor_report_filename)) else: report, precision, f1, accuracy = get_evaluation_metrics( merged_targets, merged_predictions, output_dict=False, exclude_label=NO_ENTITY, ) if isinstance(report, str): log_evaluation_table(report, precision, f1, accuracy) result[extractor] = { "report": report, "precision": precision, "f1_score": f1, "accuracy": accuracy, } return result
def persist(self, file_name: Text, model_dir: Text) -> Dict[Text, Any]: """Persist this model into the passed directory. Return the metadata necessary to load the model again. """ file_name = file_name + ".json" keyword_file = os.path.join(model_dir, file_name) utils.write_json_to_file(keyword_file, self.intent_keyword_map) return {"file": file_name}
def persist(self, file_name: Text, model_dir: Text) -> Optional[Dict[Text, Any]]: """Persist this component to disk for future loading.""" if self.regex_feature: file_name = file_name + ".json" regex_feature_file = os.path.join(model_dir, file_name) write_json_to_file( regex_feature_file, self.regex_feature, separators=(",", ": ")) return {"file": file_name} else: return {"file": None}
def persist(self, file_name: Text, model_dir: Text) -> Optional[Dict[Text, Any]]: # type: (Text) -> Optional[Dict[Text, Any]] if self.entity_customize: entity_customize_file = os.path.join(model_dir, file_name) write_json_to_file(entity_customize_file, self.entity_customize, separators=(',', ': ')) return {"customize_file": file_name} else: return {"customize_file": None}
def persist(self, model_dir: Text) -> Optional[Dict[Text, Any]]: if self.synonyms: entity_synonyms_file = os.path.join(model_dir, ENTITY_SYNONYMS_FILE_NAME) write_json_to_file(entity_synonyms_file, self.synonyms, separators=(',', ': ')) return {"synonyms_file": ENTITY_SYNONYMS_FILE_NAME} else: return {"synonyms_file": None}
def fake_model_dir(empty_model_dir): metadata_file = "metadata.json" metadata_content = {"pipeline": "pretrained_embeddings_spacy", "language": "en"} metadata_path = os.path.join(empty_model_dir, metadata_file) utils.write_json_to_file(metadata_path, metadata_content) fake_obj = {"Fake", "model"} fake_obj_path = os.path.join(empty_model_dir, "component.pkl") with open(fake_obj_path, "wb") as f: pickle.dump(fake_obj, f) return empty_model_dir # not empty anymore ;)
def persist(self, file_name: Text, model_dir: Text) -> Optional[Dict[Text, Any]]: if self.synonyms: file_name = file_name + ".json" entity_synonyms_file = os.path.join(model_dir, file_name) write_json_to_file( entity_synonyms_file, self.synonyms, separators=(",", ": ") ) return {"file": file_name} else: return {"file": None}
def test_remove_model_with_files(empty_model_dir): metadata_file = "metadata.json" metadata_content = {"pipeline": "pretrained_embeddings_spacy", "language": "en"} metadata_path = os.path.join(empty_model_dir, metadata_file) write_json_to_file(metadata_path, metadata_content) fake_obj = {"Fake", "model"} fake_obj_path = os.path.join(empty_model_dir, "component.pkl") with io.open(fake_obj_path, "wb") as f: pickle.dump(fake_obj, f) assert remove_model(empty_model_dir)
def persist(self, model_dir: Text): """Persists the metadata of a model to a given directory.""" metadata = self.metadata.copy() metadata.update( { "trained_at": datetime.datetime.now().strftime("%Y%m%d-%H%M%S"), "rasa_version": rasa.__version__, } ) filename = os.path.join(model_dir, "metadata.json") write_json_to_file(filename, metadata, indent=4)
def persist(self, file_name: Text, model_dir: Text) -> Optional[Dict[Text, Any]]: """Persist this model into the passed directory.""" file_name = file_name + ".json" featurizer_file = os.path.join(model_dir, file_name) data = { "all_ngrams": self.all_ngrams, "best_num_ngrams": self.best_num_ngrams } write_json_to_file(featurizer_file, data, separators=(",", ": ")) return {"file": file_name}
def persist(self, file_name: Text, model_dir: Text) -> Optional[Dict[Text, Any]]: """Persist this model into the passed directory. Args: file_name: Prefix to add to all files stored as part of this component. model_dir: Path where files should be stored. Returns: Metadata necessary to load the model again. """ patterns_file_name = file_name + ".patterns.pkl" regex_file = Path(model_dir) / patterns_file_name utils.write_json_to_file(regex_file, self.known_patterns, indent=4) return {"file": file_name}
def write_incorrect_entity_predictions( entity_results: List[EntityEvaluationResult], merged_targets: List[Text], merged_predictions: List[Text], error_filename: Text, ): errors = collect_incorrect_entity_predictions(entity_results, merged_predictions, merged_targets) if errors: utils.write_json_to_file(error_filename, errors) logger.info(f"Incorrect entity predictions saved to {error_filename}.") logger.debug("\n\nThese intent examples could not be classified " "correctly: \n{}".format(errors)) else: logger.info("Your model predicted all entities successfully.")
def evaluate_entities(entity_results, interpreter, report_folder): # pragma: no cover """Creates summary statistics for each entity extractor. Logs precision, recall, and F1 per entity type for each extractor.""" extractors = get_entity_extractors(interpreter) aligned_predictions = align_all_entity_predictions(entity_results, extractors) merged_targets = merge_labels(aligned_predictions) merged_targets = substitute_labels(merged_targets, "O", "no_entity") result = {} for extractor in extractors: merged_predictions = merge_labels(aligned_predictions, extractor) merged_predictions = substitute_labels(merged_predictions, "O", "no_entity") logger.info("Evaluation for entity extractor: {} ".format(extractor)) if report_folder: report, precision, f1, accuracy = get_evaluation_metrics( merged_targets, merged_predictions, output_dict=True) report_filename = extractor + "_report.json" extractor_report_filename = os.path.join(report_folder, report_filename) utils.write_json_to_file(extractor_report_filename, report) logger.info("Classification report for '{}' saved to '{}'." "".format(extractor, extractor_report_filename)) else: report, precision, f1, accuracy = get_evaluation_metrics( merged_targets, merged_predictions) log_evaluation_table(report, precision, f1, accuracy) result[extractor] = { "report": report, "precision": precision, "f1_score": f1, "accuracy": accuracy, } return result
def compare_nlu_models( configs: List[Text], nlu: Text, output: Text, runs: int, exclusion_percentages: List[int], ): """Trains multiple models, compares them and saves the results.""" from rasa.nlu.test import drop_intents_below_freq from rasa.nlu.training_data import load_data from rasa.nlu.utils import write_json_to_file from rasa.utils.io import create_path from rasa.nlu.test import compare_nlu from rasa.core.test import plot_nlu_results data = load_data(nlu) data = drop_intents_below_freq(data, cutoff=5) create_path(output) bases = [os.path.basename(nlu_config) for nlu_config in configs] model_names = [os.path.splitext(base)[0] for base in bases] f1_score_results = { model_name: [[] for _ in range(runs)] for model_name in model_names } training_examples_per_run = compare_nlu( configs, data, exclusion_percentages, f1_score_results, model_names, output, runs, ) f1_path = os.path.join(output, RESULTS_FILE) write_json_to_file(f1_path, f1_score_results) plot_nlu_results(output, training_examples_per_run)
def write_successful_entity_predictions( entity_results: List[EntityEvaluationResult], merged_targets: List[Text], merged_predictions: List[Text], successes_filename: Text, ): successes = collect_successful_entity_predictions(entity_results, merged_predictions, merged_targets) if successes: utils.write_json_to_file(successes_filename, successes) logger.info("Successful entity predictions saved to {}.".format( successes_filename)) logger.debug( "\n\nSuccessfully predicted the following entities: \n{}".format( successes)) else: logger.info("No successful entity prediction found.")
def collect_nlu_errors(intent_results, errors_filename): """Log messages which result in wrong predictions and save them to file""" errors = [{ "text": r.message, "intent": r.intent_target, "intent_prediction": { "name": r.intent_prediction, "confidence": r.confidence, }, } for r in intent_results if r.intent_target != r.intent_prediction] if errors: utils.write_json_to_file(errors_filename, errors) logger.info( "Model prediction errors saved to {}.".format(errors_filename)) logger.debug("\n\nThese intent examples could not be classified " "correctly: \n{}".format(errors)) else: logger.info("Your model made no errors")
def collect_nlu_successes(intent_results, successes_filename): """Log messages which result in successful predictions and save them to file""" successes = [{ "text": r.message, "intent": r.intent_target, "intent_prediction": { "name": r.intent_prediction, "confidence": r.confidence, }, } for r in intent_results if r.intent_target == r.intent_prediction] if successes: utils.write_json_to_file(successes_filename, successes) logger.info("Model prediction successes saved to {}.".format( successes_filename)) logger.debug( "\n\nSuccessfully predicted the following intents: \n{}".format( successes)) else: logger.info("Your model made no successful predictions")
def save_result_by_group(datasets_results, n_fold, out_config_directory, datasets_names): big_results = [] big_names = [] medium_results = [] medium_names = [] small_results = [] small_names = [] size = len(datasets_results) for i in range(size): size = int(datasets_results[i]['intent_evaluation']['report'] ['weighted avg']['support']) * int(n_fold) if size < 300: small_results.append(datasets_results[i]) small_names.append(datasets_names[i]) elif size < 700: medium_results.append(datasets_results[i]) medium_names.append(datasets_names[i]) else: big_results.append(datasets_results[i]) big_names.append(datasets_names[i]) if len(small_results) > 0: small_result = sum_results(small_results) small_result['datasets'] = small_names utils.write_json_to_file( out_config_directory + 'Small_Datasets_Mean_Result', small_result) if len(medium_results) > 0: medium_result = sum_results(medium_results) medium_result['datasets'] = medium_names utils.write_json_to_file( out_config_directory + 'Medium_Datasets_Mean_Result', medium_result) if len(big_results) > 0: big_result = sum_results(big_results) big_result['datasets'] = big_names utils.write_json_to_file( out_config_directory + 'Big_Datasets_Mean_Result', big_result)
def evaluate_entities( entity_results: List[EntityEvaluationResult], extractors: Set[Text], output_directory: Optional[Text], successes: bool = False, errors: bool = False, ) -> Dict: # pragma: no cover """Creates summary statistics for each entity extractor. Logs precision, recall, and F1 per entity type for each extractor.""" aligned_predictions = align_all_entity_predictions(entity_results, extractors) merged_targets = merge_labels(aligned_predictions) merged_targets = substitute_labels(merged_targets, "O", NO_ENTITY) result = {} for extractor in extractors: merged_predictions = merge_labels(aligned_predictions, extractor) merged_predictions = substitute_labels(merged_predictions, "O", NO_ENTITY) logger.info("Evaluation for entity extractor: {} ".format(extractor)) if output_directory: report_filename = "{}_report.json".format(extractor) extractor_report_filename = os.path.join(output_directory, report_filename) report, precision, f1, accuracy = get_evaluation_metrics( merged_targets, merged_predictions, output_dict=True, exclude_label=NO_ENTITY, ) utils.write_json_to_file(extractor_report_filename, report) logger.info("Classification report for '{}' saved to '{}'." "".format(extractor, extractor_report_filename)) else: report, precision, f1, accuracy = get_evaluation_metrics( merged_targets, merged_predictions, output_dict=False, exclude_label=NO_ENTITY, ) if isinstance(report, str): log_evaluation_table(report, precision, f1, accuracy) if successes: successes_filename = "{}_successes.json".format(extractor) if output_directory: successes_filename = os.path.join(output_directory, successes_filename) # save classified samples to file for debugging write_successful_entity_predictions(entity_results, merged_targets, merged_predictions, successes_filename) if errors: errors_filename = "{}_errors.json".format(extractor) if output_directory: errors_filename = os.path.join(output_directory, errors_filename) # log and save misclassified samples to file for debugging write_incorrect_entity_predictions(entity_results, merged_targets, merged_predictions, errors_filename) result[extractor] = { "report": report, "precision": precision, "f1_score": f1, "accuracy": accuracy, } return result
def evaluate_intents( intent_results: List[IntentEvaluationResult], report_folder: Optional[Text], successes_filename: Optional[Text], errors_filename: Optional[Text], confmat_filename: Optional[Text], intent_hist_filename: Optional[Text], ) -> Dict: # pragma: no cover """Creates a confusion matrix and summary statistics for intent predictions. Log samples which could not be classified correctly and save them to file. Creates a confidence histogram which is saved to file. Wrong and correct prediction confidences will be plotted in separate bars of the same histogram plot. Only considers those examples with a set intent. Others are filtered out. Returns a dictionary of containing the evaluation result.""" # remove empty intent targets num_examples = len(intent_results) intent_results = remove_empty_intent_examples(intent_results) logger.info("Intent Evaluation: Only considering those " "{} examples that have a defined intent out " "of {} examples".format(len(intent_results), num_examples)) target_intents, predicted_intents = _targets_predictions_from( intent_results) if report_folder: report, precision, f1, accuracy = get_evaluation_metrics( target_intents, predicted_intents, output_dict=True) report_filename = os.path.join(report_folder, "intent_report.json") utils.write_json_to_file(report_filename, report) logger.info( "Classification report saved to {}.".format(report_filename)) else: report, precision, f1, accuracy = get_evaluation_metrics( target_intents, predicted_intents) if isinstance(report, str): log_evaluation_table(report, precision, f1, accuracy) if successes_filename: # save classified samples to file for debugging collect_nlu_successes(intent_results, successes_filename) if errors_filename: # log and save misclassified samples to file for debugging collect_nlu_errors(intent_results, errors_filename) if confmat_filename: from sklearn.metrics import confusion_matrix from sklearn.utils.multiclass import unique_labels import matplotlib.pyplot as plt cnf_matrix = confusion_matrix(target_intents, predicted_intents) labels = unique_labels(target_intents, predicted_intents) plot_confusion_matrix( cnf_matrix, classes=labels, title="Intent Confusion matrix", out=confmat_filename, ) plt.show(block=False) plot_intent_confidences(intent_results, intent_hist_filename) plt.show(block=False) predictions = [{ "text": res.message, "intent": res.intent_target, "predicted": res.intent_prediction, "confidence": res.confidence, } for res in intent_results] return { "predictions": predictions, "report": report, "precision": precision, "f1_score": f1, "accuracy": accuracy, }
def persist(self) -> None: """Persist this model into the passed directory.""" with self._model_storage.write_to(self._resource) as model_dir: file_name = f"{self.__class__.__name__}.json" keyword_file = model_dir / file_name utils.write_json_to_file(keyword_file.name, self.intent_keyword_map)