def test_empty_intent_removal(): intent_results = [ IntentEvaluationResult("", "restaurant_search", "I am hungry", 0.12345), IntentEvaluationResult("greet", "greet", "hello", 0.98765), ] intent_results = remove_empty_intent_examples(intent_results) assert len(intent_results) == 1 assert intent_results[0].intent_target == "greet" assert intent_results[0].intent_prediction == "greet" assert intent_results[0].confidence == 0.98765 assert intent_results[0].message == "hello"
def evaluate_intents(intent_results): # pragma: no cover """Creates a confusion matrix and summary statistics for intent predictions. Log samples which could not be classified correctly and save them to file. Creates a confidence histogram which is saved to file. Wrong and correct prediction confidences will be plotted in separate bars of the same histogram plot. Only considers those examples with a set intent. Others are filtered out. Returns a dictionary of containing the evaluation result. """ # remove empty intent targets intent_results = remove_empty_intent_examples(intent_results) target_intents, predicted_intents = _targets_predictions_from( intent_results, "intent_target", "intent_prediction" ) report, precision, f1, accuracy = get_evaluation_metrics( target_intents, predicted_intents, output_dict=True ) log = collect_nlu_errors(intent_results) + collect_nlu_successes(intent_results) predictions = [ { "text": res.message, "intent": res.intent_target, "predicted": res.intent_prediction, "confidence": res.confidence, } for res in intent_results ] return { "predictions": predictions, "report": report, "precision": precision, "f1_score": f1, "accuracy": accuracy, "log": log, }
def plot_and_save_charts(update, intent_results): # pragma: no cover import io import boto3 import matplotlib as mpl mpl.use("Agg") import matplotlib.pyplot as plt from sklearn.metrics import confusion_matrix from sklearn.utils.multiclass import unique_labels from botocore.exceptions import ClientError from decouple import config aws_access_endpoint_url = config("BOTHUB_NLP_AWS_ACCESS_ENDPOINT_URL", default=None) aws_access_key_id = config("BOTHUB_NLP_AWS_ACCESS_KEY_ID", default="") aws_secret_access_key = config("BOTHUB_NLP_AWS_SECRET_ACCESS_KEY", default="") aws_bucket_name = config("BOTHUB_NLP_AWS_S3_BUCKET_NAME", default="") aws_region_name = config("BOTHUB_NLP_AWS_REGION_NAME", "us-east-1") confmat_url = "" intent_hist_url = "" if all([aws_access_key_id, aws_secret_access_key, aws_bucket_name]): confmat_filename = "repository_{}/confmat_{}.png".format( update, uuid.uuid4()) intent_hist_filename = "repository_{}/intent_hist_{}.png".format( update, uuid.uuid4()) intent_results = remove_empty_intent_examples(intent_results) targets, predictions = _targets_predictions_from( intent_results, "intent_target", "intent_prediction") cnf_matrix = confusion_matrix(targets, predictions) labels = unique_labels(targets, predictions) plot_confusion_matrix(cnf_matrix, classes=labels, title="Intent Confusion matrix") chart = io.BytesIO() fig = plt.gcf() fig.set_size_inches(20, 20) fig.savefig(chart, format="png", bbox_inches="tight") chart.seek(0) s3_client = boto3.client( "s3", endpoint_url=aws_access_endpoint_url, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, region_name=aws_region_name, ) try: s3_client.upload_fileobj( chart, aws_bucket_name, confmat_filename, ExtraArgs={"ContentType": "image/png"}, ) confmat_url = "{}/{}/{}".format(s3_client.meta.endpoint_url, aws_bucket_name, confmat_filename) except ClientError as e: logger.error(e) plot_attribute_confidences(intent_results, None, "intent_target", "intent_prediction") chart = io.BytesIO() fig = plt.gcf() fig.set_size_inches(10, 10) fig.savefig(chart, format="png", bbox_inches="tight") chart.seek(0) try: s3_client.upload_fileobj( chart, aws_bucket_name, intent_hist_filename, ExtraArgs={"ContentType": "image/png"}, ) intent_hist_url = "{}/{}/{}".format(s3_client.meta.endpoint_url, aws_bucket_name, intent_hist_filename) except ClientError as e: logger.error(e) return {"matrix_chart": confmat_url, "confidence_chart": intent_hist_url}