def plot_story_evaluation(test_y, predictions, report, precision, f1, accuracy, in_training_data_fraction, out_directory): """Plot the results of story evaluation""" from sklearn.metrics import confusion_matrix from sklearn.utils.multiclass import unique_labels import matplotlib.pyplot as plt from rasa.nlu.test import plot_confusion_matrix log_evaluation_table(test_y, "ACTION", report, precision, f1, accuracy, in_training_data_fraction, include_report=True) cnf_matrix = confusion_matrix(test_y, predictions) plot_confusion_matrix(cnf_matrix, classes=unique_labels(test_y, predictions), title='Action Confusion matrix') fig = plt.gcf() fig.set_size_inches(int(20), int(20)) fig.savefig(os.path.join(out_directory, "story_confmat.pdf"), bbox_inches='tight')
def compute_confusion_matrix(self,save_path): from sklearn.metrics import confusion_matrix from sklearn.utils.multiclass import unique_labels import matplotlib.pyplot as plt from rasa.nlu.test import plot_confusion_matrix plt.gcf().clear() # compute confusion matrix cnf_matrix = confusion_matrix(self.intent_true, self.intent_pred) # get list of unique labels from target and predicted labels = unique_labels(self.intent_true, self.intent_pred) plot_confusion_matrix( cnf_matrix, classes=labels, title="Intent Confusion matrix", out=save_path, )
def compute_confusion_matrix(self, save_path): from sklearn.metrics import confusion_matrix from sklearn.utils.multiclass import unique_labels import matplotlib.pyplot as plt from rasa.nlu.test import _targets_predictions_from from rasa.nlu.test import plot_confusion_matrix plt.gcf().clear() # extract target and predicted labels from results self.target_intents, self.predicted_intents = _targets_predictions_from( self.intent_results, "intent_target", "intent_prediction") # compute confusion matrix cnf_matrix = confusion_matrix(self.target_intents, self.predicted_intents) # get list of unique labels from target and predicted labels = unique_labels(self.target_intents, self.predicted_intents) plot_confusion_matrix( cnf_matrix, classes=labels, title="Intent Confusion matrix", out=save_path, )
def plot_and_save_charts(update, intent_results): # pragma: no cover import io import boto3 import matplotlib as mpl mpl.use("Agg") import matplotlib.pyplot as plt from sklearn.metrics import confusion_matrix from sklearn.utils.multiclass import unique_labels from botocore.exceptions import ClientError from decouple import config aws_access_endpoint_url = config("BOTHUB_NLP_AWS_ACCESS_ENDPOINT_URL", default=None) aws_access_key_id = config("BOTHUB_NLP_AWS_ACCESS_KEY_ID", default="") aws_secret_access_key = config("BOTHUB_NLP_AWS_SECRET_ACCESS_KEY", default="") aws_bucket_name = config("BOTHUB_NLP_AWS_S3_BUCKET_NAME", default="") aws_region_name = config("BOTHUB_NLP_AWS_REGION_NAME", "us-east-1") confmat_url = "" intent_hist_url = "" if all([aws_access_key_id, aws_secret_access_key, aws_bucket_name]): confmat_filename = "repository_{}/confmat_{}.png".format( update, uuid.uuid4()) intent_hist_filename = "repository_{}/intent_hist_{}.png".format( update, uuid.uuid4()) intent_results = remove_empty_intent_examples(intent_results) targets, predictions = _targets_predictions_from( intent_results, "intent_target", "intent_prediction") cnf_matrix = confusion_matrix(targets, predictions) labels = unique_labels(targets, predictions) plot_confusion_matrix(cnf_matrix, classes=labels, title="Intent Confusion matrix") chart = io.BytesIO() fig = plt.gcf() fig.set_size_inches(20, 20) fig.savefig(chart, format="png", bbox_inches="tight") chart.seek(0) s3_client = boto3.client( "s3", endpoint_url=aws_access_endpoint_url, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, region_name=aws_region_name, ) try: s3_client.upload_fileobj( chart, aws_bucket_name, confmat_filename, ExtraArgs={"ContentType": "image/png"}, ) confmat_url = "{}/{}/{}".format(s3_client.meta.endpoint_url, aws_bucket_name, confmat_filename) except ClientError as e: logger.error(e) plot_attribute_confidences(intent_results, None, "intent_target", "intent_prediction") chart = io.BytesIO() fig = plt.gcf() fig.set_size_inches(10, 10) fig.savefig(chart, format="png", bbox_inches="tight") chart.seek(0) try: s3_client.upload_fileobj( chart, aws_bucket_name, intent_hist_filename, ExtraArgs={"ContentType": "image/png"}, ) intent_hist_url = "{}/{}/{}".format(s3_client.meta.endpoint_url, aws_bucket_name, intent_hist_filename) except ClientError as e: logger.error(e) return {"matrix_chart": confmat_url, "confidence_chart": intent_hist_url}