def plot_story_evaluation(test_y, predictions, report, precision, f1, accuracy,
in_training_data_fraction, out_directory):
"""Plot the results of story evaluation"""
from sklearn.metrics import confusion_matrix
from sklearn.utils.multiclass import unique_labels
import matplotlib.pyplot as plt
from rasa.nlu.test import plot_confusion_matrix
log_evaluation_table(test_y,
"ACTION",
report,
precision,
f1,
accuracy,
in_training_data_fraction,
include_report=True)
cnf_matrix = confusion_matrix(test_y, predictions)
plot_confusion_matrix(cnf_matrix,
classes=unique_labels(test_y, predictions),
title='Action Confusion matrix')
fig = plt.gcf()
fig.set_size_inches(int(20), int(20))
fig.savefig(os.path.join(out_directory, "story_confmat.pdf"),
bbox_inches='tight')
def compute_confusion_matrix(self,save_path):
from sklearn.metrics import confusion_matrix
from sklearn.utils.multiclass import unique_labels
import matplotlib.pyplot as plt
from rasa.nlu.test import plot_confusion_matrix
plt.gcf().clear()
# compute confusion matrix
cnf_matrix = confusion_matrix(self.intent_true, self.intent_pred)
# get list of unique labels from target and predicted
labels = unique_labels(self.intent_true, self.intent_pred)
plot_confusion_matrix(
cnf_matrix,
classes=labels,
title="Intent Confusion matrix",
out=save_path,
)
def compute_confusion_matrix(self, save_path):
from sklearn.metrics import confusion_matrix
from sklearn.utils.multiclass import unique_labels
import matplotlib.pyplot as plt
from rasa.nlu.test import _targets_predictions_from
from rasa.nlu.test import plot_confusion_matrix
plt.gcf().clear()
# extract target and predicted labels from results
self.target_intents, self.predicted_intents = _targets_predictions_from(
self.intent_results, "intent_target", "intent_prediction")
# compute confusion matrix
cnf_matrix = confusion_matrix(self.target_intents,
self.predicted_intents)
# get list of unique labels from target and predicted
labels = unique_labels(self.target_intents, self.predicted_intents)
plot_confusion_matrix(
cnf_matrix,
classes=labels,
title="Intent Confusion matrix",
out=save_path,
)
def plot_and_save_charts(update, intent_results): # pragma: no cover
import io
import boto3
import matplotlib as mpl
mpl.use("Agg")
import matplotlib.pyplot as plt
from sklearn.metrics import confusion_matrix
from sklearn.utils.multiclass import unique_labels
from botocore.exceptions import ClientError
from decouple import config
aws_access_endpoint_url = config("BOTHUB_NLP_AWS_ACCESS_ENDPOINT_URL",
default=None)
aws_access_key_id = config("BOTHUB_NLP_AWS_ACCESS_KEY_ID", default="")
aws_secret_access_key = config("BOTHUB_NLP_AWS_SECRET_ACCESS_KEY",
default="")
aws_bucket_name = config("BOTHUB_NLP_AWS_S3_BUCKET_NAME", default="")
aws_region_name = config("BOTHUB_NLP_AWS_REGION_NAME", "us-east-1")
confmat_url = ""
intent_hist_url = ""
if all([aws_access_key_id, aws_secret_access_key, aws_bucket_name]):
confmat_filename = "repository_{}/confmat_{}.png".format(
update, uuid.uuid4())
intent_hist_filename = "repository_{}/intent_hist_{}.png".format(
update, uuid.uuid4())
intent_results = remove_empty_intent_examples(intent_results)
targets, predictions = _targets_predictions_from(
intent_results, "intent_target", "intent_prediction")
cnf_matrix = confusion_matrix(targets, predictions)
labels = unique_labels(targets, predictions)
plot_confusion_matrix(cnf_matrix,
classes=labels,
title="Intent Confusion matrix")
chart = io.BytesIO()
fig = plt.gcf()
fig.set_size_inches(20, 20)
fig.savefig(chart, format="png", bbox_inches="tight")
chart.seek(0)
s3_client = boto3.client(
"s3",
endpoint_url=aws_access_endpoint_url,
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
region_name=aws_region_name,
)
try:
s3_client.upload_fileobj(
chart,
aws_bucket_name,
confmat_filename,
ExtraArgs={"ContentType": "image/png"},
)
confmat_url = "{}/{}/{}".format(s3_client.meta.endpoint_url,
aws_bucket_name, confmat_filename)
except ClientError as e:
logger.error(e)
plot_attribute_confidences(intent_results, None, "intent_target",
"intent_prediction")
chart = io.BytesIO()
fig = plt.gcf()
fig.set_size_inches(10, 10)
fig.savefig(chart, format="png", bbox_inches="tight")
chart.seek(0)
try:
s3_client.upload_fileobj(
chart,
aws_bucket_name,
intent_hist_filename,
ExtraArgs={"ContentType": "image/png"},
)
intent_hist_url = "{}/{}/{}".format(s3_client.meta.endpoint_url,
aws_bucket_name,
intent_hist_filename)
except ClientError as e:
logger.error(e)
return {"matrix_chart": confmat_url, "confidence_chart": intent_hist_url}
