def test_empty_intent_removal():
intent_results = [
IntentEvaluationResult("", "restaurant_search", "I am hungry", 0.12345),
IntentEvaluationResult("greet", "greet", "hello", 0.98765),
]
intent_results = remove_empty_intent_examples(intent_results)
assert len(intent_results) == 1
assert intent_results[0].intent_target == "greet"
assert intent_results[0].intent_prediction == "greet"
assert intent_results[0].confidence == 0.98765
assert intent_results[0].message == "hello"
def evaluate_intents(intent_results): # pragma: no cover
"""Creates a confusion matrix and summary statistics for intent predictions.
Log samples which could not be classified correctly and save them to file.
Creates a confidence histogram which is saved to file.
Wrong and correct prediction confidences will be
plotted in separate bars of the same histogram plot.
Only considers those examples with a set intent.
Others are filtered out. Returns a dictionary of containing the
evaluation result.
"""
# remove empty intent targets
intent_results = remove_empty_intent_examples(intent_results)
target_intents, predicted_intents = _targets_predictions_from(
intent_results, "intent_target", "intent_prediction"
)
report, precision, f1, accuracy = get_evaluation_metrics(
target_intents, predicted_intents, output_dict=True
)
log = collect_nlu_errors(intent_results) + collect_nlu_successes(intent_results)
predictions = [
{
"text": res.message,
"intent": res.intent_target,
"predicted": res.intent_prediction,
"confidence": res.confidence,
}
for res in intent_results
]
return {
"predictions": predictions,
"report": report,
"precision": precision,
"f1_score": f1,
"accuracy": accuracy,
"log": log,
}
def plot_and_save_charts(update, intent_results): # pragma: no cover
import io
import boto3
import matplotlib as mpl
mpl.use("Agg")
import matplotlib.pyplot as plt
from sklearn.metrics import confusion_matrix
from sklearn.utils.multiclass import unique_labels
from botocore.exceptions import ClientError
from decouple import config
aws_access_endpoint_url = config("BOTHUB_NLP_AWS_ACCESS_ENDPOINT_URL",
default=None)
aws_access_key_id = config("BOTHUB_NLP_AWS_ACCESS_KEY_ID", default="")
aws_secret_access_key = config("BOTHUB_NLP_AWS_SECRET_ACCESS_KEY",
default="")
aws_bucket_name = config("BOTHUB_NLP_AWS_S3_BUCKET_NAME", default="")
aws_region_name = config("BOTHUB_NLP_AWS_REGION_NAME", "us-east-1")
confmat_url = ""
intent_hist_url = ""
if all([aws_access_key_id, aws_secret_access_key, aws_bucket_name]):
confmat_filename = "repository_{}/confmat_{}.png".format(
update, uuid.uuid4())
intent_hist_filename = "repository_{}/intent_hist_{}.png".format(
update, uuid.uuid4())
intent_results = remove_empty_intent_examples(intent_results)
targets, predictions = _targets_predictions_from(
intent_results, "intent_target", "intent_prediction")
cnf_matrix = confusion_matrix(targets, predictions)
labels = unique_labels(targets, predictions)
plot_confusion_matrix(cnf_matrix,
classes=labels,
title="Intent Confusion matrix")
chart = io.BytesIO()
fig = plt.gcf()
fig.set_size_inches(20, 20)
fig.savefig(chart, format="png", bbox_inches="tight")
chart.seek(0)
s3_client = boto3.client(
"s3",
endpoint_url=aws_access_endpoint_url,
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
region_name=aws_region_name,
)
try:
s3_client.upload_fileobj(
chart,
aws_bucket_name,
confmat_filename,
ExtraArgs={"ContentType": "image/png"},
)
confmat_url = "{}/{}/{}".format(s3_client.meta.endpoint_url,
aws_bucket_name, confmat_filename)
except ClientError as e:
logger.error(e)
plot_attribute_confidences(intent_results, None, "intent_target",
"intent_prediction")
chart = io.BytesIO()
fig = plt.gcf()
fig.set_size_inches(10, 10)
fig.savefig(chart, format="png", bbox_inches="tight")
chart.seek(0)
try:
s3_client.upload_fileobj(
chart,
aws_bucket_name,
intent_hist_filename,
ExtraArgs={"ContentType": "image/png"},
)
intent_hist_url = "{}/{}/{}".format(s3_client.meta.endpoint_url,
aws_bucket_name,
intent_hist_filename)
except ClientError as e:
logger.error(e)
return {"matrix_chart": confmat_url, "confidence_chart": intent_hist_url}