def compute_log_metrics(clf, x_val, y_val):
"""Compute and log metrics."""
y_prob = clf.predict_proba(x_val)[:, 1]
y_pred = (y_prob > 0.5).astype(int)
acc = metrics.accuracy_score(y_val, y_pred)
precision = metrics.precision_score(y_val, y_pred)
recall = metrics.recall_score(y_val, y_pred)
f1_score = metrics.f1_score(y_val, y_pred)
roc_auc = metrics.roc_auc_score(y_val, y_prob)
avg_prc = metrics.average_precision_score(y_val, y_prob)
print("Evaluation\n"
f" Accuracy = {acc:.4f}\n"
f" Precision = {precision:.4f}\n"
f" Recall = {recall:.4f}\n"
f" F1 score = {f1_score:.4f}\n"
f" ROC AUC = {roc_auc:.4f}\n"
f" Average precision = {avg_prc:.4f}")
# Log metrics
bedrock = BedrockApi(logging.getLogger(__name__))
bedrock.log_metric("Accuracy", acc)
bedrock.log_metric("ROC AUC", roc_auc)
bedrock.log_metric("Avg precision", avg_prc)
bedrock.log_chart_data(y_val.astype(int).tolist(),
y_prob.flatten().tolist())
# Calculate and upload xafai metrics
analyzer = ModelAnalyzer(clf, "tree_model", model_type=ModelTypes.TREE).test_features(x_val)
analyzer.fairness_config(PROTECTED_FEATURES).test_labels(y_val).test_inference(y_pred)
analyzer.analyze()
def compute_log_metrics(clf, x_val, y_val):
"""Compute and log metrics."""
y_prob = clf.predict_proba(x_val)[:, 1]
# select best threshold
fpr, tpr, thresholds = metrics.roc_curve(y_val, y_prob)
best_threshold = thresholds[np.argmax(tpr-fpr)]
y_pred = (y_prob > best_threshold).astype(int)
acc = metrics.accuracy_score(y_val, y_pred)
roc_auc = metrics.roc_auc_score(y_val, y_prob)
avg_prc = metrics.average_precision_score(y_val, y_prob)
print("Evaluation\n"
f" Accuracy = {acc:.4f}\n"
f" ROC AUC = {roc_auc:.4f}\n"
f" Average precision = {avg_prc:.4f}\n")
print(metrics.classification_report(y_val, y_pred, digits=4))
# Log metrics
bedrock = BedrockApi(logging.getLogger(__name__))
bedrock.log_metric("Accuracy", acc)
bedrock.log_metric("ROC AUC", roc_auc)
bedrock.log_metric("Avg precision", avg_prc)
bedrock.log_chart_data(y_val.astype(int).tolist(),
y_prob.flatten().tolist())
# Calculate and upload xafai metrics
analyzer = ModelAnalyzer(clf, "tree_model", model_type=ModelTypes.TREE).test_features(x_val)
analyzer.fairness_config(CONFIG_FAI).test_labels(y_val).test_inference(y_pred)
analyzer.analyze()
def compute_log_metrics(model,
x_train,
x_test,
y_test,
best_th=0.5,
model_name="tree_model",
model_type=ModelTypes.TREE):
"""Compute and log metrics."""
test_prob = model.predict_proba(x_test)
test_pred = model.predict(x_test)
acc = metrics.accuracy_score(y_test, test_pred)
precision = metrics.precision_score(y_test, test_pred, average='macro')
recall = metrics.recall_score(y_test, test_pred, average='macro')
f1_score = metrics.f1_score(y_test, test_pred, average='macro')
roc_auc = metrics.roc_auc_score(y_test, test_prob, multi_class='ovr')
avg_prc = metrics.average_precision_score(y_test, test_prob)
print("Evaluation\n"
f" Accuracy = {acc:.4f}\n"
f" Precision = {precision:.4f}\n"
f" Recall = {recall:.4f}\n"
f" F1 score = {f1_score:.4f}\n"
f" ROC AUC = {roc_auc:.4f}\n"
f" Average precision = {avg_prc:.4f}")
# Bedrock Logger: captures model metrics
bedrock = BedrockApi(logging.getLogger(__name__))
bedrock.log_chart_data(
y_test.astype(int).tolist(),
test_prob.flatten().tolist())
bedrock.log_metric("Accuracy", acc)
# TODO - Bedrock model monitoring: Fill in the blanks
# Add ROC AUC and Avg precision
bedrock.log_metric("Precision", precision)
bedrock.log_metric("Recall", recall)
bedrock.log_metric("F1 Score", f1_score)
bedrock.log_metric("ROC AUC", roc_auc)
bedrock.log_metric("Avg precision", avg_prc)
# TODO - Explainability metrics: Fill in the blanks
# Bedrock Model Analyzer: generates model explainability and fairness metrics
# Requires model object from pipeline to be passed in
analyzer = ModelAnalyzer(model[1], model_name=model_name, model_type=model_type)\
.train_features(x_train)\
.test_features(x_test)
# TODO - Fairness metrics: Fill in the blanks
# Apply fairness config to the Bedrock Model Analyzer instance
analyzer.fairness_config(CONFIG_FAI)\
.test_labels(y_test)\
.test_inference(test_pred)
# Return the 4 metrics
return analyzer.analyze()