def get_confusion_matrix(test_list, prediction_list):
privileged_groups = [{'sex': 1}]
unprivileged_groups = [{'sex': 0}]
model_metric = ClassificationMetric(
test_list,
prediction_list,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
priv_conf_mat = model_metric.binary_confusion_matrix(privileged=True)
unpriv_conf_mat = model_metric.binary_confusion_matrix(privileged=False)
print('Confusion Matrix for Men {}'.format(priv_conf_mat))
print('Confusion Matrix for Women {}'.format(unpriv_conf_mat))
return priv_conf_mat, unpriv_conf_mat
def test_multiclass_confusion_matrix():
data = np.array([[0, 1], [0, 0], [1, 0], [1, 1], [1, 0], [1, 2], [2, 1],
[2, 0], [2, 2], [2, 1]])
pred = data.copy()
pred[3, 1] = 0
pred[4, 1] = 2
df = pd.DataFrame(data, columns=['feat', 'label'])
df2 = pd.DataFrame(pred, columns=['feat', 'label'])
favorable_values = [0, 1]
unfavorable_values = [2]
mcld = MulticlassLabelDataset(favorable_label=favorable_values,
unfavorable_label=unfavorable_values,
df=df,
label_names=['label'],
protected_attribute_names=['feat'])
mcld2 = MulticlassLabelDataset(favorable_label=favorable_values,
unfavorable_label=unfavorable_values,
df=df2,
label_names=['label'],
protected_attribute_names=['feat'])
cm = ClassificationMetric(mcld,
mcld2,
unprivileged_groups=[{
'feat': 2
}],
privileged_groups=[{
'feat': 0
}, {
'feat': 1
}])
confusion_matrix = cm.binary_confusion_matrix()
actual_labels_df = df[['label']].values
actual_labels_df2 = df2[['label']].values
assert np.all(actual_labels_df == mcld.labels)
assert np.all(actual_labels_df2 == mcld2.labels)
assert confusion_matrix == {'TP': 7.0, 'FN': 1.0, 'TN': 2.0, 'FP': 0.0}
fnr = cm.false_negative_rate_difference()
assert fnr == -0.2
#%%
# =============================================================================
# TESTING FAIRNESS METRIC FUNCTIONALITY OF AIF360
# =============================================================================
# Compute the classification metrics for the black vs white comparison
fairness_metrics = ClassificationMetric(dataset=bld_true,
classified_dataset=bld_pred,
unprivileged_groups=unprivileged_map,
privileged_groups=privileged_map)
fairness_metrics.generalized_entropy_index()
# Confusion matrices for total pop, whites, blacks
conf_mat_total = fairness_metrics.binary_confusion_matrix(privileged=None)
conf_mat_white = fairness_metrics.binary_confusion_matrix(privileged=True)
conf_mat_black = fairness_metrics.binary_confusion_matrix(privileged=False)
for i, j in conf_mat_total.items():
print('The number of ' + i + 's is ' + str(j))
type(conf_mat_total)
conf_mat_total.keys()
conf_mat_total.values()
conf_mat_keys_ordered = ['TN', 'FP', 'FN', 'TP']
conf_mat_values_ordered = []
for i in range(len(conf_mat_keys_ordered)):
print(i)
conf_mat_values_ordered[i] = conf_mat_total.get(conf_mat_keys_ordered[i])
