def get_confusion_matrix(test_list, prediction_list): privileged_groups = [{'sex': 1}] unprivileged_groups = [{'sex': 0}] model_metric = ClassificationMetric( test_list, prediction_list, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) priv_conf_mat = model_metric.binary_confusion_matrix(privileged=True) unpriv_conf_mat = model_metric.binary_confusion_matrix(privileged=False) print('Confusion Matrix for Men {}'.format(priv_conf_mat)) print('Confusion Matrix for Women {}'.format(unpriv_conf_mat)) return priv_conf_mat, unpriv_conf_mat
def test_multiclass_confusion_matrix(): data = np.array([[0, 1], [0, 0], [1, 0], [1, 1], [1, 0], [1, 2], [2, 1], [2, 0], [2, 2], [2, 1]]) pred = data.copy() pred[3, 1] = 0 pred[4, 1] = 2 df = pd.DataFrame(data, columns=['feat', 'label']) df2 = pd.DataFrame(pred, columns=['feat', 'label']) favorable_values = [0, 1] unfavorable_values = [2] mcld = MulticlassLabelDataset(favorable_label=favorable_values, unfavorable_label=unfavorable_values, df=df, label_names=['label'], protected_attribute_names=['feat']) mcld2 = MulticlassLabelDataset(favorable_label=favorable_values, unfavorable_label=unfavorable_values, df=df2, label_names=['label'], protected_attribute_names=['feat']) cm = ClassificationMetric(mcld, mcld2, unprivileged_groups=[{ 'feat': 2 }], privileged_groups=[{ 'feat': 0 }, { 'feat': 1 }]) confusion_matrix = cm.binary_confusion_matrix() actual_labels_df = df[['label']].values actual_labels_df2 = df2[['label']].values assert np.all(actual_labels_df == mcld.labels) assert np.all(actual_labels_df2 == mcld2.labels) assert confusion_matrix == {'TP': 7.0, 'FN': 1.0, 'TN': 2.0, 'FP': 0.0} fnr = cm.false_negative_rate_difference() assert fnr == -0.2
#%% # ============================================================================= # TESTING FAIRNESS METRIC FUNCTIONALITY OF AIF360 # ============================================================================= # Compute the classification metrics for the black vs white comparison fairness_metrics = ClassificationMetric(dataset=bld_true, classified_dataset=bld_pred, unprivileged_groups=unprivileged_map, privileged_groups=privileged_map) fairness_metrics.generalized_entropy_index() # Confusion matrices for total pop, whites, blacks conf_mat_total = fairness_metrics.binary_confusion_matrix(privileged=None) conf_mat_white = fairness_metrics.binary_confusion_matrix(privileged=True) conf_mat_black = fairness_metrics.binary_confusion_matrix(privileged=False) for i, j in conf_mat_total.items(): print('The number of ' + i + 's is ' + str(j)) type(conf_mat_total) conf_mat_total.keys() conf_mat_total.values() conf_mat_keys_ordered = ['TN', 'FP', 'FN', 'TP'] conf_mat_values_ordered = [] for i in range(len(conf_mat_keys_ordered)): print(i) conf_mat_values_ordered[i] = conf_mat_total.get(conf_mat_keys_ordered[i])