def compute_metrics(model, X_test, y_test, X_train, y_train, dataset_test,
unprivileged_groups, privileged_groups, protect_attribute,
print_result):
"""
Calculate and return: model accuracy and fairness metrics
Parameters
----------
model: scikit-learn classifier
X_test: numpy 2d array
y_test: numpy 1d array
X_train: numpy 2d array
y_train: numpy 1d array
dataset_test: aif360.datasets.BinaryLabelDataset
unprivileged_groups: list<dict>
Dictionary where the key is the name of the sensitive column in the
dataset, and the value is the value of the unprivileged group in the
dataset
privileged_groups: list<dict>
Dictionary where the key is the name of the sensitive column in the
dataset, and the value is the value of the privileged group in the
dataset
protect_attribute
print_result
"""
result = {}
y_pred_test = model.predict(X_test)
result['acc_test'] = accuracy_score(y_true=y_test, y_pred=y_pred_test)
y_pred_train = model.predict(X_train)
result['acc_train'] = accuracy_score(y_true=y_train, y_pred=y_pred_train)
dataset_pred = dataset_test.copy()
dataset_pred.labels = y_pred_test
bin_metric = BinaryLabelDatasetMetric(dataset_pred,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
result['disp_impact'] = bin_metric.disparate_impact()
result['stat_parity'] = bin_metric.mean_difference()
classif_metric = ClassificationMetric(dataset_test, dataset_pred,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
result['avg_odds'] = classif_metric.average_odds_difference()
result['equal_opport'] = classif_metric.equal_opportunity_difference()
result['false_discovery_rate'] = classif_metric.false_discovery_rate_difference()
result['entropy_index'] = classif_metric.generalized_entropy_index()
result['acc_test_clf'] = classif_metric.accuracy(privileged=None)
result['acc_test_priv'] = classif_metric.accuracy(privileged=True)
result['acc_test_unpriv'] = classif_metric.accuracy(privileged=False)
result['consistency'] = consitency(X_test, y_pred_test, protect_attribute, n_neighbors=5)
result['counterfactual'] = counterfactual(X_test, model, protect_attribute)
if print_result:
print("Train accuracy: ", result['acc_train'])
print("Test accuracy: ", result['acc_test'])
print("Test accuracy clf: ", result['acc_test_clf'])
print("Test accuracy priv.: ", result['acc_test_priv'])
print("Test accuracy unpriv.: ", result['acc_test_unpriv'])
print('Disparate impact: ', result['disp_impact'])
print('Mean difference: ', result['stat_parity'])
print('Average odds difference:', result['avg_odds'])
print('Equality of opportunity:', result['equal_opport'])
print('False discovery rate difference:', result['false_discovery_rate'])
print('Generalized entropy index:', result['entropy_index'])
print('Consistency: ', result['consistency'])
print('Counterfactual fairness: ', result['counterfactual'])
return result
def compute_metrics(model, X_test, y_test, X_train, y_train, dataset_test,
dataset_name, model_name, unprivileged_groups,
privileged_groups, position):
"""
Calculate and return: model accuracy and fairness metrics
Parameters
----------
model: scikit-learn classifier
X_test: numpy 2d array
y_test: numpy 1d array
X_train: numpy 2d array
y_train: numpy 1d array
dataset_test: aif360.datasets.BinaryLabelDataset
dataset_name: string
Dataset name used in the analysis
model_name: string
unprivileged_groups: list<dict>
Dictionary where the key is the name of the sensitive column in the
dataset, and the value is the value of the unprivileged group in the
dataset
privileged_groups: list<dict>
Dictionary where the key is the name of the sensitive column in the
dataset, and the value is the value of the privileged group in the
dataset
position: int
Column position of the sensitive group in the dataset
"""
y_pred_test = model.predict(X_test)
acc_test = accuracy_score(y_true=y_test, y_pred=y_pred_test)
print("Test accuracy: ", acc_test)
y_pred_train = model.predict(X_train)
acc_train = accuracy_score(y_true=y_train, y_pred=y_pred_train)
print("Train accuracy: ", acc_train)
dataset_pred = dataset_test.copy()
dataset_pred.labels = y_pred_test
bin_metric = BinaryLabelDatasetMetric(
dataset_pred,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
disparate_impact_bin = bin_metric.disparate_impact()
print('Disparate impact: ', disparate_impact_bin)
mean_difference = bin_metric.mean_difference()
print('Mean difference: ', mean_difference)
classif_metric = ClassificationMetric(
dataset_test,
dataset_pred,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
classif_disparete_impact = classif_metric.disparate_impact()
avg_odds = classif_metric.average_odds_difference()
print('Average odds difference:', avg_odds)
equal_opport = classif_metric.equal_opportunity_difference()
print('Equality of opportunity:', equal_opport)
false_discovery_rate = classif_metric.false_discovery_rate_difference()
print('False discovery rate difference:', false_discovery_rate)
entropy_index = classif_metric.generalized_entropy_index()
print('Generalized entropy index:', entropy_index)
cons_comp = consitency_mod(bin_metric, position, n_neighbors=5)
print('Consistency: ', cons_comp)
result = (dataset_name, model_name, acc_test, disparate_impact_bin,
mean_difference, classif_disparete_impact, avg_odds,
equal_opport, false_discovery_rate, entropy_index, cons_comp)
return result
