def calc_mean_diff(data, target_variable, protected_variable, unprivileged_input):
df_aif = BinaryLabelDataset(df=data, label_names=[target_variable],
protected_attribute_names=[protected_variable])
privileged_group = []
for v in data[protected_variable].unique()[data[protected_variable].unique() != unprivileged_input]:
privileged_group.append({protected_variable: v})
unprivileged_group = [{protected_variable: unprivileged_input}] #female=0
metric_orig = BinaryLabelDatasetMetric(df_aif, unprivileged_group, privileged_group)
print(metric_orig.mean_difference().round(3))
if abs(metric_orig.mean_difference().round(3)) < 0.2:
print('The algorithm can be considered to be not biased')
else:
print('There is a potential bias')
def calculate_bias_measures(data_orig_train, data_orig_vt, unprivileged_groups,
privileged_groups):
model = RandomForestClassifier().fit(
data_orig_train.features,
data_orig_train.labels.ravel(),
sample_weight=data_orig_train.instance_weights)
dataset = data_orig_vt
dataset_pred = dataset.copy()
dataset_pred.labels = model.predict(data_orig_vt.features)
classified_metric_race = ClassificationMetric(
dataset,
dataset_pred,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
metric_pred_race = BinaryLabelDatasetMetric(
dataset_pred,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
print("Mean difference {}".format(metric_pred_race.mean_difference()))
print("Disparate Metric {}".format(metric_pred_race.disparate_impact()))
print("Equal Opportunity Difference {}".format(
classified_metric_race.equal_opportunity_difference()))
print("Average Abs Odds Difference {}".format(
classified_metric_race.average_abs_odds_difference()))
print("Theil index {}".format(classified_metric_race.theil_index()))
def get_metrics(dataset_train):
metric_orig_train = BinaryLabelDatasetMetric(
dataset_train,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
#print("Difference in mean outcomes between unprivileged and privileged groups = %f" % metric_orig_train.mean_difference())
return metric_orig_train.mean_difference()
def mean_diff_values(data, target_variable, protected_variable, unprivileged_input):
df_aif = BinaryLabelDataset(df=data, label_names=[target_variable],
protected_attribute_names=[protected_variable])
privileged_group = []
for v in data[protected_variable].unique()[data[protected_variable].unique() != unprivileged_input]:
privileged_group.append({protected_variable: v})
unprivileged_group = [{protected_variable: unprivileged_input}] #female=0
metric_orig = BinaryLabelDatasetMetric(df_aif, unprivileged_group, privileged_group)
return abs(metric_orig.mean_difference().round(3))
def main():
print('Calculate bias')
np.random.seed(1)
protected_attribute = 'ethnicity'
dataset = load_preproc_data_heart([protected_attribute])
privileged_groups = [{protected_attribute: 1}]
unprivileged_groups = [{
protected_attribute: 2
}, {
protected_attribute: 3
}, {
protected_attribute: 4
}, {
protected_attribute: 5
}, {
protected_attribute: 6
}]
data_orig_train, data_orig_vt = dataset.split([0.7], shuffle=True)
data_orig_valid, data_orig_test = data_orig_vt.split([0.5], shuffle=True)
metric_orig_train = BinaryLabelDatasetMetric(
data_orig_train,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
print("Mean {}".format(metric_orig_train.mean_difference()))
rw = Reweighing(unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
data_transf_train = rw.fit_transform(data_orig_train)
metric_transf_train = BinaryLabelDatasetMetric(
data_transf_train,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
print("Mean difference after transformation =%f " %
metric_transf_train.mean_difference())
calculate_bias_measures(data_orig_train, data_orig_vt, unprivileged_groups,
privileged_groups)
calculate_bias_measures(data_orig_valid, data_orig_test,
unprivileged_groups, privileged_groups)
def fair_metrics(dataset, pred, pred_is_dataset=False):
if pred_is_dataset:
dataset_pred = pred
else:
dataset_pred = dataset.copy()
dataset_pred.labels = pred
cols = [
'statistical_parity_difference', 'equal_opportunity_difference',
'average_abs_odds_difference', 'disparate_impact', 'theil_index'
]
obj_fairness = [[0, 0, 0, 1, 0]]
fair_metrics = pd.DataFrame(data=obj_fairness,
index=['objective'],
columns=cols)
for attr in dataset_pred.protected_attribute_names:
idx = dataset_pred.protected_attribute_names.index(attr)
privileged_groups = [{
attr:
dataset_pred.privileged_protected_attributes[idx][0]
}]
unprivileged_groups = [{
attr:
dataset_pred.unprivileged_protected_attributes[idx][0]
}]
classified_metric = ClassificationMetric(
dataset,
dataset_pred,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
metric_pred = BinaryLabelDatasetMetric(
dataset_pred,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
acc = classified_metric.accuracy()
row = pd.DataFrame([[
metric_pred.mean_difference(),
classified_metric.equal_opportunity_difference(),
classified_metric.average_abs_odds_difference(),
metric_pred.disparate_impact(),
classified_metric.theil_index()
]],
columns=cols,
index=[attr])
fair_metrics = fair_metrics.append(row)
fair_metrics = fair_metrics.replace([-np.inf, np.inf], 2)
return fair_metrics
def get_dataset_metrics_list(binary_dataset_list):
#Set privileged and unprivileged groups
privileged_groups= [{'sex':1}]
unprivileged_groups= [{'sex': 0}]
mean_diff_list = []
disp_imp_list = []
for dataset in binary_dataset_list:
metrics = BinaryLabelDatasetMetric(dataset,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
mean_diff_list.append(metrics.mean_difference())
disp_imp_list.append(1 - metrics.disparate_impact())
return mean_diff_list, disp_imp_list
def nondebiased_classifier(train, test, privileged_groups,
unprivileged_groups):
sess = tf.Session()
NN_model = AdversarialDebiasing(privileged_groups,
unprivileged_groups,
scope_name='nondebiased_classifier',
debias=False,
sess=sess)
NN_model.fit(train)
# predict outcome using the test set
pred_NNmodel = NN_model.predict(test)
sess.close()
tf.reset_default_graph()
# calculate accuracy
accuracy = accuracy_score(y_true=test.labels, y_pred=pred_NNmodel.labels)
# calculate fairness metrics
metric_test = BinaryLabelDatasetMetric(
pred_NNmodel,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
acc_test = ClassificationMetric(test,
pred_NNmodel,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
equal_opportunity_difference = equal_opp_diff(test,
pred_NNmodel,
'sex',
privileged=1,
unprivileged=0,
favourable=1,
unfavourable=0)
average_odds_difference = avg_odds_diff(test,
pred_NNmodel,
'sex',
privileged=1,
unprivileged=0,
favourable=1,
unfavourable=0)
metrics = [
metric_test.mean_difference(),
acc_test.disparate_impact(), equal_opportunity_difference,
average_odds_difference,
acc_test.theil_index()
]
return pred_NNmodel, accuracy, metrics
def ensemble(test, pred_adversarial, pred_prejudice, pred_nondebiased,
unprivileged_groups, privileged_groups):
pred_labels = []
for i in range(0, len(test.features)):
arr = mode([
pred_adversarial.labels[i], pred_prejudice.labels[i],
pred_nondebiased.labels[i]
])
pred_labels.append(arr[0][0])
pred_ensemble = test.copy()
pred_ensemble.labels = np.array(pred_labels)
accuracy = accuracy_score(y_true=test.labels, y_pred=pred_ensemble.labels)
metric_test = BinaryLabelDatasetMetric(
pred_ensemble,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
acc_test = ClassificationMetric(test,
pred_ensemble,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
equal_opportunity_difference = equal_opp_diff(test,
pred_ensemble,
'sex',
privileged=1,
unprivileged=0,
favourable=1,
unfavourable=0)
average_odds_difference = avg_odds_diff(test,
pred_ensemble,
'sex',
privileged=1,
unprivileged=0,
favourable=1,
unfavourable=0)
metrics = [
metric_test.mean_difference(),
acc_test.disparate_impact(), equal_opportunity_difference,
average_odds_difference,
acc_test.theil_index()
]
return accuracy, metrics
def prejudice(train, test, unprivileged_groups, privileged_groups):
prejudice_model = PrejudiceRemover(eta=100, sensitive_attr='sex')
prejudice_model.fit(train)
# predict outcome using the test set
pred_prejudice = prejudice_model.predict(test)
# calculate accuracy
accuracy = accuracy_score(y_true=test.labels, y_pred=pred_prejudice.labels)
# calculate fairness metrics
metric_test = BinaryLabelDatasetMetric(
pred_prejudice,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
acc_test = ClassificationMetric(test,
pred_prejudice,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
equal_opportunity_difference = equal_opp_diff(test,
pred_prejudice,
'sex',
privileged=1,
unprivileged=0,
favourable=1,
unfavourable=0)
average_odds_difference = avg_odds_diff(test,
pred_prejudice,
'sex',
privileged=1,
unprivileged=0,
favourable=1,
unfavourable=0)
if acc_test.disparate_impact() == math.inf:
disparate_impact = 5.0
else:
disparate_impact = acc_test.disparate_impact()
metrics = [
metric_test.mean_difference(), disparate_impact,
equal_opportunity_difference, average_odds_difference,
acc_test.theil_index()
]
return pred_prejudice, accuracy, metrics
def plot_using_aif(df_predict,df_true):
predict_list, true_list = [], []
unpriv_label_list , priv_label_list = [], []
for (u,p) in zip(unpriv_list,priv_list):
cur_predict, cur_true = [], []
unpriv_label = '+'.join(['-'.join([prot_attr_dict[key][u_el[key]] for key in u_el]) for u_el in u])
priv_label = '+'.join(['-'.join([prot_attr_dict[key][p_el[key]] for key in p_el]) for p_el in p])
print('-------------------------------------------------------------------')
print('unpriv_label:-->',unpriv_label)
print('-------------------------------------------------------------------')
print('priv_label :-->',priv_label)
print('-------------------------------------------------------------------')
print('\n\n')
for i,label in enumerate(rating_names):
#print('Fairness Metric for the label------>',label.upper())
predict_dataset = StandardDataset(df=predict_df_list[i], label_name=label, favorable_classes=[1.0,1.0],
protected_attribute_names=protected_attribute_names, privileged_classes=privileged_classes)
true_dataset = StandardDataset(df=true_df_list[i], label_name=label, favorable_classes=[1.0,1.0],
protected_attribute_names=protected_attribute_names, privileged_classes=privileged_classes)
predict_dataset_metric = BinaryLabelDatasetMetric(predict_dataset, unprivileged_groups=u, privileged_groups=p)
true_dataset_metric = BinaryLabelDatasetMetric(true_dataset, unprivileged_groups=u, privileged_groups=p)
#classfication_metric = ClassificationMetric(true_dataset, predict_dataset, unprivileged_groups=u, privileged_groups=p)
#x=classfication_metric.generalized_entropy_index()
#print(label,': -->','predicted : -->',abs(predict_dataset_metric.disparate_impact()),'true : -->',abs(true_dataset_metric.disparate_impact()))
print(label,': -->','predicted : -->',abs(predict_dataset_metric.mean_difference()),'true : -->',abs(true_dataset_metric.mean_difference()))
def compute_metrics(model, X_test, y_test, X_train, y_train, dataset_test,
dataset_name, model_name, unprivileged_groups,
privileged_groups, position):
"""
Calculate and return: model accuracy and fairness metrics
Parameters
----------
model: scikit-learn classifier
X_test: numpy 2d array
y_test: numpy 1d array
X_train: numpy 2d array
y_train: numpy 1d array
dataset_test: aif360.datasets.BinaryLabelDataset
dataset_name: string
Dataset name used in the analysis
model_name: string
unprivileged_groups: list<dict>
Dictionary where the key is the name of the sensitive column in the
dataset, and the value is the value of the unprivileged group in the
dataset
privileged_groups: list<dict>
Dictionary where the key is the name of the sensitive column in the
dataset, and the value is the value of the privileged group in the
dataset
position: int
Column position of the sensitive group in the dataset
"""
y_pred_test = model.predict(X_test)
acc_test = accuracy_score(y_true=y_test, y_pred=y_pred_test)
print("Test accuracy: ", acc_test)
y_pred_train = model.predict(X_train)
acc_train = accuracy_score(y_true=y_train, y_pred=y_pred_train)
print("Train accuracy: ", acc_train)
dataset_pred = dataset_test.copy()
dataset_pred.labels = y_pred_test
bin_metric = BinaryLabelDatasetMetric(
dataset_pred,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
disparate_impact_bin = bin_metric.disparate_impact()
print('Disparate impact: ', disparate_impact_bin)
mean_difference = bin_metric.mean_difference()
print('Mean difference: ', mean_difference)
classif_metric = ClassificationMetric(
dataset_test,
dataset_pred,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
classif_disparete_impact = classif_metric.disparate_impact()
avg_odds = classif_metric.average_odds_difference()
print('Average odds difference:', avg_odds)
equal_opport = classif_metric.equal_opportunity_difference()
print('Equality of opportunity:', equal_opport)
false_discovery_rate = classif_metric.false_discovery_rate_difference()
print('False discovery rate difference:', false_discovery_rate)
entropy_index = classif_metric.generalized_entropy_index()
print('Generalized entropy index:', entropy_index)
cons_comp = consitency_mod(bin_metric, position, n_neighbors=5)
print('Consistency: ', cons_comp)
result = (dataset_name, model_name, acc_test, disparate_impact_bin,
mean_difference, classif_disparete_impact, avg_odds,
equal_opport, false_discovery_rate, entropy_index, cons_comp)
return result
def train():
privileged_groups = [{'race': 1}]
unprivileged_groups = [{'race': 0}]
dataset_orig = load_preproc_data_compas(['race'])
optim_options = {
"distortion_fun": get_distortion_compas,
"epsilon": 0.05,
"clist": [0.99, 1.99, 2.99],
"dlist": [.1, 0.05, 0]
}
dataset_orig_train, dataset_orig_vt = dataset_orig.split([0.7],
shuffle=True)
metric_transf_train = BinaryLabelDatasetMetric(
dataset_orig_train,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
OP = OptimPreproc(OptTools,
optim_options,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
OP = OP.fit(dataset_orig_train)
dataset_transf_cat_test = OP.transform(dataset_orig_vt, transform_Y=True)
dataset_transf_cat_test = dataset_orig_vt.align_datasets(
dataset_transf_cat_test)
dataset_transf_cat_train = OP.transform(dataset_orig_train,
transform_Y=True)
dataset_transf_cat_train = dataset_orig_train.align_datasets(
dataset_transf_cat_train)
scale_transf = StandardScaler()
X_train = scale_transf.fit_transform(dataset_transf_cat_train.features)
y_train = dataset_transf_cat_train.labels.ravel()
X_test = scale_transf.fit_transform(dataset_transf_cat_test.features)
lmod = LogisticRegression()
lmod.fit(X_train, y_train)
y_pred = lmod.predict(X_test)
print('Without reweight')
print('Accuracy')
print(accuracy_score(dataset_orig_vt.labels, y_pred))
dataset_orig_vt_copy1 = dataset_orig_vt.copy()
dataset_orig_vt_copy1.labels = y_pred
metric_transf_train1 = BinaryLabelDatasetMetric(
dataset_orig_vt_copy1,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
print('p-rule')
print(metric_transf_train1.disparate_impact())
print('CV')
print(metric_transf_train1.mean_difference())
print('FPR for unpriv')
orig_sens_att = dataset_orig_vt.protected_attributes.ravel()
print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[
orig_sens_att == 0], 0))
print("FNR for unpriv")
print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[
orig_sens_att == 0], 0))
print('FPR for priv')
orig_sens_att = dataset_orig_vt.protected_attributes.ravel()
print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[
orig_sens_att == 1], 0))
print("FNR for priv")
print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[
orig_sens_att == 1], 0))
df_weight = dataset_orig_train.convert_to_dataframe()[0]
df_weight['weight'] = 1
df_weight['is_missing'] = 0
df_weight['tmp'] = ''
tmp_result = []
for i, j in zip(df_weight['race'], df_weight['two_year_recid']):
tmp_result.append(str(i) + str(j))
df_weight['tmp'] = tmp_result
df_weight.loc[df_weight['priors_count=missing'] == 1, 'is_missing'] = 1
for i in df_weight['tmp'].unique():
df_weight.loc[
(df_weight['tmp'] == i) & (df_weight['is_missing'] == 0),
'weight'] = len(
df_weight.loc[(df_weight['tmp'] == i), :].index) / len(
df_weight.loc[(df_weight['tmp'] == i) &
(df_weight['is_missing'] == 0), :].index)
df_weight.loc[(df_weight['tmp'] == i) & (df_weight['is_missing'] == 1),
'weight'] = len(df_weight.loc[
(df_weight['tmp'] == i) &
(df_weight['is_missing'] == 0), :].index) / len(
df_weight.loc[(df_weight['tmp'] == i), :].index)
dataset_orig_train.instance_weights = np.array(df_weight['weight'])
scale_transf = StandardScaler()
#X_train = scale_transf.fit_transform(dataset_transf_cat_train.features[:,1:])
X_train = scale_transf.fit_transform(dataset_transf_cat_train.features)
y_train = dataset_transf_cat_train.labels.ravel()
#X_test = scale_transf.fit_transform(dataset_transf_cat_test.features[:,1:])
X_test = scale_transf.fit_transform(dataset_transf_cat_test.features)
lmod = LogisticRegression()
lmod.fit(X_train,
y_train,
sample_weight=dataset_orig_train.instance_weights)
y_pred = lmod.predict(X_test)
print('With reweight')
print('Accuracy')
print(accuracy_score(dataset_orig_vt.labels, y_pred))
dataset_orig_vt_copy1 = dataset_orig_vt.copy()
dataset_orig_vt_copy1.labels = y_pred
metric_transf_train1 = BinaryLabelDatasetMetric(
dataset_orig_vt_copy1,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
print('p-rule')
print(metric_transf_train1.disparate_impact())
print('CV')
print(metric_transf_train1.mean_difference())
print('FPR for unpriv')
orig_sens_att = dataset_orig_vt.protected_attributes.ravel()
print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[
orig_sens_att == 0], 0))
print("FNR for unpriv")
print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[
orig_sens_att == 0], 0))
print('FPR for priv')
orig_sens_att = dataset_orig_vt.protected_attributes.ravel()
print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[
orig_sens_att == 1], 0))
print("FNR for priv")
print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[
orig_sens_att == 1], 0))
def checkClassifierFairnessAndReweighData(frame,
dpoints,
mname,
x_columns,
verbose=True,
pre=True):
''' Measure fairness according to the metric using the value of A and the classification outcome.
Results get added to a dictionary used to pass them to a function to generate graphs of the results.
If we have not performed intervention, perform intervention and return post intervention data.'''
xay_columns = copy.deepcopy(x_columns)
xay_columns.extend(["A", "Y"])
ycols = copy.deepcopy(frame["Y"])
tempframe = copy.deepcopy(frame)
tempframe.drop(["Y"], axis=1, inplace=True)
aifdf = BinaryLabelDataset(favorable_label=1.0,
unfavorable_label=0.0,
df=tempframe,
label_names=['Ya'],
protected_attribute_names=['A'])
privileged_groups = [{'A': 1}]
unprivileged_groups = [{'A': 0}]
RW = Reweighing(unprivileged_groups=[{
'A': 0
}],
privileged_groups=[{
'A': 1
}])
metric_aifdf_train = BinaryLabelDatasetMetric(
aifdf,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
if pre:
if verbose:
print("\n\tINTERVENTION: {}\n".format(type(RW).__name__))
print("\t######### PRE {} ###########".format(type(RW).__name__))
print(
"\tDifference in mean outcomes between unprivileged and privileged groups = {}\n"
.format(metric_aifdf_train.mean_difference()))
dpoints[mname]['PRE'][type(
RW).__name__]['FAIR'] = metric_aifdf_train.mean_difference()
print("PRE CLASSIFICATION MATRIX")
print("----------------")
print(" |Y'=0 | Y'=1 |")
print("----------------")
print("A=0| {0} | {1} |".format(
metric_aifdf_train.num_negatives(False),
metric_aifdf_train.num_positives(False)))
print("A=1| {0} | {1} |".format(
metric_aifdf_train.num_negatives(True),
metric_aifdf_train.num_positives(True)))
print("----------------")
dataset_transf_train = RW.fit_transform(aifdf)
fairdf = dataset_transf_train.convert_to_dataframe()[0]
fairdf.drop(['Ya'], axis=1, inplace=True)
ycols.reset_index(drop=True, inplace=True)
fairdf.reset_index(drop=True, inplace=True)
fairdf.insert(0, "Y", ycols)
fairdf[xay_columns] = fairdf[xay_columns].astype(int)
fairdf.insert(loc=len(fairdf.columns),
column="weights",
value=dataset_transf_train.instance_weights)
return fairdf
else:
if verbose:
print(
"\tDifference in mean outcomes between unprivileged and privileged groups = {}\n"
.format(metric_aifdf_train.mean_difference()))
dpoints[mname]['POST'][type(
RW).__name__]['FAIR'] = metric_aifdf_train.mean_difference()
print("POST CLASSIFICATION MATRIX")
print("----------------")
print(" |Y'=0 | Y'=1 |")
print("----------------")
print("A=0| {0} | {1} |".format(
metric_aifdf_train.num_negatives(False),
metric_aifdf_train.num_positives(False)))
print("A=1| {0} | {1} |".format(
metric_aifdf_train.num_negatives(True),
metric_aifdf_train.num_positives(True)))
print("----------------")
return frame
def compute_metrics(model, X_test, y_test, X_train, y_train, dataset_test,
unprivileged_groups, privileged_groups, protect_attribute,
print_result):
"""
Calculate and return: model accuracy and fairness metrics
Parameters
----------
model: scikit-learn classifier
X_test: numpy 2d array
y_test: numpy 1d array
X_train: numpy 2d array
y_train: numpy 1d array
dataset_test: aif360.datasets.BinaryLabelDataset
unprivileged_groups: list<dict>
Dictionary where the key is the name of the sensitive column in the
dataset, and the value is the value of the unprivileged group in the
dataset
privileged_groups: list<dict>
Dictionary where the key is the name of the sensitive column in the
dataset, and the value is the value of the privileged group in the
dataset
protect_attribute
print_result
"""
result = {}
y_pred_test = model.predict(X_test)
result['acc_test'] = accuracy_score(y_true=y_test, y_pred=y_pred_test)
y_pred_train = model.predict(X_train)
result['acc_train'] = accuracy_score(y_true=y_train, y_pred=y_pred_train)
dataset_pred = dataset_test.copy()
dataset_pred.labels = y_pred_test
bin_metric = BinaryLabelDatasetMetric(dataset_pred,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
result['disp_impact'] = bin_metric.disparate_impact()
result['stat_parity'] = bin_metric.mean_difference()
classif_metric = ClassificationMetric(dataset_test, dataset_pred,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
result['avg_odds'] = classif_metric.average_odds_difference()
result['equal_opport'] = classif_metric.equal_opportunity_difference()
result['false_discovery_rate'] = classif_metric.false_discovery_rate_difference()
result['entropy_index'] = classif_metric.generalized_entropy_index()
result['acc_test_clf'] = classif_metric.accuracy(privileged=None)
result['acc_test_priv'] = classif_metric.accuracy(privileged=True)
result['acc_test_unpriv'] = classif_metric.accuracy(privileged=False)
result['consistency'] = consitency(X_test, y_pred_test, protect_attribute, n_neighbors=5)
result['counterfactual'] = counterfactual(X_test, model, protect_attribute)
if print_result:
print("Train accuracy: ", result['acc_train'])
print("Test accuracy: ", result['acc_test'])
print("Test accuracy clf: ", result['acc_test_clf'])
print("Test accuracy priv.: ", result['acc_test_priv'])
print("Test accuracy unpriv.: ", result['acc_test_unpriv'])
print('Disparate impact: ', result['disp_impact'])
print('Mean difference: ', result['stat_parity'])
print('Average odds difference:', result['avg_odds'])
print('Equality of opportunity:', result['equal_opport'])
print('False discovery rate difference:', result['false_discovery_rate'])
print('Generalized entropy index:', result['entropy_index'])
print('Consistency: ', result['consistency'])
print('Counterfactual fairness: ', result['counterfactual'])
return result
test_biased = BinaryLabelDataset(df=pd.concat((x_test, y_test),
axis=1),
label_names=['medium_to_high_risk'],
protected_attribute_names=['sex_Male'],
favorable_label=1,
unfavorable_label=0)
# Create the metric object for training set
metric_train_biased = BinaryLabelDatasetMetric(train_biased,
unprivileged_groups=sex_unprivileged_groups,
privileged_groups=sex_privileged_groups)
display(Markdown("#### Original training dataset"))
print("Sex biased - Difference in mean outcomes between unprivileged and privileged sex groups = %f" % metric_train_biased.mean_difference())
# Create the metric object for testing set
metric_test_biased = BinaryLabelDatasetMetric(test_biased,
unprivileged_groups=sex_unprivileged_groups,
privileged_groups=sex_privileged_groups)
display(Markdown("#### Original training dataset"))
print("Sex biased - Difference in mean outcomes between unprivileged and privileged sex groups = %f" % metric_test_biased.mean_difference())
#debias with the reweighing method
RW = Reweighing(unprivileged_groups=sex_unprivileged_groups,
privileged_groups=sex_privileged_groups)
RW.fit(train_biased)
dataset_transf_train_f = RW.fit_transform(train_biased)
def train(request):
df = pd.read_csv('./training/resume_data_5000.csv')
df = df.drop(df.columns[0], axis=1)
dataset_orig = StandardDataset(df,
label_name='Accepted',
favorable_classes=[1],
protected_attribute_names=['Gender'],
privileged_classes=[[1]],
categorical_features=['School'],
features_to_drop=['Name'])
dataset_orig_train, dataset_orig_vt = dataset_orig.split([0.7],
shuffle=True)
dataset_orig_valid, dataset_orig_test = dataset_orig_vt.split([0.5],
shuffle=True)
privileged_groups = [{'Gender': 1}]
unprivileged_groups = [{'Gender': 0}]
metric_orig_train = BinaryLabelDatasetMetric(
dataset_orig_train,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
orig_mean_difference = metric_orig_train.mean_difference()
with open('./training/orig_mean_difference.pkl', 'wb') as f:
pickle.dump(orig_mean_difference, f)
RW = Reweighing(unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
dataset_transf_train = RW.fit_transform(dataset_orig_train)
metric_transf_train = BinaryLabelDatasetMetric(
dataset_transf_train,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
transf_mean_difference = metric_transf_train.mean_difference()
with open('./training/transf_mean_difference.pkl', 'wb') as f:
pickle.dump(transf_mean_difference, f)
# Logistic regression classifier and predictions
scale_orig = StandardScaler()
X_train = scale_orig.fit_transform(dataset_orig_train.features)
y_train = dataset_orig_train.labels.ravel()
w_train = dataset_orig_train.instance_weights.ravel()
with open('./training/scaler.pkl', 'wb') as f:
pickle.dump(scale_orig, f)
lmod_orig = LogisticRegression(solver='lbfgs')
lmod_orig.fit(X_train,
y_train,
sample_weight=dataset_orig_train.instance_weights)
y_train_pred = lmod_orig.predict(X_train)
pos_ind = np.where(
lmod_orig.classes_ == dataset_orig_train.favorable_label)[0][0]
dataset_orig_train_pred = dataset_orig_train.copy()
dataset_orig_train_pred.labels = y_train_pred
dataset_orig_valid_pred = dataset_orig_valid.copy(deepcopy=True)
X_valid = scale_orig.transform(dataset_orig_valid_pred.features)
y_valid = dataset_orig_valid_pred.labels
dataset_orig_valid_pred.scores = lmod_orig.predict_proba(
X_valid)[:, pos_ind].reshape(-1, 1)
dataset_orig_test_pred = dataset_orig_test.copy(deepcopy=True)
X_test = scale_orig.transform(dataset_orig_test_pred.features)
y_test = dataset_orig_test_pred.labels
dataset_orig_test_pred.scores = lmod_orig.predict_proba(
X_test)[:, pos_ind].reshape(-1, 1)
num_thresh = 100
ba_arr = np.zeros(num_thresh)
class_thresh_arr = np.linspace(0.01, 0.99, num_thresh)
for idx, class_thresh in enumerate(class_thresh_arr):
fav_inds = dataset_orig_valid_pred.scores > class_thresh
dataset_orig_valid_pred.labels[
fav_inds] = dataset_orig_valid_pred.favorable_label
dataset_orig_valid_pred.labels[
~fav_inds] = dataset_orig_valid_pred.unfavorable_label
classified_metric_orig_valid = ClassificationMetric(
dataset_orig_valid,
dataset_orig_valid_pred,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
ba_arr[idx] = 0.5*(classified_metric_orig_valid.true_positive_rate()\
+classified_metric_orig_valid.true_negative_rate())
best_ind = np.where(ba_arr == np.max(ba_arr))[0][0]
best_class_thresh = class_thresh_arr[best_ind]
bal_acc_arr_orig = []
disp_imp_arr_orig = []
avg_odds_diff_arr_orig = []
for thresh in tqdm(class_thresh_arr):
fav_inds = dataset_orig_test_pred.scores > thresh
dataset_orig_test_pred.labels[
fav_inds] = dataset_orig_test_pred.favorable_label
dataset_orig_test_pred.labels[
~fav_inds] = dataset_orig_test_pred.unfavorable_label
metric_test_bef = compute_metrics(dataset_orig_test,
dataset_orig_test_pred,
unprivileged_groups,
privileged_groups,
disp=False)
if thresh == best_class_thresh:
with open('./training/metrics_orig.pkl', 'wb') as f:
pickle.dump(metric_test_bef,
f,
protocol=pickle.HIGHEST_PROTOCOL)
bal_acc_arr_orig.append(metric_test_bef["Balanced accuracy"])
avg_odds_diff_arr_orig.append(
metric_test_bef["Average odds difference"])
disp_imp_arr_orig.append(metric_test_bef["Disparate impact"])
scale_transf = StandardScaler()
X_train = scale_transf.fit_transform(dataset_transf_train.features)
y_train = dataset_transf_train.labels.ravel()
lmod_transf = LogisticRegression(solver='lbfgs')
lmod_transf.fit(X_train,
y_train,
sample_weight=dataset_transf_train.instance_weights)
y_train_pred = lmod_transf.predict(X_train)
dataset_transf_test_pred = dataset_orig_test.copy(deepcopy=True)
X_test = scale_transf.fit_transform(dataset_transf_test_pred.features)
y_test = dataset_transf_test_pred.labels
dataset_transf_test_pred.scores = lmod_transf.predict_proba(
X_test)[:, pos_ind].reshape(-1, 1)
bal_acc_arr_transf = []
disp_imp_arr_transf = []
avg_odds_diff_arr_transf = []
for thresh in tqdm(class_thresh_arr):
fav_inds = dataset_transf_test_pred.scores > thresh
dataset_transf_test_pred.labels[
fav_inds] = dataset_transf_test_pred.favorable_label
dataset_transf_test_pred.labels[
~fav_inds] = dataset_transf_test_pred.unfavorable_label
metric_test_aft = compute_metrics(dataset_orig_test,
dataset_transf_test_pred,
unprivileged_groups,
privileged_groups,
disp=False)
if thresh == best_class_thresh:
with open('./training/metrics_transf.pkl', 'wb') as f:
pickle.dump(metric_test_aft,
f,
protocol=pickle.HIGHEST_PROTOCOL)
bal_acc_arr_transf.append(metric_test_aft["Balanced accuracy"])
avg_odds_diff_arr_transf.append(
metric_test_aft["Average odds difference"])
disp_imp_arr_transf.append(metric_test_aft["Disparate impact"])
with open('./training/model_orig.pkl', 'wb') as f:
pickle.dump(lmod_orig, f)
with open('./training/model_transf.pkl', 'wb') as f:
pickle.dump(lmod_transf, f)
return HttpResponse('Model trained')
new_dataset = BinaryLabelDataset(favorable_label=favorable_label,
unfavorable_label=unfavorable_label,
df=new_b_train,
label_names=['Attrition'],
protected_attribute_names=['Gender'],
unprivileged_protected_attributes=unprivileged_groups)
# In[ ]:
metric_orig_train = BinaryLabelDatasetMetric(new_dataset,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
display(Markdown("#### Original training dataset"))
print("Difference in mean outcomes between unprivileged and privileged groups = %f" % metric_orig_train.mean_difference())
# In[ ]:
new_dataset.protected_attribute_names
# In[ ]:
RW = Reweighing(unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
RW.fit(new_dataset)
def train():
privileged_groups = [{'race': 1}]
unprivileged_groups = [{'race': 0}]
dataset_orig_train = load_preproc_data_compas(['race'])
optim_options = {
"distortion_fun": get_distortion_compas,
"epsilon": 0.1,
"clist": [0.99, 1.99, 2.99],
"dlist": [.1, 0.05, 0]
}
# dataset_orig_train, dataset_orig_vt = dataset_orig.split([0.7], shuffle=True)
metric_transf_train = BinaryLabelDatasetMetric(dataset_orig_train,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
OP = OptimPreproc(OptTools, optim_options,
unprivileged_groups = unprivileged_groups,
privileged_groups = privileged_groups)
OP = OP.fit(dataset_orig_train)
dataset_transf_cat_test = OP.transform(dataset_orig_vt, transform_Y = True)
dataset_transf_cat_test = dataset_orig_vt.align_datasets(dataset_transf_cat_test)
dataset_transf_cat_train = OP.transform(dataset_orig_train, transform_Y = True)
dataset_transf_cat_train = dataset_orig_train.align_datasets(dataset_transf_cat_train)
scale_transf = StandardScaler()
X_train = scale_transf.fit_transform(dataset_transf_cat_train.features)
y_train = dataset_transf_cat_train.labels.ravel()
X_test = scale_transf.fit_transform(dataset_transf_cat_test.features)
lmod = LogisticRegression()
lmod.fit(X_train, y_train)
y_pred = lmod.predict(X_test)
print('Without applying fixing algorithms')
print('Accuracy')
print(accuracy_score(dataset_orig_vt.labels, y_pred))
dataset_orig_vt_copy1 = dataset_orig_vt.copy()
dataset_orig_vt_copy1.labels = y_pred
metric_transf_train1 = BinaryLabelDatasetMetric(dataset_orig_vt_copy1,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
print('p-rule')
print(metric_transf_train1.disparate_impact())
print('CV')
print(metric_transf_train1.mean_difference())
print('FPR for unpriv')
orig_sens_att = dataset_orig_vt.protected_attributes.ravel()
print(1-TNR(dataset_orig_vt.labels.ravel()[orig_sens_att==0],y_pred[orig_sens_att==0],0))
print("FNR for unpriv")
print(1-TPR(dataset_orig_vt.labels.ravel()[orig_sens_att==0],y_pred[orig_sens_att==0],0))
print('FPR for priv')
orig_sens_att = dataset_orig_vt.protected_attributes.ravel()
print(1-TNR(dataset_orig_vt.labels.ravel()[orig_sens_att==1],y_pred[orig_sens_att==1],0))
print("FNR for priv")
print(1-TPR(dataset_orig_vt.labels.ravel()[orig_sens_att==1],y_pred[orig_sens_att==1],0))
def main() -> None:
# read from inventor
filepath = ait_input.get_inventory_path('Data')
# prepare column names as given by german.doc
column_names = [
'status', 'month', 'credit_history', 'purpose', 'credit_amount',
'savings', 'employment', 'investment_as_income_percentage',
'personal_status', 'other_debtors', 'residence_since', 'property',
'age', 'installment_plans', 'housing', 'number_of_credits',
'skill_level', 'people_liable_for', 'telephone', 'foreign_worker',
'credit'
]
# load into a dataframe
df = data_loading(filepath=filepath,
column_names=column_names,
na_values=None)
# prepare for mappings
mappings = {
'label_maps': [{
1.0: 'Good Credit',
2.0: 'Bad Credit'
}],
'protected_attribute_maps': [{
1.0: 'Male',
0.0: 'Female'
}, {
1.0: 'Old',
0.0: 'Young'
}]
}
# prepare for categorical features
categorical_features = [
'status', 'credit_history', 'purpose', 'savings', 'employment',
'other_debtors', 'property', 'installment_plans', 'housing',
'skill_level', 'telephone', 'foreign_worker'
]
# load param
protected_attribute = ait_input.get_method_param_value(
'protected_attribute')
privileged_classes = ait_input.get_method_param_value('privileged_classes')
# input check
ait_input_check(protected_attribute, privileged_classes)
# prepare for structure from dataframe and edit data features setting
dataset = StandardDataset(
df=df,
label_name='credit',
favorable_classes=[1],
protected_attribute_names=[protected_attribute],
privileged_classes=[lambda x: x >= privileged_classes],
instance_weights_name=None,
categorical_features=categorical_features,
features_to_keep=None,
features_to_drop=['personal_status', 'sex'],
na_values=None,
custom_preprocessing=preprocessing,
metadata=mappings)
# set two variables for the privileged (1) and unprivileged (0) values for the age attribute.
privileged_groups = [{protected_attribute: 1}]
unprivileged_groups = [{protected_attribute: 0}]
# compute fairness metric on original training dataset
metric_fairness = BinaryLabelDatasetMetric(
dataset,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
print("Original training dataset: German Credit Data")
print(
"Difference in mean outcomes between unprivileged and privileged groups = %f"
% metric_fairness.mean_difference())
print("unprivileged groups = %f" %
metric_fairness.base_rate(privileged=False))
print("privileged groups = %f" %
metric_fairness.base_rate(privileged=True))
# resource observed_predicted_plot
save_metric_fairness_plot(metric_fairness, protected_attribute)
# measures
measure_mean_difference(metric_fairness.mean_difference())
# ait.log
move_log()
def run_trial():
# stores each run (4 algos without reweighing, 4 algos with reweighing) as a sublist
# number of sublists = number of runs
# each sublist has four elements
# we ONLY predict on the testing data
########## WITHOUT REWEIGHING #############
stat_par = []
disp_imp = []
eq_opp_diff = []
avg_odds_diff = []
theil = []
acc = []
########## WITH REWEIGHING #############
stat_par_reweigh = []
disp_imp_reweigh = []
eq_opp_diff_reweigh = []
avg_odds_diff_reweigh = []
theil_reweigh = []
acc_reweigh = []
###########################################
for i in range(10):
###########################################
privileged_groups = [{'sex': 1}]
unprivileged_groups = [{'sex': 0}]
dataset_orig = load_preproc_data_adult()
# train1, test1 are the original dataset
train1, test1 = dataset_orig.split([0.7], shuffle=True)
RW = Reweighing(unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
RW.fit(train1)
# dataset_transf_train, test1 are for the reweighed dataset
dataset_transf_train = RW.transform(train1)
###########################################
# change weights to whole numbers
for i in range(dataset_transf_train.instance_weights.size):
dataset_transf_train.instance_weights[i] = (round(
dataset_transf_train.instance_weights[i] / 0.1) * 0.1) * 10
weights = copy.deepcopy(dataset_transf_train.instance_weights)
# change dataset_transf_train.features and dataset_transf_train.labels and dataset_transf_train.protected_attributes according to the weights of each instance
sum_weights = 0
for i in range(dataset_transf_train.features.shape[0]):
row = copy.deepcopy(dataset_transf_train.features[i])
row_label = copy.deepcopy(dataset_transf_train.labels[i])
row_protected_attributes = copy.deepcopy(
dataset_transf_train.protected_attributes[i])
row_protected_attributes.resize(1, 2)
row.resize(1, 18)
row_label.resize(1, 1)
weight = int(weights[i])
for j in range(weight - 1):
dataset_transf_train.features = np.concatenate(
(dataset_transf_train.features, row))
dataset_transf_train.labels = np.concatenate(
(dataset_transf_train.labels, row_label))
dataset_transf_train.protected_attributes = np.concatenate(
(dataset_transf_train.protected_attributes,
row_protected_attributes))
# change the dataset_transf_train to a numpy array of ones to match number of rows in features
dataset_transf_train.instance_weights = np.ones(
dataset_transf_train.features.shape[0])
################## without reweighing ##########################
temp_stat_par = []
temp_disp_imp = []
temp_eq_opp_diff = []
temp_avg_odds_diff = []
temp_theil = []
temp_acc = []
##################### adversarial debiasing #####################
sess = tf.Session()
debiased_model = AdversarialDebiasing(
privileged_groups=privileged_groups,
unprivileged_groups=unprivileged_groups,
scope_name='debiased_classifier',
debias=True,
sess=sess)
debiased_model.fit(train1)
dataset_debiasing_test = debiased_model.predict(test1)
sess.close()
tf.reset_default_graph()
##################### metrics #####################
metric_test = BinaryLabelDatasetMetric(
dataset_debiasing_test,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
acc_test = ClassificationMetric(
test1,
dataset_debiasing_test,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
temp_stat_par.append(metric_test.mean_difference())
temp_disp_imp.append(metric_test.disparate_impact())
temp_eq_opp_diff.append(metric_test.equal_opportunity_difference())
temp_avg_odds_diff.append(metric_test.average_odds_difference())
temp_theil.append(metric_test.theil_index())
temp_acc.append(acc_test.accuracy())
##################### prejudice remover #####################
prejudice_model = PrejudiceRemover(eta=100, sensitive_attr='sex')
prejudice_model.fit(train1)
dataset_prejudice_test = prejudice_model.predict(test1)
##################### metrics #####################
metric_test = BinaryLabelDatasetMetric(
dataset_prejudice_test,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
acc_test = ClassificationMetric(
test1,
dataset_prejudice_test,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
temp_stat_par.append(metric_test.mean_difference())
temp_disp_imp.append(metric_test.disparate_impact())
temp_eq_opp_diff.append(metric_test.equal_opportunity_difference())
temp_avg_odds_diff.append(metric_test.average_odds_difference())
temp_theil.append(metric_test.theil_index())
temp_acc.append(acc_test.accuracy())
##################### normal neural net #####################
sess = tf.Session()
neural_model = AdversarialDebiasing(
privileged_groups=privileged_groups,
unprivileged_groups=unprivileged_groups,
scope_name='debiased_classifier',
debias=False,
sess=sess)
neural_model.fit(train1)
dataset_neural_test = neural_model.predict(test1)
sess.close()
tf.reset_default_graph()
##################### metrics #####################
metric_test = BinaryLabelDatasetMetric(
dataset_neural_test,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
acc_test = ClassificationMetric(
test1,
dataset_neural_test,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
temp_stat_par.append(metric_test.mean_difference())
temp_disp_imp.append(metric_test.disparate_impact())
temp_eq_opp_diff.append(metric_test.equal_opportunity_difference())
temp_avg_odds_diff.append(metric_test.average_odds_difference())
temp_theil.append(metric_test.theil_index())
temp_acc.append(acc_test.accuracy())
##################### ensemble #####################
pred_labels_test = []
for i in range(0, len(test1.features)):
arr_test = mode([
dataset_debiasing_test[i], dataset_prejudice_test[i],
dataset_neural_test[i]
])
pred_labels_test.append(arr_test[0][0])
dataset_ensemble_test = test1.copy()
dataset_ensemble_test.labels = np.array(pred_labels_test)
##################### metrics #####################
metric_test = BinaryLabelDatasetMetric(
dataset_ensemble_train,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
acc_test = ClassificationMetric(
test1,
dataset_ensemble_train,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
temp_stat_par.append(metric_test.mean_difference())
temp_disp_imp.append(metric_test.disparate_impact())
temp_eq_opp_diff.append(metric_test.equal_opportunity_difference())
temp_avg_odds_diff.append(metric_test.average_odds_difference())
temp_theil.append(metric_test.theil_index())
temp_acc.append(acc_test.accuracy())
######### DUMP SHIT ###########
stat_par.append(temp_stat_par)
disp_imp.append(temp_disp_imp)
eq_opp_diff.append(temp_eq_opp_diff)
avg_odds_diff.append(temp_avg_odds_diff)
theil.append(temp_theil)
acc.append(temp_acc)
################## with reweighing ##########################
temp_stat_par = []
temp_disp_imp = []
temp_eq_opp_diff = []
temp_avg_odds_diff = []
temp_theil = []
temp_acc = []
################## adversarial debiasing ##################
sess = tf.Session()
debiased_model_reweighing = AdversarialDebiasing(
privileged_groups=privileged_groups,
unprivileged_groups=unprivileged_groups,
scope_name='debiased_classifier',
debias=True,
sess=sess)
debiased_model_reweighing.fit(dataset_transf_train)
dataset_debiasing_test_reweighing = debiased_model_reweighing.predict(
test1)
sess.close()
tf.reset_default_graph()
##################### metrics #####################
metric_test = BinaryLabelDatasetMetric(
dataset_debiasing_test_reweighing,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
acc_test = ClassificationMetric(
test1,
dataset_debiasing_test_reweighing,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
temp_stat_par.append(metric_test.mean_difference())
temp_disp_imp.append(metric_test.disparate_impact())
temp_eq_opp_diff.append(metric_test.equal_opportunity_difference())
temp_avg_odds_diff.append(metric_test.average_odds_difference())
temp_theil.append(metric_test.theil_index())
temp_acc.append(acc_test.accuracy())
##################### prejudice remover #####################
prejudice_model_reweighing = PrejudiceRemover(eta=100,
sensitive_attr='sex')
prejudice_model_reweighing.fit(dataset_transf_train)
dataset_prejudice_test_reweighing = prejudice_model_reweighing.predict(
test1)
##################### metrics #####################
metric_test = BinaryLabelDatasetMetric(
dataset_prejudice_test_reweighing,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
acc_test = ClassificationMetric(
test1,
dataset_prejudice_test_reweighing,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
temp_stat_par.append(metric_test.mean_difference())
temp_disp_imp.append(metric_test.disparate_impact())
temp_eq_opp_diff.append(metric_test.equal_opportunity_difference())
temp_avg_odds_diff.append(metric_test.average_odds_difference())
temp_theil.append(metric_test.theil_index())
temp_acc.append(acc_test.accuracy())
##################### normal neural net #####################
sess = tf.Session()
neural_model = AdversarialDebiasing(
privileged_groups=privileged_groups,
unprivileged_groups=unprivileged_groups,
scope_name='debiased_classifier',
debias=False,
sess=sess)
neural_model.fit(dataset_transf_train)
dataset_neural_test = neural_model.predict(test1)
sess.close()
tf.reset_default_graph()
##################### metrics #####################
metric_test = BinaryLabelDatasetMetric(
dataset_neural_test,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
acc_test = ClassificationMetric(
test1,
dataset_neural_test,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
temp_stat_par.append(metric_test.mean_difference())
temp_disp_imp.append(metric_test.disparate_impact())
temp_eq_opp_diff.append(metric_test.equal_opportunity_difference())
temp_avg_odds_diff.append(metric_test.average_odds_difference())
temp_theil.append(metric_test.theil_index())
temp_acc.append(acc_test.accuracy())
##################### ensemble #####################
pred_labels_test = []
for i in range(0, len(test1.features)):
arr_test = mode([
dataset_debiasing_test[i], dataset_prejudice_test[i],
dataset_neural_test[i]
])
pred_labels_test.append(arr_test[0][0])
dataset_ensemble_test = test1.copy()
dataset_ensemble_test.labels = np.array(pred_labels_test)
##################### metrics #####################
metric_test = BinaryLabelDatasetMetric(
dataset_ensemble_test,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
acc_test = ClassificationMetric(
test1,
dataset_ensemble_test,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
temp_stat_par.append(metric_test.mean_difference())
temp_disp_imp.append(metric_test.disparate_impact())
temp_eq_opp_diff.append(metric_test.equal_opportunity_difference())
temp_avg_odds_diff.append(metric_test.average_odds_difference())
temp_theil.append(metric_test.theil_index())
temp_acc.append(acc_test.accuracy())
######### DUMP SHIT ###########
stat_par_reweigh.append(temp_stat_par)
disp_imp_reweigh.append(temp_disp_imp)
eq_opp_diff_reweigh.append(temp_eq_opp_diff)
avg_odds_diff_reweigh.append(temp_avg_odds_diff)
theil_reweigh.append(temp_theil)
acc_reweigh.append(temp_acc)
without_reweighing = [
stat_par, disp_imp, eq_opp_diff, avg_odds_diff, theil, acc
]
with_reweighing = [
stat_par_reweigh, disp_imp_reweigh, eq_opp_diff_reweigh,
avg_odds_diff_reweigh, theil_reweigh, acc_reweigh
]
for metric in range(len(without_reweighing)):
name = "metric" + str(metric)
sublist = without_reweighing[metric]
with open(name, "wb") as csv_file:
writer = csv.writer(csv_file)
writer.writerows(sublist)
for metric in range(len(with_reweighing)):
name = "metric" + str(metric) + "reweigh"
sublist = with_reweighing[metric]
with open(name, "wb") as csv_file:
writer = csv.writer(csv_file)
writer.writerows(sublist)
def main():
import sys
sys.path.insert(1, "../")
import numpy as np
np.random.seed(0)
#pip install numba==0.43.0
#pip install --ignore-installed llvmlite==0.32.1
from aif360.datasets import GermanDataset
from aif360.metrics import BinaryLabelDatasetMetric, ClassificationMetric as CM
from aif360.algorithms.preprocessing import Reweighing
from IPython.display import Markdown, display
from sklearn.ensemble import RandomForestClassifier as RF
from sklearn.datasets import make_classification as mc
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score
#Step 2 Load dataset, specifying protected attribute, and split dataset into train and test
dataset_orig = GermanDataset(
protected_attribute_names=[
'age'
], # this dataset also contains protected attribute for "sex"
# which we do not consider in this evaluation
privileged_classes=[lambda x: x >= 25
], # age >=25 is considered privileged
features_to_drop=['personal_status',
'sex'] # ignore sex-related attributes
)
dataset_orig_train, dataset_orig_test = dataset_orig.split([0.7],
shuffle=True)
dataset_orig_test_pred = dataset_orig_test.copy(deepcopy=True)
privileged_groups = [{'age': 1}]
unprivileged_groups = [{'age': 0}]
#Step 3 Compute fairness metric on original training dataset
metric_orig_train = BinaryLabelDatasetMetric(
dataset_orig_train, #mean difference
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
display(Markdown("#### Original training dataset"))
print(
"Difference in mean outcomes between unprivileged and privileged groups = %f. AKA the privileged group is getting .17 more positive outcomes in the training dataset."
% metric_orig_train.mean_difference()) #
print()
#metrics
clf = RF()
clf.fit(dataset_orig_train.features, dataset_orig_train.labels)
predictions = clf.predict(dataset_orig_test.features)
proba_predictions = clf.predict_proba(dataset_orig_test.features)
dataset_orig_test_pred.scores = proba_predictions[:, 0].reshape(-1, 1)
dataset_orig_test_pred.labels = predictions.reshape(-1, 1)
cm_pred_valid = CM(dataset_orig_test,
dataset_orig_test_pred,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
cm = ["precision", "recall", "accuracy"]
metrics = {}
for c in cm:
metric = eval("cm_pred_valid." + c + "()")
metrics[c] = metric
metrics["recall"], metrics["accuracy"], metrics["precision"]
print("AIF360 metrics")
for key in ["recall", "accuracy", "precision"]:
print("{} score is: {}".format(key, metrics[key]))
#Step 4 Mitigate bias by transforming the original dataset
RW = Reweighing(
unprivileged_groups=
unprivileged_groups, #pre-processing mitigation algorithm
privileged_groups=privileged_groups)
dataset_transf_train = RW.fit_transform(dataset_orig_train)
#Step 5 Compute fairness metric on transformed dataset
metric_transf_train = BinaryLabelDatasetMetric(
dataset_transf_train,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
display(Markdown("#### Transformed training dataset"))
print(
"Difference in mean outcomes between unprivileged and privileged groups = %f"
% metric_transf_train.mean_difference()) #
#metrics
#split
dataset_transf_train, dataset_transf_test = dataset_transf_train.split(
[0.7], shuffle=True)
dataset_transf_test_pred = dataset_transf_test.copy(deepcopy=True)
clf = RF()
clf.fit(dataset_transf_train.features, dataset_transf_train.labels)
predictions = clf.predict(dataset_transf_test.features)
proba_predictions = clf.predict_proba(dataset_transf_test.features)
dataset_transf_test_pred.scores = proba_predictions[:, 0].reshape(-1, 1)
dataset_transf_test_pred.labels = predictions.reshape(-1, 1)
cm_pred_valid = CM(dataset_transf_test,
dataset_transf_test_pred,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
cm = ["precision", "recall", "accuracy"]
metrics = {}
for c in cm:
metric = eval("cm_pred_valid." + c + "()")
metrics[c] = metric
metrics["recall"], metrics["accuracy"], metrics["precision"]
print("AIF360 metrics")
for key in ["recall", "accuracy", "precision"]:
print("{} score is: {}".format(key, metrics[key]))
de_dummy_code=True, sep='=', set_category=True)[0]
# valid classification
dataset_transf_valid = di.fit_transform(dataset_orig_valid)
out_valid = dataset_transf_valid.convert_to_dataframe(
de_dummy_code=True, sep='=', set_category=True)[0]
# test classification
dataset_transf_test = di.fit_transform(dataset_orig_test)
out_test = dataset_transf_test.convert_to_dataframe(
de_dummy_code=True, sep='=', set_category=True)[0]
metric_transf_train = BinaryLabelDatasetMetric(
dataset_transf_train,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
print(
m +
" achieved a statistical parity difference between unprivileged and privileged groups = %f"
% metric_transf_train.mean_difference())
out_train.to_csv(output_path + 'taiwan_pre_train_' + m + '.csv',
index=None,
header=True)
out_valid.to_csv(output_path + 'taiwan_pre_valid_' + m + '.csv',
index=None,
header=True)
out_test.to_csv(output_path + 'taiwan_pre_test_' + m + '.csv',
index=None,
header=True)
sess = tf.Session()
debiased_model = AdversarialDebiasing(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups,
scope_name='debiased_classifier',
debias=True,
sess=sess)
debiased_model.fit(train1)
dataset_debiasing_train = debiased_model.predict(train1)
dataset_debiasing_test = debiased_model.predict(test1)
metric_debiasing_train = BinaryLabelDatasetMetric(dataset_debiasing_train,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
metric_debiasing_test = BinaryLabelDatasetMetric(dataset_debiasing_test,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
metric_dataset_debiasing_train.append(metric_debiasing_train.mean_difference())
metric_dataset_debiasing_test.append(metric_debiasing_test.mean_difference())
sess.close()
tf.reset_default_graph()
sess = tf.Session()
debiased_model = AdversarialDebiasing(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups,
scope_name='debiased_classifier',
debias=True,
sess=sess)
debiased_model.fit(dataset_transf_train)
dataset_reweigh_train = debiased_model.predict(dataset_transf_train)
dataset_reweigh_test = debiased_model.predict(test1)
def train():
df = pd.read_csv('syn_train.csv')
df_test = pd.read_csv('syn_test.csv')
df_neg = df.loc[df['two_year_recid'] == 1, :]
df_neg_priv = df_neg.loc[(df_neg['two_year_recid'] == 1) &
(df_neg['race'] == 1), :]
df_neg_unpriv = df_neg.loc[(df_neg['two_year_recid'] == 1) &
(df_neg['race'] == 0), :]
df_neg = df.loc[df['two_year_recid'] == 1, :]
df_neg_priv = df_neg.loc[(df_neg['two_year_recid'] == 1) &
(df_neg['race'] == 1), :]
df_neg_unpriv = df_neg.loc[(df_neg['two_year_recid'] == 1) &
(df_neg['race'] == 0), :]
_, df_neg_priv_test = train_test_split(df_neg_priv,
test_size=1200,
random_state=1)
_, df_neg_unpriv_test = train_test_split(df_neg_unpriv,
test_size=2800,
random_state=1)
df_neg_test = df_neg_priv_test.append(df_neg_unpriv_test)
print('negative outcome, unpriv before resampling')
print(len(df_neg_unpriv_test.index))
print('negative outcome, priv before resampling')
print(len(df_neg_priv_test.index))
df_pos = df.loc[df['two_year_recid'] == 0, :]
df_pos_priv = df_pos.loc[(df_pos['two_year_recid'] == 0) &
(df_pos['race'] == 1), :]
df_pos_unpriv = df_pos.loc[(df_pos['two_year_recid'] == 0) &
(df_pos['race'] == 0), :]
_, df_pos_priv_test = train_test_split(df_pos_priv,
test_size=2000,
random_state=1)
_, df_pos_unpriv_test = train_test_split(df_pos_unpriv,
test_size=2000,
random_state=1)
df_pos_test = df_pos_priv_test.append(df_pos_unpriv_test)
df = df_pos_test.append(df_neg_test)
print('positive outcome, unpriv before resampling')
print(len(df_pos_unpriv_test.index))
print('positive outcome, priv before resampling')
print(len(df_pos_priv_test.index))
df = df_pos_test.append(df_neg_test)
N = 8000
df_result = pd.DataFrame()
# unif sampling
for i in df['two_year_recid'].unique():
for j in df['race'].unique():
orig_df = df.loc[(df['two_year_recid'] == i) &
(df['race'] == j), :]
real_count = len(orig_df.index)
exp_count = int(
(len(df.loc[(df['two_year_recid'] == i), :].index) /
len(df.index)) *
(len(df.loc[(df['race'] == j), :].index) / len(df.index)) * N)
if real_count >= exp_count:
_, df_toapp = train_test_split(orig_df,
test_size=exp_count,
random_state=10)
else:
df_toapp = resample(orig_df,
replace=True,
n_samples=exp_count - real_count,
random_state=10)
df_toapp = df_toapp.append(orig_df)
if len(df_result.index) == 0:
df_result = df_toapp.copy()
else:
df_result = df_result.append(df_toapp)
df = df_result
df_train = df
privileged_groups = [{'race': 1}]
unprivileged_groups = [{'race': 0}]
all_protected_attribute_maps = {"race": {0.0: 0, 1.0: 1}}
D_features = ['race']
dataset_orig_train = CustomDataset(
label_name='two_year_recid',
favorable_classes=[0],
protected_attribute_names=['race'],
privileged_classes=[[1]],
categorical_features=[
'priors_count', 'c_charge_degree', 'age_cat', 'score_text'
],
features_to_keep=[
'priors_count', 'c_charge_degree', 'race', 'age_cat', 'score_text'
],
df=df_train,
metadata={
'label_maps': [{
1: 'Did recid.',
0: 'No recid.'
}],
'protected_attribute_maps':
[all_protected_attribute_maps[x] for x in D_features]
})
dataset_orig_vt = CustomDataset(
label_name='two_year_recid',
favorable_classes=[0],
protected_attribute_names=['race'],
privileged_classes=[[1]],
categorical_features=[
'priors_count', 'c_charge_degree', 'age_cat', 'score_text'
],
features_to_keep=[
'priors_count', 'c_charge_degree', 'race', 'age_cat', 'score_text'
],
df=df_test,
metadata={
'label_maps': [{
1: 'Did recid.',
0: 'No recid.'
}],
'protected_attribute_maps':
[all_protected_attribute_maps[x] for x in D_features]
})
optim_options = {
"distortion_fun": get_distortion_compas,
"epsilon": 0.04,
"clist": [0.99, 1.99, 2.99],
"dlist": [.1, 0.05, 0]
}
metric_transf_train = BinaryLabelDatasetMetric(
dataset_orig_train,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
OP = OptimPreproc(OptTools,
optim_options,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
OP = OP.fit(dataset_orig_train)
dataset_transf_cat_test = OP.transform(dataset_orig_vt, transform_Y=True)
dataset_transf_cat_test = dataset_orig_vt.align_datasets(
dataset_transf_cat_test)
dataset_transf_cat_train = OP.transform(dataset_orig_train,
transform_Y=True)
dataset_transf_cat_train = dataset_orig_train.align_datasets(
dataset_transf_cat_train)
scale_transf = StandardScaler()
X_train = dataset_orig_train.features
y_train = dataset_orig_train.labels.ravel()
X_test = scale_transf.fit_transform(dataset_orig_vt.features)
lmod = LogisticRegression()
lmod.fit(X_train, y_train)
y_pred = lmod.predict(X_test)
print('With resampling')
print('Accuracy')
print(accuracy_score(dataset_orig_vt.labels, y_pred))
dataset_orig_vt_copy = dataset_orig_vt.copy()
dataset_orig_vt_copy.labels = y_pred
metric_transf_train = BinaryLabelDatasetMetric(
dataset_orig_vt_copy,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
print('p-rule')
print(metric_transf_train.disparate_impact())
print('CV')
print(metric_transf_train.mean_difference())
print('FPR for unpriv')
orig_sens_att = dataset_orig_vt.protected_attributes.ravel()
print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[
orig_sens_att == 0], 0))
print('FNR for unpriv')
print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[
orig_sens_att == 0], 0))
print('FPR for priv')
orig_sens_att = dataset_orig_vt.protected_attributes.ravel()
print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[
orig_sens_att == 1], 0))
print('FNR for priv')
print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[
orig_sens_att == 1], 0))
def fit(self, dataset_true, dataset_pred):
"""Estimates the optimal classification threshold and margin for reject
option classification that optimizes the metric provided.
Note:
The `fit` function is a no-op for this algorithm.
Args:
dataset_true (BinaryLabelDataset): Dataset containing the true
`labels`.
dataset_pred (BinaryLabelDataset): Dataset containing the predicted
`scores`.
Returns:
RejectOptionClassification: Returns self.
"""
fair_metric_arr = np.zeros(self.num_class_thresh*self.num_ROC_margin)
balanced_acc_arr = np.zeros_like(fair_metric_arr)
ROC_margin_arr = np.zeros_like(fair_metric_arr)
class_thresh_arr = np.zeros_like(fair_metric_arr)
cnt = 0
# Iterate through class thresholds
for class_thresh in np.linspace(self.low_class_thresh,
self.high_class_thresh,
self.num_class_thresh):
self.classification_threshold = class_thresh
if class_thresh <= 0.5:
low_ROC_margin = 0.0
high_ROC_margin = class_thresh
else:
low_ROC_margin = 0.0
high_ROC_margin = (1.0-class_thresh)
# Iterate through ROC margins
for ROC_margin in np.linspace(
low_ROC_margin,
high_ROC_margin,
self.num_ROC_margin):
self.ROC_margin = ROC_margin
# Predict using the current threshold and margin
dataset_transf_pred = self.predict(dataset_pred)
dataset_transf_metric_pred = BinaryLabelDatasetMetric(
dataset_transf_pred,
unprivileged_groups=self.unprivileged_groups,
privileged_groups=self.privileged_groups)
classified_transf_metric = ClassificationMetric(
dataset_true,
dataset_transf_pred,
unprivileged_groups=self.unprivileged_groups,
privileged_groups=self.privileged_groups)
ROC_margin_arr[cnt] = self.ROC_margin
class_thresh_arr[cnt] = self.classification_threshold
# Balanced accuracy and fairness metric computations
balanced_acc_arr[cnt] = 0.5*(classified_transf_metric.true_positive_rate()\
+classified_transf_metric.true_negative_rate())
if self.metric_name == "Statistical parity difference":
fair_metric_arr[cnt] = dataset_transf_metric_pred.mean_difference()
elif self.metric_name == "Average odds difference":
fair_metric_arr[cnt] = classified_transf_metric.average_odds_difference()
elif self.metric_name == "Equal opportunity difference":
fair_metric_arr[cnt] = classified_transf_metric.equal_opportunity_difference()
cnt += 1
rel_inds = np.logical_and(fair_metric_arr >= self.metric_lb,
fair_metric_arr <= self.metric_ub)
if any(rel_inds):
best_ind = np.where(balanced_acc_arr[rel_inds]
== np.max(balanced_acc_arr[rel_inds]))[0][0]
else:
warn("Unable to satisy fairness constraints")
rel_inds = np.ones(len(fair_metric_arr), dtype=bool)
best_ind = np.where(fair_metric_arr[rel_inds]
== np.min(fair_metric_arr[rel_inds]))[0][0]
self.ROC_margin = ROC_margin_arr[rel_inds][best_ind]
self.classification_threshold = class_thresh_arr[rel_inds][best_ind]
return self
def get_fair_metrics(dataset, pred, pred_is_dataset=False):
"""
Measure fairness metrics.
Parameters:
dataset (pandas dataframe): Dataset
pred (array): Model predictions
pred_is_dataset, optional (bool): True if prediction is already part of the dataset, column name 'labels'.
Returns:
fair_metrics: Fairness metrics.
"""
if pred_is_dataset:
dataset_pred = pred
else:
dataset_pred = dataset.copy()
dataset_pred.labels = pred
cols = [
'statistical_parity_difference', 'equal_opportunity_difference',
'average_abs_odds_difference', 'disparate_impact', 'theil_index'
]
obj_fairness = [[0, 0, 0, 1, 0]]
fair_metrics = pd.DataFrame(data=obj_fairness,
index=['objective'],
columns=cols)
for attr in dataset_pred.protected_attribute_names:
idx = dataset_pred.protected_attribute_names.index(attr)
privileged_groups = [{
attr:
dataset_pred.privileged_protected_attributes[idx][0]
}]
unprivileged_groups = [{
attr:
dataset_pred.unprivileged_protected_attributes[idx][0]
}]
classified_metric = ClassificationMetric(
dataset,
dataset_pred,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
metric_pred = BinaryLabelDatasetMetric(
dataset_pred,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
acc = classified_metric.accuracy()
row = pd.DataFrame([[
metric_pred.mean_difference(),
classified_metric.equal_opportunity_difference(),
classified_metric.average_abs_odds_difference(),
metric_pred.disparate_impact(),
classified_metric.theil_index()
]],
columns=cols,
index=[attr])
fair_metrics = fair_metrics.append(row)
fair_metrics = fair_metrics.replace([-np.inf, np.inf], 2)
return fair_metrics
def train():
np.random.seed(10)
def quantizePrior1(x):
if x <= 0:
return 0
elif 1 <= x <= 3:
return 1
else:
return 2
def quantizeLOS(x):
if x <= 7:
return 0
if 8 < x <= 93:
return 1
else:
return 2
def group_race(x):
if x == "Caucasian":
return 1.0
else:
return 0.0
filepath = 'AIF360/aif360/data/raw/compas/compas-scores-two-years.csv'
df = pd.read_csv(filepath, index_col='id', na_values=[])
df['age_cat'] = df['age_cat'].replace('Greater than 45', 2)
df['age_cat'] = df['age_cat'].replace('25 - 45', 1)
df['age_cat'] = df['age_cat'].replace('Less than 25', 0)
df['score_text'] = df['score_text'].replace('High', 1)
df['score_text'] = df['score_text'].replace('Medium', 1)
df['score_text'] = df['score_text'].replace('Low', 0)
df['priors_count'] = df['priors_count'].apply(lambda x: quantizePrior1(x))
df['length_of_stay'] = (
pd.to_datetime(df['c_jail_out']) -
pd.to_datetime(df['c_jail_in'])).apply(lambda x: x.days)
df['length_of_stay'] = df['length_of_stay'].apply(lambda x: quantizeLOS(x))
df = df.loc[
~df['race'].isin(['Native American', 'Hispanic', 'Asian', 'Other']), :]
df['c_charge_degree'] = df['c_charge_degree'].replace({'F': 0, 'M': 1})
df1 = df[[
'priors_count', 'c_charge_degree', 'race', 'age_cat', 'score_text',
'two_year_recid'
]]
feature_list = []
for index, row in df1.iterrows():
feature_list.append('\t'.join(row.astype(str).to_list()))
df1['feature_list'] = feature_list
df3 = df1.groupby('feature_list').count() / len(df1.index)
df2 = pd.DataFrame()
df2['feature_list'] = list(df3.index)
df2['prob_list'] = list(df3.priors_count)
for index, row in df2.iterrows():
if row['feature_list'][0] == '0' and row['feature_list'][
-1] == '1' and 'African' in row['feature_list']:
row['prob_list'] = row['prob_list'] * 10
elif row['feature_list'][0] == '0' and row['feature_list'][-1] == '1':
row['prob_list'] = row['prob_list'] * 7
elif row['feature_list'][0] == '2' and row['feature_list'][-1] == '0':
row['prob_list'] = row['prob_list'] * 7
prob_list = list(df2.prob_list)
df_new = pd.DataFrame()
rng = np.random.default_rng()
prob_list = np.array(prob_list)
prob_list = prob_list / prob_list.sum()
feature_list = rng.choice(list(df2.feature_list),
len(df1.index),
p=prob_list)
var_list = [
'priors_count', 'c_charge_degree', 'race', 'age_cat', 'score_text',
'two_year_recid'
]
for i in var_list:
vars()[i] = []
for i in feature_list:
tmp = i.split('\t')
for j in range(len(var_list)):
vars()[var_list[j]].append(tmp[j])
for i in var_list:
df_new[i] = vars()[i]
df = df_new
df1 = df[[
'priors_count', 'c_charge_degree', 'race', 'age_cat', 'score_text',
'two_year_recid'
]]
tot = []
for index, row in df1.iterrows():
result = ''
for j in df1.columns:
result = result + str(row[j])
tot.append(result)
df['tmp_feature'] = tot
df['mis_prob'] = 0
for i in df['tmp_feature'].unique():
if 'African' in i and i[-1] == '0':
df.loc[df['tmp_feature'] == i, 'mis_prob'] = 0.1
elif 'African' in i:
df.loc[df['tmp_feature'] == i, 'mis_prob'] = 0.02
elif 'African' not in i and i[-1] == '0':
df.loc[df['tmp_feature'] == i, 'mis_prob'] = 0.02
else:
df.loc[df['tmp_feature'] == i, 'mis_prob'] = 0.02
new_label = []
for i, j in zip(df['mis_prob'], df['priors_count']):
if np.random.binomial(1, i, 1)[0] == 1:
new_label.append(3)
else:
new_label.append(j)
df['priors_count'] = new_label
print(len(df.loc[df['priors_count'] == 3, :].index))
print(len(df.index))
df['priors_count'] = df['priors_count'].astype(int)
df['score_text'] = df['score_text'].astype(int)
df['age_cat'] = df['age_cat'].astype(int)
df['score_text'] = df['score_text'].astype(int)
df['c_charge_degree'] = df['c_charge_degree'].astype(int)
df['two_year_recid'] = df['two_year_recid'].astype(int)
df['c_charge_degree'] = df['c_charge_degree'].replace({0: 'F', 1: 'M'})
def quantizePrior(x):
if x == 0:
return '0'
elif x == 1:
return '1 to 3'
elif x == 2:
return 'More than 3'
else:
return 'missing'
# Quantize length of stay
def quantizeLOS(x):
if x == 0:
return '<week'
if x == 1:
return '<3months'
else:
return '>3 months'
# Quantize length of stay
def adjustAge(x):
if x == 0:
return '25 to 45'
elif x == 1:
return 'Greater than 45'
elif x == 2:
return 'Less than 25'
def quantizeScore(x):
if x == 1:
return 'MediumHigh'
else:
return 'Low'
def group_race(x):
if x == "Caucasian":
return 1.0
else:
return 0.0
df['priors_count'] = df['priors_count'].apply(lambda x: quantizePrior(x))
df['score_text'] = df['score_text'].apply(lambda x: quantizeScore(x))
df['age_cat'] = df['age_cat'].apply(lambda x: adjustAge(x))
# Recode sex and race
df['race'] = df['race'].apply(lambda x: group_race(x))
df['race'] = df['race'].astype(int)
df['two_year_recid'] = df['two_year_recid'].astype(int)
df = df[[
'priors_count', 'c_charge_degree', 'race', 'age_cat', 'score_text',
'two_year_recid'
]]
df_train, df_test = train_test_split(df, test_size=0.3, random_state=10)
all_protected_attribute_maps = {"race": {0.0: 0, 1.0: 1}}
D_features = ['race']
dataset_orig_train = CustomDataset(
label_name='two_year_recid',
favorable_classes=[0],
protected_attribute_names=['race'],
privileged_classes=[[1]],
categorical_features=[
'priors_count', 'c_charge_degree', 'age_cat', 'score_text'
],
features_to_keep=[
'priors_count', 'c_charge_degree', 'race', 'age_cat', 'score_text'
],
df=df_train,
metadata={
'label_maps': [{
1: 'Did recid.',
0: 'No recid.'
}],
'protected_attribute_maps':
[all_protected_attribute_maps[x] for x in D_features]
})
dataset_orig_vt = CustomDataset(
label_name='two_year_recid',
favorable_classes=[0],
protected_attribute_names=['race'],
privileged_classes=[[1]],
categorical_features=[
'priors_count', 'c_charge_degree', 'age_cat', 'score_text'
],
features_to_keep=[
'priors_count', 'c_charge_degree', 'race', 'age_cat', 'score_text'
],
df=df_test,
metadata={
'label_maps': [{
1: 'Did recid.',
0: 'No recid.'
}],
'protected_attribute_maps':
[all_protected_attribute_maps[x] for x in D_features]
})
privileged_groups = [{'race': 1}]
unprivileged_groups = [{'race': 0}]
optim_options = {
"distortion_fun": get_distortion_compas,
"epsilon": 0.04,
"clist": [0.99, 1.99, 2.99],
"dlist": [.1, 0.05, 0]
}
metric_transf_train = BinaryLabelDatasetMetric(
dataset_orig_train,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
OP = OptimPreproc(OptTools,
optim_options,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
OP = OP.fit(dataset_orig_train)
dataset_transf_cat_test = OP.transform(dataset_orig_vt, transform_Y=True)
dataset_transf_cat_test = dataset_orig_vt.align_datasets(
dataset_transf_cat_test)
dataset_transf_cat_train = OP.transform(dataset_orig_train,
transform_Y=True)
dataset_transf_cat_train = dataset_orig_train.align_datasets(
dataset_transf_cat_train)
scale_transf = StandardScaler()
X_train = scale_transf.fit_transform(dataset_transf_cat_train.features)
y_train = dataset_transf_cat_train.labels.ravel()
X_test = scale_transf.fit_transform(dataset_transf_cat_test.features)
lmod = LogisticRegression()
lmod.fit(X_train, y_train)
y_pred = lmod.predict(X_test)
print('Without reweight')
print('Accuracy')
print(accuracy_score(dataset_orig_vt.labels, y_pred))
dataset_orig_vt_copy1 = dataset_orig_vt.copy()
dataset_orig_vt_copy1.labels = y_pred
metric_transf_train1 = BinaryLabelDatasetMetric(
dataset_orig_vt_copy1,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
print('p-rule')
print(metric_transf_train1.disparate_impact())
print('CV')
print(metric_transf_train1.mean_difference())
print('FPR for unpriv')
orig_sens_att = dataset_orig_vt.protected_attributes.ravel()
print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[
orig_sens_att == 0], 0))
print("FNR for unpriv")
print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[
orig_sens_att == 0], 0))
print('FPR for priv')
orig_sens_att = dataset_orig_vt.protected_attributes.ravel()
print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[
orig_sens_att == 1], 0))
print("FNR for priv")
print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[
orig_sens_att == 1], 0))
df_weight = dataset_orig_train.convert_to_dataframe()[0]
df_weight['weight'] = 1
df_weight['is_missing'] = 0
df_weight['tmp'] = ''
tmp_result = []
for i, j in zip(df_weight['race'], df_weight['two_year_recid']):
tmp_result.append(str(i) + str(j))
df_weight['tmp'] = tmp_result
df_weight.loc[df_weight['priors_count=missing'] == 1, 'is_missing'] = 1
for i in df_weight['tmp'].unique():
df_weight.loc[
(df_weight['tmp'] == i) & (df_weight['is_missing'] == 0),
'weight'] = len(
df_weight.loc[(df_weight['tmp'] == i), :].index) / len(
df_weight.loc[(df_weight['tmp'] == i) &
(df_weight['is_missing'] == 0), :].index)
df_weight.loc[(df_weight['tmp'] == i) & (df_weight['is_missing'] == 1),
'weight'] = len(df_weight.loc[
(df_weight['tmp'] == i) &
(df_weight['is_missing'] == 0), :].index) / len(
df_weight.loc[(df_weight['tmp'] == i), :].index)
dataset_orig_train.instance_weights = np.array(df_weight['weight'])
scale_transf = StandardScaler()
X_train = scale_transf.fit_transform(dataset_transf_cat_train.features)
y_train = dataset_transf_cat_train.labels.ravel()
X_test = scale_transf.fit_transform(dataset_transf_cat_test.features)
lmod = LogisticRegression()
lmod.fit(X_train,
y_train,
sample_weight=dataset_orig_train.instance_weights)
y_pred = lmod.predict(X_test)
print('With reweight')
print('Accuracy')
print(accuracy_score(dataset_orig_vt.labels, y_pred))
dataset_orig_vt_copy1 = dataset_orig_vt.copy()
dataset_orig_vt_copy1.labels = y_pred
metric_transf_train1 = BinaryLabelDatasetMetric(
dataset_orig_vt_copy1,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
print('p-rule')
print(metric_transf_train1.disparate_impact())
print('CV')
print(metric_transf_train1.mean_difference())
print('FPR for unpriv')
orig_sens_att = dataset_orig_vt.protected_attributes.ravel()
print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[
orig_sens_att == 0], 0))
print("FNR for unpriv")
print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[
orig_sens_att == 0], 0))
print('FPR for priv')
orig_sens_att = dataset_orig_vt.protected_attributes.ravel()
print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[
orig_sens_att == 1], 0))
print("FNR for priv")
print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[
orig_sens_att == 1], 0))
def train_before_resample():
dataset_orig_train = load_preproc_data_adult(['sex'])
optim_options = {
"distortion_fun": get_distortion_adult,
"epsilon": 0.05,
"clist": [0.99, 1.99, 2.99],
"dlist": [.1, 0.05, 0]
}
OP = OptimPreproc(OptTools,
optim_options,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
OP = OP.fit(dataset_orig_train)
dataset_transf_cat_test = OP.transform(dataset_orig_vt, transform_Y=True)
dataset_transf_cat_test = dataset_orig_vt.align_datasets(
dataset_transf_cat_test)
dataset_transf_cat_train = OP.transform(dataset_orig_train,
transform_Y=True)
dataset_transf_cat_train = dataset_orig_train.align_datasets(
dataset_transf_cat_train)
scale_transf = StandardScaler()
#X_train = scale_transf.fit_transform(dataset_orig_train.features[:,1:])
#X_train = scale_transf.fit_transform(dataset_orig_train.features)
X_train = dataset_orig_train.features
y_train = dataset_orig_train.labels.ravel()
X_test = scale_transf.fit_transform(dataset_orig_vt.features)
#X_test = scale_transf.fit_transform(dataset_orig_vt.features[:,1:])
lmod = LogisticRegression()
#lmod = LogisticRegression()
#lmod.fit(X_train, y_train,sample_weight=dataset_orig_train.instance_weights)
lmod.fit(X_train, y_train)
y_pred = lmod.predict(X_test)
print('Accuracy and fairness results before resampling')
print('accuracy')
print(accuracy_score(dataset_orig_vt.labels, y_pred))
dataset_orig_vt_copy = dataset_orig_vt.copy()
dataset_orig_vt_copy.labels = y_pred
metric_transf_train = BinaryLabelDatasetMetric(
dataset_orig_vt_copy,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
print('p-rule')
print(metric_transf_train.disparate_impact())
print('CV')
print(metric_transf_train.mean_difference())
print('FPR for priv')
orig_sens_att = dataset_orig_vt.protected_attributes.ravel()
print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[
orig_sens_att == 0], 1))
print("FNR for priv")
print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[
orig_sens_att == 0], 1))
print("BCR")
print(
get_BCR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0],
y_pred[orig_sens_att == 0], 1))
print('FPR for unpriv')
orig_sens_att = dataset_orig_vt.protected_attributes.ravel()
print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[
orig_sens_att == 1], 1))
print("FNR for unpriv")
print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[
orig_sens_att == 1], 1))
