def get_evaluation(dataset_orig_vt, y_pred, privileged_groups,
unprivileged_groups, unpriv_val, priv_val, pos_label):
print('Accuracy')
print(accuracy_score(dataset_orig_vt.labels, y_pred))
dataset_orig_vt_copy1 = dataset_orig_vt.copy()
dataset_orig_vt_copy1.labels = y_pred
metric_transf_train1 = BinaryLabelDatasetMetric(
dataset_orig_vt_copy1,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
print('p-rule')
print(
min(metric_transf_train1.disparate_impact(),
1 / metric_transf_train1.disparate_impact()))
print('FPR for unpriv group')
orig_sens_att = dataset_orig_vt.protected_attributes.ravel()
print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == unpriv_val],
y_pred[orig_sens_att == unpriv_val], pos_label))
print("FNR for unpriv group")
print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == unpriv_val],
y_pred[orig_sens_att == unpriv_val], pos_label))
print('FPR for priv group')
orig_sens_att = dataset_orig_vt.protected_attributes.ravel()
print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == priv_val],
y_pred[orig_sens_att == priv_val], pos_label))
print("FNR for priv group")
print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == priv_val],
y_pred[orig_sens_att == priv_val], pos_label))
def calc_disparity_index(data, target_variable, protected_variable, unprivileged_input):
df_aif = BinaryLabelDataset(df=data, label_names=[target_variable],
protected_attribute_names=[protected_variable])
privileged_group = []
for v in data[protected_variable].unique()[data[protected_variable].unique() != unprivileged_input]:
privileged_group.append({protected_variable: v})
unprivileged_group = [{protected_variable: unprivileged_input}] #female=0
metric_orig = BinaryLabelDatasetMetric(df_aif, unprivileged_group, privileged_group)
print('1-min(DI, 1/DI):', get_disparity_index(metric_orig.disparate_impact()).round(3))
if get_disparity_index(metric_orig.disparate_impact()).round(3) < 0.2:
print('The algorithm can be considered to be not biased')
else:
print('There is a potential bias')
def calculate_bias_measures(data_orig_train, data_orig_vt, unprivileged_groups,
privileged_groups):
model = RandomForestClassifier().fit(
data_orig_train.features,
data_orig_train.labels.ravel(),
sample_weight=data_orig_train.instance_weights)
dataset = data_orig_vt
dataset_pred = dataset.copy()
dataset_pred.labels = model.predict(data_orig_vt.features)
classified_metric_race = ClassificationMetric(
dataset,
dataset_pred,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
metric_pred_race = BinaryLabelDatasetMetric(
dataset_pred,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
print("Mean difference {}".format(metric_pred_race.mean_difference()))
print("Disparate Metric {}".format(metric_pred_race.disparate_impact()))
print("Equal Opportunity Difference {}".format(
classified_metric_race.equal_opportunity_difference()))
print("Average Abs Odds Difference {}".format(
classified_metric_race.average_abs_odds_difference()))
print("Theil index {}".format(classified_metric_race.theil_index()))
def disparity_values(data, target_variable, protected_variable, unprivileged_input):
df_aif = BinaryLabelDataset(df=data, label_names=[target_variable],
protected_attribute_names=[protected_variable])
privileged_group = []
for v in data[protected_variable].unique()[data[protected_variable].unique() != unprivileged_input]:
privileged_group.append({protected_variable: v})
unprivileged_group = [{protected_variable: unprivileged_input}] #female=0
metric_orig = BinaryLabelDatasetMetric(df_aif, unprivileged_group, privileged_group)
return abs(get_disparity_index(metric_orig.disparate_impact()).round(3))
def fair_metrics(dataset, pred, pred_is_dataset=False):
if pred_is_dataset:
dataset_pred = pred
else:
dataset_pred = dataset.copy()
dataset_pred.labels = pred
cols = [
'statistical_parity_difference', 'equal_opportunity_difference',
'average_abs_odds_difference', 'disparate_impact', 'theil_index'
]
obj_fairness = [[0, 0, 0, 1, 0]]
fair_metrics = pd.DataFrame(data=obj_fairness,
index=['objective'],
columns=cols)
for attr in dataset_pred.protected_attribute_names:
idx = dataset_pred.protected_attribute_names.index(attr)
privileged_groups = [{
attr:
dataset_pred.privileged_protected_attributes[idx][0]
}]
unprivileged_groups = [{
attr:
dataset_pred.unprivileged_protected_attributes[idx][0]
}]
classified_metric = ClassificationMetric(
dataset,
dataset_pred,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
metric_pred = BinaryLabelDatasetMetric(
dataset_pred,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
acc = classified_metric.accuracy()
row = pd.DataFrame([[
metric_pred.mean_difference(),
classified_metric.equal_opportunity_difference(),
classified_metric.average_abs_odds_difference(),
metric_pred.disparate_impact(),
classified_metric.theil_index()
]],
columns=cols,
index=[attr])
fair_metrics = fair_metrics.append(row)
fair_metrics = fair_metrics.replace([-np.inf, np.inf], 2)
return fair_metrics
def get_dataset_metrics_list(binary_dataset_list):
#Set privileged and unprivileged groups
privileged_groups= [{'sex':1}]
unprivileged_groups= [{'sex': 0}]
mean_diff_list = []
disp_imp_list = []
for dataset in binary_dataset_list:
metrics = BinaryLabelDatasetMetric(dataset,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
mean_diff_list.append(metrics.mean_difference())
disp_imp_list.append(1 - metrics.disparate_impact())
return mean_diff_list, disp_imp_list
def explain(self, request: Dict) -> Dict:
inputs = request["instances"]
predictions = np.array(request["outputs"])
dataframe_predicted = pd.DataFrame(inputs, columns=self.feature_names)
dataframe_predicted[self.label_names[0]] = predictions
dataset_predicted = BinaryLabelDataset(
favorable_label=self.favorable_label,
unfavorable_label=self.unfavorable_label,
df=dataframe_predicted,
label_names=self.label_names,
protected_attribute_names=['age'])
metrics = BinaryLabelDatasetMetric(
dataset_predicted,
unprivileged_groups=self.unprivileged_groups,
privileged_groups=self.privileged_groups)
return {
"predictions": predictions.tolist(),
"metrics": {
"base_rate":
metrics.base_rate(),
"consistency":
metrics.consistency().tolist(),
"disparate_impact":
metrics.disparate_impact(),
"num_instances":
metrics.num_instances(),
"num_negatives":
metrics.num_negatives(),
"num_positives":
metrics.num_positives(),
"statistical_parity_difference":
metrics.statistical_parity_difference(),
}
}
def checkClassifierFairnessAndRemoveDI(frame,
dpoints,
mname,
x_columns,
verbose=True,
pre=True):
''' Measure fairness according to the metric using the value of A and the classification outcome.
Results get added to a dictionary used to pass them to a function to generate graphs of the results.
If we have not performed intervention, perform intervention and return post intervention data.'''
xay_columns = copy.deepcopy(x_columns)
xay_columns.extend(["A", "Y"])
ycols = copy.deepcopy(frame["Y"])
tempframe = copy.deepcopy(frame)
tempframe.drop(["Y"], axis=1, inplace=True)
aifdf = BinaryLabelDataset(favorable_label=1.0,
unfavorable_label=0.0,
df=tempframe,
label_names=['Ya'],
protected_attribute_names=['A'])
privileged_groups = [{'A': 1}]
unprivileged_groups = [{'A': 0}]
DIR = DisparateImpactRemover(sensitive_attribute='A')
metric_aifdf_train = BinaryLabelDatasetMetric(
aifdf,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
if pre:
if verbose:
print("\n\tINTERVENTION: {}\n".format(type(DIR).__name__))
print("\t######### PRE {} ###########".format(type(DIR).__name__))
print(
"\tDisparate impact between unprivileged and privileged groups = {}\n"
.format(metric_aifdf_train.disparate_impact()))
dpoints[mname]['PRE'][type(
DIR).__name__]['FAIR'] = metric_aifdf_train.disparate_impact()
print("PRE CLASSIFICATION MATRIX")
print("----------------")
print(" |Y'=0 | Y'=1 |")
print("----------------")
print("A=0| {0} | {1} |".format(
metric_aifdf_train.num_negatives(False),
metric_aifdf_train.num_positives(False)))
print("A=1| {0} | {1} |".format(
metric_aifdf_train.num_negatives(True),
metric_aifdf_train.num_positives(True)))
print("----------------")
dataset_transf_train = DIR.fit_transform(aifdf)
fairdf = dataset_transf_train.convert_to_dataframe()[0]
fairdf.drop(['Ya'], axis=1, inplace=True)
ycols.reset_index(drop=True, inplace=True)
fairdf.reset_index(drop=True, inplace=True)
fairdf.insert(0, "Y", ycols)
fairdf[xay_columns] = fairdf[xay_columns]
return fairdf
else:
if verbose:
print(
"\tDisparate impact between unprivileged and privileged groups = {}\n"
.format(metric_aifdf_train.disparate_impact()))
dpoints[mname]['POST'][type(
DIR).__name__]['FAIR'] = metric_aifdf_train.disparate_impact()
print("POST CLASSIFICATION MATRIX")
print("----------------")
print(" |Y'=0 | Y'=1|")
print("----------------")
print("A=0| {0} | {1} |".format(
metric_aifdf_train.num_negatives(False),
metric_aifdf_train.num_positives(False)))
print("A=1| {0} | {1} |".format(
metric_aifdf_train.num_negatives(True),
metric_aifdf_train.num_positives(True)))
print("----------------")
return frame
train_dataset.features = scaler.fit_transform(train_dataset.features)
test_dataset.features = scaler.fit_transform(test_dataset.features)
index = [
test_dataset.feature_names.index(x)
for x in protected_attribute_names
]
#Metric of Original Data
train_dataset_metric = BinaryLabelDatasetMetric(train_dataset,
unprivileged_groups=u,
privileged_groups=p)
test_dataset_metric = BinaryLabelDatasetMetric(test_dataset,
unprivileged_groups=u,
privileged_groups=p)
di_orig_list.append(test_dataset_metric.disparate_impact())
xaxis = np.arange(14)
width = 0.2
baseline = 1
fig, ax = plt.subplots()
rects1 = ax.bar(xaxis,
np.array(di_orig_list) - baseline,
width,
bottom=baseline,
label='Original')
# rects2 = ax.bar(xaxis + width, np.array(di_pred_orig_list)- baseline, width,bottom=baseline, label='Predicted Original')
# rects3 = ax.bar(xaxis + width+width, np.array(di_pred_trans_list)- baseline, width,bottom=baseline, label='Predicted Transformed')
# Add some text for labels, title and custom x-axis tick labels, etc.
ax.set_ylabel('Disparate Impact')
def compute_metrics(model, X_test, y_test, X_train, y_train, dataset_test,
unprivileged_groups, privileged_groups, protect_attribute,
print_result):
"""
Calculate and return: model accuracy and fairness metrics
Parameters
----------
model: scikit-learn classifier
X_test: numpy 2d array
y_test: numpy 1d array
X_train: numpy 2d array
y_train: numpy 1d array
dataset_test: aif360.datasets.BinaryLabelDataset
unprivileged_groups: list<dict>
Dictionary where the key is the name of the sensitive column in the
dataset, and the value is the value of the unprivileged group in the
dataset
privileged_groups: list<dict>
Dictionary where the key is the name of the sensitive column in the
dataset, and the value is the value of the privileged group in the
dataset
protect_attribute
print_result
"""
result = {}
y_pred_test = model.predict(X_test)
result['acc_test'] = accuracy_score(y_true=y_test, y_pred=y_pred_test)
y_pred_train = model.predict(X_train)
result['acc_train'] = accuracy_score(y_true=y_train, y_pred=y_pred_train)
dataset_pred = dataset_test.copy()
dataset_pred.labels = y_pred_test
bin_metric = BinaryLabelDatasetMetric(dataset_pred,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
result['disp_impact'] = bin_metric.disparate_impact()
result['stat_parity'] = bin_metric.mean_difference()
classif_metric = ClassificationMetric(dataset_test, dataset_pred,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
result['avg_odds'] = classif_metric.average_odds_difference()
result['equal_opport'] = classif_metric.equal_opportunity_difference()
result['false_discovery_rate'] = classif_metric.false_discovery_rate_difference()
result['entropy_index'] = classif_metric.generalized_entropy_index()
result['acc_test_clf'] = classif_metric.accuracy(privileged=None)
result['acc_test_priv'] = classif_metric.accuracy(privileged=True)
result['acc_test_unpriv'] = classif_metric.accuracy(privileged=False)
result['consistency'] = consitency(X_test, y_pred_test, protect_attribute, n_neighbors=5)
result['counterfactual'] = counterfactual(X_test, model, protect_attribute)
if print_result:
print("Train accuracy: ", result['acc_train'])
print("Test accuracy: ", result['acc_test'])
print("Test accuracy clf: ", result['acc_test_clf'])
print("Test accuracy priv.: ", result['acc_test_priv'])
print("Test accuracy unpriv.: ", result['acc_test_unpriv'])
print('Disparate impact: ', result['disp_impact'])
print('Mean difference: ', result['stat_parity'])
print('Average odds difference:', result['avg_odds'])
print('Equality of opportunity:', result['equal_opport'])
print('False discovery rate difference:', result['false_discovery_rate'])
print('Generalized entropy index:', result['entropy_index'])
print('Consistency: ', result['consistency'])
print('Counterfactual fairness: ', result['counterfactual'])
return result
def compute_metrics(model, X_test, y_test, X_train, y_train, dataset_test,
dataset_name, model_name, unprivileged_groups,
privileged_groups, position):
"""
Calculate and return: model accuracy and fairness metrics
Parameters
----------
model: scikit-learn classifier
X_test: numpy 2d array
y_test: numpy 1d array
X_train: numpy 2d array
y_train: numpy 1d array
dataset_test: aif360.datasets.BinaryLabelDataset
dataset_name: string
Dataset name used in the analysis
model_name: string
unprivileged_groups: list<dict>
Dictionary where the key is the name of the sensitive column in the
dataset, and the value is the value of the unprivileged group in the
dataset
privileged_groups: list<dict>
Dictionary where the key is the name of the sensitive column in the
dataset, and the value is the value of the privileged group in the
dataset
position: int
Column position of the sensitive group in the dataset
"""
y_pred_test = model.predict(X_test)
acc_test = accuracy_score(y_true=y_test, y_pred=y_pred_test)
print("Test accuracy: ", acc_test)
y_pred_train = model.predict(X_train)
acc_train = accuracy_score(y_true=y_train, y_pred=y_pred_train)
print("Train accuracy: ", acc_train)
dataset_pred = dataset_test.copy()
dataset_pred.labels = y_pred_test
bin_metric = BinaryLabelDatasetMetric(
dataset_pred,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
disparate_impact_bin = bin_metric.disparate_impact()
print('Disparate impact: ', disparate_impact_bin)
mean_difference = bin_metric.mean_difference()
print('Mean difference: ', mean_difference)
classif_metric = ClassificationMetric(
dataset_test,
dataset_pred,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
classif_disparete_impact = classif_metric.disparate_impact()
avg_odds = classif_metric.average_odds_difference()
print('Average odds difference:', avg_odds)
equal_opport = classif_metric.equal_opportunity_difference()
print('Equality of opportunity:', equal_opport)
false_discovery_rate = classif_metric.false_discovery_rate_difference()
print('False discovery rate difference:', false_discovery_rate)
entropy_index = classif_metric.generalized_entropy_index()
print('Generalized entropy index:', entropy_index)
cons_comp = consitency_mod(bin_metric, position, n_neighbors=5)
print('Consistency: ', cons_comp)
result = (dataset_name, model_name, acc_test, disparate_impact_bin,
mean_difference, classif_disparete_impact, avg_odds,
equal_opport, false_discovery_rate, entropy_index, cons_comp)
return result
def train():
privileged_groups = [{'race': 1}]
unprivileged_groups = [{'race': 0}]
dataset_orig_train = load_preproc_data_compas(['race'])
optim_options = {
"distortion_fun": get_distortion_compas,
"epsilon": 0.1,
"clist": [0.99, 1.99, 2.99],
"dlist": [.1, 0.05, 0]
}
# dataset_orig_train, dataset_orig_vt = dataset_orig.split([0.7], shuffle=True)
metric_transf_train = BinaryLabelDatasetMetric(dataset_orig_train,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
OP = OptimPreproc(OptTools, optim_options,
unprivileged_groups = unprivileged_groups,
privileged_groups = privileged_groups)
OP = OP.fit(dataset_orig_train)
dataset_transf_cat_test = OP.transform(dataset_orig_vt, transform_Y = True)
dataset_transf_cat_test = dataset_orig_vt.align_datasets(dataset_transf_cat_test)
dataset_transf_cat_train = OP.transform(dataset_orig_train, transform_Y = True)
dataset_transf_cat_train = dataset_orig_train.align_datasets(dataset_transf_cat_train)
scale_transf = StandardScaler()
X_train = scale_transf.fit_transform(dataset_transf_cat_train.features)
y_train = dataset_transf_cat_train.labels.ravel()
X_test = scale_transf.fit_transform(dataset_transf_cat_test.features)
lmod = LogisticRegression()
lmod.fit(X_train, y_train)
y_pred = lmod.predict(X_test)
print('Without applying fixing algorithms')
print('Accuracy')
print(accuracy_score(dataset_orig_vt.labels, y_pred))
dataset_orig_vt_copy1 = dataset_orig_vt.copy()
dataset_orig_vt_copy1.labels = y_pred
metric_transf_train1 = BinaryLabelDatasetMetric(dataset_orig_vt_copy1,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
print('p-rule')
print(metric_transf_train1.disparate_impact())
print('CV')
print(metric_transf_train1.mean_difference())
print('FPR for unpriv')
orig_sens_att = dataset_orig_vt.protected_attributes.ravel()
print(1-TNR(dataset_orig_vt.labels.ravel()[orig_sens_att==0],y_pred[orig_sens_att==0],0))
print("FNR for unpriv")
print(1-TPR(dataset_orig_vt.labels.ravel()[orig_sens_att==0],y_pred[orig_sens_att==0],0))
print('FPR for priv')
orig_sens_att = dataset_orig_vt.protected_attributes.ravel()
print(1-TNR(dataset_orig_vt.labels.ravel()[orig_sens_att==1],y_pred[orig_sens_att==1],0))
print("FNR for priv")
print(1-TPR(dataset_orig_vt.labels.ravel()[orig_sens_att==1],y_pred[orig_sens_att==1],0))
def train_before_resample():
dataset_orig_train = load_preproc_data_adult(['sex'])
optim_options = {
"distortion_fun": get_distortion_adult,
"epsilon": 0.05,
"clist": [0.99, 1.99, 2.99],
"dlist": [.1, 0.05, 0]
}
OP = OptimPreproc(OptTools,
optim_options,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
OP = OP.fit(dataset_orig_train)
dataset_transf_cat_test = OP.transform(dataset_orig_vt, transform_Y=True)
dataset_transf_cat_test = dataset_orig_vt.align_datasets(
dataset_transf_cat_test)
dataset_transf_cat_train = OP.transform(dataset_orig_train,
transform_Y=True)
dataset_transf_cat_train = dataset_orig_train.align_datasets(
dataset_transf_cat_train)
scale_transf = StandardScaler()
#X_train = scale_transf.fit_transform(dataset_orig_train.features[:,1:])
#X_train = scale_transf.fit_transform(dataset_orig_train.features)
X_train = dataset_orig_train.features
y_train = dataset_orig_train.labels.ravel()
X_test = scale_transf.fit_transform(dataset_orig_vt.features)
#X_test = scale_transf.fit_transform(dataset_orig_vt.features[:,1:])
lmod = LogisticRegression()
#lmod = LogisticRegression()
#lmod.fit(X_train, y_train,sample_weight=dataset_orig_train.instance_weights)
lmod.fit(X_train, y_train)
y_pred = lmod.predict(X_test)
print('Accuracy and fairness results before resampling')
print('accuracy')
print(accuracy_score(dataset_orig_vt.labels, y_pred))
dataset_orig_vt_copy = dataset_orig_vt.copy()
dataset_orig_vt_copy.labels = y_pred
metric_transf_train = BinaryLabelDatasetMetric(
dataset_orig_vt_copy,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
print('p-rule')
print(metric_transf_train.disparate_impact())
print('CV')
print(metric_transf_train.mean_difference())
print('FPR for priv')
orig_sens_att = dataset_orig_vt.protected_attributes.ravel()
print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[
orig_sens_att == 0], 1))
print("FNR for priv")
print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[
orig_sens_att == 0], 1))
print("BCR")
print(
get_BCR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0],
y_pred[orig_sens_att == 0], 1))
print('FPR for unpriv')
orig_sens_att = dataset_orig_vt.protected_attributes.ravel()
print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[
orig_sens_att == 1], 1))
print("FNR for unpriv")
print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[
orig_sens_att == 1], 1))
def run_trial():
# stores each run (4 algos without reweighing, 4 algos with reweighing) as a sublist
# number of sublists = number of runs
# each sublist has four elements
# we ONLY predict on the testing data
########## WITHOUT REWEIGHING #############
stat_par = []
disp_imp = []
eq_opp_diff = []
avg_odds_diff = []
theil = []
acc = []
########## WITH REWEIGHING #############
stat_par_reweigh = []
disp_imp_reweigh = []
eq_opp_diff_reweigh = []
avg_odds_diff_reweigh = []
theil_reweigh = []
acc_reweigh = []
###########################################
for i in range(10):
###########################################
privileged_groups = [{'sex': 1}]
unprivileged_groups = [{'sex': 0}]
dataset_orig = load_preproc_data_adult()
# train1, test1 are the original dataset
train1, test1 = dataset_orig.split([0.7], shuffle=True)
RW = Reweighing(unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
RW.fit(train1)
# dataset_transf_train, test1 are for the reweighed dataset
dataset_transf_train = RW.transform(train1)
###########################################
# change weights to whole numbers
for i in range(dataset_transf_train.instance_weights.size):
dataset_transf_train.instance_weights[i] = (round(
dataset_transf_train.instance_weights[i] / 0.1) * 0.1) * 10
weights = copy.deepcopy(dataset_transf_train.instance_weights)
# change dataset_transf_train.features and dataset_transf_train.labels and dataset_transf_train.protected_attributes according to the weights of each instance
sum_weights = 0
for i in range(dataset_transf_train.features.shape[0]):
row = copy.deepcopy(dataset_transf_train.features[i])
row_label = copy.deepcopy(dataset_transf_train.labels[i])
row_protected_attributes = copy.deepcopy(
dataset_transf_train.protected_attributes[i])
row_protected_attributes.resize(1, 2)
row.resize(1, 18)
row_label.resize(1, 1)
weight = int(weights[i])
for j in range(weight - 1):
dataset_transf_train.features = np.concatenate(
(dataset_transf_train.features, row))
dataset_transf_train.labels = np.concatenate(
(dataset_transf_train.labels, row_label))
dataset_transf_train.protected_attributes = np.concatenate(
(dataset_transf_train.protected_attributes,
row_protected_attributes))
# change the dataset_transf_train to a numpy array of ones to match number of rows in features
dataset_transf_train.instance_weights = np.ones(
dataset_transf_train.features.shape[0])
################## without reweighing ##########################
temp_stat_par = []
temp_disp_imp = []
temp_eq_opp_diff = []
temp_avg_odds_diff = []
temp_theil = []
temp_acc = []
##################### adversarial debiasing #####################
sess = tf.Session()
debiased_model = AdversarialDebiasing(
privileged_groups=privileged_groups,
unprivileged_groups=unprivileged_groups,
scope_name='debiased_classifier',
debias=True,
sess=sess)
debiased_model.fit(train1)
dataset_debiasing_test = debiased_model.predict(test1)
sess.close()
tf.reset_default_graph()
##################### metrics #####################
metric_test = BinaryLabelDatasetMetric(
dataset_debiasing_test,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
acc_test = ClassificationMetric(
test1,
dataset_debiasing_test,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
temp_stat_par.append(metric_test.mean_difference())
temp_disp_imp.append(metric_test.disparate_impact())
temp_eq_opp_diff.append(metric_test.equal_opportunity_difference())
temp_avg_odds_diff.append(metric_test.average_odds_difference())
temp_theil.append(metric_test.theil_index())
temp_acc.append(acc_test.accuracy())
##################### prejudice remover #####################
prejudice_model = PrejudiceRemover(eta=100, sensitive_attr='sex')
prejudice_model.fit(train1)
dataset_prejudice_test = prejudice_model.predict(test1)
##################### metrics #####################
metric_test = BinaryLabelDatasetMetric(
dataset_prejudice_test,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
acc_test = ClassificationMetric(
test1,
dataset_prejudice_test,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
temp_stat_par.append(metric_test.mean_difference())
temp_disp_imp.append(metric_test.disparate_impact())
temp_eq_opp_diff.append(metric_test.equal_opportunity_difference())
temp_avg_odds_diff.append(metric_test.average_odds_difference())
temp_theil.append(metric_test.theil_index())
temp_acc.append(acc_test.accuracy())
##################### normal neural net #####################
sess = tf.Session()
neural_model = AdversarialDebiasing(
privileged_groups=privileged_groups,
unprivileged_groups=unprivileged_groups,
scope_name='debiased_classifier',
debias=False,
sess=sess)
neural_model.fit(train1)
dataset_neural_test = neural_model.predict(test1)
sess.close()
tf.reset_default_graph()
##################### metrics #####################
metric_test = BinaryLabelDatasetMetric(
dataset_neural_test,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
acc_test = ClassificationMetric(
test1,
dataset_neural_test,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
temp_stat_par.append(metric_test.mean_difference())
temp_disp_imp.append(metric_test.disparate_impact())
temp_eq_opp_diff.append(metric_test.equal_opportunity_difference())
temp_avg_odds_diff.append(metric_test.average_odds_difference())
temp_theil.append(metric_test.theil_index())
temp_acc.append(acc_test.accuracy())
##################### ensemble #####################
pred_labels_test = []
for i in range(0, len(test1.features)):
arr_test = mode([
dataset_debiasing_test[i], dataset_prejudice_test[i],
dataset_neural_test[i]
])
pred_labels_test.append(arr_test[0][0])
dataset_ensemble_test = test1.copy()
dataset_ensemble_test.labels = np.array(pred_labels_test)
##################### metrics #####################
metric_test = BinaryLabelDatasetMetric(
dataset_ensemble_train,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
acc_test = ClassificationMetric(
test1,
dataset_ensemble_train,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
temp_stat_par.append(metric_test.mean_difference())
temp_disp_imp.append(metric_test.disparate_impact())
temp_eq_opp_diff.append(metric_test.equal_opportunity_difference())
temp_avg_odds_diff.append(metric_test.average_odds_difference())
temp_theil.append(metric_test.theil_index())
temp_acc.append(acc_test.accuracy())
######### DUMP SHIT ###########
stat_par.append(temp_stat_par)
disp_imp.append(temp_disp_imp)
eq_opp_diff.append(temp_eq_opp_diff)
avg_odds_diff.append(temp_avg_odds_diff)
theil.append(temp_theil)
acc.append(temp_acc)
################## with reweighing ##########################
temp_stat_par = []
temp_disp_imp = []
temp_eq_opp_diff = []
temp_avg_odds_diff = []
temp_theil = []
temp_acc = []
################## adversarial debiasing ##################
sess = tf.Session()
debiased_model_reweighing = AdversarialDebiasing(
privileged_groups=privileged_groups,
unprivileged_groups=unprivileged_groups,
scope_name='debiased_classifier',
debias=True,
sess=sess)
debiased_model_reweighing.fit(dataset_transf_train)
dataset_debiasing_test_reweighing = debiased_model_reweighing.predict(
test1)
sess.close()
tf.reset_default_graph()
##################### metrics #####################
metric_test = BinaryLabelDatasetMetric(
dataset_debiasing_test_reweighing,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
acc_test = ClassificationMetric(
test1,
dataset_debiasing_test_reweighing,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
temp_stat_par.append(metric_test.mean_difference())
temp_disp_imp.append(metric_test.disparate_impact())
temp_eq_opp_diff.append(metric_test.equal_opportunity_difference())
temp_avg_odds_diff.append(metric_test.average_odds_difference())
temp_theil.append(metric_test.theil_index())
temp_acc.append(acc_test.accuracy())
##################### prejudice remover #####################
prejudice_model_reweighing = PrejudiceRemover(eta=100,
sensitive_attr='sex')
prejudice_model_reweighing.fit(dataset_transf_train)
dataset_prejudice_test_reweighing = prejudice_model_reweighing.predict(
test1)
##################### metrics #####################
metric_test = BinaryLabelDatasetMetric(
dataset_prejudice_test_reweighing,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
acc_test = ClassificationMetric(
test1,
dataset_prejudice_test_reweighing,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
temp_stat_par.append(metric_test.mean_difference())
temp_disp_imp.append(metric_test.disparate_impact())
temp_eq_opp_diff.append(metric_test.equal_opportunity_difference())
temp_avg_odds_diff.append(metric_test.average_odds_difference())
temp_theil.append(metric_test.theil_index())
temp_acc.append(acc_test.accuracy())
##################### normal neural net #####################
sess = tf.Session()
neural_model = AdversarialDebiasing(
privileged_groups=privileged_groups,
unprivileged_groups=unprivileged_groups,
scope_name='debiased_classifier',
debias=False,
sess=sess)
neural_model.fit(dataset_transf_train)
dataset_neural_test = neural_model.predict(test1)
sess.close()
tf.reset_default_graph()
##################### metrics #####################
metric_test = BinaryLabelDatasetMetric(
dataset_neural_test,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
acc_test = ClassificationMetric(
test1,
dataset_neural_test,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
temp_stat_par.append(metric_test.mean_difference())
temp_disp_imp.append(metric_test.disparate_impact())
temp_eq_opp_diff.append(metric_test.equal_opportunity_difference())
temp_avg_odds_diff.append(metric_test.average_odds_difference())
temp_theil.append(metric_test.theil_index())
temp_acc.append(acc_test.accuracy())
##################### ensemble #####################
pred_labels_test = []
for i in range(0, len(test1.features)):
arr_test = mode([
dataset_debiasing_test[i], dataset_prejudice_test[i],
dataset_neural_test[i]
])
pred_labels_test.append(arr_test[0][0])
dataset_ensemble_test = test1.copy()
dataset_ensemble_test.labels = np.array(pred_labels_test)
##################### metrics #####################
metric_test = BinaryLabelDatasetMetric(
dataset_ensemble_test,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
acc_test = ClassificationMetric(
test1,
dataset_ensemble_test,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
temp_stat_par.append(metric_test.mean_difference())
temp_disp_imp.append(metric_test.disparate_impact())
temp_eq_opp_diff.append(metric_test.equal_opportunity_difference())
temp_avg_odds_diff.append(metric_test.average_odds_difference())
temp_theil.append(metric_test.theil_index())
temp_acc.append(acc_test.accuracy())
######### DUMP SHIT ###########
stat_par_reweigh.append(temp_stat_par)
disp_imp_reweigh.append(temp_disp_imp)
eq_opp_diff_reweigh.append(temp_eq_opp_diff)
avg_odds_diff_reweigh.append(temp_avg_odds_diff)
theil_reweigh.append(temp_theil)
acc_reweigh.append(temp_acc)
without_reweighing = [
stat_par, disp_imp, eq_opp_diff, avg_odds_diff, theil, acc
]
with_reweighing = [
stat_par_reweigh, disp_imp_reweigh, eq_opp_diff_reweigh,
avg_odds_diff_reweigh, theil_reweigh, acc_reweigh
]
for metric in range(len(without_reweighing)):
name = "metric" + str(metric)
sublist = without_reweighing[metric]
with open(name, "wb") as csv_file:
writer = csv.writer(csv_file)
writer.writerows(sublist)
for metric in range(len(with_reweighing)):
name = "metric" + str(metric) + "reweigh"
sublist = with_reweighing[metric]
with open(name, "wb") as csv_file:
writer = csv.writer(csv_file)
writer.writerows(sublist)
def get_bldm_metrics():
metric_BLDM = BinaryLabelDatasetMetric(
dataset, unprivileged_group, privileged_group)
return {"Statistical Parity Difference": metric_BLDM.statistical_parity_difference(), "Disparate Impact": metric_BLDM.disparate_impact()}
prot.append(validation_comp.columns[i])
priv_dict = {validation_comp.columns[i]: 1}
else:
priv.append([])
stdDs = StandardDataset(validation_comp, 'is_violent_recid', [0], prot, priv)
stdPred = StandardDataset(validation_pred, 'is_violent_recid', [0], prot, priv)
bi_met = BinaryLabelDatasetMetric(stdDs,
privileged_groups=[priv_dict],
unprivileged_groups=[unpriv_dict])
class_met = ClassificationMetric(stdDs,
stdPred,
unprivileged_groups=[unpriv_dict],
privileged_groups=[priv_dict])
disparate_impact = bi_met.disparate_impact()
#error_rate_ratio = class_met.error_rate_ratio()
eq_diff = class_met.equal_opportunity_difference()
#Create 2 Bar Graphs
x = [1]
di_y = [disparate_impact]
er_y = [error_rate_ratio]
plt.ylim(bottom=0, top=2)
plt.xlim(left=0, right=2)
ax = plt.gca()
ax.axes.xaxis.set_visible(False)
plt.bar(x, di_y, width=0.6)
plt.axhline(y=1.25, xmin=0, xmax=2, linestyle='--', color='black')
def train():
privileged_groups = [{'race': 1}]
unprivileged_groups = [{'race': 0}]
dataset_orig = load_preproc_data_compas(['race'])
optim_options = {
"distortion_fun": get_distortion_compas,
"epsilon": 0.05,
"clist": [0.99, 1.99, 2.99],
"dlist": [.1, 0.05, 0]
}
dataset_orig_train, dataset_orig_vt = dataset_orig.split([0.7],
shuffle=True)
metric_transf_train = BinaryLabelDatasetMetric(
dataset_orig_train,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
OP = OptimPreproc(OptTools,
optim_options,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
OP = OP.fit(dataset_orig_train)
dataset_transf_cat_test = OP.transform(dataset_orig_vt, transform_Y=True)
dataset_transf_cat_test = dataset_orig_vt.align_datasets(
dataset_transf_cat_test)
dataset_transf_cat_train = OP.transform(dataset_orig_train,
transform_Y=True)
dataset_transf_cat_train = dataset_orig_train.align_datasets(
dataset_transf_cat_train)
scale_transf = StandardScaler()
X_train = scale_transf.fit_transform(dataset_transf_cat_train.features)
y_train = dataset_transf_cat_train.labels.ravel()
X_test = scale_transf.fit_transform(dataset_transf_cat_test.features)
lmod = LogisticRegression()
lmod.fit(X_train, y_train)
y_pred = lmod.predict(X_test)
print('Without reweight')
print('Accuracy')
print(accuracy_score(dataset_orig_vt.labels, y_pred))
dataset_orig_vt_copy1 = dataset_orig_vt.copy()
dataset_orig_vt_copy1.labels = y_pred
metric_transf_train1 = BinaryLabelDatasetMetric(
dataset_orig_vt_copy1,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
print('p-rule')
print(metric_transf_train1.disparate_impact())
print('CV')
print(metric_transf_train1.mean_difference())
print('FPR for unpriv')
orig_sens_att = dataset_orig_vt.protected_attributes.ravel()
print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[
orig_sens_att == 0], 0))
print("FNR for unpriv")
print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[
orig_sens_att == 0], 0))
print('FPR for priv')
orig_sens_att = dataset_orig_vt.protected_attributes.ravel()
print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[
orig_sens_att == 1], 0))
print("FNR for priv")
print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[
orig_sens_att == 1], 0))
df_weight = dataset_orig_train.convert_to_dataframe()[0]
df_weight['weight'] = 1
df_weight['is_missing'] = 0
df_weight['tmp'] = ''
tmp_result = []
for i, j in zip(df_weight['race'], df_weight['two_year_recid']):
tmp_result.append(str(i) + str(j))
df_weight['tmp'] = tmp_result
df_weight.loc[df_weight['priors_count=missing'] == 1, 'is_missing'] = 1
for i in df_weight['tmp'].unique():
df_weight.loc[
(df_weight['tmp'] == i) & (df_weight['is_missing'] == 0),
'weight'] = len(
df_weight.loc[(df_weight['tmp'] == i), :].index) / len(
df_weight.loc[(df_weight['tmp'] == i) &
(df_weight['is_missing'] == 0), :].index)
df_weight.loc[(df_weight['tmp'] == i) & (df_weight['is_missing'] == 1),
'weight'] = len(df_weight.loc[
(df_weight['tmp'] == i) &
(df_weight['is_missing'] == 0), :].index) / len(
df_weight.loc[(df_weight['tmp'] == i), :].index)
dataset_orig_train.instance_weights = np.array(df_weight['weight'])
scale_transf = StandardScaler()
#X_train = scale_transf.fit_transform(dataset_transf_cat_train.features[:,1:])
X_train = scale_transf.fit_transform(dataset_transf_cat_train.features)
y_train = dataset_transf_cat_train.labels.ravel()
#X_test = scale_transf.fit_transform(dataset_transf_cat_test.features[:,1:])
X_test = scale_transf.fit_transform(dataset_transf_cat_test.features)
lmod = LogisticRegression()
lmod.fit(X_train,
y_train,
sample_weight=dataset_orig_train.instance_weights)
y_pred = lmod.predict(X_test)
print('With reweight')
print('Accuracy')
print(accuracy_score(dataset_orig_vt.labels, y_pred))
dataset_orig_vt_copy1 = dataset_orig_vt.copy()
dataset_orig_vt_copy1.labels = y_pred
metric_transf_train1 = BinaryLabelDatasetMetric(
dataset_orig_vt_copy1,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
print('p-rule')
print(metric_transf_train1.disparate_impact())
print('CV')
print(metric_transf_train1.mean_difference())
print('FPR for unpriv')
orig_sens_att = dataset_orig_vt.protected_attributes.ravel()
print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[
orig_sens_att == 0], 0))
print("FNR for unpriv")
print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[
orig_sens_att == 0], 0))
print('FPR for priv')
orig_sens_att = dataset_orig_vt.protected_attributes.ravel()
print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[
orig_sens_att == 1], 0))
print("FNR for priv")
print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[
orig_sens_att == 1], 0))
def train():
df = pd.read_csv('syn_train.csv')
df_test = pd.read_csv('syn_test.csv')
df_neg = df.loc[df['two_year_recid'] == 1, :]
df_neg_priv = df_neg.loc[(df_neg['two_year_recid'] == 1) &
(df_neg['race'] == 1), :]
df_neg_unpriv = df_neg.loc[(df_neg['two_year_recid'] == 1) &
(df_neg['race'] == 0), :]
df_neg = df.loc[df['two_year_recid'] == 1, :]
df_neg_priv = df_neg.loc[(df_neg['two_year_recid'] == 1) &
(df_neg['race'] == 1), :]
df_neg_unpriv = df_neg.loc[(df_neg['two_year_recid'] == 1) &
(df_neg['race'] == 0), :]
_, df_neg_priv_test = train_test_split(df_neg_priv,
test_size=1200,
random_state=1)
_, df_neg_unpriv_test = train_test_split(df_neg_unpriv,
test_size=2800,
random_state=1)
df_neg_test = df_neg_priv_test.append(df_neg_unpriv_test)
print('negative outcome, unpriv before resampling')
print(len(df_neg_unpriv_test.index))
print('negative outcome, priv before resampling')
print(len(df_neg_priv_test.index))
df_pos = df.loc[df['two_year_recid'] == 0, :]
df_pos_priv = df_pos.loc[(df_pos['two_year_recid'] == 0) &
(df_pos['race'] == 1), :]
df_pos_unpriv = df_pos.loc[(df_pos['two_year_recid'] == 0) &
(df_pos['race'] == 0), :]
_, df_pos_priv_test = train_test_split(df_pos_priv,
test_size=2000,
random_state=1)
_, df_pos_unpriv_test = train_test_split(df_pos_unpriv,
test_size=2000,
random_state=1)
df_pos_test = df_pos_priv_test.append(df_pos_unpriv_test)
df = df_pos_test.append(df_neg_test)
print('positive outcome, unpriv before resampling')
print(len(df_pos_unpriv_test.index))
print('positive outcome, priv before resampling')
print(len(df_pos_priv_test.index))
df = df_pos_test.append(df_neg_test)
N = 8000
df_result = pd.DataFrame()
# unif sampling
for i in df['two_year_recid'].unique():
for j in df['race'].unique():
orig_df = df.loc[(df['two_year_recid'] == i) &
(df['race'] == j), :]
real_count = len(orig_df.index)
exp_count = int(
(len(df.loc[(df['two_year_recid'] == i), :].index) /
len(df.index)) *
(len(df.loc[(df['race'] == j), :].index) / len(df.index)) * N)
if real_count >= exp_count:
_, df_toapp = train_test_split(orig_df,
test_size=exp_count,
random_state=10)
else:
df_toapp = resample(orig_df,
replace=True,
n_samples=exp_count - real_count,
random_state=10)
df_toapp = df_toapp.append(orig_df)
if len(df_result.index) == 0:
df_result = df_toapp.copy()
else:
df_result = df_result.append(df_toapp)
df = df_result
df_train = df
privileged_groups = [{'race': 1}]
unprivileged_groups = [{'race': 0}]
all_protected_attribute_maps = {"race": {0.0: 0, 1.0: 1}}
D_features = ['race']
dataset_orig_train = CustomDataset(
label_name='two_year_recid',
favorable_classes=[0],
protected_attribute_names=['race'],
privileged_classes=[[1]],
categorical_features=[
'priors_count', 'c_charge_degree', 'age_cat', 'score_text'
],
features_to_keep=[
'priors_count', 'c_charge_degree', 'race', 'age_cat', 'score_text'
],
df=df_train,
metadata={
'label_maps': [{
1: 'Did recid.',
0: 'No recid.'
}],
'protected_attribute_maps':
[all_protected_attribute_maps[x] for x in D_features]
})
dataset_orig_vt = CustomDataset(
label_name='two_year_recid',
favorable_classes=[0],
protected_attribute_names=['race'],
privileged_classes=[[1]],
categorical_features=[
'priors_count', 'c_charge_degree', 'age_cat', 'score_text'
],
features_to_keep=[
'priors_count', 'c_charge_degree', 'race', 'age_cat', 'score_text'
],
df=df_test,
metadata={
'label_maps': [{
1: 'Did recid.',
0: 'No recid.'
}],
'protected_attribute_maps':
[all_protected_attribute_maps[x] for x in D_features]
})
optim_options = {
"distortion_fun": get_distortion_compas,
"epsilon": 0.04,
"clist": [0.99, 1.99, 2.99],
"dlist": [.1, 0.05, 0]
}
metric_transf_train = BinaryLabelDatasetMetric(
dataset_orig_train,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
OP = OptimPreproc(OptTools,
optim_options,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
OP = OP.fit(dataset_orig_train)
dataset_transf_cat_test = OP.transform(dataset_orig_vt, transform_Y=True)
dataset_transf_cat_test = dataset_orig_vt.align_datasets(
dataset_transf_cat_test)
dataset_transf_cat_train = OP.transform(dataset_orig_train,
transform_Y=True)
dataset_transf_cat_train = dataset_orig_train.align_datasets(
dataset_transf_cat_train)
scale_transf = StandardScaler()
X_train = dataset_orig_train.features
y_train = dataset_orig_train.labels.ravel()
X_test = scale_transf.fit_transform(dataset_orig_vt.features)
lmod = LogisticRegression()
lmod.fit(X_train, y_train)
y_pred = lmod.predict(X_test)
print('With resampling')
print('Accuracy')
print(accuracy_score(dataset_orig_vt.labels, y_pred))
dataset_orig_vt_copy = dataset_orig_vt.copy()
dataset_orig_vt_copy.labels = y_pred
metric_transf_train = BinaryLabelDatasetMetric(
dataset_orig_vt_copy,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
print('p-rule')
print(metric_transf_train.disparate_impact())
print('CV')
print(metric_transf_train.mean_difference())
print('FPR for unpriv')
orig_sens_att = dataset_orig_vt.protected_attributes.ravel()
print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[
orig_sens_att == 0], 0))
print('FNR for unpriv')
print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[
orig_sens_att == 0], 0))
print('FPR for priv')
orig_sens_att = dataset_orig_vt.protected_attributes.ravel()
print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[
orig_sens_att == 1], 0))
print('FNR for priv')
print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[
orig_sens_att == 1], 0))
def train():
np.random.seed(10)
def quantizePrior1(x):
if x <= 0:
return 0
elif 1 <= x <= 3:
return 1
else:
return 2
def quantizeLOS(x):
if x <= 7:
return 0
if 8 < x <= 93:
return 1
else:
return 2
def group_race(x):
if x == "Caucasian":
return 1.0
else:
return 0.0
filepath = 'AIF360/aif360/data/raw/compas/compas-scores-two-years.csv'
df = pd.read_csv(filepath, index_col='id', na_values=[])
df['age_cat'] = df['age_cat'].replace('Greater than 45', 2)
df['age_cat'] = df['age_cat'].replace('25 - 45', 1)
df['age_cat'] = df['age_cat'].replace('Less than 25', 0)
df['score_text'] = df['score_text'].replace('High', 1)
df['score_text'] = df['score_text'].replace('Medium', 1)
df['score_text'] = df['score_text'].replace('Low', 0)
df['priors_count'] = df['priors_count'].apply(lambda x: quantizePrior1(x))
df['length_of_stay'] = (
pd.to_datetime(df['c_jail_out']) -
pd.to_datetime(df['c_jail_in'])).apply(lambda x: x.days)
df['length_of_stay'] = df['length_of_stay'].apply(lambda x: quantizeLOS(x))
df = df.loc[
~df['race'].isin(['Native American', 'Hispanic', 'Asian', 'Other']), :]
df['c_charge_degree'] = df['c_charge_degree'].replace({'F': 0, 'M': 1})
df1 = df[[
'priors_count', 'c_charge_degree', 'race', 'age_cat', 'score_text',
'two_year_recid'
]]
feature_list = []
for index, row in df1.iterrows():
feature_list.append('\t'.join(row.astype(str).to_list()))
df1['feature_list'] = feature_list
df3 = df1.groupby('feature_list').count() / len(df1.index)
df2 = pd.DataFrame()
df2['feature_list'] = list(df3.index)
df2['prob_list'] = list(df3.priors_count)
for index, row in df2.iterrows():
if row['feature_list'][0] == '0' and row['feature_list'][
-1] == '1' and 'African' in row['feature_list']:
row['prob_list'] = row['prob_list'] * 10
elif row['feature_list'][0] == '0' and row['feature_list'][-1] == '1':
row['prob_list'] = row['prob_list'] * 7
elif row['feature_list'][0] == '2' and row['feature_list'][-1] == '0':
row['prob_list'] = row['prob_list'] * 7
prob_list = list(df2.prob_list)
df_new = pd.DataFrame()
rng = np.random.default_rng()
prob_list = np.array(prob_list)
prob_list = prob_list / prob_list.sum()
feature_list = rng.choice(list(df2.feature_list),
len(df1.index),
p=prob_list)
var_list = [
'priors_count', 'c_charge_degree', 'race', 'age_cat', 'score_text',
'two_year_recid'
]
for i in var_list:
vars()[i] = []
for i in feature_list:
tmp = i.split('\t')
for j in range(len(var_list)):
vars()[var_list[j]].append(tmp[j])
for i in var_list:
df_new[i] = vars()[i]
df = df_new
df1 = df[[
'priors_count', 'c_charge_degree', 'race', 'age_cat', 'score_text',
'two_year_recid'
]]
tot = []
for index, row in df1.iterrows():
result = ''
for j in df1.columns:
result = result + str(row[j])
tot.append(result)
df['tmp_feature'] = tot
df['mis_prob'] = 0
for i in df['tmp_feature'].unique():
if 'African' in i and i[-1] == '0':
df.loc[df['tmp_feature'] == i, 'mis_prob'] = 0.1
elif 'African' in i:
df.loc[df['tmp_feature'] == i, 'mis_prob'] = 0.02
elif 'African' not in i and i[-1] == '0':
df.loc[df['tmp_feature'] == i, 'mis_prob'] = 0.02
else:
df.loc[df['tmp_feature'] == i, 'mis_prob'] = 0.02
new_label = []
for i, j in zip(df['mis_prob'], df['priors_count']):
if np.random.binomial(1, i, 1)[0] == 1:
new_label.append(3)
else:
new_label.append(j)
df['priors_count'] = new_label
print(len(df.loc[df['priors_count'] == 3, :].index))
print(len(df.index))
df['priors_count'] = df['priors_count'].astype(int)
df['score_text'] = df['score_text'].astype(int)
df['age_cat'] = df['age_cat'].astype(int)
df['score_text'] = df['score_text'].astype(int)
df['c_charge_degree'] = df['c_charge_degree'].astype(int)
df['two_year_recid'] = df['two_year_recid'].astype(int)
df['c_charge_degree'] = df['c_charge_degree'].replace({0: 'F', 1: 'M'})
def quantizePrior(x):
if x == 0:
return '0'
elif x == 1:
return '1 to 3'
elif x == 2:
return 'More than 3'
else:
return 'missing'
# Quantize length of stay
def quantizeLOS(x):
if x == 0:
return '<week'
if x == 1:
return '<3months'
else:
return '>3 months'
# Quantize length of stay
def adjustAge(x):
if x == 0:
return '25 to 45'
elif x == 1:
return 'Greater than 45'
elif x == 2:
return 'Less than 25'
def quantizeScore(x):
if x == 1:
return 'MediumHigh'
else:
return 'Low'
def group_race(x):
if x == "Caucasian":
return 1.0
else:
return 0.0
df['priors_count'] = df['priors_count'].apply(lambda x: quantizePrior(x))
df['score_text'] = df['score_text'].apply(lambda x: quantizeScore(x))
df['age_cat'] = df['age_cat'].apply(lambda x: adjustAge(x))
# Recode sex and race
df['race'] = df['race'].apply(lambda x: group_race(x))
df['race'] = df['race'].astype(int)
df['two_year_recid'] = df['two_year_recid'].astype(int)
df = df[[
'priors_count', 'c_charge_degree', 'race', 'age_cat', 'score_text',
'two_year_recid'
]]
df_train, df_test = train_test_split(df, test_size=0.3, random_state=10)
all_protected_attribute_maps = {"race": {0.0: 0, 1.0: 1}}
D_features = ['race']
dataset_orig_train = CustomDataset(
label_name='two_year_recid',
favorable_classes=[0],
protected_attribute_names=['race'],
privileged_classes=[[1]],
categorical_features=[
'priors_count', 'c_charge_degree', 'age_cat', 'score_text'
],
features_to_keep=[
'priors_count', 'c_charge_degree', 'race', 'age_cat', 'score_text'
],
df=df_train,
metadata={
'label_maps': [{
1: 'Did recid.',
0: 'No recid.'
}],
'protected_attribute_maps':
[all_protected_attribute_maps[x] for x in D_features]
})
dataset_orig_vt = CustomDataset(
label_name='two_year_recid',
favorable_classes=[0],
protected_attribute_names=['race'],
privileged_classes=[[1]],
categorical_features=[
'priors_count', 'c_charge_degree', 'age_cat', 'score_text'
],
features_to_keep=[
'priors_count', 'c_charge_degree', 'race', 'age_cat', 'score_text'
],
df=df_test,
metadata={
'label_maps': [{
1: 'Did recid.',
0: 'No recid.'
}],
'protected_attribute_maps':
[all_protected_attribute_maps[x] for x in D_features]
})
privileged_groups = [{'race': 1}]
unprivileged_groups = [{'race': 0}]
optim_options = {
"distortion_fun": get_distortion_compas,
"epsilon": 0.04,
"clist": [0.99, 1.99, 2.99],
"dlist": [.1, 0.05, 0]
}
metric_transf_train = BinaryLabelDatasetMetric(
dataset_orig_train,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
OP = OptimPreproc(OptTools,
optim_options,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
OP = OP.fit(dataset_orig_train)
dataset_transf_cat_test = OP.transform(dataset_orig_vt, transform_Y=True)
dataset_transf_cat_test = dataset_orig_vt.align_datasets(
dataset_transf_cat_test)
dataset_transf_cat_train = OP.transform(dataset_orig_train,
transform_Y=True)
dataset_transf_cat_train = dataset_orig_train.align_datasets(
dataset_transf_cat_train)
scale_transf = StandardScaler()
X_train = scale_transf.fit_transform(dataset_transf_cat_train.features)
y_train = dataset_transf_cat_train.labels.ravel()
X_test = scale_transf.fit_transform(dataset_transf_cat_test.features)
lmod = LogisticRegression()
lmod.fit(X_train, y_train)
y_pred = lmod.predict(X_test)
print('Without reweight')
print('Accuracy')
print(accuracy_score(dataset_orig_vt.labels, y_pred))
dataset_orig_vt_copy1 = dataset_orig_vt.copy()
dataset_orig_vt_copy1.labels = y_pred
metric_transf_train1 = BinaryLabelDatasetMetric(
dataset_orig_vt_copy1,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
print('p-rule')
print(metric_transf_train1.disparate_impact())
print('CV')
print(metric_transf_train1.mean_difference())
print('FPR for unpriv')
orig_sens_att = dataset_orig_vt.protected_attributes.ravel()
print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[
orig_sens_att == 0], 0))
print("FNR for unpriv")
print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[
orig_sens_att == 0], 0))
print('FPR for priv')
orig_sens_att = dataset_orig_vt.protected_attributes.ravel()
print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[
orig_sens_att == 1], 0))
print("FNR for priv")
print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[
orig_sens_att == 1], 0))
df_weight = dataset_orig_train.convert_to_dataframe()[0]
df_weight['weight'] = 1
df_weight['is_missing'] = 0
df_weight['tmp'] = ''
tmp_result = []
for i, j in zip(df_weight['race'], df_weight['two_year_recid']):
tmp_result.append(str(i) + str(j))
df_weight['tmp'] = tmp_result
df_weight.loc[df_weight['priors_count=missing'] == 1, 'is_missing'] = 1
for i in df_weight['tmp'].unique():
df_weight.loc[
(df_weight['tmp'] == i) & (df_weight['is_missing'] == 0),
'weight'] = len(
df_weight.loc[(df_weight['tmp'] == i), :].index) / len(
df_weight.loc[(df_weight['tmp'] == i) &
(df_weight['is_missing'] == 0), :].index)
df_weight.loc[(df_weight['tmp'] == i) & (df_weight['is_missing'] == 1),
'weight'] = len(df_weight.loc[
(df_weight['tmp'] == i) &
(df_weight['is_missing'] == 0), :].index) / len(
df_weight.loc[(df_weight['tmp'] == i), :].index)
dataset_orig_train.instance_weights = np.array(df_weight['weight'])
scale_transf = StandardScaler()
X_train = scale_transf.fit_transform(dataset_transf_cat_train.features)
y_train = dataset_transf_cat_train.labels.ravel()
X_test = scale_transf.fit_transform(dataset_transf_cat_test.features)
lmod = LogisticRegression()
lmod.fit(X_train,
y_train,
sample_weight=dataset_orig_train.instance_weights)
y_pred = lmod.predict(X_test)
print('With reweight')
print('Accuracy')
print(accuracy_score(dataset_orig_vt.labels, y_pred))
dataset_orig_vt_copy1 = dataset_orig_vt.copy()
dataset_orig_vt_copy1.labels = y_pred
metric_transf_train1 = BinaryLabelDatasetMetric(
dataset_orig_vt_copy1,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
print('p-rule')
print(metric_transf_train1.disparate_impact())
print('CV')
print(metric_transf_train1.mean_difference())
print('FPR for unpriv')
orig_sens_att = dataset_orig_vt.protected_attributes.ravel()
print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[
orig_sens_att == 0], 0))
print("FNR for unpriv")
print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[
orig_sens_att == 0], 0))
print('FPR for priv')
orig_sens_att = dataset_orig_vt.protected_attributes.ravel()
print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[
orig_sens_att == 1], 0))
print("FNR for priv")
print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[
orig_sens_att == 1], 0))
def get_fair_metrics(dataset, pred, pred_is_dataset=False):
"""
Measure fairness metrics.
Parameters:
dataset (pandas dataframe): Dataset
pred (array): Model predictions
pred_is_dataset, optional (bool): True if prediction is already part of the dataset, column name 'labels'.
Returns:
fair_metrics: Fairness metrics.
"""
if pred_is_dataset:
dataset_pred = pred
else:
dataset_pred = dataset.copy()
dataset_pred.labels = pred
cols = [
'statistical_parity_difference', 'equal_opportunity_difference',
'average_abs_odds_difference', 'disparate_impact', 'theil_index'
]
obj_fairness = [[0, 0, 0, 1, 0]]
fair_metrics = pd.DataFrame(data=obj_fairness,
index=['objective'],
columns=cols)
for attr in dataset_pred.protected_attribute_names:
idx = dataset_pred.protected_attribute_names.index(attr)
privileged_groups = [{
attr:
dataset_pred.privileged_protected_attributes[idx][0]
}]
unprivileged_groups = [{
attr:
dataset_pred.unprivileged_protected_attributes[idx][0]
}]
classified_metric = ClassificationMetric(
dataset,
dataset_pred,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
metric_pred = BinaryLabelDatasetMetric(
dataset_pred,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
acc = classified_metric.accuracy()
row = pd.DataFrame([[
metric_pred.mean_difference(),
classified_metric.equal_opportunity_difference(),
classified_metric.average_abs_odds_difference(),
metric_pred.disparate_impact(),
classified_metric.theil_index()
]],
columns=cols,
index=[attr])
fair_metrics = fair_metrics.append(row)
fair_metrics = fair_metrics.replace([-np.inf, np.inf], 2)
return fair_metrics
