def __init__(self, sensitive, repair_level, n_est = 100, min_sam_leaf = 25):
if repair_level<0:
repair_level = 0
elif repair_level>1:
repair_level = 1
self.model_reweight = DisparateImpactRemover(sensitive_attribute = sensitive, repair_level=repair_level)
self.model = RandomForestClassifier(n_estimators=n_est, min_samples_leaf=min_sam_leaf)
class Fair_DI_NN():
def __init__(self, sensitive, repair_level, inp_size, num_layers_y,
step_y):
if repair_level < 0:
repair_level = 0
elif repair_level > 1:
repair_level = 1
self.model_reweight = DisparateImpactRemover(
sensitive_attribute=sensitive, repair_level=repair_level)
self.model = FairClass(inp_size, num_layers_y, step_y)
def fit(self, data, labels, prot):
ds = BinaryLabelDataset(df=data,
label_names=labels,
protected_attribute_names=prot)
self.prot = prot
x = self.model_reweight.fit_transform(ds)
index = x.feature_names.index(prot[0])
x_train = np.delete(x.features, index, 1)
y_train = x.labels
x_train = torch.tensor(x_train).type('torch.FloatTensor')
y_train = torch.tensor(y_train).type('torch.FloatTensor')
self.model.fit(x_train, y_train)
def predict_proba(self, data_test):
x = self.model_reweight.fit_transform(data_test)
index = x.feature_names.index(self.prot[0])
x_test = np.delete(x.features, index, 1)
x_test = torch.tensor(x_test).type('torch.FloatTensor')
y = self.model.predict_proba(x_test)
return y
class Fair_DI_RF():
def __init__(self, sensitive, repair_level, n_est = 100, min_sam_leaf = 25):
if repair_level<0:
repair_level = 0
elif repair_level>1:
repair_level = 1
self.model_reweight = DisparateImpactRemover(sensitive_attribute = sensitive, repair_level=repair_level)
self.model = RandomForestClassifier(n_estimators=n_est, min_samples_leaf=min_sam_leaf)
def fit(self, data, labels, prot):
ds = BinaryLabelDataset(df = data, label_names = labels,
protected_attribute_names= prot)
self.prot = prot
x = self.model_reweight.fit_transform(ds)
index = x.feature_names.index(prot[0])
x_train = np.delete(x.features,index,1)
y_train = x.labels.ravel()
self.model.fit(x_train, y_train)
def predict_proba(self, data_test):
x = self.model_reweight.fit_transform(data_test)
index = x.feature_names.index(self.prot[0])
x_test = np.delete(x.features,index,1)
y = self.model.predict_proba(x_test)[:,1]
return y
def test_repair0():
ad = AdultDataset(protected_attribute_names=['sex'],
privileged_classes=[['Male']],
categorical_features=[],
features_to_keep=['age', 'education-num'])
di = DisparateImpactRemover(repair_level=0.)
ad_repd = di.fit_transform(ad)
assert ad_repd == ad
def __init__(self, sensitive, repair_level, inp_size, num_layers_y,
step_y):
if repair_level < 0:
repair_level = 0
elif repair_level > 1:
repair_level = 1
self.model_reweight = DisparateImpactRemover(
sensitive_attribute=sensitive, repair_level=repair_level)
self.model = FairClass(inp_size, num_layers_y, step_y)
def __init__(self, df, target_col, sensitive_att, repair_level):
"""
:param df: pandas dataframe, stores the data to fit the scaler.
:param target_col: str, the name of the target variable in above data.
:param target_positive_value: str, the value of above target variable that represents positive outcome. default is 1.
:param sensitive_att: str, the name of a sensitive attribute in above data. If none, call auto_detection to update. Value 0 represent protected.
"""
if repair_level is None or not isinstance(repair_level, float):
print(
"Input repair_level is not valid! Should be float within [0,1]!"
)
raise ValueError
else:
if repair_level < 0 or repair_level > 1:
print(
"Input repair_level is not valid! Should be float within [0,1]!"
)
raise ValueError
self.repair_level = repair_level
cur_step = DisparateImpactRemover(repair_level=repair_level,
sensitive_attribute=sensitive_att)
super().__init__("@".join(["AIF_DIRemover", sensitive_att]),
df,
step=cur_step,
fit_flag=False,
sensitive_att=sensitive_att,
target_col=target_col,
fair_aware=True)
def disparate_impact_remover(structured_data):
"""
Perform disparate impact removal from dataset and convert to pandas dataframe.
Parameters:
aif_standard_data (aif360.datasets.standard_dataset.StandardDataset): Structured dataset.
Returns:
data_transf_df (pandas dataframe): Pandas dataframe.
"""
DIR = DisparateImpactRemover()
data_transf = DIR.fit_transform(structured_data)
data_transf_df = convert_to_pd_dataframe(data_transf)
return data_transf_df
def train_svm_dir(training_data, C, gamma, keep_features, sensitive_attribute,
max_iter, svm_seed):
"""
Train the SVM classifier with Disparate Impact Remover preprocessing on specified data set,
with provided parameters, and calculate fitness scores.
:param training_data: The training data set to run the classifier on
:param C: The C parameter for SVC
:param gamma: The gamma parameter for SVC
:param keep_features: The features to keep for SVC
:param sensitive_attribute: The sensitive attribute in the dataset
:param max_iter: Max iterations for SVM
:param svm_seed: Seed used for RNG in SVM
:return: The trained classifier and the scaler
"""
dataset_orig_train = training_data
# Run Disparate Impact Remover
di = DisparateImpactRemover(repair_level=0.8,
sensitive_attribute=sensitive_attribute)
dataset_transf_train = di.fit_transform(dataset_orig_train)
# Prepare data
scale = StandardScaler()
X_train = scale.fit_transform(dataset_transf_train.features)
y_train = dataset_transf_train.labels.ravel()
w_train = dataset_transf_train.instance_weights
if len(keep_features) > 0: # If keep_features empty, use all features
X_train = X_train[:, keep_features]
# Train
clf = SVC(C=C,
gamma=gamma,
kernel='rbf',
probability=True,
max_iter=max_iter,
random_state=svm_seed)
clf.fit(X_train, y_train, sample_weight=w_train)
return clf, scale
def test_adult():
protected = 'sex'
ad = AdultDataset(protected_attribute_names=[protected],
privileged_classes=[['Male']],
categorical_features=[],
features_to_keep=[
'age', 'education-num', 'capital-gain',
'capital-loss', 'hours-per-week'
])
scaler = MinMaxScaler(copy=False)
# ad.features = scaler.fit_transform(ad.features)
train, test = ad.split([32561])
assert np.any(test.labels)
train.features = scaler.fit_transform(train.features)
test.features = scaler.transform(test.features)
index = train.feature_names.index(protected)
X_tr = np.delete(train.features, index, axis=1)
X_te = np.delete(test.features, index, axis=1)
y_tr = train.labels.ravel()
di = DisparateImpactRemover(repair_level=1.0)
train_repd = di.fit_transform(train)
# train_repd2 = di.fit_transform(train)
# assert train_repd == train_repd2
test_repd = di.fit_transform(test)
assert np.all(
train_repd.protected_attributes == train.protected_attributes)
lmod = LogisticRegression(class_weight='balanced')
# lmod = SVM(class_weight='balanced')
lmod.fit(X_tr, y_tr)
test_pred = test.copy()
test_pred.labels = lmod.predict(X_te)
X_tr_repd = np.delete(train_repd.features, index, axis=1)
X_te_repd = np.delete(test_repd.features, index, axis=1)
y_tr_repd = train_repd.labels.ravel()
assert (y_tr == y_tr_repd).all()
lmod.fit(X_tr_repd, y_tr_repd)
test_repd_pred = test_repd.copy()
test_repd_pred.labels = lmod.predict(X_te_repd)
p = [{protected: 1}]
u = [{protected: 0}]
cm = ClassificationMetric(test,
test_pred,
privileged_groups=p,
unprivileged_groups=u)
before = cm.disparate_impact()
# print('Disparate impact: {:.4}'.format(before))
# print('Acc overall: {:.4}'.format(cm.accuracy()))
repaired_cm = ClassificationMetric(test_repd,
test_repd_pred,
privileged_groups=p,
unprivileged_groups=u)
after = repaired_cm.disparate_impact()
# print('Disparate impact: {:.4}'.format(after))
# print('Acc overall: {:.4}'.format(repaired_cm.accuracy()))
assert after > before
assert abs(1 - after) <= 0.2
def runWysiwygDIAnalysis(fname="wysiwygdata.csv",
trainwithA=True,
xcolumns=["X1", "X2", "X3", "X4", "X5", "X6"]):
''' run the full experiment, based on a parameters like train classifier with A (sensitive attribute)'''
c_columns = ["C1", "C2", "C3", "C4", "C5", "C6"]
x_columns = xcolumns
wysiwygframe = prepareWysiwygData(fname=fname, c_columns=c_columns)
LR = LogisticRegression(solver='liblinear', random_state=1)
SVM = SVC(random_state=1, gamma='scale')
RF = RandomForestClassifier(n_estimators=100, random_state=1)
DIRmodels = [LR, SVM, RF]
DIR = DisparateImpactRemover(sensitive_attribute='A')
dirname = type(DIR).__name__
wysiwygpoints = {}
for model in DIRmodels:
modelname = type(model).__name__
wysiwygpoints[modelname] = {}
wysiwygpoints[modelname]['PRE'] = {}
wysiwygpoints[modelname]['POST'] = {}
wysiwygpoints[modelname]['PRE'][dirname] = {}
wysiwygpoints[modelname]['PRE'][dirname]['ACC'] = 0.0
wysiwygpoints[modelname]['PRE'][dirname]['ACCA0'] = 0.0
wysiwygpoints[modelname]['PRE'][dirname]['ACCA1'] = 0.0
wysiwygpoints[modelname]['PRE'][dirname]['FAIR'] = 0.0
wysiwygpoints[modelname]['POST'][dirname] = {}
wysiwygpoints[modelname]['POST'][dirname]['ACC'] = 0.0
wysiwygpoints[modelname]['POST'][dirname]['ACCA0'] = 0.0
wysiwygpoints[modelname]['POST'][dirname]['ACCA1'] = 0.0
wysiwygpoints[modelname]['POST'][dirname]['FAIR'] = 0.0
for model in DIRmodels:
modelname = type(model).__name__
print("####### {} #########\n".format(modelname))
preaccs = trainWysiwygDataFromFrame(wysiwygframe,
model,
x_columns=x_columns,
trainwithA=trainwithA)
wysiwygpoints[modelname]['PRE'][dirname]['ACC'] = preaccs[1]
wysiwygpoints[modelname]['PRE'][dirname]['ACCA0'] = preaccs[2]
wysiwygpoints[modelname]['PRE'][dirname]['ACCA1'] = preaccs[3]
dirdata = checkClassifierFairnessAndRemoveDI(preaccs[4],
x_columns=x_columns,
dpoints=wysiwygpoints,
mname=modelname,
verbose=True,
pre=True)
print("\t######### POST {} ###########".format(dirname))
postaccs = trainWysiwygDataFromFrame(dirdata,
model,
x_columns=x_columns,
trainwithA=trainwithA)
wysiwygpoints[modelname]['POST'][dirname]['ACC'] = postaccs[1]
wysiwygpoints[modelname]['POST'][dirname]['ACCA0'] = postaccs[2]
wysiwygpoints[modelname]['POST'][dirname]['ACCA1'] = postaccs[3]
checkClassifierFairnessAndRemoveDI(postaccs[4],
x_columns=x_columns,
dpoints=wysiwygpoints,
mname=modelname,
verbose=True,
pre=False)
return wysiwygpoints
def checkClassifierFairnessAndRemoveDI(frame,
dpoints,
mname,
x_columns,
verbose=True,
pre=True):
''' Measure fairness according to the metric using the value of A and the classification outcome.
Results get added to a dictionary used to pass them to a function to generate graphs of the results.
If we have not performed intervention, perform intervention and return post intervention data.'''
xay_columns = copy.deepcopy(x_columns)
xay_columns.extend(["A", "Y"])
ycols = copy.deepcopy(frame["Y"])
tempframe = copy.deepcopy(frame)
tempframe.drop(["Y"], axis=1, inplace=True)
aifdf = BinaryLabelDataset(favorable_label=1.0,
unfavorable_label=0.0,
df=tempframe,
label_names=['Ya'],
protected_attribute_names=['A'])
privileged_groups = [{'A': 1}]
unprivileged_groups = [{'A': 0}]
DIR = DisparateImpactRemover(sensitive_attribute='A')
metric_aifdf_train = BinaryLabelDatasetMetric(
aifdf,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
if pre:
if verbose:
print("\n\tINTERVENTION: {}\n".format(type(DIR).__name__))
print("\t######### PRE {} ###########".format(type(DIR).__name__))
print(
"\tDisparate impact between unprivileged and privileged groups = {}\n"
.format(metric_aifdf_train.disparate_impact()))
dpoints[mname]['PRE'][type(
DIR).__name__]['FAIR'] = metric_aifdf_train.disparate_impact()
print("PRE CLASSIFICATION MATRIX")
print("----------------")
print(" |Y'=0 | Y'=1 |")
print("----------------")
print("A=0| {0} | {1} |".format(
metric_aifdf_train.num_negatives(False),
metric_aifdf_train.num_positives(False)))
print("A=1| {0} | {1} |".format(
metric_aifdf_train.num_negatives(True),
metric_aifdf_train.num_positives(True)))
print("----------------")
dataset_transf_train = DIR.fit_transform(aifdf)
fairdf = dataset_transf_train.convert_to_dataframe()[0]
fairdf.drop(['Ya'], axis=1, inplace=True)
ycols.reset_index(drop=True, inplace=True)
fairdf.reset_index(drop=True, inplace=True)
fairdf.insert(0, "Y", ycols)
fairdf[xay_columns] = fairdf[xay_columns]
return fairdf
else:
if verbose:
print(
"\tDisparate impact between unprivileged and privileged groups = {}\n"
.format(metric_aifdf_train.disparate_impact()))
dpoints[mname]['POST'][type(
DIR).__name__]['FAIR'] = metric_aifdf_train.disparate_impact()
print("POST CLASSIFICATION MATRIX")
print("----------------")
print(" |Y'=0 | Y'=1|")
print("----------------")
print("A=0| {0} | {1} |".format(
metric_aifdf_train.num_negatives(False),
metric_aifdf_train.num_positives(False)))
print("A=1| {0} | {1} |".format(
metric_aifdf_train.num_negatives(True),
metric_aifdf_train.num_positives(True)))
print("----------------")
return frame
dataset_orig_train.instance_weights.sum()) < 1e-6
elif m == "lfr":
TR = LFR(unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
TR = TR.fit(dataset_orig_train)
dataset_transf_train = TR.transform(dataset_orig_train, threshold=0.8)
out = dataset_transf_train.convert_to_dataframe(de_dummy_code=True,
sep='=',
set_category=True)[0]
elif m == "disp_impact_remover":
# Test if scaling changes something --> but then also export a scaled test set
# scaler = MinMaxScaler(copy=False)
di = DisparateImpactRemover(repair_level=1, sensitive_attribute='AGE')
dataset_transf_train = di.fit_transform(dataset_orig_train)
out_train = dataset_transf_train.convert_to_dataframe(
de_dummy_code=True, sep='=', set_category=True)[0]
# valid classification
dataset_transf_valid = di.fit_transform(dataset_orig_valid)
out_valid = dataset_transf_valid.convert_to_dataframe(
de_dummy_code=True, sep='=', set_category=True)[0]
# test classification
dataset_transf_test = di.fit_transform(dataset_orig_test)
out_test = dataset_transf_test.convert_to_dataframe(
de_dummy_code=True, sep='=', set_category=True)[0]
metric_transf_train = BinaryLabelDatasetMetric(
def svm_dir(training_data, test_data, fairness_metric, accuracy_metric, C,
gamma, keep_features, privileged_groups, unprivileged_groups,
max_iter, svm_seed):
"""
Run SVM classifier with Disparate Impact Remover preprocessing on specified data set,
with provided parameters, and calculate fitness scores.
:param training_data: The training data set to run the classifier on
:param test_data: The test data set to test the classifier on
:param fairness_metric: The fairness metric to calculate
:param accuracy_metric: The accuracy metric to calculate
:param C: The C parameter for SVC
:param gamma: The gamma parameter for SVC
:param keep_features: The features to keep for SVC
:param privileged_groups: The privileged group in the data set
:param unprivileged_groups: The unprivileged group in the data set
:param max_iter: Max iterations for SVM
:param svm_seed: Seed used for RNG in SVM
:return: Return the accuracy and fairness score for the classifier
"""
dataset_orig_train, dataset_orig_test = training_data, test_data
# Run Disparate Impact Remover
sensitive_attribute = list(privileged_groups[0].keys())[0]
di = DisparateImpactRemover(repair_level=0.8,
sensitive_attribute=sensitive_attribute)
dataset_transf_train = di.fit_transform(dataset_orig_train)
# Prepare data
scale = StandardScaler()
X_train = scale.fit_transform(dataset_transf_train.features)
y_train = dataset_transf_train.labels.ravel()
w_train = dataset_transf_train.instance_weights
dataset_transf_test_pred = dataset_orig_test.copy(deepcopy=True)
X_test = scale.fit_transform(dataset_transf_test_pred.features)
if len(keep_features) > 0: # If keep_features empty, use all features
X_train = X_train[:, keep_features]
X_test = X_test[:, keep_features]
# Train
clf = SVC(C=C,
gamma=gamma,
kernel='rbf',
probability=True,
max_iter=max_iter,
random_state=svm_seed)
clf.fit(X_train, y_train, sample_weight=w_train)
# Test
pos_ind = np.where(clf.classes_ == dataset_orig_train.favorable_label)[0][
0] # positive class index
dataset_transf_test_pred.scores = clf.predict_proba(
X_test)[:, pos_ind].reshape(-1, 1)
# Assign labels
fav_inds = dataset_transf_test_pred.scores > 0.5
dataset_transf_test_pred.labels[
fav_inds] = dataset_transf_test_pred.favorable_label
dataset_transf_test_pred.labels[
~fav_inds] = dataset_transf_test_pred.unfavorable_label
# Calculate metrics
cm = ClassificationMetric(dataset_orig_test,
dataset_transf_test_pred,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
accuracy_score = accuracy_metric(cm)
fairness_score = fairness_metric(cm)
return accuracy_score, fairness_score
def pre_process(self, annotated_data, privileged_groups,
unprivileged_groups):
disparate_impact_remover = DisparateImpactRemover(
repair_level=self.repair_level)
return disparate_impact_remover.fit_transform(annotated_data)