def calc_mean_diff(data, target_variable, protected_variable, unprivileged_input): df_aif = BinaryLabelDataset(df=data, label_names=[target_variable], protected_attribute_names=[protected_variable]) privileged_group = [] for v in data[protected_variable].unique()[data[protected_variable].unique() != unprivileged_input]: privileged_group.append({protected_variable: v}) unprivileged_group = [{protected_variable: unprivileged_input}] #female=0 metric_orig = BinaryLabelDatasetMetric(df_aif, unprivileged_group, privileged_group) print(metric_orig.mean_difference().round(3)) if abs(metric_orig.mean_difference().round(3)) < 0.2: print('The algorithm can be considered to be not biased') else: print('There is a potential bias')
def calculate_bias_measures(data_orig_train, data_orig_vt, unprivileged_groups, privileged_groups): model = RandomForestClassifier().fit( data_orig_train.features, data_orig_train.labels.ravel(), sample_weight=data_orig_train.instance_weights) dataset = data_orig_vt dataset_pred = dataset.copy() dataset_pred.labels = model.predict(data_orig_vt.features) classified_metric_race = ClassificationMetric( dataset, dataset_pred, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) metric_pred_race = BinaryLabelDatasetMetric( dataset_pred, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) print("Mean difference {}".format(metric_pred_race.mean_difference())) print("Disparate Metric {}".format(metric_pred_race.disparate_impact())) print("Equal Opportunity Difference {}".format( classified_metric_race.equal_opportunity_difference())) print("Average Abs Odds Difference {}".format( classified_metric_race.average_abs_odds_difference())) print("Theil index {}".format(classified_metric_race.theil_index()))
def get_metrics(dataset_train): metric_orig_train = BinaryLabelDatasetMetric( dataset_train, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) #print("Difference in mean outcomes between unprivileged and privileged groups = %f" % metric_orig_train.mean_difference()) return metric_orig_train.mean_difference()
def mean_diff_values(data, target_variable, protected_variable, unprivileged_input): df_aif = BinaryLabelDataset(df=data, label_names=[target_variable], protected_attribute_names=[protected_variable]) privileged_group = [] for v in data[protected_variable].unique()[data[protected_variable].unique() != unprivileged_input]: privileged_group.append({protected_variable: v}) unprivileged_group = [{protected_variable: unprivileged_input}] #female=0 metric_orig = BinaryLabelDatasetMetric(df_aif, unprivileged_group, privileged_group) return abs(metric_orig.mean_difference().round(3))
def main(): print('Calculate bias') np.random.seed(1) protected_attribute = 'ethnicity' dataset = load_preproc_data_heart([protected_attribute]) privileged_groups = [{protected_attribute: 1}] unprivileged_groups = [{ protected_attribute: 2 }, { protected_attribute: 3 }, { protected_attribute: 4 }, { protected_attribute: 5 }, { protected_attribute: 6 }] data_orig_train, data_orig_vt = dataset.split([0.7], shuffle=True) data_orig_valid, data_orig_test = data_orig_vt.split([0.5], shuffle=True) metric_orig_train = BinaryLabelDatasetMetric( data_orig_train, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) print("Mean {}".format(metric_orig_train.mean_difference())) rw = Reweighing(unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) data_transf_train = rw.fit_transform(data_orig_train) metric_transf_train = BinaryLabelDatasetMetric( data_transf_train, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) print("Mean difference after transformation =%f " % metric_transf_train.mean_difference()) calculate_bias_measures(data_orig_train, data_orig_vt, unprivileged_groups, privileged_groups) calculate_bias_measures(data_orig_valid, data_orig_test, unprivileged_groups, privileged_groups)
def fair_metrics(dataset, pred, pred_is_dataset=False): if pred_is_dataset: dataset_pred = pred else: dataset_pred = dataset.copy() dataset_pred.labels = pred cols = [ 'statistical_parity_difference', 'equal_opportunity_difference', 'average_abs_odds_difference', 'disparate_impact', 'theil_index' ] obj_fairness = [[0, 0, 0, 1, 0]] fair_metrics = pd.DataFrame(data=obj_fairness, index=['objective'], columns=cols) for attr in dataset_pred.protected_attribute_names: idx = dataset_pred.protected_attribute_names.index(attr) privileged_groups = [{ attr: dataset_pred.privileged_protected_attributes[idx][0] }] unprivileged_groups = [{ attr: dataset_pred.unprivileged_protected_attributes[idx][0] }] classified_metric = ClassificationMetric( dataset, dataset_pred, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) metric_pred = BinaryLabelDatasetMetric( dataset_pred, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) acc = classified_metric.accuracy() row = pd.DataFrame([[ metric_pred.mean_difference(), classified_metric.equal_opportunity_difference(), classified_metric.average_abs_odds_difference(), metric_pred.disparate_impact(), classified_metric.theil_index() ]], columns=cols, index=[attr]) fair_metrics = fair_metrics.append(row) fair_metrics = fair_metrics.replace([-np.inf, np.inf], 2) return fair_metrics
def get_dataset_metrics_list(binary_dataset_list): #Set privileged and unprivileged groups privileged_groups= [{'sex':1}] unprivileged_groups= [{'sex': 0}] mean_diff_list = [] disp_imp_list = [] for dataset in binary_dataset_list: metrics = BinaryLabelDatasetMetric(dataset, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) mean_diff_list.append(metrics.mean_difference()) disp_imp_list.append(1 - metrics.disparate_impact()) return mean_diff_list, disp_imp_list
def nondebiased_classifier(train, test, privileged_groups, unprivileged_groups): sess = tf.Session() NN_model = AdversarialDebiasing(privileged_groups, unprivileged_groups, scope_name='nondebiased_classifier', debias=False, sess=sess) NN_model.fit(train) # predict outcome using the test set pred_NNmodel = NN_model.predict(test) sess.close() tf.reset_default_graph() # calculate accuracy accuracy = accuracy_score(y_true=test.labels, y_pred=pred_NNmodel.labels) # calculate fairness metrics metric_test = BinaryLabelDatasetMetric( pred_NNmodel, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) acc_test = ClassificationMetric(test, pred_NNmodel, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) equal_opportunity_difference = equal_opp_diff(test, pred_NNmodel, 'sex', privileged=1, unprivileged=0, favourable=1, unfavourable=0) average_odds_difference = avg_odds_diff(test, pred_NNmodel, 'sex', privileged=1, unprivileged=0, favourable=1, unfavourable=0) metrics = [ metric_test.mean_difference(), acc_test.disparate_impact(), equal_opportunity_difference, average_odds_difference, acc_test.theil_index() ] return pred_NNmodel, accuracy, metrics
def ensemble(test, pred_adversarial, pred_prejudice, pred_nondebiased, unprivileged_groups, privileged_groups): pred_labels = [] for i in range(0, len(test.features)): arr = mode([ pred_adversarial.labels[i], pred_prejudice.labels[i], pred_nondebiased.labels[i] ]) pred_labels.append(arr[0][0]) pred_ensemble = test.copy() pred_ensemble.labels = np.array(pred_labels) accuracy = accuracy_score(y_true=test.labels, y_pred=pred_ensemble.labels) metric_test = BinaryLabelDatasetMetric( pred_ensemble, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) acc_test = ClassificationMetric(test, pred_ensemble, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) equal_opportunity_difference = equal_opp_diff(test, pred_ensemble, 'sex', privileged=1, unprivileged=0, favourable=1, unfavourable=0) average_odds_difference = avg_odds_diff(test, pred_ensemble, 'sex', privileged=1, unprivileged=0, favourable=1, unfavourable=0) metrics = [ metric_test.mean_difference(), acc_test.disparate_impact(), equal_opportunity_difference, average_odds_difference, acc_test.theil_index() ] return accuracy, metrics
def prejudice(train, test, unprivileged_groups, privileged_groups): prejudice_model = PrejudiceRemover(eta=100, sensitive_attr='sex') prejudice_model.fit(train) # predict outcome using the test set pred_prejudice = prejudice_model.predict(test) # calculate accuracy accuracy = accuracy_score(y_true=test.labels, y_pred=pred_prejudice.labels) # calculate fairness metrics metric_test = BinaryLabelDatasetMetric( pred_prejudice, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) acc_test = ClassificationMetric(test, pred_prejudice, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) equal_opportunity_difference = equal_opp_diff(test, pred_prejudice, 'sex', privileged=1, unprivileged=0, favourable=1, unfavourable=0) average_odds_difference = avg_odds_diff(test, pred_prejudice, 'sex', privileged=1, unprivileged=0, favourable=1, unfavourable=0) if acc_test.disparate_impact() == math.inf: disparate_impact = 5.0 else: disparate_impact = acc_test.disparate_impact() metrics = [ metric_test.mean_difference(), disparate_impact, equal_opportunity_difference, average_odds_difference, acc_test.theil_index() ] return pred_prejudice, accuracy, metrics
def plot_using_aif(df_predict,df_true): predict_list, true_list = [], [] unpriv_label_list , priv_label_list = [], [] for (u,p) in zip(unpriv_list,priv_list): cur_predict, cur_true = [], [] unpriv_label = '+'.join(['-'.join([prot_attr_dict[key][u_el[key]] for key in u_el]) for u_el in u]) priv_label = '+'.join(['-'.join([prot_attr_dict[key][p_el[key]] for key in p_el]) for p_el in p]) print('-------------------------------------------------------------------') print('unpriv_label:-->',unpriv_label) print('-------------------------------------------------------------------') print('priv_label :-->',priv_label) print('-------------------------------------------------------------------') print('\n\n') for i,label in enumerate(rating_names): #print('Fairness Metric for the label------>',label.upper()) predict_dataset = StandardDataset(df=predict_df_list[i], label_name=label, favorable_classes=[1.0,1.0], protected_attribute_names=protected_attribute_names, privileged_classes=privileged_classes) true_dataset = StandardDataset(df=true_df_list[i], label_name=label, favorable_classes=[1.0,1.0], protected_attribute_names=protected_attribute_names, privileged_classes=privileged_classes) predict_dataset_metric = BinaryLabelDatasetMetric(predict_dataset, unprivileged_groups=u, privileged_groups=p) true_dataset_metric = BinaryLabelDatasetMetric(true_dataset, unprivileged_groups=u, privileged_groups=p) #classfication_metric = ClassificationMetric(true_dataset, predict_dataset, unprivileged_groups=u, privileged_groups=p) #x=classfication_metric.generalized_entropy_index() #print(label,': -->','predicted : -->',abs(predict_dataset_metric.disparate_impact()),'true : -->',abs(true_dataset_metric.disparate_impact())) print(label,': -->','predicted : -->',abs(predict_dataset_metric.mean_difference()),'true : -->',abs(true_dataset_metric.mean_difference()))
def compute_metrics(model, X_test, y_test, X_train, y_train, dataset_test, dataset_name, model_name, unprivileged_groups, privileged_groups, position): """ Calculate and return: model accuracy and fairness metrics Parameters ---------- model: scikit-learn classifier X_test: numpy 2d array y_test: numpy 1d array X_train: numpy 2d array y_train: numpy 1d array dataset_test: aif360.datasets.BinaryLabelDataset dataset_name: string Dataset name used in the analysis model_name: string unprivileged_groups: list<dict> Dictionary where the key is the name of the sensitive column in the dataset, and the value is the value of the unprivileged group in the dataset privileged_groups: list<dict> Dictionary where the key is the name of the sensitive column in the dataset, and the value is the value of the privileged group in the dataset position: int Column position of the sensitive group in the dataset """ y_pred_test = model.predict(X_test) acc_test = accuracy_score(y_true=y_test, y_pred=y_pred_test) print("Test accuracy: ", acc_test) y_pred_train = model.predict(X_train) acc_train = accuracy_score(y_true=y_train, y_pred=y_pred_train) print("Train accuracy: ", acc_train) dataset_pred = dataset_test.copy() dataset_pred.labels = y_pred_test bin_metric = BinaryLabelDatasetMetric( dataset_pred, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) disparate_impact_bin = bin_metric.disparate_impact() print('Disparate impact: ', disparate_impact_bin) mean_difference = bin_metric.mean_difference() print('Mean difference: ', mean_difference) classif_metric = ClassificationMetric( dataset_test, dataset_pred, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) classif_disparete_impact = classif_metric.disparate_impact() avg_odds = classif_metric.average_odds_difference() print('Average odds difference:', avg_odds) equal_opport = classif_metric.equal_opportunity_difference() print('Equality of opportunity:', equal_opport) false_discovery_rate = classif_metric.false_discovery_rate_difference() print('False discovery rate difference:', false_discovery_rate) entropy_index = classif_metric.generalized_entropy_index() print('Generalized entropy index:', entropy_index) cons_comp = consitency_mod(bin_metric, position, n_neighbors=5) print('Consistency: ', cons_comp) result = (dataset_name, model_name, acc_test, disparate_impact_bin, mean_difference, classif_disparete_impact, avg_odds, equal_opport, false_discovery_rate, entropy_index, cons_comp) return result
def train(): privileged_groups = [{'race': 1}] unprivileged_groups = [{'race': 0}] dataset_orig = load_preproc_data_compas(['race']) optim_options = { "distortion_fun": get_distortion_compas, "epsilon": 0.05, "clist": [0.99, 1.99, 2.99], "dlist": [.1, 0.05, 0] } dataset_orig_train, dataset_orig_vt = dataset_orig.split([0.7], shuffle=True) metric_transf_train = BinaryLabelDatasetMetric( dataset_orig_train, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) OP = OptimPreproc(OptTools, optim_options, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) OP = OP.fit(dataset_orig_train) dataset_transf_cat_test = OP.transform(dataset_orig_vt, transform_Y=True) dataset_transf_cat_test = dataset_orig_vt.align_datasets( dataset_transf_cat_test) dataset_transf_cat_train = OP.transform(dataset_orig_train, transform_Y=True) dataset_transf_cat_train = dataset_orig_train.align_datasets( dataset_transf_cat_train) scale_transf = StandardScaler() X_train = scale_transf.fit_transform(dataset_transf_cat_train.features) y_train = dataset_transf_cat_train.labels.ravel() X_test = scale_transf.fit_transform(dataset_transf_cat_test.features) lmod = LogisticRegression() lmod.fit(X_train, y_train) y_pred = lmod.predict(X_test) print('Without reweight') print('Accuracy') print(accuracy_score(dataset_orig_vt.labels, y_pred)) dataset_orig_vt_copy1 = dataset_orig_vt.copy() dataset_orig_vt_copy1.labels = y_pred metric_transf_train1 = BinaryLabelDatasetMetric( dataset_orig_vt_copy1, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) print('p-rule') print(metric_transf_train1.disparate_impact()) print('CV') print(metric_transf_train1.mean_difference()) print('FPR for unpriv') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[ orig_sens_att == 0], 0)) print("FNR for unpriv") print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[ orig_sens_att == 0], 0)) print('FPR for priv') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[ orig_sens_att == 1], 0)) print("FNR for priv") print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[ orig_sens_att == 1], 0)) df_weight = dataset_orig_train.convert_to_dataframe()[0] df_weight['weight'] = 1 df_weight['is_missing'] = 0 df_weight['tmp'] = '' tmp_result = [] for i, j in zip(df_weight['race'], df_weight['two_year_recid']): tmp_result.append(str(i) + str(j)) df_weight['tmp'] = tmp_result df_weight.loc[df_weight['priors_count=missing'] == 1, 'is_missing'] = 1 for i in df_weight['tmp'].unique(): df_weight.loc[ (df_weight['tmp'] == i) & (df_weight['is_missing'] == 0), 'weight'] = len( df_weight.loc[(df_weight['tmp'] == i), :].index) / len( df_weight.loc[(df_weight['tmp'] == i) & (df_weight['is_missing'] == 0), :].index) df_weight.loc[(df_weight['tmp'] == i) & (df_weight['is_missing'] == 1), 'weight'] = len(df_weight.loc[ (df_weight['tmp'] == i) & (df_weight['is_missing'] == 0), :].index) / len( df_weight.loc[(df_weight['tmp'] == i), :].index) dataset_orig_train.instance_weights = np.array(df_weight['weight']) scale_transf = StandardScaler() #X_train = scale_transf.fit_transform(dataset_transf_cat_train.features[:,1:]) X_train = scale_transf.fit_transform(dataset_transf_cat_train.features) y_train = dataset_transf_cat_train.labels.ravel() #X_test = scale_transf.fit_transform(dataset_transf_cat_test.features[:,1:]) X_test = scale_transf.fit_transform(dataset_transf_cat_test.features) lmod = LogisticRegression() lmod.fit(X_train, y_train, sample_weight=dataset_orig_train.instance_weights) y_pred = lmod.predict(X_test) print('With reweight') print('Accuracy') print(accuracy_score(dataset_orig_vt.labels, y_pred)) dataset_orig_vt_copy1 = dataset_orig_vt.copy() dataset_orig_vt_copy1.labels = y_pred metric_transf_train1 = BinaryLabelDatasetMetric( dataset_orig_vt_copy1, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) print('p-rule') print(metric_transf_train1.disparate_impact()) print('CV') print(metric_transf_train1.mean_difference()) print('FPR for unpriv') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[ orig_sens_att == 0], 0)) print("FNR for unpriv") print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[ orig_sens_att == 0], 0)) print('FPR for priv') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[ orig_sens_att == 1], 0)) print("FNR for priv") print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[ orig_sens_att == 1], 0))
def checkClassifierFairnessAndReweighData(frame, dpoints, mname, x_columns, verbose=True, pre=True): ''' Measure fairness according to the metric using the value of A and the classification outcome. Results get added to a dictionary used to pass them to a function to generate graphs of the results. If we have not performed intervention, perform intervention and return post intervention data.''' xay_columns = copy.deepcopy(x_columns) xay_columns.extend(["A", "Y"]) ycols = copy.deepcopy(frame["Y"]) tempframe = copy.deepcopy(frame) tempframe.drop(["Y"], axis=1, inplace=True) aifdf = BinaryLabelDataset(favorable_label=1.0, unfavorable_label=0.0, df=tempframe, label_names=['Ya'], protected_attribute_names=['A']) privileged_groups = [{'A': 1}] unprivileged_groups = [{'A': 0}] RW = Reweighing(unprivileged_groups=[{ 'A': 0 }], privileged_groups=[{ 'A': 1 }]) metric_aifdf_train = BinaryLabelDatasetMetric( aifdf, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) if pre: if verbose: print("\n\tINTERVENTION: {}\n".format(type(RW).__name__)) print("\t######### PRE {} ###########".format(type(RW).__name__)) print( "\tDifference in mean outcomes between unprivileged and privileged groups = {}\n" .format(metric_aifdf_train.mean_difference())) dpoints[mname]['PRE'][type( RW).__name__]['FAIR'] = metric_aifdf_train.mean_difference() print("PRE CLASSIFICATION MATRIX") print("----------------") print(" |Y'=0 | Y'=1 |") print("----------------") print("A=0| {0} | {1} |".format( metric_aifdf_train.num_negatives(False), metric_aifdf_train.num_positives(False))) print("A=1| {0} | {1} |".format( metric_aifdf_train.num_negatives(True), metric_aifdf_train.num_positives(True))) print("----------------") dataset_transf_train = RW.fit_transform(aifdf) fairdf = dataset_transf_train.convert_to_dataframe()[0] fairdf.drop(['Ya'], axis=1, inplace=True) ycols.reset_index(drop=True, inplace=True) fairdf.reset_index(drop=True, inplace=True) fairdf.insert(0, "Y", ycols) fairdf[xay_columns] = fairdf[xay_columns].astype(int) fairdf.insert(loc=len(fairdf.columns), column="weights", value=dataset_transf_train.instance_weights) return fairdf else: if verbose: print( "\tDifference in mean outcomes between unprivileged and privileged groups = {}\n" .format(metric_aifdf_train.mean_difference())) dpoints[mname]['POST'][type( RW).__name__]['FAIR'] = metric_aifdf_train.mean_difference() print("POST CLASSIFICATION MATRIX") print("----------------") print(" |Y'=0 | Y'=1 |") print("----------------") print("A=0| {0} | {1} |".format( metric_aifdf_train.num_negatives(False), metric_aifdf_train.num_positives(False))) print("A=1| {0} | {1} |".format( metric_aifdf_train.num_negatives(True), metric_aifdf_train.num_positives(True))) print("----------------") return frame
def compute_metrics(model, X_test, y_test, X_train, y_train, dataset_test, unprivileged_groups, privileged_groups, protect_attribute, print_result): """ Calculate and return: model accuracy and fairness metrics Parameters ---------- model: scikit-learn classifier X_test: numpy 2d array y_test: numpy 1d array X_train: numpy 2d array y_train: numpy 1d array dataset_test: aif360.datasets.BinaryLabelDataset unprivileged_groups: list<dict> Dictionary where the key is the name of the sensitive column in the dataset, and the value is the value of the unprivileged group in the dataset privileged_groups: list<dict> Dictionary where the key is the name of the sensitive column in the dataset, and the value is the value of the privileged group in the dataset protect_attribute print_result """ result = {} y_pred_test = model.predict(X_test) result['acc_test'] = accuracy_score(y_true=y_test, y_pred=y_pred_test) y_pred_train = model.predict(X_train) result['acc_train'] = accuracy_score(y_true=y_train, y_pred=y_pred_train) dataset_pred = dataset_test.copy() dataset_pred.labels = y_pred_test bin_metric = BinaryLabelDatasetMetric(dataset_pred, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) result['disp_impact'] = bin_metric.disparate_impact() result['stat_parity'] = bin_metric.mean_difference() classif_metric = ClassificationMetric(dataset_test, dataset_pred, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) result['avg_odds'] = classif_metric.average_odds_difference() result['equal_opport'] = classif_metric.equal_opportunity_difference() result['false_discovery_rate'] = classif_metric.false_discovery_rate_difference() result['entropy_index'] = classif_metric.generalized_entropy_index() result['acc_test_clf'] = classif_metric.accuracy(privileged=None) result['acc_test_priv'] = classif_metric.accuracy(privileged=True) result['acc_test_unpriv'] = classif_metric.accuracy(privileged=False) result['consistency'] = consitency(X_test, y_pred_test, protect_attribute, n_neighbors=5) result['counterfactual'] = counterfactual(X_test, model, protect_attribute) if print_result: print("Train accuracy: ", result['acc_train']) print("Test accuracy: ", result['acc_test']) print("Test accuracy clf: ", result['acc_test_clf']) print("Test accuracy priv.: ", result['acc_test_priv']) print("Test accuracy unpriv.: ", result['acc_test_unpriv']) print('Disparate impact: ', result['disp_impact']) print('Mean difference: ', result['stat_parity']) print('Average odds difference:', result['avg_odds']) print('Equality of opportunity:', result['equal_opport']) print('False discovery rate difference:', result['false_discovery_rate']) print('Generalized entropy index:', result['entropy_index']) print('Consistency: ', result['consistency']) print('Counterfactual fairness: ', result['counterfactual']) return result
test_biased = BinaryLabelDataset(df=pd.concat((x_test, y_test), axis=1), label_names=['medium_to_high_risk'], protected_attribute_names=['sex_Male'], favorable_label=1, unfavorable_label=0) # Create the metric object for training set metric_train_biased = BinaryLabelDatasetMetric(train_biased, unprivileged_groups=sex_unprivileged_groups, privileged_groups=sex_privileged_groups) display(Markdown("#### Original training dataset")) print("Sex biased - Difference in mean outcomes between unprivileged and privileged sex groups = %f" % metric_train_biased.mean_difference()) # Create the metric object for testing set metric_test_biased = BinaryLabelDatasetMetric(test_biased, unprivileged_groups=sex_unprivileged_groups, privileged_groups=sex_privileged_groups) display(Markdown("#### Original training dataset")) print("Sex biased - Difference in mean outcomes between unprivileged and privileged sex groups = %f" % metric_test_biased.mean_difference()) #debias with the reweighing method RW = Reweighing(unprivileged_groups=sex_unprivileged_groups, privileged_groups=sex_privileged_groups) RW.fit(train_biased) dataset_transf_train_f = RW.fit_transform(train_biased)
def train(request): df = pd.read_csv('./training/resume_data_5000.csv') df = df.drop(df.columns[0], axis=1) dataset_orig = StandardDataset(df, label_name='Accepted', favorable_classes=[1], protected_attribute_names=['Gender'], privileged_classes=[[1]], categorical_features=['School'], features_to_drop=['Name']) dataset_orig_train, dataset_orig_vt = dataset_orig.split([0.7], shuffle=True) dataset_orig_valid, dataset_orig_test = dataset_orig_vt.split([0.5], shuffle=True) privileged_groups = [{'Gender': 1}] unprivileged_groups = [{'Gender': 0}] metric_orig_train = BinaryLabelDatasetMetric( dataset_orig_train, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) orig_mean_difference = metric_orig_train.mean_difference() with open('./training/orig_mean_difference.pkl', 'wb') as f: pickle.dump(orig_mean_difference, f) RW = Reweighing(unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) dataset_transf_train = RW.fit_transform(dataset_orig_train) metric_transf_train = BinaryLabelDatasetMetric( dataset_transf_train, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) transf_mean_difference = metric_transf_train.mean_difference() with open('./training/transf_mean_difference.pkl', 'wb') as f: pickle.dump(transf_mean_difference, f) # Logistic regression classifier and predictions scale_orig = StandardScaler() X_train = scale_orig.fit_transform(dataset_orig_train.features) y_train = dataset_orig_train.labels.ravel() w_train = dataset_orig_train.instance_weights.ravel() with open('./training/scaler.pkl', 'wb') as f: pickle.dump(scale_orig, f) lmod_orig = LogisticRegression(solver='lbfgs') lmod_orig.fit(X_train, y_train, sample_weight=dataset_orig_train.instance_weights) y_train_pred = lmod_orig.predict(X_train) pos_ind = np.where( lmod_orig.classes_ == dataset_orig_train.favorable_label)[0][0] dataset_orig_train_pred = dataset_orig_train.copy() dataset_orig_train_pred.labels = y_train_pred dataset_orig_valid_pred = dataset_orig_valid.copy(deepcopy=True) X_valid = scale_orig.transform(dataset_orig_valid_pred.features) y_valid = dataset_orig_valid_pred.labels dataset_orig_valid_pred.scores = lmod_orig.predict_proba( X_valid)[:, pos_ind].reshape(-1, 1) dataset_orig_test_pred = dataset_orig_test.copy(deepcopy=True) X_test = scale_orig.transform(dataset_orig_test_pred.features) y_test = dataset_orig_test_pred.labels dataset_orig_test_pred.scores = lmod_orig.predict_proba( X_test)[:, pos_ind].reshape(-1, 1) num_thresh = 100 ba_arr = np.zeros(num_thresh) class_thresh_arr = np.linspace(0.01, 0.99, num_thresh) for idx, class_thresh in enumerate(class_thresh_arr): fav_inds = dataset_orig_valid_pred.scores > class_thresh dataset_orig_valid_pred.labels[ fav_inds] = dataset_orig_valid_pred.favorable_label dataset_orig_valid_pred.labels[ ~fav_inds] = dataset_orig_valid_pred.unfavorable_label classified_metric_orig_valid = ClassificationMetric( dataset_orig_valid, dataset_orig_valid_pred, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) ba_arr[idx] = 0.5*(classified_metric_orig_valid.true_positive_rate()\ +classified_metric_orig_valid.true_negative_rate()) best_ind = np.where(ba_arr == np.max(ba_arr))[0][0] best_class_thresh = class_thresh_arr[best_ind] bal_acc_arr_orig = [] disp_imp_arr_orig = [] avg_odds_diff_arr_orig = [] for thresh in tqdm(class_thresh_arr): fav_inds = dataset_orig_test_pred.scores > thresh dataset_orig_test_pred.labels[ fav_inds] = dataset_orig_test_pred.favorable_label dataset_orig_test_pred.labels[ ~fav_inds] = dataset_orig_test_pred.unfavorable_label metric_test_bef = compute_metrics(dataset_orig_test, dataset_orig_test_pred, unprivileged_groups, privileged_groups, disp=False) if thresh == best_class_thresh: with open('./training/metrics_orig.pkl', 'wb') as f: pickle.dump(metric_test_bef, f, protocol=pickle.HIGHEST_PROTOCOL) bal_acc_arr_orig.append(metric_test_bef["Balanced accuracy"]) avg_odds_diff_arr_orig.append( metric_test_bef["Average odds difference"]) disp_imp_arr_orig.append(metric_test_bef["Disparate impact"]) scale_transf = StandardScaler() X_train = scale_transf.fit_transform(dataset_transf_train.features) y_train = dataset_transf_train.labels.ravel() lmod_transf = LogisticRegression(solver='lbfgs') lmod_transf.fit(X_train, y_train, sample_weight=dataset_transf_train.instance_weights) y_train_pred = lmod_transf.predict(X_train) dataset_transf_test_pred = dataset_orig_test.copy(deepcopy=True) X_test = scale_transf.fit_transform(dataset_transf_test_pred.features) y_test = dataset_transf_test_pred.labels dataset_transf_test_pred.scores = lmod_transf.predict_proba( X_test)[:, pos_ind].reshape(-1, 1) bal_acc_arr_transf = [] disp_imp_arr_transf = [] avg_odds_diff_arr_transf = [] for thresh in tqdm(class_thresh_arr): fav_inds = dataset_transf_test_pred.scores > thresh dataset_transf_test_pred.labels[ fav_inds] = dataset_transf_test_pred.favorable_label dataset_transf_test_pred.labels[ ~fav_inds] = dataset_transf_test_pred.unfavorable_label metric_test_aft = compute_metrics(dataset_orig_test, dataset_transf_test_pred, unprivileged_groups, privileged_groups, disp=False) if thresh == best_class_thresh: with open('./training/metrics_transf.pkl', 'wb') as f: pickle.dump(metric_test_aft, f, protocol=pickle.HIGHEST_PROTOCOL) bal_acc_arr_transf.append(metric_test_aft["Balanced accuracy"]) avg_odds_diff_arr_transf.append( metric_test_aft["Average odds difference"]) disp_imp_arr_transf.append(metric_test_aft["Disparate impact"]) with open('./training/model_orig.pkl', 'wb') as f: pickle.dump(lmod_orig, f) with open('./training/model_transf.pkl', 'wb') as f: pickle.dump(lmod_transf, f) return HttpResponse('Model trained')
new_dataset = BinaryLabelDataset(favorable_label=favorable_label, unfavorable_label=unfavorable_label, df=new_b_train, label_names=['Attrition'], protected_attribute_names=['Gender'], unprivileged_protected_attributes=unprivileged_groups) # In[ ]: metric_orig_train = BinaryLabelDatasetMetric(new_dataset, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) display(Markdown("#### Original training dataset")) print("Difference in mean outcomes between unprivileged and privileged groups = %f" % metric_orig_train.mean_difference()) # In[ ]: new_dataset.protected_attribute_names # In[ ]: RW = Reweighing(unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) RW.fit(new_dataset)
def train(): privileged_groups = [{'race': 1}] unprivileged_groups = [{'race': 0}] dataset_orig_train = load_preproc_data_compas(['race']) optim_options = { "distortion_fun": get_distortion_compas, "epsilon": 0.1, "clist": [0.99, 1.99, 2.99], "dlist": [.1, 0.05, 0] } # dataset_orig_train, dataset_orig_vt = dataset_orig.split([0.7], shuffle=True) metric_transf_train = BinaryLabelDatasetMetric(dataset_orig_train, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) OP = OptimPreproc(OptTools, optim_options, unprivileged_groups = unprivileged_groups, privileged_groups = privileged_groups) OP = OP.fit(dataset_orig_train) dataset_transf_cat_test = OP.transform(dataset_orig_vt, transform_Y = True) dataset_transf_cat_test = dataset_orig_vt.align_datasets(dataset_transf_cat_test) dataset_transf_cat_train = OP.transform(dataset_orig_train, transform_Y = True) dataset_transf_cat_train = dataset_orig_train.align_datasets(dataset_transf_cat_train) scale_transf = StandardScaler() X_train = scale_transf.fit_transform(dataset_transf_cat_train.features) y_train = dataset_transf_cat_train.labels.ravel() X_test = scale_transf.fit_transform(dataset_transf_cat_test.features) lmod = LogisticRegression() lmod.fit(X_train, y_train) y_pred = lmod.predict(X_test) print('Without applying fixing algorithms') print('Accuracy') print(accuracy_score(dataset_orig_vt.labels, y_pred)) dataset_orig_vt_copy1 = dataset_orig_vt.copy() dataset_orig_vt_copy1.labels = y_pred metric_transf_train1 = BinaryLabelDatasetMetric(dataset_orig_vt_copy1, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) print('p-rule') print(metric_transf_train1.disparate_impact()) print('CV') print(metric_transf_train1.mean_difference()) print('FPR for unpriv') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1-TNR(dataset_orig_vt.labels.ravel()[orig_sens_att==0],y_pred[orig_sens_att==0],0)) print("FNR for unpriv") print(1-TPR(dataset_orig_vt.labels.ravel()[orig_sens_att==0],y_pred[orig_sens_att==0],0)) print('FPR for priv') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1-TNR(dataset_orig_vt.labels.ravel()[orig_sens_att==1],y_pred[orig_sens_att==1],0)) print("FNR for priv") print(1-TPR(dataset_orig_vt.labels.ravel()[orig_sens_att==1],y_pred[orig_sens_att==1],0))
def main() -> None: # read from inventor filepath = ait_input.get_inventory_path('Data') # prepare column names as given by german.doc column_names = [ 'status', 'month', 'credit_history', 'purpose', 'credit_amount', 'savings', 'employment', 'investment_as_income_percentage', 'personal_status', 'other_debtors', 'residence_since', 'property', 'age', 'installment_plans', 'housing', 'number_of_credits', 'skill_level', 'people_liable_for', 'telephone', 'foreign_worker', 'credit' ] # load into a dataframe df = data_loading(filepath=filepath, column_names=column_names, na_values=None) # prepare for mappings mappings = { 'label_maps': [{ 1.0: 'Good Credit', 2.0: 'Bad Credit' }], 'protected_attribute_maps': [{ 1.0: 'Male', 0.0: 'Female' }, { 1.0: 'Old', 0.0: 'Young' }] } # prepare for categorical features categorical_features = [ 'status', 'credit_history', 'purpose', 'savings', 'employment', 'other_debtors', 'property', 'installment_plans', 'housing', 'skill_level', 'telephone', 'foreign_worker' ] # load param protected_attribute = ait_input.get_method_param_value( 'protected_attribute') privileged_classes = ait_input.get_method_param_value('privileged_classes') # input check ait_input_check(protected_attribute, privileged_classes) # prepare for structure from dataframe and edit data features setting dataset = StandardDataset( df=df, label_name='credit', favorable_classes=[1], protected_attribute_names=[protected_attribute], privileged_classes=[lambda x: x >= privileged_classes], instance_weights_name=None, categorical_features=categorical_features, features_to_keep=None, features_to_drop=['personal_status', 'sex'], na_values=None, custom_preprocessing=preprocessing, metadata=mappings) # set two variables for the privileged (1) and unprivileged (0) values for the age attribute. privileged_groups = [{protected_attribute: 1}] unprivileged_groups = [{protected_attribute: 0}] # compute fairness metric on original training dataset metric_fairness = BinaryLabelDatasetMetric( dataset, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) print("Original training dataset: German Credit Data") print( "Difference in mean outcomes between unprivileged and privileged groups = %f" % metric_fairness.mean_difference()) print("unprivileged groups = %f" % metric_fairness.base_rate(privileged=False)) print("privileged groups = %f" % metric_fairness.base_rate(privileged=True)) # resource observed_predicted_plot save_metric_fairness_plot(metric_fairness, protected_attribute) # measures measure_mean_difference(metric_fairness.mean_difference()) # ait.log move_log()
def run_trial(): # stores each run (4 algos without reweighing, 4 algos with reweighing) as a sublist # number of sublists = number of runs # each sublist has four elements # we ONLY predict on the testing data ########## WITHOUT REWEIGHING ############# stat_par = [] disp_imp = [] eq_opp_diff = [] avg_odds_diff = [] theil = [] acc = [] ########## WITH REWEIGHING ############# stat_par_reweigh = [] disp_imp_reweigh = [] eq_opp_diff_reweigh = [] avg_odds_diff_reweigh = [] theil_reweigh = [] acc_reweigh = [] ########################################### for i in range(10): ########################################### privileged_groups = [{'sex': 1}] unprivileged_groups = [{'sex': 0}] dataset_orig = load_preproc_data_adult() # train1, test1 are the original dataset train1, test1 = dataset_orig.split([0.7], shuffle=True) RW = Reweighing(unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) RW.fit(train1) # dataset_transf_train, test1 are for the reweighed dataset dataset_transf_train = RW.transform(train1) ########################################### # change weights to whole numbers for i in range(dataset_transf_train.instance_weights.size): dataset_transf_train.instance_weights[i] = (round( dataset_transf_train.instance_weights[i] / 0.1) * 0.1) * 10 weights = copy.deepcopy(dataset_transf_train.instance_weights) # change dataset_transf_train.features and dataset_transf_train.labels and dataset_transf_train.protected_attributes according to the weights of each instance sum_weights = 0 for i in range(dataset_transf_train.features.shape[0]): row = copy.deepcopy(dataset_transf_train.features[i]) row_label = copy.deepcopy(dataset_transf_train.labels[i]) row_protected_attributes = copy.deepcopy( dataset_transf_train.protected_attributes[i]) row_protected_attributes.resize(1, 2) row.resize(1, 18) row_label.resize(1, 1) weight = int(weights[i]) for j in range(weight - 1): dataset_transf_train.features = np.concatenate( (dataset_transf_train.features, row)) dataset_transf_train.labels = np.concatenate( (dataset_transf_train.labels, row_label)) dataset_transf_train.protected_attributes = np.concatenate( (dataset_transf_train.protected_attributes, row_protected_attributes)) # change the dataset_transf_train to a numpy array of ones to match number of rows in features dataset_transf_train.instance_weights = np.ones( dataset_transf_train.features.shape[0]) ################## without reweighing ########################## temp_stat_par = [] temp_disp_imp = [] temp_eq_opp_diff = [] temp_avg_odds_diff = [] temp_theil = [] temp_acc = [] ##################### adversarial debiasing ##################### sess = tf.Session() debiased_model = AdversarialDebiasing( privileged_groups=privileged_groups, unprivileged_groups=unprivileged_groups, scope_name='debiased_classifier', debias=True, sess=sess) debiased_model.fit(train1) dataset_debiasing_test = debiased_model.predict(test1) sess.close() tf.reset_default_graph() ##################### metrics ##################### metric_test = BinaryLabelDatasetMetric( dataset_debiasing_test, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) acc_test = ClassificationMetric( test1, dataset_debiasing_test, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) temp_stat_par.append(metric_test.mean_difference()) temp_disp_imp.append(metric_test.disparate_impact()) temp_eq_opp_diff.append(metric_test.equal_opportunity_difference()) temp_avg_odds_diff.append(metric_test.average_odds_difference()) temp_theil.append(metric_test.theil_index()) temp_acc.append(acc_test.accuracy()) ##################### prejudice remover ##################### prejudice_model = PrejudiceRemover(eta=100, sensitive_attr='sex') prejudice_model.fit(train1) dataset_prejudice_test = prejudice_model.predict(test1) ##################### metrics ##################### metric_test = BinaryLabelDatasetMetric( dataset_prejudice_test, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) acc_test = ClassificationMetric( test1, dataset_prejudice_test, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) temp_stat_par.append(metric_test.mean_difference()) temp_disp_imp.append(metric_test.disparate_impact()) temp_eq_opp_diff.append(metric_test.equal_opportunity_difference()) temp_avg_odds_diff.append(metric_test.average_odds_difference()) temp_theil.append(metric_test.theil_index()) temp_acc.append(acc_test.accuracy()) ##################### normal neural net ##################### sess = tf.Session() neural_model = AdversarialDebiasing( privileged_groups=privileged_groups, unprivileged_groups=unprivileged_groups, scope_name='debiased_classifier', debias=False, sess=sess) neural_model.fit(train1) dataset_neural_test = neural_model.predict(test1) sess.close() tf.reset_default_graph() ##################### metrics ##################### metric_test = BinaryLabelDatasetMetric( dataset_neural_test, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) acc_test = ClassificationMetric( test1, dataset_neural_test, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) temp_stat_par.append(metric_test.mean_difference()) temp_disp_imp.append(metric_test.disparate_impact()) temp_eq_opp_diff.append(metric_test.equal_opportunity_difference()) temp_avg_odds_diff.append(metric_test.average_odds_difference()) temp_theil.append(metric_test.theil_index()) temp_acc.append(acc_test.accuracy()) ##################### ensemble ##################### pred_labels_test = [] for i in range(0, len(test1.features)): arr_test = mode([ dataset_debiasing_test[i], dataset_prejudice_test[i], dataset_neural_test[i] ]) pred_labels_test.append(arr_test[0][0]) dataset_ensemble_test = test1.copy() dataset_ensemble_test.labels = np.array(pred_labels_test) ##################### metrics ##################### metric_test = BinaryLabelDatasetMetric( dataset_ensemble_train, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) acc_test = ClassificationMetric( test1, dataset_ensemble_train, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) temp_stat_par.append(metric_test.mean_difference()) temp_disp_imp.append(metric_test.disparate_impact()) temp_eq_opp_diff.append(metric_test.equal_opportunity_difference()) temp_avg_odds_diff.append(metric_test.average_odds_difference()) temp_theil.append(metric_test.theil_index()) temp_acc.append(acc_test.accuracy()) ######### DUMP SHIT ########### stat_par.append(temp_stat_par) disp_imp.append(temp_disp_imp) eq_opp_diff.append(temp_eq_opp_diff) avg_odds_diff.append(temp_avg_odds_diff) theil.append(temp_theil) acc.append(temp_acc) ################## with reweighing ########################## temp_stat_par = [] temp_disp_imp = [] temp_eq_opp_diff = [] temp_avg_odds_diff = [] temp_theil = [] temp_acc = [] ################## adversarial debiasing ################## sess = tf.Session() debiased_model_reweighing = AdversarialDebiasing( privileged_groups=privileged_groups, unprivileged_groups=unprivileged_groups, scope_name='debiased_classifier', debias=True, sess=sess) debiased_model_reweighing.fit(dataset_transf_train) dataset_debiasing_test_reweighing = debiased_model_reweighing.predict( test1) sess.close() tf.reset_default_graph() ##################### metrics ##################### metric_test = BinaryLabelDatasetMetric( dataset_debiasing_test_reweighing, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) acc_test = ClassificationMetric( test1, dataset_debiasing_test_reweighing, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) temp_stat_par.append(metric_test.mean_difference()) temp_disp_imp.append(metric_test.disparate_impact()) temp_eq_opp_diff.append(metric_test.equal_opportunity_difference()) temp_avg_odds_diff.append(metric_test.average_odds_difference()) temp_theil.append(metric_test.theil_index()) temp_acc.append(acc_test.accuracy()) ##################### prejudice remover ##################### prejudice_model_reweighing = PrejudiceRemover(eta=100, sensitive_attr='sex') prejudice_model_reweighing.fit(dataset_transf_train) dataset_prejudice_test_reweighing = prejudice_model_reweighing.predict( test1) ##################### metrics ##################### metric_test = BinaryLabelDatasetMetric( dataset_prejudice_test_reweighing, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) acc_test = ClassificationMetric( test1, dataset_prejudice_test_reweighing, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) temp_stat_par.append(metric_test.mean_difference()) temp_disp_imp.append(metric_test.disparate_impact()) temp_eq_opp_diff.append(metric_test.equal_opportunity_difference()) temp_avg_odds_diff.append(metric_test.average_odds_difference()) temp_theil.append(metric_test.theil_index()) temp_acc.append(acc_test.accuracy()) ##################### normal neural net ##################### sess = tf.Session() neural_model = AdversarialDebiasing( privileged_groups=privileged_groups, unprivileged_groups=unprivileged_groups, scope_name='debiased_classifier', debias=False, sess=sess) neural_model.fit(dataset_transf_train) dataset_neural_test = neural_model.predict(test1) sess.close() tf.reset_default_graph() ##################### metrics ##################### metric_test = BinaryLabelDatasetMetric( dataset_neural_test, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) acc_test = ClassificationMetric( test1, dataset_neural_test, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) temp_stat_par.append(metric_test.mean_difference()) temp_disp_imp.append(metric_test.disparate_impact()) temp_eq_opp_diff.append(metric_test.equal_opportunity_difference()) temp_avg_odds_diff.append(metric_test.average_odds_difference()) temp_theil.append(metric_test.theil_index()) temp_acc.append(acc_test.accuracy()) ##################### ensemble ##################### pred_labels_test = [] for i in range(0, len(test1.features)): arr_test = mode([ dataset_debiasing_test[i], dataset_prejudice_test[i], dataset_neural_test[i] ]) pred_labels_test.append(arr_test[0][0]) dataset_ensemble_test = test1.copy() dataset_ensemble_test.labels = np.array(pred_labels_test) ##################### metrics ##################### metric_test = BinaryLabelDatasetMetric( dataset_ensemble_test, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) acc_test = ClassificationMetric( test1, dataset_ensemble_test, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) temp_stat_par.append(metric_test.mean_difference()) temp_disp_imp.append(metric_test.disparate_impact()) temp_eq_opp_diff.append(metric_test.equal_opportunity_difference()) temp_avg_odds_diff.append(metric_test.average_odds_difference()) temp_theil.append(metric_test.theil_index()) temp_acc.append(acc_test.accuracy()) ######### DUMP SHIT ########### stat_par_reweigh.append(temp_stat_par) disp_imp_reweigh.append(temp_disp_imp) eq_opp_diff_reweigh.append(temp_eq_opp_diff) avg_odds_diff_reweigh.append(temp_avg_odds_diff) theil_reweigh.append(temp_theil) acc_reweigh.append(temp_acc) without_reweighing = [ stat_par, disp_imp, eq_opp_diff, avg_odds_diff, theil, acc ] with_reweighing = [ stat_par_reweigh, disp_imp_reweigh, eq_opp_diff_reweigh, avg_odds_diff_reweigh, theil_reweigh, acc_reweigh ] for metric in range(len(without_reweighing)): name = "metric" + str(metric) sublist = without_reweighing[metric] with open(name, "wb") as csv_file: writer = csv.writer(csv_file) writer.writerows(sublist) for metric in range(len(with_reweighing)): name = "metric" + str(metric) + "reweigh" sublist = with_reweighing[metric] with open(name, "wb") as csv_file: writer = csv.writer(csv_file) writer.writerows(sublist)
def main(): import sys sys.path.insert(1, "../") import numpy as np np.random.seed(0) #pip install numba==0.43.0 #pip install --ignore-installed llvmlite==0.32.1 from aif360.datasets import GermanDataset from aif360.metrics import BinaryLabelDatasetMetric, ClassificationMetric as CM from aif360.algorithms.preprocessing import Reweighing from IPython.display import Markdown, display from sklearn.ensemble import RandomForestClassifier as RF from sklearn.datasets import make_classification as mc from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score #Step 2 Load dataset, specifying protected attribute, and split dataset into train and test dataset_orig = GermanDataset( protected_attribute_names=[ 'age' ], # this dataset also contains protected attribute for "sex" # which we do not consider in this evaluation privileged_classes=[lambda x: x >= 25 ], # age >=25 is considered privileged features_to_drop=['personal_status', 'sex'] # ignore sex-related attributes ) dataset_orig_train, dataset_orig_test = dataset_orig.split([0.7], shuffle=True) dataset_orig_test_pred = dataset_orig_test.copy(deepcopy=True) privileged_groups = [{'age': 1}] unprivileged_groups = [{'age': 0}] #Step 3 Compute fairness metric on original training dataset metric_orig_train = BinaryLabelDatasetMetric( dataset_orig_train, #mean difference unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) display(Markdown("#### Original training dataset")) print( "Difference in mean outcomes between unprivileged and privileged groups = %f. AKA the privileged group is getting .17 more positive outcomes in the training dataset." % metric_orig_train.mean_difference()) # print() #metrics clf = RF() clf.fit(dataset_orig_train.features, dataset_orig_train.labels) predictions = clf.predict(dataset_orig_test.features) proba_predictions = clf.predict_proba(dataset_orig_test.features) dataset_orig_test_pred.scores = proba_predictions[:, 0].reshape(-1, 1) dataset_orig_test_pred.labels = predictions.reshape(-1, 1) cm_pred_valid = CM(dataset_orig_test, dataset_orig_test_pred, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) cm = ["precision", "recall", "accuracy"] metrics = {} for c in cm: metric = eval("cm_pred_valid." + c + "()") metrics[c] = metric metrics["recall"], metrics["accuracy"], metrics["precision"] print("AIF360 metrics") for key in ["recall", "accuracy", "precision"]: print("{} score is: {}".format(key, metrics[key])) #Step 4 Mitigate bias by transforming the original dataset RW = Reweighing( unprivileged_groups= unprivileged_groups, #pre-processing mitigation algorithm privileged_groups=privileged_groups) dataset_transf_train = RW.fit_transform(dataset_orig_train) #Step 5 Compute fairness metric on transformed dataset metric_transf_train = BinaryLabelDatasetMetric( dataset_transf_train, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) display(Markdown("#### Transformed training dataset")) print( "Difference in mean outcomes between unprivileged and privileged groups = %f" % metric_transf_train.mean_difference()) # #metrics #split dataset_transf_train, dataset_transf_test = dataset_transf_train.split( [0.7], shuffle=True) dataset_transf_test_pred = dataset_transf_test.copy(deepcopy=True) clf = RF() clf.fit(dataset_transf_train.features, dataset_transf_train.labels) predictions = clf.predict(dataset_transf_test.features) proba_predictions = clf.predict_proba(dataset_transf_test.features) dataset_transf_test_pred.scores = proba_predictions[:, 0].reshape(-1, 1) dataset_transf_test_pred.labels = predictions.reshape(-1, 1) cm_pred_valid = CM(dataset_transf_test, dataset_transf_test_pred, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) cm = ["precision", "recall", "accuracy"] metrics = {} for c in cm: metric = eval("cm_pred_valid." + c + "()") metrics[c] = metric metrics["recall"], metrics["accuracy"], metrics["precision"] print("AIF360 metrics") for key in ["recall", "accuracy", "precision"]: print("{} score is: {}".format(key, metrics[key]))
de_dummy_code=True, sep='=', set_category=True)[0] # valid classification dataset_transf_valid = di.fit_transform(dataset_orig_valid) out_valid = dataset_transf_valid.convert_to_dataframe( de_dummy_code=True, sep='=', set_category=True)[0] # test classification dataset_transf_test = di.fit_transform(dataset_orig_test) out_test = dataset_transf_test.convert_to_dataframe( de_dummy_code=True, sep='=', set_category=True)[0] metric_transf_train = BinaryLabelDatasetMetric( dataset_transf_train, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) print( m + " achieved a statistical parity difference between unprivileged and privileged groups = %f" % metric_transf_train.mean_difference()) out_train.to_csv(output_path + 'taiwan_pre_train_' + m + '.csv', index=None, header=True) out_valid.to_csv(output_path + 'taiwan_pre_valid_' + m + '.csv', index=None, header=True) out_test.to_csv(output_path + 'taiwan_pre_test_' + m + '.csv', index=None, header=True)
sess = tf.Session() debiased_model = AdversarialDebiasing(privileged_groups = privileged_groups, unprivileged_groups = unprivileged_groups, scope_name='debiased_classifier', debias=True, sess=sess) debiased_model.fit(train1) dataset_debiasing_train = debiased_model.predict(train1) dataset_debiasing_test = debiased_model.predict(test1) metric_debiasing_train = BinaryLabelDatasetMetric(dataset_debiasing_train, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) metric_debiasing_test = BinaryLabelDatasetMetric(dataset_debiasing_test, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) metric_dataset_debiasing_train.append(metric_debiasing_train.mean_difference()) metric_dataset_debiasing_test.append(metric_debiasing_test.mean_difference()) sess.close() tf.reset_default_graph() sess = tf.Session() debiased_model = AdversarialDebiasing(privileged_groups = privileged_groups, unprivileged_groups = unprivileged_groups, scope_name='debiased_classifier', debias=True, sess=sess) debiased_model.fit(dataset_transf_train) dataset_reweigh_train = debiased_model.predict(dataset_transf_train) dataset_reweigh_test = debiased_model.predict(test1)
def train(): df = pd.read_csv('syn_train.csv') df_test = pd.read_csv('syn_test.csv') df_neg = df.loc[df['two_year_recid'] == 1, :] df_neg_priv = df_neg.loc[(df_neg['two_year_recid'] == 1) & (df_neg['race'] == 1), :] df_neg_unpriv = df_neg.loc[(df_neg['two_year_recid'] == 1) & (df_neg['race'] == 0), :] df_neg = df.loc[df['two_year_recid'] == 1, :] df_neg_priv = df_neg.loc[(df_neg['two_year_recid'] == 1) & (df_neg['race'] == 1), :] df_neg_unpriv = df_neg.loc[(df_neg['two_year_recid'] == 1) & (df_neg['race'] == 0), :] _, df_neg_priv_test = train_test_split(df_neg_priv, test_size=1200, random_state=1) _, df_neg_unpriv_test = train_test_split(df_neg_unpriv, test_size=2800, random_state=1) df_neg_test = df_neg_priv_test.append(df_neg_unpriv_test) print('negative outcome, unpriv before resampling') print(len(df_neg_unpriv_test.index)) print('negative outcome, priv before resampling') print(len(df_neg_priv_test.index)) df_pos = df.loc[df['two_year_recid'] == 0, :] df_pos_priv = df_pos.loc[(df_pos['two_year_recid'] == 0) & (df_pos['race'] == 1), :] df_pos_unpriv = df_pos.loc[(df_pos['two_year_recid'] == 0) & (df_pos['race'] == 0), :] _, df_pos_priv_test = train_test_split(df_pos_priv, test_size=2000, random_state=1) _, df_pos_unpriv_test = train_test_split(df_pos_unpriv, test_size=2000, random_state=1) df_pos_test = df_pos_priv_test.append(df_pos_unpriv_test) df = df_pos_test.append(df_neg_test) print('positive outcome, unpriv before resampling') print(len(df_pos_unpriv_test.index)) print('positive outcome, priv before resampling') print(len(df_pos_priv_test.index)) df = df_pos_test.append(df_neg_test) N = 8000 df_result = pd.DataFrame() # unif sampling for i in df['two_year_recid'].unique(): for j in df['race'].unique(): orig_df = df.loc[(df['two_year_recid'] == i) & (df['race'] == j), :] real_count = len(orig_df.index) exp_count = int( (len(df.loc[(df['two_year_recid'] == i), :].index) / len(df.index)) * (len(df.loc[(df['race'] == j), :].index) / len(df.index)) * N) if real_count >= exp_count: _, df_toapp = train_test_split(orig_df, test_size=exp_count, random_state=10) else: df_toapp = resample(orig_df, replace=True, n_samples=exp_count - real_count, random_state=10) df_toapp = df_toapp.append(orig_df) if len(df_result.index) == 0: df_result = df_toapp.copy() else: df_result = df_result.append(df_toapp) df = df_result df_train = df privileged_groups = [{'race': 1}] unprivileged_groups = [{'race': 0}] all_protected_attribute_maps = {"race": {0.0: 0, 1.0: 1}} D_features = ['race'] dataset_orig_train = CustomDataset( label_name='two_year_recid', favorable_classes=[0], protected_attribute_names=['race'], privileged_classes=[[1]], categorical_features=[ 'priors_count', 'c_charge_degree', 'age_cat', 'score_text' ], features_to_keep=[ 'priors_count', 'c_charge_degree', 'race', 'age_cat', 'score_text' ], df=df_train, metadata={ 'label_maps': [{ 1: 'Did recid.', 0: 'No recid.' }], 'protected_attribute_maps': [all_protected_attribute_maps[x] for x in D_features] }) dataset_orig_vt = CustomDataset( label_name='two_year_recid', favorable_classes=[0], protected_attribute_names=['race'], privileged_classes=[[1]], categorical_features=[ 'priors_count', 'c_charge_degree', 'age_cat', 'score_text' ], features_to_keep=[ 'priors_count', 'c_charge_degree', 'race', 'age_cat', 'score_text' ], df=df_test, metadata={ 'label_maps': [{ 1: 'Did recid.', 0: 'No recid.' }], 'protected_attribute_maps': [all_protected_attribute_maps[x] for x in D_features] }) optim_options = { "distortion_fun": get_distortion_compas, "epsilon": 0.04, "clist": [0.99, 1.99, 2.99], "dlist": [.1, 0.05, 0] } metric_transf_train = BinaryLabelDatasetMetric( dataset_orig_train, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) OP = OptimPreproc(OptTools, optim_options, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) OP = OP.fit(dataset_orig_train) dataset_transf_cat_test = OP.transform(dataset_orig_vt, transform_Y=True) dataset_transf_cat_test = dataset_orig_vt.align_datasets( dataset_transf_cat_test) dataset_transf_cat_train = OP.transform(dataset_orig_train, transform_Y=True) dataset_transf_cat_train = dataset_orig_train.align_datasets( dataset_transf_cat_train) scale_transf = StandardScaler() X_train = dataset_orig_train.features y_train = dataset_orig_train.labels.ravel() X_test = scale_transf.fit_transform(dataset_orig_vt.features) lmod = LogisticRegression() lmod.fit(X_train, y_train) y_pred = lmod.predict(X_test) print('With resampling') print('Accuracy') print(accuracy_score(dataset_orig_vt.labels, y_pred)) dataset_orig_vt_copy = dataset_orig_vt.copy() dataset_orig_vt_copy.labels = y_pred metric_transf_train = BinaryLabelDatasetMetric( dataset_orig_vt_copy, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) print('p-rule') print(metric_transf_train.disparate_impact()) print('CV') print(metric_transf_train.mean_difference()) print('FPR for unpriv') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[ orig_sens_att == 0], 0)) print('FNR for unpriv') print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[ orig_sens_att == 0], 0)) print('FPR for priv') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[ orig_sens_att == 1], 0)) print('FNR for priv') print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[ orig_sens_att == 1], 0))
def fit(self, dataset_true, dataset_pred): """Estimates the optimal classification threshold and margin for reject option classification that optimizes the metric provided. Note: The `fit` function is a no-op for this algorithm. Args: dataset_true (BinaryLabelDataset): Dataset containing the true `labels`. dataset_pred (BinaryLabelDataset): Dataset containing the predicted `scores`. Returns: RejectOptionClassification: Returns self. """ fair_metric_arr = np.zeros(self.num_class_thresh*self.num_ROC_margin) balanced_acc_arr = np.zeros_like(fair_metric_arr) ROC_margin_arr = np.zeros_like(fair_metric_arr) class_thresh_arr = np.zeros_like(fair_metric_arr) cnt = 0 # Iterate through class thresholds for class_thresh in np.linspace(self.low_class_thresh, self.high_class_thresh, self.num_class_thresh): self.classification_threshold = class_thresh if class_thresh <= 0.5: low_ROC_margin = 0.0 high_ROC_margin = class_thresh else: low_ROC_margin = 0.0 high_ROC_margin = (1.0-class_thresh) # Iterate through ROC margins for ROC_margin in np.linspace( low_ROC_margin, high_ROC_margin, self.num_ROC_margin): self.ROC_margin = ROC_margin # Predict using the current threshold and margin dataset_transf_pred = self.predict(dataset_pred) dataset_transf_metric_pred = BinaryLabelDatasetMetric( dataset_transf_pred, unprivileged_groups=self.unprivileged_groups, privileged_groups=self.privileged_groups) classified_transf_metric = ClassificationMetric( dataset_true, dataset_transf_pred, unprivileged_groups=self.unprivileged_groups, privileged_groups=self.privileged_groups) ROC_margin_arr[cnt] = self.ROC_margin class_thresh_arr[cnt] = self.classification_threshold # Balanced accuracy and fairness metric computations balanced_acc_arr[cnt] = 0.5*(classified_transf_metric.true_positive_rate()\ +classified_transf_metric.true_negative_rate()) if self.metric_name == "Statistical parity difference": fair_metric_arr[cnt] = dataset_transf_metric_pred.mean_difference() elif self.metric_name == "Average odds difference": fair_metric_arr[cnt] = classified_transf_metric.average_odds_difference() elif self.metric_name == "Equal opportunity difference": fair_metric_arr[cnt] = classified_transf_metric.equal_opportunity_difference() cnt += 1 rel_inds = np.logical_and(fair_metric_arr >= self.metric_lb, fair_metric_arr <= self.metric_ub) if any(rel_inds): best_ind = np.where(balanced_acc_arr[rel_inds] == np.max(balanced_acc_arr[rel_inds]))[0][0] else: warn("Unable to satisy fairness constraints") rel_inds = np.ones(len(fair_metric_arr), dtype=bool) best_ind = np.where(fair_metric_arr[rel_inds] == np.min(fair_metric_arr[rel_inds]))[0][0] self.ROC_margin = ROC_margin_arr[rel_inds][best_ind] self.classification_threshold = class_thresh_arr[rel_inds][best_ind] return self
def get_fair_metrics(dataset, pred, pred_is_dataset=False): """ Measure fairness metrics. Parameters: dataset (pandas dataframe): Dataset pred (array): Model predictions pred_is_dataset, optional (bool): True if prediction is already part of the dataset, column name 'labels'. Returns: fair_metrics: Fairness metrics. """ if pred_is_dataset: dataset_pred = pred else: dataset_pred = dataset.copy() dataset_pred.labels = pred cols = [ 'statistical_parity_difference', 'equal_opportunity_difference', 'average_abs_odds_difference', 'disparate_impact', 'theil_index' ] obj_fairness = [[0, 0, 0, 1, 0]] fair_metrics = pd.DataFrame(data=obj_fairness, index=['objective'], columns=cols) for attr in dataset_pred.protected_attribute_names: idx = dataset_pred.protected_attribute_names.index(attr) privileged_groups = [{ attr: dataset_pred.privileged_protected_attributes[idx][0] }] unprivileged_groups = [{ attr: dataset_pred.unprivileged_protected_attributes[idx][0] }] classified_metric = ClassificationMetric( dataset, dataset_pred, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) metric_pred = BinaryLabelDatasetMetric( dataset_pred, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) acc = classified_metric.accuracy() row = pd.DataFrame([[ metric_pred.mean_difference(), classified_metric.equal_opportunity_difference(), classified_metric.average_abs_odds_difference(), metric_pred.disparate_impact(), classified_metric.theil_index() ]], columns=cols, index=[attr]) fair_metrics = fair_metrics.append(row) fair_metrics = fair_metrics.replace([-np.inf, np.inf], 2) return fair_metrics
def train(): np.random.seed(10) def quantizePrior1(x): if x <= 0: return 0 elif 1 <= x <= 3: return 1 else: return 2 def quantizeLOS(x): if x <= 7: return 0 if 8 < x <= 93: return 1 else: return 2 def group_race(x): if x == "Caucasian": return 1.0 else: return 0.0 filepath = 'AIF360/aif360/data/raw/compas/compas-scores-two-years.csv' df = pd.read_csv(filepath, index_col='id', na_values=[]) df['age_cat'] = df['age_cat'].replace('Greater than 45', 2) df['age_cat'] = df['age_cat'].replace('25 - 45', 1) df['age_cat'] = df['age_cat'].replace('Less than 25', 0) df['score_text'] = df['score_text'].replace('High', 1) df['score_text'] = df['score_text'].replace('Medium', 1) df['score_text'] = df['score_text'].replace('Low', 0) df['priors_count'] = df['priors_count'].apply(lambda x: quantizePrior1(x)) df['length_of_stay'] = ( pd.to_datetime(df['c_jail_out']) - pd.to_datetime(df['c_jail_in'])).apply(lambda x: x.days) df['length_of_stay'] = df['length_of_stay'].apply(lambda x: quantizeLOS(x)) df = df.loc[ ~df['race'].isin(['Native American', 'Hispanic', 'Asian', 'Other']), :] df['c_charge_degree'] = df['c_charge_degree'].replace({'F': 0, 'M': 1}) df1 = df[[ 'priors_count', 'c_charge_degree', 'race', 'age_cat', 'score_text', 'two_year_recid' ]] feature_list = [] for index, row in df1.iterrows(): feature_list.append('\t'.join(row.astype(str).to_list())) df1['feature_list'] = feature_list df3 = df1.groupby('feature_list').count() / len(df1.index) df2 = pd.DataFrame() df2['feature_list'] = list(df3.index) df2['prob_list'] = list(df3.priors_count) for index, row in df2.iterrows(): if row['feature_list'][0] == '0' and row['feature_list'][ -1] == '1' and 'African' in row['feature_list']: row['prob_list'] = row['prob_list'] * 10 elif row['feature_list'][0] == '0' and row['feature_list'][-1] == '1': row['prob_list'] = row['prob_list'] * 7 elif row['feature_list'][0] == '2' and row['feature_list'][-1] == '0': row['prob_list'] = row['prob_list'] * 7 prob_list = list(df2.prob_list) df_new = pd.DataFrame() rng = np.random.default_rng() prob_list = np.array(prob_list) prob_list = prob_list / prob_list.sum() feature_list = rng.choice(list(df2.feature_list), len(df1.index), p=prob_list) var_list = [ 'priors_count', 'c_charge_degree', 'race', 'age_cat', 'score_text', 'two_year_recid' ] for i in var_list: vars()[i] = [] for i in feature_list: tmp = i.split('\t') for j in range(len(var_list)): vars()[var_list[j]].append(tmp[j]) for i in var_list: df_new[i] = vars()[i] df = df_new df1 = df[[ 'priors_count', 'c_charge_degree', 'race', 'age_cat', 'score_text', 'two_year_recid' ]] tot = [] for index, row in df1.iterrows(): result = '' for j in df1.columns: result = result + str(row[j]) tot.append(result) df['tmp_feature'] = tot df['mis_prob'] = 0 for i in df['tmp_feature'].unique(): if 'African' in i and i[-1] == '0': df.loc[df['tmp_feature'] == i, 'mis_prob'] = 0.1 elif 'African' in i: df.loc[df['tmp_feature'] == i, 'mis_prob'] = 0.02 elif 'African' not in i and i[-1] == '0': df.loc[df['tmp_feature'] == i, 'mis_prob'] = 0.02 else: df.loc[df['tmp_feature'] == i, 'mis_prob'] = 0.02 new_label = [] for i, j in zip(df['mis_prob'], df['priors_count']): if np.random.binomial(1, i, 1)[0] == 1: new_label.append(3) else: new_label.append(j) df['priors_count'] = new_label print(len(df.loc[df['priors_count'] == 3, :].index)) print(len(df.index)) df['priors_count'] = df['priors_count'].astype(int) df['score_text'] = df['score_text'].astype(int) df['age_cat'] = df['age_cat'].astype(int) df['score_text'] = df['score_text'].astype(int) df['c_charge_degree'] = df['c_charge_degree'].astype(int) df['two_year_recid'] = df['two_year_recid'].astype(int) df['c_charge_degree'] = df['c_charge_degree'].replace({0: 'F', 1: 'M'}) def quantizePrior(x): if x == 0: return '0' elif x == 1: return '1 to 3' elif x == 2: return 'More than 3' else: return 'missing' # Quantize length of stay def quantizeLOS(x): if x == 0: return '<week' if x == 1: return '<3months' else: return '>3 months' # Quantize length of stay def adjustAge(x): if x == 0: return '25 to 45' elif x == 1: return 'Greater than 45' elif x == 2: return 'Less than 25' def quantizeScore(x): if x == 1: return 'MediumHigh' else: return 'Low' def group_race(x): if x == "Caucasian": return 1.0 else: return 0.0 df['priors_count'] = df['priors_count'].apply(lambda x: quantizePrior(x)) df['score_text'] = df['score_text'].apply(lambda x: quantizeScore(x)) df['age_cat'] = df['age_cat'].apply(lambda x: adjustAge(x)) # Recode sex and race df['race'] = df['race'].apply(lambda x: group_race(x)) df['race'] = df['race'].astype(int) df['two_year_recid'] = df['two_year_recid'].astype(int) df = df[[ 'priors_count', 'c_charge_degree', 'race', 'age_cat', 'score_text', 'two_year_recid' ]] df_train, df_test = train_test_split(df, test_size=0.3, random_state=10) all_protected_attribute_maps = {"race": {0.0: 0, 1.0: 1}} D_features = ['race'] dataset_orig_train = CustomDataset( label_name='two_year_recid', favorable_classes=[0], protected_attribute_names=['race'], privileged_classes=[[1]], categorical_features=[ 'priors_count', 'c_charge_degree', 'age_cat', 'score_text' ], features_to_keep=[ 'priors_count', 'c_charge_degree', 'race', 'age_cat', 'score_text' ], df=df_train, metadata={ 'label_maps': [{ 1: 'Did recid.', 0: 'No recid.' }], 'protected_attribute_maps': [all_protected_attribute_maps[x] for x in D_features] }) dataset_orig_vt = CustomDataset( label_name='two_year_recid', favorable_classes=[0], protected_attribute_names=['race'], privileged_classes=[[1]], categorical_features=[ 'priors_count', 'c_charge_degree', 'age_cat', 'score_text' ], features_to_keep=[ 'priors_count', 'c_charge_degree', 'race', 'age_cat', 'score_text' ], df=df_test, metadata={ 'label_maps': [{ 1: 'Did recid.', 0: 'No recid.' }], 'protected_attribute_maps': [all_protected_attribute_maps[x] for x in D_features] }) privileged_groups = [{'race': 1}] unprivileged_groups = [{'race': 0}] optim_options = { "distortion_fun": get_distortion_compas, "epsilon": 0.04, "clist": [0.99, 1.99, 2.99], "dlist": [.1, 0.05, 0] } metric_transf_train = BinaryLabelDatasetMetric( dataset_orig_train, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) OP = OptimPreproc(OptTools, optim_options, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) OP = OP.fit(dataset_orig_train) dataset_transf_cat_test = OP.transform(dataset_orig_vt, transform_Y=True) dataset_transf_cat_test = dataset_orig_vt.align_datasets( dataset_transf_cat_test) dataset_transf_cat_train = OP.transform(dataset_orig_train, transform_Y=True) dataset_transf_cat_train = dataset_orig_train.align_datasets( dataset_transf_cat_train) scale_transf = StandardScaler() X_train = scale_transf.fit_transform(dataset_transf_cat_train.features) y_train = dataset_transf_cat_train.labels.ravel() X_test = scale_transf.fit_transform(dataset_transf_cat_test.features) lmod = LogisticRegression() lmod.fit(X_train, y_train) y_pred = lmod.predict(X_test) print('Without reweight') print('Accuracy') print(accuracy_score(dataset_orig_vt.labels, y_pred)) dataset_orig_vt_copy1 = dataset_orig_vt.copy() dataset_orig_vt_copy1.labels = y_pred metric_transf_train1 = BinaryLabelDatasetMetric( dataset_orig_vt_copy1, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) print('p-rule') print(metric_transf_train1.disparate_impact()) print('CV') print(metric_transf_train1.mean_difference()) print('FPR for unpriv') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[ orig_sens_att == 0], 0)) print("FNR for unpriv") print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[ orig_sens_att == 0], 0)) print('FPR for priv') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[ orig_sens_att == 1], 0)) print("FNR for priv") print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[ orig_sens_att == 1], 0)) df_weight = dataset_orig_train.convert_to_dataframe()[0] df_weight['weight'] = 1 df_weight['is_missing'] = 0 df_weight['tmp'] = '' tmp_result = [] for i, j in zip(df_weight['race'], df_weight['two_year_recid']): tmp_result.append(str(i) + str(j)) df_weight['tmp'] = tmp_result df_weight.loc[df_weight['priors_count=missing'] == 1, 'is_missing'] = 1 for i in df_weight['tmp'].unique(): df_weight.loc[ (df_weight['tmp'] == i) & (df_weight['is_missing'] == 0), 'weight'] = len( df_weight.loc[(df_weight['tmp'] == i), :].index) / len( df_weight.loc[(df_weight['tmp'] == i) & (df_weight['is_missing'] == 0), :].index) df_weight.loc[(df_weight['tmp'] == i) & (df_weight['is_missing'] == 1), 'weight'] = len(df_weight.loc[ (df_weight['tmp'] == i) & (df_weight['is_missing'] == 0), :].index) / len( df_weight.loc[(df_weight['tmp'] == i), :].index) dataset_orig_train.instance_weights = np.array(df_weight['weight']) scale_transf = StandardScaler() X_train = scale_transf.fit_transform(dataset_transf_cat_train.features) y_train = dataset_transf_cat_train.labels.ravel() X_test = scale_transf.fit_transform(dataset_transf_cat_test.features) lmod = LogisticRegression() lmod.fit(X_train, y_train, sample_weight=dataset_orig_train.instance_weights) y_pred = lmod.predict(X_test) print('With reweight') print('Accuracy') print(accuracy_score(dataset_orig_vt.labels, y_pred)) dataset_orig_vt_copy1 = dataset_orig_vt.copy() dataset_orig_vt_copy1.labels = y_pred metric_transf_train1 = BinaryLabelDatasetMetric( dataset_orig_vt_copy1, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) print('p-rule') print(metric_transf_train1.disparate_impact()) print('CV') print(metric_transf_train1.mean_difference()) print('FPR for unpriv') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[ orig_sens_att == 0], 0)) print("FNR for unpriv") print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[ orig_sens_att == 0], 0)) print('FPR for priv') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[ orig_sens_att == 1], 0)) print("FNR for priv") print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[ orig_sens_att == 1], 0))
def train_before_resample(): dataset_orig_train = load_preproc_data_adult(['sex']) optim_options = { "distortion_fun": get_distortion_adult, "epsilon": 0.05, "clist": [0.99, 1.99, 2.99], "dlist": [.1, 0.05, 0] } OP = OptimPreproc(OptTools, optim_options, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) OP = OP.fit(dataset_orig_train) dataset_transf_cat_test = OP.transform(dataset_orig_vt, transform_Y=True) dataset_transf_cat_test = dataset_orig_vt.align_datasets( dataset_transf_cat_test) dataset_transf_cat_train = OP.transform(dataset_orig_train, transform_Y=True) dataset_transf_cat_train = dataset_orig_train.align_datasets( dataset_transf_cat_train) scale_transf = StandardScaler() #X_train = scale_transf.fit_transform(dataset_orig_train.features[:,1:]) #X_train = scale_transf.fit_transform(dataset_orig_train.features) X_train = dataset_orig_train.features y_train = dataset_orig_train.labels.ravel() X_test = scale_transf.fit_transform(dataset_orig_vt.features) #X_test = scale_transf.fit_transform(dataset_orig_vt.features[:,1:]) lmod = LogisticRegression() #lmod = LogisticRegression() #lmod.fit(X_train, y_train,sample_weight=dataset_orig_train.instance_weights) lmod.fit(X_train, y_train) y_pred = lmod.predict(X_test) print('Accuracy and fairness results before resampling') print('accuracy') print(accuracy_score(dataset_orig_vt.labels, y_pred)) dataset_orig_vt_copy = dataset_orig_vt.copy() dataset_orig_vt_copy.labels = y_pred metric_transf_train = BinaryLabelDatasetMetric( dataset_orig_vt_copy, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) print('p-rule') print(metric_transf_train.disparate_impact()) print('CV') print(metric_transf_train.mean_difference()) print('FPR for priv') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[ orig_sens_att == 0], 1)) print("FNR for priv") print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[ orig_sens_att == 0], 1)) print("BCR") print( get_BCR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[orig_sens_att == 0], 1)) print('FPR for unpriv') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[ orig_sens_att == 1], 1)) print("FNR for unpriv") print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[ orig_sens_att == 1], 1))