def get_evaluation(dataset_orig_vt, y_pred, privileged_groups, unprivileged_groups, unpriv_val, priv_val, pos_label): print('Accuracy') print(accuracy_score(dataset_orig_vt.labels, y_pred)) dataset_orig_vt_copy1 = dataset_orig_vt.copy() dataset_orig_vt_copy1.labels = y_pred metric_transf_train1 = BinaryLabelDatasetMetric( dataset_orig_vt_copy1, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) print('p-rule') print( min(metric_transf_train1.disparate_impact(), 1 / metric_transf_train1.disparate_impact())) print('FPR for unpriv group') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == unpriv_val], y_pred[orig_sens_att == unpriv_val], pos_label)) print("FNR for unpriv group") print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == unpriv_val], y_pred[orig_sens_att == unpriv_val], pos_label)) print('FPR for priv group') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == priv_val], y_pred[orig_sens_att == priv_val], pos_label)) print("FNR for priv group") print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == priv_val], y_pred[orig_sens_att == priv_val], pos_label))
def calc_disparity_index(data, target_variable, protected_variable, unprivileged_input): df_aif = BinaryLabelDataset(df=data, label_names=[target_variable], protected_attribute_names=[protected_variable]) privileged_group = [] for v in data[protected_variable].unique()[data[protected_variable].unique() != unprivileged_input]: privileged_group.append({protected_variable: v}) unprivileged_group = [{protected_variable: unprivileged_input}] #female=0 metric_orig = BinaryLabelDatasetMetric(df_aif, unprivileged_group, privileged_group) print('1-min(DI, 1/DI):', get_disparity_index(metric_orig.disparate_impact()).round(3)) if get_disparity_index(metric_orig.disparate_impact()).round(3) < 0.2: print('The algorithm can be considered to be not biased') else: print('There is a potential bias')
def calculate_bias_measures(data_orig_train, data_orig_vt, unprivileged_groups, privileged_groups): model = RandomForestClassifier().fit( data_orig_train.features, data_orig_train.labels.ravel(), sample_weight=data_orig_train.instance_weights) dataset = data_orig_vt dataset_pred = dataset.copy() dataset_pred.labels = model.predict(data_orig_vt.features) classified_metric_race = ClassificationMetric( dataset, dataset_pred, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) metric_pred_race = BinaryLabelDatasetMetric( dataset_pred, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) print("Mean difference {}".format(metric_pred_race.mean_difference())) print("Disparate Metric {}".format(metric_pred_race.disparate_impact())) print("Equal Opportunity Difference {}".format( classified_metric_race.equal_opportunity_difference())) print("Average Abs Odds Difference {}".format( classified_metric_race.average_abs_odds_difference())) print("Theil index {}".format(classified_metric_race.theil_index()))
def disparity_values(data, target_variable, protected_variable, unprivileged_input): df_aif = BinaryLabelDataset(df=data, label_names=[target_variable], protected_attribute_names=[protected_variable]) privileged_group = [] for v in data[protected_variable].unique()[data[protected_variable].unique() != unprivileged_input]: privileged_group.append({protected_variable: v}) unprivileged_group = [{protected_variable: unprivileged_input}] #female=0 metric_orig = BinaryLabelDatasetMetric(df_aif, unprivileged_group, privileged_group) return abs(get_disparity_index(metric_orig.disparate_impact()).round(3))
def fair_metrics(dataset, pred, pred_is_dataset=False): if pred_is_dataset: dataset_pred = pred else: dataset_pred = dataset.copy() dataset_pred.labels = pred cols = [ 'statistical_parity_difference', 'equal_opportunity_difference', 'average_abs_odds_difference', 'disparate_impact', 'theil_index' ] obj_fairness = [[0, 0, 0, 1, 0]] fair_metrics = pd.DataFrame(data=obj_fairness, index=['objective'], columns=cols) for attr in dataset_pred.protected_attribute_names: idx = dataset_pred.protected_attribute_names.index(attr) privileged_groups = [{ attr: dataset_pred.privileged_protected_attributes[idx][0] }] unprivileged_groups = [{ attr: dataset_pred.unprivileged_protected_attributes[idx][0] }] classified_metric = ClassificationMetric( dataset, dataset_pred, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) metric_pred = BinaryLabelDatasetMetric( dataset_pred, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) acc = classified_metric.accuracy() row = pd.DataFrame([[ metric_pred.mean_difference(), classified_metric.equal_opportunity_difference(), classified_metric.average_abs_odds_difference(), metric_pred.disparate_impact(), classified_metric.theil_index() ]], columns=cols, index=[attr]) fair_metrics = fair_metrics.append(row) fair_metrics = fair_metrics.replace([-np.inf, np.inf], 2) return fair_metrics
def get_dataset_metrics_list(binary_dataset_list): #Set privileged and unprivileged groups privileged_groups= [{'sex':1}] unprivileged_groups= [{'sex': 0}] mean_diff_list = [] disp_imp_list = [] for dataset in binary_dataset_list: metrics = BinaryLabelDatasetMetric(dataset, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) mean_diff_list.append(metrics.mean_difference()) disp_imp_list.append(1 - metrics.disparate_impact()) return mean_diff_list, disp_imp_list
def explain(self, request: Dict) -> Dict: inputs = request["instances"] predictions = np.array(request["outputs"]) dataframe_predicted = pd.DataFrame(inputs, columns=self.feature_names) dataframe_predicted[self.label_names[0]] = predictions dataset_predicted = BinaryLabelDataset( favorable_label=self.favorable_label, unfavorable_label=self.unfavorable_label, df=dataframe_predicted, label_names=self.label_names, protected_attribute_names=['age']) metrics = BinaryLabelDatasetMetric( dataset_predicted, unprivileged_groups=self.unprivileged_groups, privileged_groups=self.privileged_groups) return { "predictions": predictions.tolist(), "metrics": { "base_rate": metrics.base_rate(), "consistency": metrics.consistency().tolist(), "disparate_impact": metrics.disparate_impact(), "num_instances": metrics.num_instances(), "num_negatives": metrics.num_negatives(), "num_positives": metrics.num_positives(), "statistical_parity_difference": metrics.statistical_parity_difference(), } }
def checkClassifierFairnessAndRemoveDI(frame, dpoints, mname, x_columns, verbose=True, pre=True): ''' Measure fairness according to the metric using the value of A and the classification outcome. Results get added to a dictionary used to pass them to a function to generate graphs of the results. If we have not performed intervention, perform intervention and return post intervention data.''' xay_columns = copy.deepcopy(x_columns) xay_columns.extend(["A", "Y"]) ycols = copy.deepcopy(frame["Y"]) tempframe = copy.deepcopy(frame) tempframe.drop(["Y"], axis=1, inplace=True) aifdf = BinaryLabelDataset(favorable_label=1.0, unfavorable_label=0.0, df=tempframe, label_names=['Ya'], protected_attribute_names=['A']) privileged_groups = [{'A': 1}] unprivileged_groups = [{'A': 0}] DIR = DisparateImpactRemover(sensitive_attribute='A') metric_aifdf_train = BinaryLabelDatasetMetric( aifdf, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) if pre: if verbose: print("\n\tINTERVENTION: {}\n".format(type(DIR).__name__)) print("\t######### PRE {} ###########".format(type(DIR).__name__)) print( "\tDisparate impact between unprivileged and privileged groups = {}\n" .format(metric_aifdf_train.disparate_impact())) dpoints[mname]['PRE'][type( DIR).__name__]['FAIR'] = metric_aifdf_train.disparate_impact() print("PRE CLASSIFICATION MATRIX") print("----------------") print(" |Y'=0 | Y'=1 |") print("----------------") print("A=0| {0} | {1} |".format( metric_aifdf_train.num_negatives(False), metric_aifdf_train.num_positives(False))) print("A=1| {0} | {1} |".format( metric_aifdf_train.num_negatives(True), metric_aifdf_train.num_positives(True))) print("----------------") dataset_transf_train = DIR.fit_transform(aifdf) fairdf = dataset_transf_train.convert_to_dataframe()[0] fairdf.drop(['Ya'], axis=1, inplace=True) ycols.reset_index(drop=True, inplace=True) fairdf.reset_index(drop=True, inplace=True) fairdf.insert(0, "Y", ycols) fairdf[xay_columns] = fairdf[xay_columns] return fairdf else: if verbose: print( "\tDisparate impact between unprivileged and privileged groups = {}\n" .format(metric_aifdf_train.disparate_impact())) dpoints[mname]['POST'][type( DIR).__name__]['FAIR'] = metric_aifdf_train.disparate_impact() print("POST CLASSIFICATION MATRIX") print("----------------") print(" |Y'=0 | Y'=1|") print("----------------") print("A=0| {0} | {1} |".format( metric_aifdf_train.num_negatives(False), metric_aifdf_train.num_positives(False))) print("A=1| {0} | {1} |".format( metric_aifdf_train.num_negatives(True), metric_aifdf_train.num_positives(True))) print("----------------") return frame
train_dataset.features = scaler.fit_transform(train_dataset.features) test_dataset.features = scaler.fit_transform(test_dataset.features) index = [ test_dataset.feature_names.index(x) for x in protected_attribute_names ] #Metric of Original Data train_dataset_metric = BinaryLabelDatasetMetric(train_dataset, unprivileged_groups=u, privileged_groups=p) test_dataset_metric = BinaryLabelDatasetMetric(test_dataset, unprivileged_groups=u, privileged_groups=p) di_orig_list.append(test_dataset_metric.disparate_impact()) xaxis = np.arange(14) width = 0.2 baseline = 1 fig, ax = plt.subplots() rects1 = ax.bar(xaxis, np.array(di_orig_list) - baseline, width, bottom=baseline, label='Original') # rects2 = ax.bar(xaxis + width, np.array(di_pred_orig_list)- baseline, width,bottom=baseline, label='Predicted Original') # rects3 = ax.bar(xaxis + width+width, np.array(di_pred_trans_list)- baseline, width,bottom=baseline, label='Predicted Transformed') # Add some text for labels, title and custom x-axis tick labels, etc. ax.set_ylabel('Disparate Impact')
def compute_metrics(model, X_test, y_test, X_train, y_train, dataset_test, unprivileged_groups, privileged_groups, protect_attribute, print_result): """ Calculate and return: model accuracy and fairness metrics Parameters ---------- model: scikit-learn classifier X_test: numpy 2d array y_test: numpy 1d array X_train: numpy 2d array y_train: numpy 1d array dataset_test: aif360.datasets.BinaryLabelDataset unprivileged_groups: list<dict> Dictionary where the key is the name of the sensitive column in the dataset, and the value is the value of the unprivileged group in the dataset privileged_groups: list<dict> Dictionary where the key is the name of the sensitive column in the dataset, and the value is the value of the privileged group in the dataset protect_attribute print_result """ result = {} y_pred_test = model.predict(X_test) result['acc_test'] = accuracy_score(y_true=y_test, y_pred=y_pred_test) y_pred_train = model.predict(X_train) result['acc_train'] = accuracy_score(y_true=y_train, y_pred=y_pred_train) dataset_pred = dataset_test.copy() dataset_pred.labels = y_pred_test bin_metric = BinaryLabelDatasetMetric(dataset_pred, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) result['disp_impact'] = bin_metric.disparate_impact() result['stat_parity'] = bin_metric.mean_difference() classif_metric = ClassificationMetric(dataset_test, dataset_pred, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) result['avg_odds'] = classif_metric.average_odds_difference() result['equal_opport'] = classif_metric.equal_opportunity_difference() result['false_discovery_rate'] = classif_metric.false_discovery_rate_difference() result['entropy_index'] = classif_metric.generalized_entropy_index() result['acc_test_clf'] = classif_metric.accuracy(privileged=None) result['acc_test_priv'] = classif_metric.accuracy(privileged=True) result['acc_test_unpriv'] = classif_metric.accuracy(privileged=False) result['consistency'] = consitency(X_test, y_pred_test, protect_attribute, n_neighbors=5) result['counterfactual'] = counterfactual(X_test, model, protect_attribute) if print_result: print("Train accuracy: ", result['acc_train']) print("Test accuracy: ", result['acc_test']) print("Test accuracy clf: ", result['acc_test_clf']) print("Test accuracy priv.: ", result['acc_test_priv']) print("Test accuracy unpriv.: ", result['acc_test_unpriv']) print('Disparate impact: ', result['disp_impact']) print('Mean difference: ', result['stat_parity']) print('Average odds difference:', result['avg_odds']) print('Equality of opportunity:', result['equal_opport']) print('False discovery rate difference:', result['false_discovery_rate']) print('Generalized entropy index:', result['entropy_index']) print('Consistency: ', result['consistency']) print('Counterfactual fairness: ', result['counterfactual']) return result
def compute_metrics(model, X_test, y_test, X_train, y_train, dataset_test, dataset_name, model_name, unprivileged_groups, privileged_groups, position): """ Calculate and return: model accuracy and fairness metrics Parameters ---------- model: scikit-learn classifier X_test: numpy 2d array y_test: numpy 1d array X_train: numpy 2d array y_train: numpy 1d array dataset_test: aif360.datasets.BinaryLabelDataset dataset_name: string Dataset name used in the analysis model_name: string unprivileged_groups: list<dict> Dictionary where the key is the name of the sensitive column in the dataset, and the value is the value of the unprivileged group in the dataset privileged_groups: list<dict> Dictionary where the key is the name of the sensitive column in the dataset, and the value is the value of the privileged group in the dataset position: int Column position of the sensitive group in the dataset """ y_pred_test = model.predict(X_test) acc_test = accuracy_score(y_true=y_test, y_pred=y_pred_test) print("Test accuracy: ", acc_test) y_pred_train = model.predict(X_train) acc_train = accuracy_score(y_true=y_train, y_pred=y_pred_train) print("Train accuracy: ", acc_train) dataset_pred = dataset_test.copy() dataset_pred.labels = y_pred_test bin_metric = BinaryLabelDatasetMetric( dataset_pred, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) disparate_impact_bin = bin_metric.disparate_impact() print('Disparate impact: ', disparate_impact_bin) mean_difference = bin_metric.mean_difference() print('Mean difference: ', mean_difference) classif_metric = ClassificationMetric( dataset_test, dataset_pred, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) classif_disparete_impact = classif_metric.disparate_impact() avg_odds = classif_metric.average_odds_difference() print('Average odds difference:', avg_odds) equal_opport = classif_metric.equal_opportunity_difference() print('Equality of opportunity:', equal_opport) false_discovery_rate = classif_metric.false_discovery_rate_difference() print('False discovery rate difference:', false_discovery_rate) entropy_index = classif_metric.generalized_entropy_index() print('Generalized entropy index:', entropy_index) cons_comp = consitency_mod(bin_metric, position, n_neighbors=5) print('Consistency: ', cons_comp) result = (dataset_name, model_name, acc_test, disparate_impact_bin, mean_difference, classif_disparete_impact, avg_odds, equal_opport, false_discovery_rate, entropy_index, cons_comp) return result
def train(): privileged_groups = [{'race': 1}] unprivileged_groups = [{'race': 0}] dataset_orig_train = load_preproc_data_compas(['race']) optim_options = { "distortion_fun": get_distortion_compas, "epsilon": 0.1, "clist": [0.99, 1.99, 2.99], "dlist": [.1, 0.05, 0] } # dataset_orig_train, dataset_orig_vt = dataset_orig.split([0.7], shuffle=True) metric_transf_train = BinaryLabelDatasetMetric(dataset_orig_train, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) OP = OptimPreproc(OptTools, optim_options, unprivileged_groups = unprivileged_groups, privileged_groups = privileged_groups) OP = OP.fit(dataset_orig_train) dataset_transf_cat_test = OP.transform(dataset_orig_vt, transform_Y = True) dataset_transf_cat_test = dataset_orig_vt.align_datasets(dataset_transf_cat_test) dataset_transf_cat_train = OP.transform(dataset_orig_train, transform_Y = True) dataset_transf_cat_train = dataset_orig_train.align_datasets(dataset_transf_cat_train) scale_transf = StandardScaler() X_train = scale_transf.fit_transform(dataset_transf_cat_train.features) y_train = dataset_transf_cat_train.labels.ravel() X_test = scale_transf.fit_transform(dataset_transf_cat_test.features) lmod = LogisticRegression() lmod.fit(X_train, y_train) y_pred = lmod.predict(X_test) print('Without applying fixing algorithms') print('Accuracy') print(accuracy_score(dataset_orig_vt.labels, y_pred)) dataset_orig_vt_copy1 = dataset_orig_vt.copy() dataset_orig_vt_copy1.labels = y_pred metric_transf_train1 = BinaryLabelDatasetMetric(dataset_orig_vt_copy1, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) print('p-rule') print(metric_transf_train1.disparate_impact()) print('CV') print(metric_transf_train1.mean_difference()) print('FPR for unpriv') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1-TNR(dataset_orig_vt.labels.ravel()[orig_sens_att==0],y_pred[orig_sens_att==0],0)) print("FNR for unpriv") print(1-TPR(dataset_orig_vt.labels.ravel()[orig_sens_att==0],y_pred[orig_sens_att==0],0)) print('FPR for priv') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1-TNR(dataset_orig_vt.labels.ravel()[orig_sens_att==1],y_pred[orig_sens_att==1],0)) print("FNR for priv") print(1-TPR(dataset_orig_vt.labels.ravel()[orig_sens_att==1],y_pred[orig_sens_att==1],0))
def train_before_resample(): dataset_orig_train = load_preproc_data_adult(['sex']) optim_options = { "distortion_fun": get_distortion_adult, "epsilon": 0.05, "clist": [0.99, 1.99, 2.99], "dlist": [.1, 0.05, 0] } OP = OptimPreproc(OptTools, optim_options, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) OP = OP.fit(dataset_orig_train) dataset_transf_cat_test = OP.transform(dataset_orig_vt, transform_Y=True) dataset_transf_cat_test = dataset_orig_vt.align_datasets( dataset_transf_cat_test) dataset_transf_cat_train = OP.transform(dataset_orig_train, transform_Y=True) dataset_transf_cat_train = dataset_orig_train.align_datasets( dataset_transf_cat_train) scale_transf = StandardScaler() #X_train = scale_transf.fit_transform(dataset_orig_train.features[:,1:]) #X_train = scale_transf.fit_transform(dataset_orig_train.features) X_train = dataset_orig_train.features y_train = dataset_orig_train.labels.ravel() X_test = scale_transf.fit_transform(dataset_orig_vt.features) #X_test = scale_transf.fit_transform(dataset_orig_vt.features[:,1:]) lmod = LogisticRegression() #lmod = LogisticRegression() #lmod.fit(X_train, y_train,sample_weight=dataset_orig_train.instance_weights) lmod.fit(X_train, y_train) y_pred = lmod.predict(X_test) print('Accuracy and fairness results before resampling') print('accuracy') print(accuracy_score(dataset_orig_vt.labels, y_pred)) dataset_orig_vt_copy = dataset_orig_vt.copy() dataset_orig_vt_copy.labels = y_pred metric_transf_train = BinaryLabelDatasetMetric( dataset_orig_vt_copy, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) print('p-rule') print(metric_transf_train.disparate_impact()) print('CV') print(metric_transf_train.mean_difference()) print('FPR for priv') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[ orig_sens_att == 0], 1)) print("FNR for priv") print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[ orig_sens_att == 0], 1)) print("BCR") print( get_BCR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[orig_sens_att == 0], 1)) print('FPR for unpriv') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[ orig_sens_att == 1], 1)) print("FNR for unpriv") print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[ orig_sens_att == 1], 1))
def run_trial(): # stores each run (4 algos without reweighing, 4 algos with reweighing) as a sublist # number of sublists = number of runs # each sublist has four elements # we ONLY predict on the testing data ########## WITHOUT REWEIGHING ############# stat_par = [] disp_imp = [] eq_opp_diff = [] avg_odds_diff = [] theil = [] acc = [] ########## WITH REWEIGHING ############# stat_par_reweigh = [] disp_imp_reweigh = [] eq_opp_diff_reweigh = [] avg_odds_diff_reweigh = [] theil_reweigh = [] acc_reweigh = [] ########################################### for i in range(10): ########################################### privileged_groups = [{'sex': 1}] unprivileged_groups = [{'sex': 0}] dataset_orig = load_preproc_data_adult() # train1, test1 are the original dataset train1, test1 = dataset_orig.split([0.7], shuffle=True) RW = Reweighing(unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) RW.fit(train1) # dataset_transf_train, test1 are for the reweighed dataset dataset_transf_train = RW.transform(train1) ########################################### # change weights to whole numbers for i in range(dataset_transf_train.instance_weights.size): dataset_transf_train.instance_weights[i] = (round( dataset_transf_train.instance_weights[i] / 0.1) * 0.1) * 10 weights = copy.deepcopy(dataset_transf_train.instance_weights) # change dataset_transf_train.features and dataset_transf_train.labels and dataset_transf_train.protected_attributes according to the weights of each instance sum_weights = 0 for i in range(dataset_transf_train.features.shape[0]): row = copy.deepcopy(dataset_transf_train.features[i]) row_label = copy.deepcopy(dataset_transf_train.labels[i]) row_protected_attributes = copy.deepcopy( dataset_transf_train.protected_attributes[i]) row_protected_attributes.resize(1, 2) row.resize(1, 18) row_label.resize(1, 1) weight = int(weights[i]) for j in range(weight - 1): dataset_transf_train.features = np.concatenate( (dataset_transf_train.features, row)) dataset_transf_train.labels = np.concatenate( (dataset_transf_train.labels, row_label)) dataset_transf_train.protected_attributes = np.concatenate( (dataset_transf_train.protected_attributes, row_protected_attributes)) # change the dataset_transf_train to a numpy array of ones to match number of rows in features dataset_transf_train.instance_weights = np.ones( dataset_transf_train.features.shape[0]) ################## without reweighing ########################## temp_stat_par = [] temp_disp_imp = [] temp_eq_opp_diff = [] temp_avg_odds_diff = [] temp_theil = [] temp_acc = [] ##################### adversarial debiasing ##################### sess = tf.Session() debiased_model = AdversarialDebiasing( privileged_groups=privileged_groups, unprivileged_groups=unprivileged_groups, scope_name='debiased_classifier', debias=True, sess=sess) debiased_model.fit(train1) dataset_debiasing_test = debiased_model.predict(test1) sess.close() tf.reset_default_graph() ##################### metrics ##################### metric_test = BinaryLabelDatasetMetric( dataset_debiasing_test, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) acc_test = ClassificationMetric( test1, dataset_debiasing_test, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) temp_stat_par.append(metric_test.mean_difference()) temp_disp_imp.append(metric_test.disparate_impact()) temp_eq_opp_diff.append(metric_test.equal_opportunity_difference()) temp_avg_odds_diff.append(metric_test.average_odds_difference()) temp_theil.append(metric_test.theil_index()) temp_acc.append(acc_test.accuracy()) ##################### prejudice remover ##################### prejudice_model = PrejudiceRemover(eta=100, sensitive_attr='sex') prejudice_model.fit(train1) dataset_prejudice_test = prejudice_model.predict(test1) ##################### metrics ##################### metric_test = BinaryLabelDatasetMetric( dataset_prejudice_test, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) acc_test = ClassificationMetric( test1, dataset_prejudice_test, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) temp_stat_par.append(metric_test.mean_difference()) temp_disp_imp.append(metric_test.disparate_impact()) temp_eq_opp_diff.append(metric_test.equal_opportunity_difference()) temp_avg_odds_diff.append(metric_test.average_odds_difference()) temp_theil.append(metric_test.theil_index()) temp_acc.append(acc_test.accuracy()) ##################### normal neural net ##################### sess = tf.Session() neural_model = AdversarialDebiasing( privileged_groups=privileged_groups, unprivileged_groups=unprivileged_groups, scope_name='debiased_classifier', debias=False, sess=sess) neural_model.fit(train1) dataset_neural_test = neural_model.predict(test1) sess.close() tf.reset_default_graph() ##################### metrics ##################### metric_test = BinaryLabelDatasetMetric( dataset_neural_test, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) acc_test = ClassificationMetric( test1, dataset_neural_test, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) temp_stat_par.append(metric_test.mean_difference()) temp_disp_imp.append(metric_test.disparate_impact()) temp_eq_opp_diff.append(metric_test.equal_opportunity_difference()) temp_avg_odds_diff.append(metric_test.average_odds_difference()) temp_theil.append(metric_test.theil_index()) temp_acc.append(acc_test.accuracy()) ##################### ensemble ##################### pred_labels_test = [] for i in range(0, len(test1.features)): arr_test = mode([ dataset_debiasing_test[i], dataset_prejudice_test[i], dataset_neural_test[i] ]) pred_labels_test.append(arr_test[0][0]) dataset_ensemble_test = test1.copy() dataset_ensemble_test.labels = np.array(pred_labels_test) ##################### metrics ##################### metric_test = BinaryLabelDatasetMetric( dataset_ensemble_train, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) acc_test = ClassificationMetric( test1, dataset_ensemble_train, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) temp_stat_par.append(metric_test.mean_difference()) temp_disp_imp.append(metric_test.disparate_impact()) temp_eq_opp_diff.append(metric_test.equal_opportunity_difference()) temp_avg_odds_diff.append(metric_test.average_odds_difference()) temp_theil.append(metric_test.theil_index()) temp_acc.append(acc_test.accuracy()) ######### DUMP SHIT ########### stat_par.append(temp_stat_par) disp_imp.append(temp_disp_imp) eq_opp_diff.append(temp_eq_opp_diff) avg_odds_diff.append(temp_avg_odds_diff) theil.append(temp_theil) acc.append(temp_acc) ################## with reweighing ########################## temp_stat_par = [] temp_disp_imp = [] temp_eq_opp_diff = [] temp_avg_odds_diff = [] temp_theil = [] temp_acc = [] ################## adversarial debiasing ################## sess = tf.Session() debiased_model_reweighing = AdversarialDebiasing( privileged_groups=privileged_groups, unprivileged_groups=unprivileged_groups, scope_name='debiased_classifier', debias=True, sess=sess) debiased_model_reweighing.fit(dataset_transf_train) dataset_debiasing_test_reweighing = debiased_model_reweighing.predict( test1) sess.close() tf.reset_default_graph() ##################### metrics ##################### metric_test = BinaryLabelDatasetMetric( dataset_debiasing_test_reweighing, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) acc_test = ClassificationMetric( test1, dataset_debiasing_test_reweighing, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) temp_stat_par.append(metric_test.mean_difference()) temp_disp_imp.append(metric_test.disparate_impact()) temp_eq_opp_diff.append(metric_test.equal_opportunity_difference()) temp_avg_odds_diff.append(metric_test.average_odds_difference()) temp_theil.append(metric_test.theil_index()) temp_acc.append(acc_test.accuracy()) ##################### prejudice remover ##################### prejudice_model_reweighing = PrejudiceRemover(eta=100, sensitive_attr='sex') prejudice_model_reweighing.fit(dataset_transf_train) dataset_prejudice_test_reweighing = prejudice_model_reweighing.predict( test1) ##################### metrics ##################### metric_test = BinaryLabelDatasetMetric( dataset_prejudice_test_reweighing, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) acc_test = ClassificationMetric( test1, dataset_prejudice_test_reweighing, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) temp_stat_par.append(metric_test.mean_difference()) temp_disp_imp.append(metric_test.disparate_impact()) temp_eq_opp_diff.append(metric_test.equal_opportunity_difference()) temp_avg_odds_diff.append(metric_test.average_odds_difference()) temp_theil.append(metric_test.theil_index()) temp_acc.append(acc_test.accuracy()) ##################### normal neural net ##################### sess = tf.Session() neural_model = AdversarialDebiasing( privileged_groups=privileged_groups, unprivileged_groups=unprivileged_groups, scope_name='debiased_classifier', debias=False, sess=sess) neural_model.fit(dataset_transf_train) dataset_neural_test = neural_model.predict(test1) sess.close() tf.reset_default_graph() ##################### metrics ##################### metric_test = BinaryLabelDatasetMetric( dataset_neural_test, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) acc_test = ClassificationMetric( test1, dataset_neural_test, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) temp_stat_par.append(metric_test.mean_difference()) temp_disp_imp.append(metric_test.disparate_impact()) temp_eq_opp_diff.append(metric_test.equal_opportunity_difference()) temp_avg_odds_diff.append(metric_test.average_odds_difference()) temp_theil.append(metric_test.theil_index()) temp_acc.append(acc_test.accuracy()) ##################### ensemble ##################### pred_labels_test = [] for i in range(0, len(test1.features)): arr_test = mode([ dataset_debiasing_test[i], dataset_prejudice_test[i], dataset_neural_test[i] ]) pred_labels_test.append(arr_test[0][0]) dataset_ensemble_test = test1.copy() dataset_ensemble_test.labels = np.array(pred_labels_test) ##################### metrics ##################### metric_test = BinaryLabelDatasetMetric( dataset_ensemble_test, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) acc_test = ClassificationMetric( test1, dataset_ensemble_test, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) temp_stat_par.append(metric_test.mean_difference()) temp_disp_imp.append(metric_test.disparate_impact()) temp_eq_opp_diff.append(metric_test.equal_opportunity_difference()) temp_avg_odds_diff.append(metric_test.average_odds_difference()) temp_theil.append(metric_test.theil_index()) temp_acc.append(acc_test.accuracy()) ######### DUMP SHIT ########### stat_par_reweigh.append(temp_stat_par) disp_imp_reweigh.append(temp_disp_imp) eq_opp_diff_reweigh.append(temp_eq_opp_diff) avg_odds_diff_reweigh.append(temp_avg_odds_diff) theil_reweigh.append(temp_theil) acc_reweigh.append(temp_acc) without_reweighing = [ stat_par, disp_imp, eq_opp_diff, avg_odds_diff, theil, acc ] with_reweighing = [ stat_par_reweigh, disp_imp_reweigh, eq_opp_diff_reweigh, avg_odds_diff_reweigh, theil_reweigh, acc_reweigh ] for metric in range(len(without_reweighing)): name = "metric" + str(metric) sublist = without_reweighing[metric] with open(name, "wb") as csv_file: writer = csv.writer(csv_file) writer.writerows(sublist) for metric in range(len(with_reweighing)): name = "metric" + str(metric) + "reweigh" sublist = with_reweighing[metric] with open(name, "wb") as csv_file: writer = csv.writer(csv_file) writer.writerows(sublist)
def get_bldm_metrics(): metric_BLDM = BinaryLabelDatasetMetric( dataset, unprivileged_group, privileged_group) return {"Statistical Parity Difference": metric_BLDM.statistical_parity_difference(), "Disparate Impact": metric_BLDM.disparate_impact()}
prot.append(validation_comp.columns[i]) priv_dict = {validation_comp.columns[i]: 1} else: priv.append([]) stdDs = StandardDataset(validation_comp, 'is_violent_recid', [0], prot, priv) stdPred = StandardDataset(validation_pred, 'is_violent_recid', [0], prot, priv) bi_met = BinaryLabelDatasetMetric(stdDs, privileged_groups=[priv_dict], unprivileged_groups=[unpriv_dict]) class_met = ClassificationMetric(stdDs, stdPred, unprivileged_groups=[unpriv_dict], privileged_groups=[priv_dict]) disparate_impact = bi_met.disparate_impact() #error_rate_ratio = class_met.error_rate_ratio() eq_diff = class_met.equal_opportunity_difference() #Create 2 Bar Graphs x = [1] di_y = [disparate_impact] er_y = [error_rate_ratio] plt.ylim(bottom=0, top=2) plt.xlim(left=0, right=2) ax = plt.gca() ax.axes.xaxis.set_visible(False) plt.bar(x, di_y, width=0.6) plt.axhline(y=1.25, xmin=0, xmax=2, linestyle='--', color='black')
def train(): privileged_groups = [{'race': 1}] unprivileged_groups = [{'race': 0}] dataset_orig = load_preproc_data_compas(['race']) optim_options = { "distortion_fun": get_distortion_compas, "epsilon": 0.05, "clist": [0.99, 1.99, 2.99], "dlist": [.1, 0.05, 0] } dataset_orig_train, dataset_orig_vt = dataset_orig.split([0.7], shuffle=True) metric_transf_train = BinaryLabelDatasetMetric( dataset_orig_train, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) OP = OptimPreproc(OptTools, optim_options, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) OP = OP.fit(dataset_orig_train) dataset_transf_cat_test = OP.transform(dataset_orig_vt, transform_Y=True) dataset_transf_cat_test = dataset_orig_vt.align_datasets( dataset_transf_cat_test) dataset_transf_cat_train = OP.transform(dataset_orig_train, transform_Y=True) dataset_transf_cat_train = dataset_orig_train.align_datasets( dataset_transf_cat_train) scale_transf = StandardScaler() X_train = scale_transf.fit_transform(dataset_transf_cat_train.features) y_train = dataset_transf_cat_train.labels.ravel() X_test = scale_transf.fit_transform(dataset_transf_cat_test.features) lmod = LogisticRegression() lmod.fit(X_train, y_train) y_pred = lmod.predict(X_test) print('Without reweight') print('Accuracy') print(accuracy_score(dataset_orig_vt.labels, y_pred)) dataset_orig_vt_copy1 = dataset_orig_vt.copy() dataset_orig_vt_copy1.labels = y_pred metric_transf_train1 = BinaryLabelDatasetMetric( dataset_orig_vt_copy1, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) print('p-rule') print(metric_transf_train1.disparate_impact()) print('CV') print(metric_transf_train1.mean_difference()) print('FPR for unpriv') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[ orig_sens_att == 0], 0)) print("FNR for unpriv") print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[ orig_sens_att == 0], 0)) print('FPR for priv') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[ orig_sens_att == 1], 0)) print("FNR for priv") print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[ orig_sens_att == 1], 0)) df_weight = dataset_orig_train.convert_to_dataframe()[0] df_weight['weight'] = 1 df_weight['is_missing'] = 0 df_weight['tmp'] = '' tmp_result = [] for i, j in zip(df_weight['race'], df_weight['two_year_recid']): tmp_result.append(str(i) + str(j)) df_weight['tmp'] = tmp_result df_weight.loc[df_weight['priors_count=missing'] == 1, 'is_missing'] = 1 for i in df_weight['tmp'].unique(): df_weight.loc[ (df_weight['tmp'] == i) & (df_weight['is_missing'] == 0), 'weight'] = len( df_weight.loc[(df_weight['tmp'] == i), :].index) / len( df_weight.loc[(df_weight['tmp'] == i) & (df_weight['is_missing'] == 0), :].index) df_weight.loc[(df_weight['tmp'] == i) & (df_weight['is_missing'] == 1), 'weight'] = len(df_weight.loc[ (df_weight['tmp'] == i) & (df_weight['is_missing'] == 0), :].index) / len( df_weight.loc[(df_weight['tmp'] == i), :].index) dataset_orig_train.instance_weights = np.array(df_weight['weight']) scale_transf = StandardScaler() #X_train = scale_transf.fit_transform(dataset_transf_cat_train.features[:,1:]) X_train = scale_transf.fit_transform(dataset_transf_cat_train.features) y_train = dataset_transf_cat_train.labels.ravel() #X_test = scale_transf.fit_transform(dataset_transf_cat_test.features[:,1:]) X_test = scale_transf.fit_transform(dataset_transf_cat_test.features) lmod = LogisticRegression() lmod.fit(X_train, y_train, sample_weight=dataset_orig_train.instance_weights) y_pred = lmod.predict(X_test) print('With reweight') print('Accuracy') print(accuracy_score(dataset_orig_vt.labels, y_pred)) dataset_orig_vt_copy1 = dataset_orig_vt.copy() dataset_orig_vt_copy1.labels = y_pred metric_transf_train1 = BinaryLabelDatasetMetric( dataset_orig_vt_copy1, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) print('p-rule') print(metric_transf_train1.disparate_impact()) print('CV') print(metric_transf_train1.mean_difference()) print('FPR for unpriv') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[ orig_sens_att == 0], 0)) print("FNR for unpriv") print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[ orig_sens_att == 0], 0)) print('FPR for priv') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[ orig_sens_att == 1], 0)) print("FNR for priv") print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[ orig_sens_att == 1], 0))
def train(): df = pd.read_csv('syn_train.csv') df_test = pd.read_csv('syn_test.csv') df_neg = df.loc[df['two_year_recid'] == 1, :] df_neg_priv = df_neg.loc[(df_neg['two_year_recid'] == 1) & (df_neg['race'] == 1), :] df_neg_unpriv = df_neg.loc[(df_neg['two_year_recid'] == 1) & (df_neg['race'] == 0), :] df_neg = df.loc[df['two_year_recid'] == 1, :] df_neg_priv = df_neg.loc[(df_neg['two_year_recid'] == 1) & (df_neg['race'] == 1), :] df_neg_unpriv = df_neg.loc[(df_neg['two_year_recid'] == 1) & (df_neg['race'] == 0), :] _, df_neg_priv_test = train_test_split(df_neg_priv, test_size=1200, random_state=1) _, df_neg_unpriv_test = train_test_split(df_neg_unpriv, test_size=2800, random_state=1) df_neg_test = df_neg_priv_test.append(df_neg_unpriv_test) print('negative outcome, unpriv before resampling') print(len(df_neg_unpriv_test.index)) print('negative outcome, priv before resampling') print(len(df_neg_priv_test.index)) df_pos = df.loc[df['two_year_recid'] == 0, :] df_pos_priv = df_pos.loc[(df_pos['two_year_recid'] == 0) & (df_pos['race'] == 1), :] df_pos_unpriv = df_pos.loc[(df_pos['two_year_recid'] == 0) & (df_pos['race'] == 0), :] _, df_pos_priv_test = train_test_split(df_pos_priv, test_size=2000, random_state=1) _, df_pos_unpriv_test = train_test_split(df_pos_unpriv, test_size=2000, random_state=1) df_pos_test = df_pos_priv_test.append(df_pos_unpriv_test) df = df_pos_test.append(df_neg_test) print('positive outcome, unpriv before resampling') print(len(df_pos_unpriv_test.index)) print('positive outcome, priv before resampling') print(len(df_pos_priv_test.index)) df = df_pos_test.append(df_neg_test) N = 8000 df_result = pd.DataFrame() # unif sampling for i in df['two_year_recid'].unique(): for j in df['race'].unique(): orig_df = df.loc[(df['two_year_recid'] == i) & (df['race'] == j), :] real_count = len(orig_df.index) exp_count = int( (len(df.loc[(df['two_year_recid'] == i), :].index) / len(df.index)) * (len(df.loc[(df['race'] == j), :].index) / len(df.index)) * N) if real_count >= exp_count: _, df_toapp = train_test_split(orig_df, test_size=exp_count, random_state=10) else: df_toapp = resample(orig_df, replace=True, n_samples=exp_count - real_count, random_state=10) df_toapp = df_toapp.append(orig_df) if len(df_result.index) == 0: df_result = df_toapp.copy() else: df_result = df_result.append(df_toapp) df = df_result df_train = df privileged_groups = [{'race': 1}] unprivileged_groups = [{'race': 0}] all_protected_attribute_maps = {"race": {0.0: 0, 1.0: 1}} D_features = ['race'] dataset_orig_train = CustomDataset( label_name='two_year_recid', favorable_classes=[0], protected_attribute_names=['race'], privileged_classes=[[1]], categorical_features=[ 'priors_count', 'c_charge_degree', 'age_cat', 'score_text' ], features_to_keep=[ 'priors_count', 'c_charge_degree', 'race', 'age_cat', 'score_text' ], df=df_train, metadata={ 'label_maps': [{ 1: 'Did recid.', 0: 'No recid.' }], 'protected_attribute_maps': [all_protected_attribute_maps[x] for x in D_features] }) dataset_orig_vt = CustomDataset( label_name='two_year_recid', favorable_classes=[0], protected_attribute_names=['race'], privileged_classes=[[1]], categorical_features=[ 'priors_count', 'c_charge_degree', 'age_cat', 'score_text' ], features_to_keep=[ 'priors_count', 'c_charge_degree', 'race', 'age_cat', 'score_text' ], df=df_test, metadata={ 'label_maps': [{ 1: 'Did recid.', 0: 'No recid.' }], 'protected_attribute_maps': [all_protected_attribute_maps[x] for x in D_features] }) optim_options = { "distortion_fun": get_distortion_compas, "epsilon": 0.04, "clist": [0.99, 1.99, 2.99], "dlist": [.1, 0.05, 0] } metric_transf_train = BinaryLabelDatasetMetric( dataset_orig_train, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) OP = OptimPreproc(OptTools, optim_options, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) OP = OP.fit(dataset_orig_train) dataset_transf_cat_test = OP.transform(dataset_orig_vt, transform_Y=True) dataset_transf_cat_test = dataset_orig_vt.align_datasets( dataset_transf_cat_test) dataset_transf_cat_train = OP.transform(dataset_orig_train, transform_Y=True) dataset_transf_cat_train = dataset_orig_train.align_datasets( dataset_transf_cat_train) scale_transf = StandardScaler() X_train = dataset_orig_train.features y_train = dataset_orig_train.labels.ravel() X_test = scale_transf.fit_transform(dataset_orig_vt.features) lmod = LogisticRegression() lmod.fit(X_train, y_train) y_pred = lmod.predict(X_test) print('With resampling') print('Accuracy') print(accuracy_score(dataset_orig_vt.labels, y_pred)) dataset_orig_vt_copy = dataset_orig_vt.copy() dataset_orig_vt_copy.labels = y_pred metric_transf_train = BinaryLabelDatasetMetric( dataset_orig_vt_copy, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) print('p-rule') print(metric_transf_train.disparate_impact()) print('CV') print(metric_transf_train.mean_difference()) print('FPR for unpriv') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[ orig_sens_att == 0], 0)) print('FNR for unpriv') print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[ orig_sens_att == 0], 0)) print('FPR for priv') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[ orig_sens_att == 1], 0)) print('FNR for priv') print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[ orig_sens_att == 1], 0))
def train(): np.random.seed(10) def quantizePrior1(x): if x <= 0: return 0 elif 1 <= x <= 3: return 1 else: return 2 def quantizeLOS(x): if x <= 7: return 0 if 8 < x <= 93: return 1 else: return 2 def group_race(x): if x == "Caucasian": return 1.0 else: return 0.0 filepath = 'AIF360/aif360/data/raw/compas/compas-scores-two-years.csv' df = pd.read_csv(filepath, index_col='id', na_values=[]) df['age_cat'] = df['age_cat'].replace('Greater than 45', 2) df['age_cat'] = df['age_cat'].replace('25 - 45', 1) df['age_cat'] = df['age_cat'].replace('Less than 25', 0) df['score_text'] = df['score_text'].replace('High', 1) df['score_text'] = df['score_text'].replace('Medium', 1) df['score_text'] = df['score_text'].replace('Low', 0) df['priors_count'] = df['priors_count'].apply(lambda x: quantizePrior1(x)) df['length_of_stay'] = ( pd.to_datetime(df['c_jail_out']) - pd.to_datetime(df['c_jail_in'])).apply(lambda x: x.days) df['length_of_stay'] = df['length_of_stay'].apply(lambda x: quantizeLOS(x)) df = df.loc[ ~df['race'].isin(['Native American', 'Hispanic', 'Asian', 'Other']), :] df['c_charge_degree'] = df['c_charge_degree'].replace({'F': 0, 'M': 1}) df1 = df[[ 'priors_count', 'c_charge_degree', 'race', 'age_cat', 'score_text', 'two_year_recid' ]] feature_list = [] for index, row in df1.iterrows(): feature_list.append('\t'.join(row.astype(str).to_list())) df1['feature_list'] = feature_list df3 = df1.groupby('feature_list').count() / len(df1.index) df2 = pd.DataFrame() df2['feature_list'] = list(df3.index) df2['prob_list'] = list(df3.priors_count) for index, row in df2.iterrows(): if row['feature_list'][0] == '0' and row['feature_list'][ -1] == '1' and 'African' in row['feature_list']: row['prob_list'] = row['prob_list'] * 10 elif row['feature_list'][0] == '0' and row['feature_list'][-1] == '1': row['prob_list'] = row['prob_list'] * 7 elif row['feature_list'][0] == '2' and row['feature_list'][-1] == '0': row['prob_list'] = row['prob_list'] * 7 prob_list = list(df2.prob_list) df_new = pd.DataFrame() rng = np.random.default_rng() prob_list = np.array(prob_list) prob_list = prob_list / prob_list.sum() feature_list = rng.choice(list(df2.feature_list), len(df1.index), p=prob_list) var_list = [ 'priors_count', 'c_charge_degree', 'race', 'age_cat', 'score_text', 'two_year_recid' ] for i in var_list: vars()[i] = [] for i in feature_list: tmp = i.split('\t') for j in range(len(var_list)): vars()[var_list[j]].append(tmp[j]) for i in var_list: df_new[i] = vars()[i] df = df_new df1 = df[[ 'priors_count', 'c_charge_degree', 'race', 'age_cat', 'score_text', 'two_year_recid' ]] tot = [] for index, row in df1.iterrows(): result = '' for j in df1.columns: result = result + str(row[j]) tot.append(result) df['tmp_feature'] = tot df['mis_prob'] = 0 for i in df['tmp_feature'].unique(): if 'African' in i and i[-1] == '0': df.loc[df['tmp_feature'] == i, 'mis_prob'] = 0.1 elif 'African' in i: df.loc[df['tmp_feature'] == i, 'mis_prob'] = 0.02 elif 'African' not in i and i[-1] == '0': df.loc[df['tmp_feature'] == i, 'mis_prob'] = 0.02 else: df.loc[df['tmp_feature'] == i, 'mis_prob'] = 0.02 new_label = [] for i, j in zip(df['mis_prob'], df['priors_count']): if np.random.binomial(1, i, 1)[0] == 1: new_label.append(3) else: new_label.append(j) df['priors_count'] = new_label print(len(df.loc[df['priors_count'] == 3, :].index)) print(len(df.index)) df['priors_count'] = df['priors_count'].astype(int) df['score_text'] = df['score_text'].astype(int) df['age_cat'] = df['age_cat'].astype(int) df['score_text'] = df['score_text'].astype(int) df['c_charge_degree'] = df['c_charge_degree'].astype(int) df['two_year_recid'] = df['two_year_recid'].astype(int) df['c_charge_degree'] = df['c_charge_degree'].replace({0: 'F', 1: 'M'}) def quantizePrior(x): if x == 0: return '0' elif x == 1: return '1 to 3' elif x == 2: return 'More than 3' else: return 'missing' # Quantize length of stay def quantizeLOS(x): if x == 0: return '<week' if x == 1: return '<3months' else: return '>3 months' # Quantize length of stay def adjustAge(x): if x == 0: return '25 to 45' elif x == 1: return 'Greater than 45' elif x == 2: return 'Less than 25' def quantizeScore(x): if x == 1: return 'MediumHigh' else: return 'Low' def group_race(x): if x == "Caucasian": return 1.0 else: return 0.0 df['priors_count'] = df['priors_count'].apply(lambda x: quantizePrior(x)) df['score_text'] = df['score_text'].apply(lambda x: quantizeScore(x)) df['age_cat'] = df['age_cat'].apply(lambda x: adjustAge(x)) # Recode sex and race df['race'] = df['race'].apply(lambda x: group_race(x)) df['race'] = df['race'].astype(int) df['two_year_recid'] = df['two_year_recid'].astype(int) df = df[[ 'priors_count', 'c_charge_degree', 'race', 'age_cat', 'score_text', 'two_year_recid' ]] df_train, df_test = train_test_split(df, test_size=0.3, random_state=10) all_protected_attribute_maps = {"race": {0.0: 0, 1.0: 1}} D_features = ['race'] dataset_orig_train = CustomDataset( label_name='two_year_recid', favorable_classes=[0], protected_attribute_names=['race'], privileged_classes=[[1]], categorical_features=[ 'priors_count', 'c_charge_degree', 'age_cat', 'score_text' ], features_to_keep=[ 'priors_count', 'c_charge_degree', 'race', 'age_cat', 'score_text' ], df=df_train, metadata={ 'label_maps': [{ 1: 'Did recid.', 0: 'No recid.' }], 'protected_attribute_maps': [all_protected_attribute_maps[x] for x in D_features] }) dataset_orig_vt = CustomDataset( label_name='two_year_recid', favorable_classes=[0], protected_attribute_names=['race'], privileged_classes=[[1]], categorical_features=[ 'priors_count', 'c_charge_degree', 'age_cat', 'score_text' ], features_to_keep=[ 'priors_count', 'c_charge_degree', 'race', 'age_cat', 'score_text' ], df=df_test, metadata={ 'label_maps': [{ 1: 'Did recid.', 0: 'No recid.' }], 'protected_attribute_maps': [all_protected_attribute_maps[x] for x in D_features] }) privileged_groups = [{'race': 1}] unprivileged_groups = [{'race': 0}] optim_options = { "distortion_fun": get_distortion_compas, "epsilon": 0.04, "clist": [0.99, 1.99, 2.99], "dlist": [.1, 0.05, 0] } metric_transf_train = BinaryLabelDatasetMetric( dataset_orig_train, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) OP = OptimPreproc(OptTools, optim_options, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) OP = OP.fit(dataset_orig_train) dataset_transf_cat_test = OP.transform(dataset_orig_vt, transform_Y=True) dataset_transf_cat_test = dataset_orig_vt.align_datasets( dataset_transf_cat_test) dataset_transf_cat_train = OP.transform(dataset_orig_train, transform_Y=True) dataset_transf_cat_train = dataset_orig_train.align_datasets( dataset_transf_cat_train) scale_transf = StandardScaler() X_train = scale_transf.fit_transform(dataset_transf_cat_train.features) y_train = dataset_transf_cat_train.labels.ravel() X_test = scale_transf.fit_transform(dataset_transf_cat_test.features) lmod = LogisticRegression() lmod.fit(X_train, y_train) y_pred = lmod.predict(X_test) print('Without reweight') print('Accuracy') print(accuracy_score(dataset_orig_vt.labels, y_pred)) dataset_orig_vt_copy1 = dataset_orig_vt.copy() dataset_orig_vt_copy1.labels = y_pred metric_transf_train1 = BinaryLabelDatasetMetric( dataset_orig_vt_copy1, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) print('p-rule') print(metric_transf_train1.disparate_impact()) print('CV') print(metric_transf_train1.mean_difference()) print('FPR for unpriv') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[ orig_sens_att == 0], 0)) print("FNR for unpriv") print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[ orig_sens_att == 0], 0)) print('FPR for priv') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[ orig_sens_att == 1], 0)) print("FNR for priv") print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[ orig_sens_att == 1], 0)) df_weight = dataset_orig_train.convert_to_dataframe()[0] df_weight['weight'] = 1 df_weight['is_missing'] = 0 df_weight['tmp'] = '' tmp_result = [] for i, j in zip(df_weight['race'], df_weight['two_year_recid']): tmp_result.append(str(i) + str(j)) df_weight['tmp'] = tmp_result df_weight.loc[df_weight['priors_count=missing'] == 1, 'is_missing'] = 1 for i in df_weight['tmp'].unique(): df_weight.loc[ (df_weight['tmp'] == i) & (df_weight['is_missing'] == 0), 'weight'] = len( df_weight.loc[(df_weight['tmp'] == i), :].index) / len( df_weight.loc[(df_weight['tmp'] == i) & (df_weight['is_missing'] == 0), :].index) df_weight.loc[(df_weight['tmp'] == i) & (df_weight['is_missing'] == 1), 'weight'] = len(df_weight.loc[ (df_weight['tmp'] == i) & (df_weight['is_missing'] == 0), :].index) / len( df_weight.loc[(df_weight['tmp'] == i), :].index) dataset_orig_train.instance_weights = np.array(df_weight['weight']) scale_transf = StandardScaler() X_train = scale_transf.fit_transform(dataset_transf_cat_train.features) y_train = dataset_transf_cat_train.labels.ravel() X_test = scale_transf.fit_transform(dataset_transf_cat_test.features) lmod = LogisticRegression() lmod.fit(X_train, y_train, sample_weight=dataset_orig_train.instance_weights) y_pred = lmod.predict(X_test) print('With reweight') print('Accuracy') print(accuracy_score(dataset_orig_vt.labels, y_pred)) dataset_orig_vt_copy1 = dataset_orig_vt.copy() dataset_orig_vt_copy1.labels = y_pred metric_transf_train1 = BinaryLabelDatasetMetric( dataset_orig_vt_copy1, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) print('p-rule') print(metric_transf_train1.disparate_impact()) print('CV') print(metric_transf_train1.mean_difference()) print('FPR for unpriv') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[ orig_sens_att == 0], 0)) print("FNR for unpriv") print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[ orig_sens_att == 0], 0)) print('FPR for priv') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[ orig_sens_att == 1], 0)) print("FNR for priv") print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[ orig_sens_att == 1], 0))
def get_fair_metrics(dataset, pred, pred_is_dataset=False): """ Measure fairness metrics. Parameters: dataset (pandas dataframe): Dataset pred (array): Model predictions pred_is_dataset, optional (bool): True if prediction is already part of the dataset, column name 'labels'. Returns: fair_metrics: Fairness metrics. """ if pred_is_dataset: dataset_pred = pred else: dataset_pred = dataset.copy() dataset_pred.labels = pred cols = [ 'statistical_parity_difference', 'equal_opportunity_difference', 'average_abs_odds_difference', 'disparate_impact', 'theil_index' ] obj_fairness = [[0, 0, 0, 1, 0]] fair_metrics = pd.DataFrame(data=obj_fairness, index=['objective'], columns=cols) for attr in dataset_pred.protected_attribute_names: idx = dataset_pred.protected_attribute_names.index(attr) privileged_groups = [{ attr: dataset_pred.privileged_protected_attributes[idx][0] }] unprivileged_groups = [{ attr: dataset_pred.unprivileged_protected_attributes[idx][0] }] classified_metric = ClassificationMetric( dataset, dataset_pred, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) metric_pred = BinaryLabelDatasetMetric( dataset_pred, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) acc = classified_metric.accuracy() row = pd.DataFrame([[ metric_pred.mean_difference(), classified_metric.equal_opportunity_difference(), classified_metric.average_abs_odds_difference(), metric_pred.disparate_impact(), classified_metric.theil_index() ]], columns=cols, index=[attr]) fair_metrics = fair_metrics.append(row) fair_metrics = fair_metrics.replace([-np.inf, np.inf], 2) return fair_metrics