def test_adult(): ad = AdultDataset() # print(ad.feature_names) assert np.isclose(ad.labels.mean(), 0.2478, atol=5e-5) bldm = BinaryLabelDatasetMetric(ad) assert bldm.num_instances() == 45222
def get_evaluation(dataset_orig_vt, y_pred, privileged_groups, unprivileged_groups, unpriv_val, priv_val, pos_label): print('Accuracy') print(accuracy_score(dataset_orig_vt.labels, y_pred)) dataset_orig_vt_copy1 = dataset_orig_vt.copy() dataset_orig_vt_copy1.labels = y_pred metric_transf_train1 = BinaryLabelDatasetMetric( dataset_orig_vt_copy1, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) print('p-rule') print( min(metric_transf_train1.disparate_impact(), 1 / metric_transf_train1.disparate_impact())) print('FPR for unpriv group') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == unpriv_val], y_pred[orig_sens_att == unpriv_val], pos_label)) print("FNR for unpriv group") print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == unpriv_val], y_pred[orig_sens_att == unpriv_val], pos_label)) print('FPR for priv group') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == priv_val], y_pred[orig_sens_att == priv_val], pos_label)) print("FNR for priv group") print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == priv_val], y_pred[orig_sens_att == priv_val], pos_label))
def get_metrics(dataset_train): metric_orig_train = BinaryLabelDatasetMetric( dataset_train, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) #print("Difference in mean outcomes between unprivileged and privileged groups = %f" % metric_orig_train.mean_difference()) return metric_orig_train.mean_difference()
def test_adult_no_drop(): ad = AdultDataset(protected_attribute_names=['sex'], privileged_classes=[['Male']], categorical_features=[], features_to_keep=['age', 'education-num']) bldm = BinaryLabelDatasetMetric(ad) assert bldm.num_instances() == 48842
def calculate_bias_measures(data_orig_train, data_orig_vt, unprivileged_groups, privileged_groups): model = RandomForestClassifier().fit( data_orig_train.features, data_orig_train.labels.ravel(), sample_weight=data_orig_train.instance_weights) dataset = data_orig_vt dataset_pred = dataset.copy() dataset_pred.labels = model.predict(data_orig_vt.features) classified_metric_race = ClassificationMetric( dataset, dataset_pred, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) metric_pred_race = BinaryLabelDatasetMetric( dataset_pred, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) print("Mean difference {}".format(metric_pred_race.mean_difference())) print("Disparate Metric {}".format(metric_pred_race.disparate_impact())) print("Equal Opportunity Difference {}".format( classified_metric_race.equal_opportunity_difference())) print("Average Abs Odds Difference {}".format( classified_metric_race.average_abs_odds_difference())) print("Theil index {}".format(classified_metric_race.theil_index()))
def test_epsilon_all_groups(): def custom_preprocessing(df): # slight workaround for non-binary protected attribute # feature should be categorical but protected attribute should be numerical mapping = { 'Black': 0, 'White': 1, 'Asian-Pac-Islander': 2, 'Amer-Indian-Eskimo': 3, 'Other': 4 } df['race-num'] = df.race.map(mapping) return df.fillna('Unknown') nonbinary_ad = AdultDataset( protected_attribute_names=['sex', 'native-country', 'race-num'], privileged_classes=[['Male'], ['United-States'], [1]], categorical_features=[ 'workclass', 'education', 'marital-status', 'occupation', 'relationship', 'race' ], custom_preprocessing=custom_preprocessing) # drop redundant race feature (not relevant to this test) index = nonbinary_ad.feature_names.index('race-num') nonbinary_ad.features = np.delete(nonbinary_ad.features, index, axis=1) nonbinary_ad.feature_names = np.delete(nonbinary_ad.feature_names, index) _, nonbinary_test = nonbinary_ad.split([32561], shuffle=False) dataset_metric = BinaryLabelDatasetMetric(nonbinary_test) eps_data = dataset_metric.smoothed_empirical_differential_fairness() assert eps_data == 2.063813731996515 # verified with reference implementation
def mean_diff_values(data, target_variable, protected_variable, unprivileged_input): df_aif = BinaryLabelDataset(df=data, label_names=[target_variable], protected_attribute_names=[protected_variable]) privileged_group = [] for v in data[protected_variable].unique()[data[protected_variable].unique() != unprivileged_input]: privileged_group.append({protected_variable: v}) unprivileged_group = [{protected_variable: unprivileged_input}] #female=0 metric_orig = BinaryLabelDatasetMetric(df_aif, unprivileged_group, privileged_group) return abs(metric_orig.mean_difference().round(3))
def fair_metrics(dataset, pred, pred_is_dataset=False): if pred_is_dataset: dataset_pred = pred else: dataset_pred = dataset.copy() dataset_pred.labels = pred cols = [ 'statistical_parity_difference', 'equal_opportunity_difference', 'average_abs_odds_difference', 'disparate_impact', 'theil_index' ] obj_fairness = [[0, 0, 0, 1, 0]] fair_metrics = pd.DataFrame(data=obj_fairness, index=['objective'], columns=cols) for attr in dataset_pred.protected_attribute_names: idx = dataset_pred.protected_attribute_names.index(attr) privileged_groups = [{ attr: dataset_pred.privileged_protected_attributes[idx][0] }] unprivileged_groups = [{ attr: dataset_pred.unprivileged_protected_attributes[idx][0] }] classified_metric = ClassificationMetric( dataset, dataset_pred, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) metric_pred = BinaryLabelDatasetMetric( dataset_pred, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) acc = classified_metric.accuracy() row = pd.DataFrame([[ metric_pred.mean_difference(), classified_metric.equal_opportunity_difference(), classified_metric.average_abs_odds_difference(), metric_pred.disparate_impact(), classified_metric.theil_index() ]], columns=cols, index=[attr]) fair_metrics = fair_metrics.append(row) fair_metrics = fair_metrics.replace([-np.inf, np.inf], 2) return fair_metrics
def calc_mean_diff(data, target_variable, protected_variable, unprivileged_input): df_aif = BinaryLabelDataset(df=data, label_names=[target_variable], protected_attribute_names=[protected_variable]) privileged_group = [] for v in data[protected_variable].unique()[data[protected_variable].unique() != unprivileged_input]: privileged_group.append({protected_variable: v}) unprivileged_group = [{protected_variable: unprivileged_input}] #female=0 metric_orig = BinaryLabelDatasetMetric(df_aif, unprivileged_group, privileged_group) print(metric_orig.mean_difference().round(3)) if abs(metric_orig.mean_difference().round(3)) < 0.2: print('The algorithm can be considered to be not biased') else: print('There is a potential bias')
def calc_disparity_index(data, target_variable, protected_variable, unprivileged_input): df_aif = BinaryLabelDataset(df=data, label_names=[target_variable], protected_attribute_names=[protected_variable]) privileged_group = [] for v in data[protected_variable].unique()[data[protected_variable].unique() != unprivileged_input]: privileged_group.append({protected_variable: v}) unprivileged_group = [{protected_variable: unprivileged_input}] #female=0 metric_orig = BinaryLabelDatasetMetric(df_aif, unprivileged_group, privileged_group) print('1-min(DI, 1/DI):', get_disparity_index(metric_orig.disparate_impact()).round(3)) if get_disparity_index(metric_orig.disparate_impact()).round(3) < 0.2: print('The algorithm can be considered to be not biased') else: print('There is a potential bias')
def get_dataset_metrics_list(binary_dataset_list): #Set privileged and unprivileged groups privileged_groups= [{'sex':1}] unprivileged_groups= [{'sex': 0}] mean_diff_list = [] disp_imp_list = [] for dataset in binary_dataset_list: metrics = BinaryLabelDatasetMetric(dataset, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) mean_diff_list.append(metrics.mean_difference()) disp_imp_list.append(1 - metrics.disparate_impact()) return mean_diff_list, disp_imp_list
def nondebiased_classifier(train, test, privileged_groups, unprivileged_groups): sess = tf.Session() NN_model = AdversarialDebiasing(privileged_groups, unprivileged_groups, scope_name='nondebiased_classifier', debias=False, sess=sess) NN_model.fit(train) # predict outcome using the test set pred_NNmodel = NN_model.predict(test) sess.close() tf.reset_default_graph() # calculate accuracy accuracy = accuracy_score(y_true=test.labels, y_pred=pred_NNmodel.labels) # calculate fairness metrics metric_test = BinaryLabelDatasetMetric( pred_NNmodel, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) acc_test = ClassificationMetric(test, pred_NNmodel, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) equal_opportunity_difference = equal_opp_diff(test, pred_NNmodel, 'sex', privileged=1, unprivileged=0, favourable=1, unfavourable=0) average_odds_difference = avg_odds_diff(test, pred_NNmodel, 'sex', privileged=1, unprivileged=0, favourable=1, unfavourable=0) metrics = [ metric_test.mean_difference(), acc_test.disparate_impact(), equal_opportunity_difference, average_odds_difference, acc_test.theil_index() ] return pred_NNmodel, accuracy, metrics
def prejudice(train, test, unprivileged_groups, privileged_groups): prejudice_model = PrejudiceRemover(eta=100, sensitive_attr='sex') prejudice_model.fit(train) # predict outcome using the test set pred_prejudice = prejudice_model.predict(test) # calculate accuracy accuracy = accuracy_score(y_true=test.labels, y_pred=pred_prejudice.labels) # calculate fairness metrics metric_test = BinaryLabelDatasetMetric( pred_prejudice, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) acc_test = ClassificationMetric(test, pred_prejudice, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) equal_opportunity_difference = equal_opp_diff(test, pred_prejudice, 'sex', privileged=1, unprivileged=0, favourable=1, unfavourable=0) average_odds_difference = avg_odds_diff(test, pred_prejudice, 'sex', privileged=1, unprivileged=0, favourable=1, unfavourable=0) if acc_test.disparate_impact() == math.inf: disparate_impact = 5.0 else: disparate_impact = acc_test.disparate_impact() metrics = [ metric_test.mean_difference(), disparate_impact, equal_opportunity_difference, average_odds_difference, acc_test.theil_index() ] return pred_prejudice, accuracy, metrics
def ensemble(test, pred_adversarial, pred_prejudice, pred_nondebiased, unprivileged_groups, privileged_groups): pred_labels = [] for i in range(0, len(test.features)): arr = mode([ pred_adversarial.labels[i], pred_prejudice.labels[i], pred_nondebiased.labels[i] ]) pred_labels.append(arr[0][0]) pred_ensemble = test.copy() pred_ensemble.labels = np.array(pred_labels) accuracy = accuracy_score(y_true=test.labels, y_pred=pred_ensemble.labels) metric_test = BinaryLabelDatasetMetric( pred_ensemble, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) acc_test = ClassificationMetric(test, pred_ensemble, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) equal_opportunity_difference = equal_opp_diff(test, pred_ensemble, 'sex', privileged=1, unprivileged=0, favourable=1, unfavourable=0) average_odds_difference = avg_odds_diff(test, pred_ensemble, 'sex', privileged=1, unprivileged=0, favourable=1, unfavourable=0) metrics = [ metric_test.mean_difference(), acc_test.disparate_impact(), equal_opportunity_difference, average_odds_difference, acc_test.theil_index() ] return accuracy, metrics
def create_binary(data, target_variable, protected_variable, unprivileged_input): df_aif = BinaryLabelDataset(df=data, label_names=[target_variable], protected_attribute_names=[protected_variable]) privileged_group = [] for v in data[protected_variable].unique()[data[protected_variable].unique() != unprivileged_input]: privileged_group.append({protected_variable: v}) unprivileged_group = [{protected_variable: unprivileged_input}] #female=0 return BinaryLabelDatasetMetric(df_aif, unprivileged_groups=unprivileged_group, privileged_groups=privileged_group)
def compute_statistical_parity(data, unpriv_group, priv_group): if isinstance(data, pd.DataFrame): transformed_data = BinaryLabelDataset( df=data, label_names=["two_year_recid"], protected_attribute_names=["race"], favorable_label=0, unfavorable_label=1) else: transformed_data = data metric_test_data = BinaryLabelDatasetMetric( transformed_data, unprivileged_groups=unpriv_group, privileged_groups=priv_group) parity_difference = metric_test_data.statistical_parity_difference() print( f"Mean difference (statistical parity difference) = {parity_difference}" ) return parity_difference
def plot_using_aif(df_predict,df_true): predict_list, true_list = [], [] unpriv_label_list , priv_label_list = [], [] for (u,p) in zip(unpriv_list,priv_list): cur_predict, cur_true = [], [] unpriv_label = '+'.join(['-'.join([prot_attr_dict[key][u_el[key]] for key in u_el]) for u_el in u]) priv_label = '+'.join(['-'.join([prot_attr_dict[key][p_el[key]] for key in p_el]) for p_el in p]) print('-------------------------------------------------------------------') print('unpriv_label:-->',unpriv_label) print('-------------------------------------------------------------------') print('priv_label :-->',priv_label) print('-------------------------------------------------------------------') print('\n\n') for i,label in enumerate(rating_names): #print('Fairness Metric for the label------>',label.upper()) predict_dataset = StandardDataset(df=predict_df_list[i], label_name=label, favorable_classes=[1.0,1.0], protected_attribute_names=protected_attribute_names, privileged_classes=privileged_classes) true_dataset = StandardDataset(df=true_df_list[i], label_name=label, favorable_classes=[1.0,1.0], protected_attribute_names=protected_attribute_names, privileged_classes=privileged_classes) predict_dataset_metric = BinaryLabelDatasetMetric(predict_dataset, unprivileged_groups=u, privileged_groups=p) true_dataset_metric = BinaryLabelDatasetMetric(true_dataset, unprivileged_groups=u, privileged_groups=p) #classfication_metric = ClassificationMetric(true_dataset, predict_dataset, unprivileged_groups=u, privileged_groups=p) #x=classfication_metric.generalized_entropy_index() #print(label,': -->','predicted : -->',abs(predict_dataset_metric.disparate_impact()),'true : -->',abs(true_dataset_metric.disparate_impact())) print(label,': -->','predicted : -->',abs(predict_dataset_metric.mean_difference()),'true : -->',abs(true_dataset_metric.mean_difference()))
def show_metrics(binary_dataset_list): #Set privileged and unprivileged groups privileged_groups= [{'sex':1}] unprivileged_groups= [{'sex': 0}] for dataset in binary_dataset_list: display(Markdown("#### Model dataset metrics")) metrics = BinaryLabelDatasetMetric(dataset, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) ex_metrics = MetricTextExplainer(metrics) print(ex_metrics.mean_difference()) print('\n') print(ex_metrics.disparate_impact())
def explain(self, request: Dict) -> Dict: inputs = request["instances"] predictions = np.array(request["outputs"]) dataframe_predicted = pd.DataFrame(inputs, columns=self.feature_names) dataframe_predicted[self.label_names[0]] = predictions dataset_predicted = BinaryLabelDataset( favorable_label=self.favorable_label, unfavorable_label=self.unfavorable_label, df=dataframe_predicted, label_names=self.label_names, protected_attribute_names=['age']) metrics = BinaryLabelDatasetMetric( dataset_predicted, unprivileged_groups=self.unprivileged_groups, privileged_groups=self.privileged_groups) return { "predictions": predictions.tolist(), "metrics": { "base_rate": metrics.base_rate(), "consistency": metrics.consistency().tolist(), "disparate_impact": metrics.disparate_impact(), "num_instances": metrics.num_instances(), "num_negatives": metrics.num_negatives(), "num_positives": metrics.num_positives(), "statistical_parity_difference": metrics.statistical_parity_difference(), } }
def get_bias_amplification(train_data, prediction_data): privileged_groups= [{'sex':1}] unprivileged_groups= [{'sex': 0}] train_metrics = BinaryLabelDatasetMetric(train_data, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) prediction_metrics = BinaryLabelDatasetMetric(prediction_data, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) tedf = train_metrics.smoothed_empirical_differential_fairness() pedf = prediction_metrics.smoothed_empirical_differential_fairness() bias_amp = pedf - tedf return bias_amp
def generate_fairness_report(dataset, privileged_groups, unprivileged_groups): print(f'Shape: {dataset.features.shape}\n') print(f'Favorable label: {dataset.favorable_label}') print(f'Unfavorable label: {dataset.unfavorable_label}\n') print('Protected attribute names:') print(dataset.protected_attribute_names) print() print('Privileged attribute values:') print(dataset.privileged_protected_attributes) print('Unprivileged attribute values:') print(dataset.unprivileged_protected_attributes) print() binary_label_metric = BinaryLabelDatasetMetric( dataset, privileged_groups=privileged_groups, unprivileged_groups=unprivileged_groups) print(f'Statistical parity difference: ' f'{binary_label_metric.statistical_parity_difference()}') print(f'Disparate impact: {binary_label_metric.disparate_impact()}')
def fit(self, dataset): RW = Reweighing(unprivileged_groups=self.unprivileged_group, privileged_groups=self.privileged_group) mean_diff_metric = lambda dataset: BinaryLabelDatasetMetric( dataset, unprivileged_groups=self.unprivileged_group, privileged_groups=self.privileged_group).mean_difference() dataset_ = RW.fit_transform(dataset) print("before reweighing (meandiff):", mean_diff_metric(dataset), "after:", mean_diff_metric(dataset_)) #reg_ = LogisticRegression(solver='liblinear',max_iter=1000000000, C=1000000000000000000000.0).fit(dataset_.features, dataset_.labels.ravel()) reg = LogisticRegression(solver='liblinear', max_iter=1000000000).fit( dataset_.features, dataset_.labels.ravel(), sample_weight=dataset_.instance_weights) #print("reweighted",sorted(list(zip(dataset.feature_names,reg.coef_[0])),key=lambda x: abs(x[1]))) #print(sorted(list(zip(dataset.feature_names,reg.coef_[0])),key=lambda x: abs(x[1]))) self.h = reg
def main(): print('Calculate bias') np.random.seed(1) protected_attribute = 'ethnicity' dataset = load_preproc_data_heart([protected_attribute]) privileged_groups = [{protected_attribute: 1}] unprivileged_groups = [{ protected_attribute: 2 }, { protected_attribute: 3 }, { protected_attribute: 4 }, { protected_attribute: 5 }, { protected_attribute: 6 }] data_orig_train, data_orig_vt = dataset.split([0.7], shuffle=True) data_orig_valid, data_orig_test = data_orig_vt.split([0.5], shuffle=True) metric_orig_train = BinaryLabelDatasetMetric( data_orig_train, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) print("Mean {}".format(metric_orig_train.mean_difference())) rw = Reweighing(unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) data_transf_train = rw.fit_transform(data_orig_train) metric_transf_train = BinaryLabelDatasetMetric( data_transf_train, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) print("Mean difference after transformation =%f " % metric_transf_train.mean_difference()) calculate_bias_measures(data_orig_train, data_orig_vt, unprivileged_groups, privileged_groups) calculate_bias_measures(data_orig_valid, data_orig_test, unprivileged_groups, privileged_groups)
def train(): np.random.seed(10) def quantizePrior1(x): if x <= 0: return 0 elif 1 <= x <= 3: return 1 else: return 2 def quantizeLOS(x): if x <= 7: return 0 if 8 < x <= 93: return 1 else: return 2 def group_race(x): if x == "Caucasian": return 1.0 else: return 0.0 filepath = 'AIF360/aif360/data/raw/compas/compas-scores-two-years.csv' df = pd.read_csv(filepath, index_col='id', na_values=[]) df['age_cat'] = df['age_cat'].replace('Greater than 45', 2) df['age_cat'] = df['age_cat'].replace('25 - 45', 1) df['age_cat'] = df['age_cat'].replace('Less than 25', 0) df['score_text'] = df['score_text'].replace('High', 1) df['score_text'] = df['score_text'].replace('Medium', 1) df['score_text'] = df['score_text'].replace('Low', 0) df['priors_count'] = df['priors_count'].apply(lambda x: quantizePrior1(x)) df['length_of_stay'] = ( pd.to_datetime(df['c_jail_out']) - pd.to_datetime(df['c_jail_in'])).apply(lambda x: x.days) df['length_of_stay'] = df['length_of_stay'].apply(lambda x: quantizeLOS(x)) df = df.loc[ ~df['race'].isin(['Native American', 'Hispanic', 'Asian', 'Other']), :] df['c_charge_degree'] = df['c_charge_degree'].replace({'F': 0, 'M': 1}) df1 = df[[ 'priors_count', 'c_charge_degree', 'race', 'age_cat', 'score_text', 'two_year_recid' ]] feature_list = [] for index, row in df1.iterrows(): feature_list.append('\t'.join(row.astype(str).to_list())) df1['feature_list'] = feature_list df3 = df1.groupby('feature_list').count() / len(df1.index) df2 = pd.DataFrame() df2['feature_list'] = list(df3.index) df2['prob_list'] = list(df3.priors_count) for index, row in df2.iterrows(): if row['feature_list'][0] == '0' and row['feature_list'][ -1] == '1' and 'African' in row['feature_list']: row['prob_list'] = row['prob_list'] * 10 elif row['feature_list'][0] == '0' and row['feature_list'][-1] == '1': row['prob_list'] = row['prob_list'] * 7 elif row['feature_list'][0] == '2' and row['feature_list'][-1] == '0': row['prob_list'] = row['prob_list'] * 7 prob_list = list(df2.prob_list) df_new = pd.DataFrame() rng = np.random.default_rng() prob_list = np.array(prob_list) prob_list = prob_list / prob_list.sum() feature_list = rng.choice(list(df2.feature_list), len(df1.index), p=prob_list) var_list = [ 'priors_count', 'c_charge_degree', 'race', 'age_cat', 'score_text', 'two_year_recid' ] for i in var_list: vars()[i] = [] for i in feature_list: tmp = i.split('\t') for j in range(len(var_list)): vars()[var_list[j]].append(tmp[j]) for i in var_list: df_new[i] = vars()[i] df = df_new df1 = df[[ 'priors_count', 'c_charge_degree', 'race', 'age_cat', 'score_text', 'two_year_recid' ]] tot = [] for index, row in df1.iterrows(): result = '' for j in df1.columns: result = result + str(row[j]) tot.append(result) df['tmp_feature'] = tot df['mis_prob'] = 0 for i in df['tmp_feature'].unique(): if 'African' in i and i[-1] == '0': df.loc[df['tmp_feature'] == i, 'mis_prob'] = 0.1 elif 'African' in i: df.loc[df['tmp_feature'] == i, 'mis_prob'] = 0.02 elif 'African' not in i and i[-1] == '0': df.loc[df['tmp_feature'] == i, 'mis_prob'] = 0.02 else: df.loc[df['tmp_feature'] == i, 'mis_prob'] = 0.02 new_label = [] for i, j in zip(df['mis_prob'], df['priors_count']): if np.random.binomial(1, i, 1)[0] == 1: new_label.append(3) else: new_label.append(j) df['priors_count'] = new_label print(len(df.loc[df['priors_count'] == 3, :].index)) print(len(df.index)) df['priors_count'] = df['priors_count'].astype(int) df['score_text'] = df['score_text'].astype(int) df['age_cat'] = df['age_cat'].astype(int) df['score_text'] = df['score_text'].astype(int) df['c_charge_degree'] = df['c_charge_degree'].astype(int) df['two_year_recid'] = df['two_year_recid'].astype(int) df['c_charge_degree'] = df['c_charge_degree'].replace({0: 'F', 1: 'M'}) def quantizePrior(x): if x == 0: return '0' elif x == 1: return '1 to 3' elif x == 2: return 'More than 3' else: return 'missing' # Quantize length of stay def quantizeLOS(x): if x == 0: return '<week' if x == 1: return '<3months' else: return '>3 months' # Quantize length of stay def adjustAge(x): if x == 0: return '25 to 45' elif x == 1: return 'Greater than 45' elif x == 2: return 'Less than 25' def quantizeScore(x): if x == 1: return 'MediumHigh' else: return 'Low' def group_race(x): if x == "Caucasian": return 1.0 else: return 0.0 df['priors_count'] = df['priors_count'].apply(lambda x: quantizePrior(x)) df['score_text'] = df['score_text'].apply(lambda x: quantizeScore(x)) df['age_cat'] = df['age_cat'].apply(lambda x: adjustAge(x)) # Recode sex and race df['race'] = df['race'].apply(lambda x: group_race(x)) df['race'] = df['race'].astype(int) df['two_year_recid'] = df['two_year_recid'].astype(int) df = df[[ 'priors_count', 'c_charge_degree', 'race', 'age_cat', 'score_text', 'two_year_recid' ]] df_train, df_test = train_test_split(df, test_size=0.3, random_state=10) all_protected_attribute_maps = {"race": {0.0: 0, 1.0: 1}} D_features = ['race'] dataset_orig_train = CustomDataset( label_name='two_year_recid', favorable_classes=[0], protected_attribute_names=['race'], privileged_classes=[[1]], categorical_features=[ 'priors_count', 'c_charge_degree', 'age_cat', 'score_text' ], features_to_keep=[ 'priors_count', 'c_charge_degree', 'race', 'age_cat', 'score_text' ], df=df_train, metadata={ 'label_maps': [{ 1: 'Did recid.', 0: 'No recid.' }], 'protected_attribute_maps': [all_protected_attribute_maps[x] for x in D_features] }) dataset_orig_vt = CustomDataset( label_name='two_year_recid', favorable_classes=[0], protected_attribute_names=['race'], privileged_classes=[[1]], categorical_features=[ 'priors_count', 'c_charge_degree', 'age_cat', 'score_text' ], features_to_keep=[ 'priors_count', 'c_charge_degree', 'race', 'age_cat', 'score_text' ], df=df_test, metadata={ 'label_maps': [{ 1: 'Did recid.', 0: 'No recid.' }], 'protected_attribute_maps': [all_protected_attribute_maps[x] for x in D_features] }) privileged_groups = [{'race': 1}] unprivileged_groups = [{'race': 0}] optim_options = { "distortion_fun": get_distortion_compas, "epsilon": 0.04, "clist": [0.99, 1.99, 2.99], "dlist": [.1, 0.05, 0] } metric_transf_train = BinaryLabelDatasetMetric( dataset_orig_train, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) OP = OptimPreproc(OptTools, optim_options, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) OP = OP.fit(dataset_orig_train) dataset_transf_cat_test = OP.transform(dataset_orig_vt, transform_Y=True) dataset_transf_cat_test = dataset_orig_vt.align_datasets( dataset_transf_cat_test) dataset_transf_cat_train = OP.transform(dataset_orig_train, transform_Y=True) dataset_transf_cat_train = dataset_orig_train.align_datasets( dataset_transf_cat_train) scale_transf = StandardScaler() X_train = scale_transf.fit_transform(dataset_transf_cat_train.features) y_train = dataset_transf_cat_train.labels.ravel() X_test = scale_transf.fit_transform(dataset_transf_cat_test.features) lmod = LogisticRegression() lmod.fit(X_train, y_train) y_pred = lmod.predict(X_test) print('Without reweight') print('Accuracy') print(accuracy_score(dataset_orig_vt.labels, y_pred)) dataset_orig_vt_copy1 = dataset_orig_vt.copy() dataset_orig_vt_copy1.labels = y_pred metric_transf_train1 = BinaryLabelDatasetMetric( dataset_orig_vt_copy1, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) print('p-rule') print(metric_transf_train1.disparate_impact()) print('CV') print(metric_transf_train1.mean_difference()) print('FPR for unpriv') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[ orig_sens_att == 0], 0)) print("FNR for unpriv") print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[ orig_sens_att == 0], 0)) print('FPR for priv') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[ orig_sens_att == 1], 0)) print("FNR for priv") print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[ orig_sens_att == 1], 0)) df_weight = dataset_orig_train.convert_to_dataframe()[0] df_weight['weight'] = 1 df_weight['is_missing'] = 0 df_weight['tmp'] = '' tmp_result = [] for i, j in zip(df_weight['race'], df_weight['two_year_recid']): tmp_result.append(str(i) + str(j)) df_weight['tmp'] = tmp_result df_weight.loc[df_weight['priors_count=missing'] == 1, 'is_missing'] = 1 for i in df_weight['tmp'].unique(): df_weight.loc[ (df_weight['tmp'] == i) & (df_weight['is_missing'] == 0), 'weight'] = len( df_weight.loc[(df_weight['tmp'] == i), :].index) / len( df_weight.loc[(df_weight['tmp'] == i) & (df_weight['is_missing'] == 0), :].index) df_weight.loc[(df_weight['tmp'] == i) & (df_weight['is_missing'] == 1), 'weight'] = len(df_weight.loc[ (df_weight['tmp'] == i) & (df_weight['is_missing'] == 0), :].index) / len( df_weight.loc[(df_weight['tmp'] == i), :].index) dataset_orig_train.instance_weights = np.array(df_weight['weight']) scale_transf = StandardScaler() X_train = scale_transf.fit_transform(dataset_transf_cat_train.features) y_train = dataset_transf_cat_train.labels.ravel() X_test = scale_transf.fit_transform(dataset_transf_cat_test.features) lmod = LogisticRegression() lmod.fit(X_train, y_train, sample_weight=dataset_orig_train.instance_weights) y_pred = lmod.predict(X_test) print('With reweight') print('Accuracy') print(accuracy_score(dataset_orig_vt.labels, y_pred)) dataset_orig_vt_copy1 = dataset_orig_vt.copy() dataset_orig_vt_copy1.labels = y_pred metric_transf_train1 = BinaryLabelDatasetMetric( dataset_orig_vt_copy1, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) print('p-rule') print(metric_transf_train1.disparate_impact()) print('CV') print(metric_transf_train1.mean_difference()) print('FPR for unpriv') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[ orig_sens_att == 0], 0)) print("FNR for unpriv") print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[ orig_sens_att == 0], 0)) print('FPR for priv') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[ orig_sens_att == 1], 0)) print("FNR for priv") print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[ orig_sens_att == 1], 0))
def main() -> None: # read from inventor filepath = ait_input.get_inventory_path('Data') # prepare column names as given by german.doc column_names = [ 'status', 'month', 'credit_history', 'purpose', 'credit_amount', 'savings', 'employment', 'investment_as_income_percentage', 'personal_status', 'other_debtors', 'residence_since', 'property', 'age', 'installment_plans', 'housing', 'number_of_credits', 'skill_level', 'people_liable_for', 'telephone', 'foreign_worker', 'credit' ] # load into a dataframe df = data_loading(filepath=filepath, column_names=column_names, na_values=None) # prepare for mappings mappings = { 'label_maps': [{ 1.0: 'Good Credit', 2.0: 'Bad Credit' }], 'protected_attribute_maps': [{ 1.0: 'Male', 0.0: 'Female' }, { 1.0: 'Old', 0.0: 'Young' }] } # prepare for categorical features categorical_features = [ 'status', 'credit_history', 'purpose', 'savings', 'employment', 'other_debtors', 'property', 'installment_plans', 'housing', 'skill_level', 'telephone', 'foreign_worker' ] # load param protected_attribute = ait_input.get_method_param_value( 'protected_attribute') privileged_classes = ait_input.get_method_param_value('privileged_classes') # input check ait_input_check(protected_attribute, privileged_classes) # prepare for structure from dataframe and edit data features setting dataset = StandardDataset( df=df, label_name='credit', favorable_classes=[1], protected_attribute_names=[protected_attribute], privileged_classes=[lambda x: x >= privileged_classes], instance_weights_name=None, categorical_features=categorical_features, features_to_keep=None, features_to_drop=['personal_status', 'sex'], na_values=None, custom_preprocessing=preprocessing, metadata=mappings) # set two variables for the privileged (1) and unprivileged (0) values for the age attribute. privileged_groups = [{protected_attribute: 1}] unprivileged_groups = [{protected_attribute: 0}] # compute fairness metric on original training dataset metric_fairness = BinaryLabelDatasetMetric( dataset, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) print("Original training dataset: German Credit Data") print( "Difference in mean outcomes between unprivileged and privileged groups = %f" % metric_fairness.mean_difference()) print("unprivileged groups = %f" % metric_fairness.base_rate(privileged=False)) print("privileged groups = %f" % metric_fairness.base_rate(privileged=True)) # resource observed_predicted_plot save_metric_fairness_plot(metric_fairness, protected_attribute) # measures measure_mean_difference(metric_fairness.mean_difference()) # ait.log move_log()
def fit(self, dataset_true, dataset_pred): """Estimates the optimal classification threshold and margin for reject option classification that optimizes the metric provided. Note: The `fit` function is a no-op for this algorithm. Args: dataset_true (BinaryLabelDataset): Dataset containing the true `labels`. dataset_pred (BinaryLabelDataset): Dataset containing the predicted `scores`. Returns: RejectOptionClassification: Returns self. """ fair_metric_arr = np.zeros(self.num_class_thresh*self.num_ROC_margin) balanced_acc_arr = np.zeros_like(fair_metric_arr) ROC_margin_arr = np.zeros_like(fair_metric_arr) class_thresh_arr = np.zeros_like(fair_metric_arr) cnt = 0 # Iterate through class thresholds for class_thresh in np.linspace(self.low_class_thresh, self.high_class_thresh, self.num_class_thresh): self.classification_threshold = class_thresh if class_thresh <= 0.5: low_ROC_margin = 0.0 high_ROC_margin = class_thresh else: low_ROC_margin = 0.0 high_ROC_margin = (1.0-class_thresh) # Iterate through ROC margins for ROC_margin in np.linspace( low_ROC_margin, high_ROC_margin, self.num_ROC_margin): self.ROC_margin = ROC_margin # Predict using the current threshold and margin dataset_transf_pred = self.predict(dataset_pred) dataset_transf_metric_pred = BinaryLabelDatasetMetric( dataset_transf_pred, unprivileged_groups=self.unprivileged_groups, privileged_groups=self.privileged_groups) classified_transf_metric = ClassificationMetric( dataset_true, dataset_transf_pred, unprivileged_groups=self.unprivileged_groups, privileged_groups=self.privileged_groups) ROC_margin_arr[cnt] = self.ROC_margin class_thresh_arr[cnt] = self.classification_threshold # Balanced accuracy and fairness metric computations balanced_acc_arr[cnt] = 0.5*(classified_transf_metric.true_positive_rate()\ +classified_transf_metric.true_negative_rate()) if self.metric_name == "Statistical parity difference": fair_metric_arr[cnt] = dataset_transf_metric_pred.mean_difference() elif self.metric_name == "Average odds difference": fair_metric_arr[cnt] = classified_transf_metric.average_odds_difference() elif self.metric_name == "Equal opportunity difference": fair_metric_arr[cnt] = classified_transf_metric.equal_opportunity_difference() cnt += 1 rel_inds = np.logical_and(fair_metric_arr >= self.metric_lb, fair_metric_arr <= self.metric_ub) if any(rel_inds): best_ind = np.where(balanced_acc_arr[rel_inds] == np.max(balanced_acc_arr[rel_inds]))[0][0] else: warn("Unable to satisy fairness constraints") rel_inds = np.ones(len(fair_metric_arr), dtype=bool) best_ind = np.where(fair_metric_arr[rel_inds] == np.min(fair_metric_arr[rel_inds]))[0][0] self.ROC_margin = ROC_margin_arr[rel_inds][best_ind] self.classification_threshold = class_thresh_arr[rel_inds][best_ind] return self
# In[ ]: new_dataset = BinaryLabelDataset(favorable_label=favorable_label, unfavorable_label=unfavorable_label, df=new_b_train, label_names=['Attrition'], protected_attribute_names=['Gender'], unprivileged_protected_attributes=unprivileged_groups) # In[ ]: metric_orig_train = BinaryLabelDatasetMetric(new_dataset, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) display(Markdown("#### Original training dataset")) print("Difference in mean outcomes between unprivileged and privileged groups = %f" % metric_orig_train.mean_difference()) # In[ ]: new_dataset.protected_attribute_names # In[ ]:
train1, test1 = dataset_orig.split([0.7], shuffle=True) RW = Reweighing(unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) RW.fit(train1) dataset_transf_train = RW.transform(train1) sess = tf.Session() debiased_model = AdversarialDebiasing(privileged_groups = privileged_groups, unprivileged_groups = unprivileged_groups, scope_name='debiased_classifier', debias=True, sess=sess) debiased_model.fit(train1) dataset_debiasing_train = debiased_model.predict(train1) dataset_debiasing_test = debiased_model.predict(test1) metric_debiasing_train = BinaryLabelDatasetMetric(dataset_debiasing_train, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) metric_debiasing_test = BinaryLabelDatasetMetric(dataset_debiasing_test, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) metric_dataset_debiasing_train.append(metric_debiasing_train.mean_difference()) metric_dataset_debiasing_test.append(metric_debiasing_test.mean_difference()) sess.close() tf.reset_default_graph() sess = tf.Session() debiased_model = AdversarialDebiasing(privileged_groups = privileged_groups, unprivileged_groups = unprivileged_groups, scope_name='debiased_classifier',
def train(): privileged_groups = [{'race': 1}] unprivileged_groups = [{'race': 0}] dataset_orig = load_preproc_data_compas(['race']) optim_options = { "distortion_fun": get_distortion_compas, "epsilon": 0.05, "clist": [0.99, 1.99, 2.99], "dlist": [.1, 0.05, 0] } dataset_orig_train, dataset_orig_vt = dataset_orig.split([0.7], shuffle=True) metric_transf_train = BinaryLabelDatasetMetric( dataset_orig_train, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) OP = OptimPreproc(OptTools, optim_options, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) OP = OP.fit(dataset_orig_train) dataset_transf_cat_test = OP.transform(dataset_orig_vt, transform_Y=True) dataset_transf_cat_test = dataset_orig_vt.align_datasets( dataset_transf_cat_test) dataset_transf_cat_train = OP.transform(dataset_orig_train, transform_Y=True) dataset_transf_cat_train = dataset_orig_train.align_datasets( dataset_transf_cat_train) scale_transf = StandardScaler() X_train = scale_transf.fit_transform(dataset_transf_cat_train.features) y_train = dataset_transf_cat_train.labels.ravel() X_test = scale_transf.fit_transform(dataset_transf_cat_test.features) lmod = LogisticRegression() lmod.fit(X_train, y_train) y_pred = lmod.predict(X_test) print('Without reweight') print('Accuracy') print(accuracy_score(dataset_orig_vt.labels, y_pred)) dataset_orig_vt_copy1 = dataset_orig_vt.copy() dataset_orig_vt_copy1.labels = y_pred metric_transf_train1 = BinaryLabelDatasetMetric( dataset_orig_vt_copy1, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) print('p-rule') print(metric_transf_train1.disparate_impact()) print('CV') print(metric_transf_train1.mean_difference()) print('FPR for unpriv') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[ orig_sens_att == 0], 0)) print("FNR for unpriv") print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[ orig_sens_att == 0], 0)) print('FPR for priv') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[ orig_sens_att == 1], 0)) print("FNR for priv") print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[ orig_sens_att == 1], 0)) df_weight = dataset_orig_train.convert_to_dataframe()[0] df_weight['weight'] = 1 df_weight['is_missing'] = 0 df_weight['tmp'] = '' tmp_result = [] for i, j in zip(df_weight['race'], df_weight['two_year_recid']): tmp_result.append(str(i) + str(j)) df_weight['tmp'] = tmp_result df_weight.loc[df_weight['priors_count=missing'] == 1, 'is_missing'] = 1 for i in df_weight['tmp'].unique(): df_weight.loc[ (df_weight['tmp'] == i) & (df_weight['is_missing'] == 0), 'weight'] = len( df_weight.loc[(df_weight['tmp'] == i), :].index) / len( df_weight.loc[(df_weight['tmp'] == i) & (df_weight['is_missing'] == 0), :].index) df_weight.loc[(df_weight['tmp'] == i) & (df_weight['is_missing'] == 1), 'weight'] = len(df_weight.loc[ (df_weight['tmp'] == i) & (df_weight['is_missing'] == 0), :].index) / len( df_weight.loc[(df_weight['tmp'] == i), :].index) dataset_orig_train.instance_weights = np.array(df_weight['weight']) scale_transf = StandardScaler() #X_train = scale_transf.fit_transform(dataset_transf_cat_train.features[:,1:]) X_train = scale_transf.fit_transform(dataset_transf_cat_train.features) y_train = dataset_transf_cat_train.labels.ravel() #X_test = scale_transf.fit_transform(dataset_transf_cat_test.features[:,1:]) X_test = scale_transf.fit_transform(dataset_transf_cat_test.features) lmod = LogisticRegression() lmod.fit(X_train, y_train, sample_weight=dataset_orig_train.instance_weights) y_pred = lmod.predict(X_test) print('With reweight') print('Accuracy') print(accuracy_score(dataset_orig_vt.labels, y_pred)) dataset_orig_vt_copy1 = dataset_orig_vt.copy() dataset_orig_vt_copy1.labels = y_pred metric_transf_train1 = BinaryLabelDatasetMetric( dataset_orig_vt_copy1, unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups) print('p-rule') print(metric_transf_train1.disparate_impact()) print('CV') print(metric_transf_train1.mean_difference()) print('FPR for unpriv') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[ orig_sens_att == 0], 0)) print("FNR for unpriv") print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 0], y_pred[ orig_sens_att == 0], 0)) print('FPR for priv') orig_sens_att = dataset_orig_vt.protected_attributes.ravel() print(1 - TNR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[ orig_sens_att == 1], 0)) print("FNR for priv") print(1 - TPR(dataset_orig_vt.labels.ravel()[orig_sens_att == 1], y_pred[ orig_sens_att == 1], 0))
for i in range(validation_comp.shape[1]): f_c.append([]) if (validation_comp.columns[i].find(unprivileged_group) != -1): prot.append(validation_comp.columns[i]) unpriv_dict = {validation_comp.columns[i]: 1} if (validation_comp.columns[i].find(privileged_group) != -1): priv.append([1]) prot.append(validation_comp.columns[i]) priv_dict = {validation_comp.columns[i]: 1} else: priv.append([]) stdDs = StandardDataset(validation_comp, 'is_violent_recid', [0], prot, priv) stdPred = StandardDataset(validation_pred, 'is_violent_recid', [0], prot, priv) bi_met = BinaryLabelDatasetMetric(stdDs, privileged_groups=[priv_dict], unprivileged_groups=[unpriv_dict]) class_met = ClassificationMetric(stdDs, stdPred, unprivileged_groups=[unpriv_dict], privileged_groups=[priv_dict]) disparate_impact = bi_met.disparate_impact() #error_rate_ratio = class_met.error_rate_ratio() eq_diff = class_met.equal_opportunity_difference() #Create 2 Bar Graphs x = [1] di_y = [disparate_impact] er_y = [error_rate_ratio] plt.ylim(bottom=0, top=2)