def assess_bias(bias_df,
metrics=['fnr', 'for', 'fdr', 'tpr', 'tnr'],
min_group_size=None):
'''
This function creates bar charts for bias metrics given.
bias_df is preprocessed
'''
g = Group()
xtab, _ = g.get_crosstabs(bias_df)
aqp = Plot()
p = aqp.plot_group_metric_all(xtab,
metrics=metrics,
ncols=len(metrics),
min_group_size=min_group_size)
p.show()
return
def plot_bias(model_name, bias_df, bias_metrics = ['ppr','pprev','fnr','fpr', 'for'], min_group_size = None, output_type = None):
'''
This function creates bar charts for bias metrics given.
bias_df = dataframe with ID, label, predicted scores already taking into account the population threshold, and
'''
g = Group()
xtab, _ = g.get_crosstabs(bias_df)
aqp = Plot()
n = len(bias_metrics)
p = aqp.plot_group_metric_all(xtab, metrics=bias_metrics, ncols=n, min_group_size = min_group_size)
if output_type == 'save':
pltfile = os.path.join(config.GRAPH_FOLDER,model_name)
p.savefig(pltfile)
elif output_type == 'show':
p.show()
return
def plot_fairness(self, is_absolute=True):
fdf = self.get_model_fairness(level='value')
aqp = Plot()
fg = None
if is_absolute:
fg = aqp.plot_fairness_group_all(fdf,
ncols=5,
metrics="all",
show_figure=False)
else:
attr_cols = list(fdf['attribute_name'].unique())
fg = aqp.plot_fairness_disparity_all(fdf,
attributes=attr_cols,
significance_alpha=0.05,
min_group_size=0.01,
show_figure=False)
return fg
def plot_fairness(self, metric):
"""
Returns a plot. You may have to call plt.show() using matplotlib in
order to see anything.
Valid metrics (incomplete list):
- fpr
"""
fdf = self._fdf()
plot = Plot().plot_fairness_group(fdf, group_metric=metric, title=True)
return plot
def plot_group_metric(self, metric):
"""
Returns a plot. You may have to call plt.show() using matplotlib in
order to see anything.
Valid metrics (incomplete list):
- fpr
"""
xtab = self.xtabs()
plot = Plot().plot_group_metric(xtab, metric)
return plot
def plot_disparity(self, metric):
"""
Returns a plot. You may have to call plt.show() using matplotlib in
order to see anything.
Valid metrics (incomplete list):
- fpr_disparity
"""
bdf = self._bdf()
plot = Plot().plot_disparity(bdf,
group_metric=metric,
attribute_name='majority_demo')
return plot
hue="label_value",
data=dataset,
palette=aq_palette_label)
plt.savefig('./figures/LAW_DATA/label_sex_law.png')
plt.clf()
# Calculamos la tabla de metricas de grupo
g = Group()
xtab, _ = g.get_crosstabs(dataset)
absolute_metrics = g.list_absolute_metrics(xtab)
# Mostramos la tabla por pantalla
tabla_grupo = xtab[['attribute_name', 'attribute_value'] +
absolute_metrics].round(2)
print(tabla_grupo)
aqp = Plot()
# Plot de los valores de las metricas de grupo para FNR
fnr = aqp.plot_group_metric(xtab, 'fnr')
# Plot de los valores de las metricas de grupo para FNR eliminando poblaciones con umbral de individuos
fnr = aqp.plot_group_metric(xtab, 'fnr', min_group_size=0.05)
# Metricas de grupo para todas las elegidas
p = aqp.plot_group_metric_all(xtab,
metrics=['ppr', 'pprev', 'fnr', 'fpr'],
ncols=4)
# Calculamos las metricas de sesgo
b = Bias()
# Establecemos los atributos de referencia
bdf = b.get_disparity_predefined_groups(xtab,
original_df=dataset,
ref_groups_dict={
def audit(df, configs, model_id=1, preprocessed=False):
"""
:param df:
:param ref_groups_method:
:param model_id:
:param configs:
:param report:
:param preprocessed:
:return:
"""
if not preprocessed:
df, attr_cols_input = preprocess_input_df(df)
if not configs.attr_cols:
configs.attr_cols = attr_cols_input
g = Group()
print('Welcome to Aequitas-Audit')
print('Fairness measures requested:',
','.join(configs.fair_measures_requested))
groups_model, attr_cols = g.get_crosstabs(
df,
score_thresholds=configs.score_thresholds,
model_id=model_id,
attr_cols=configs.attr_cols)
print('audit: df shape from the crosstabs:', groups_model.shape)
b = Bias()
# todo move this to the new configs object / the attr_cols now are passed through the configs object...
ref_groups_method = configs.ref_groups_method
if ref_groups_method == 'predefined' and configs.ref_groups:
bias_df = b.get_disparity_predefined_groups(groups_model, df,
configs.ref_groups)
elif ref_groups_method == 'majority':
bias_df = b.get_disparity_major_group(groups_model, df)
else:
bias_df = b.get_disparity_min_metric(groups_model, df)
print('Any NaN?: ', bias_df.isnull().values.any())
print('bias_df shape:', bias_df.shape)
aqp = Plot()
if len(configs.plot_bias_metrics) == 1:
fig1 = aqp.plot_disparity(bias_df, metrics=configs.plot_bias_metrics)
elif len(configs.plot_bias_metrics) > 1:
fig1 = aqp.plot_disparity_all(bias_df,
metrics=configs.plot_bias_metrics)
if len(configs.plot_bias_disparities) == 1:
fig2 = aqp.plot_group_metric(bias_df,
metrics=configs.plot_bias_disparities)
elif len(configs.plot_bias_disparities) > 1:
fig2 = aqp.plot_group_metric_all(bias_df,
metrics=configs.plot_bias_disparities)
f = Fairness(tau=configs.fairness_threshold)
print('Fairness Threshold:', configs.fairness_threshold)
print('Fairness Measures:', configs.fair_measures_requested)
group_value_df = f.get_group_value_fairness(
bias_df, fair_measures_requested=configs.fair_measures_requested)
group_attribute_df = f.get_group_attribute_fairness(
group_value_df,
fair_measures_requested=configs.fair_measures_requested)
fair_results = f.get_overall_fairness(group_attribute_df)
if len(configs.plot_bias_metrics) == 1:
fig3 = aqp.plot_fairness_group(group_value_df,
metrics=configs.plot_bias_metrics)
elif len(configs.plot_bias_metrics) > 1:
fig3 = aqp.plot_fairness_group_all(group_value_df,
metrics=configs.plot_bias_metrics)
if len(configs.plot_bias_disparities) == 1:
fig4 = aqp.plot_fairness_disparity(
group_value_df, metrics=configs.plot_bias_disparities)
elif len(configs.plot_bias_metrics) > 1:
fig4 = aqp.plot_fairness_disparity_all(
group_value_df, metrics=configs.plot_bias_disparities)
print(fair_results)
report = None
if configs.report is True:
report = audit_report_markdown(configs, group_value_df,
f.fair_measures_depend, fair_results)
return group_value_df, report
df.groupby('race').describe()
g = Group()
xtab, _ = g.get_crosstabs(df)
xtab # Table of statistics by group
# View calculated absolute metrics for each sample population group
absolute_metrics = g.list_absolute_metrics(xtab)
# FNR: FN / (True Condition Negative) = FN / (FN + TP)
# (what share of the true positives did we mistakenly say were negative?)
# FOR: false omission rate: FN / (FN + TN)
# (how many of the people we labeled negative were false negatives (really postive)?)
# FDR: false discovery rate: FP / (FP + TP)
# (how many of the people we labeled positive were false positives?)
xtab[['attribute_name', 'attribute_value'] + absolute_metrics].round(2)
df['race'].value_counts() / df.shape[0]
aqp = Plot()
# Visualizing multiple user-specified absolute group metrics across all population groups
a = aqp.plot_group_metric_all(xtab, ncols=3)
a.savefig('fairness_plot.png')
# True prevalences
df.groupby('gender').describe()
# Disparity
b = Bias()
bdf = b.get_disparity_predefined_groups(xtab,
original_df=df,
ref_groups_dict={
'race': 'WHITE',
'gender': 'MALE'
},
tabla = tabla.rename(columns = {'etiqueta':'label_value'})
tabla = tabla.set_index(['center'])
######¿Qué sesgo existe en el modelo?########
g = Group()
xtab, _ = g.get_crosstabs(tabla)
absolute_metrics = g.list_absolute_metrics(xtab)
xtab[[col for col in xtab.columns if col not in absolute_metrics]]
aqp = Plot()
a = aqp.plot_group_metric_all(xtab, ncols=3)
######¿Cuál es el nivel de disparidad qué existe entre los grupos de población (categorías de referencia)########
b = Bias()
bdf = b.get_disparity_predefined_groups(xtab, original_df=tabla, ref_groups_dict={'borough':'borough_brooklyn', 'programtype':'programtype_preschool'}, alpha=0.05, mask_significance=True)
hbdf = b.get_disparity_predefined_groups(xtab, original_df=tabla,
ref_groups_dict={'borough':'borough_brooklyn', 'programtype':'programtype_preschool'},
alpha=0.05,
mask_significance=False)
"""
Scripts below adopted from https://github.com/dssg/aequitas
"""
from equitas.bias import Bias
from aequitas.plotting import Plot
aqp = Plot()
fpr_plot = aqp.plot_group_metric(xtab, 'fpr', min_group_size=0.05)
b = Bias()
bdf = b.get_dsiparity_predified_groups(
xtab,
original_df = df,
ref_groups_dict = {"race":"Caucasian", "sex":"Male", "age_cat":"25-45"},
alpha = 0.05,
check_signficance = False)
fpr_disparity = aqp.plot_disparity(bdf, group_metric='fpr_disparity',
attribute_name='race')
def run_aequitas(predictions_data_path):
'''
Check for False negative rate, chances of certain group missing out on assistance using aequitas toolkit
The functions transform the data to make it aequitas complaint and checks for series of bias and fairness metrics
Input: model prediction path for the selected model (unzip the selected file to run)
Output: plots saved in charts folder
'''
best_model_pred = pd.read_csv(predictions_data_path)
# Transform data for aquetias module compliance
aqc = [
'Other', 'White', 'African American', 'Asian', 'Hispanic',
'American Indian'
]
aqcol = [
'White alone_scale', 'Black/AfAmer alone_scale',
'AmInd/Alaskn alone_scale', 'Asian alone_scale', 'HI alone_scale',
'Some other race alone_scale', 'Hispanic or Latino_scale'
]
display(aqcol)
aqcol_label = [
'no_renew_nextpd', 'pred_class_10%',
'Median household income (1999 dollars)_scale'
] + aqcol
aqus = best_model_pred[aqcol_label]
print('Creating classes for racial and income distribution', '\n')
# Convert to binary
bin_var = [
'no_renew_nextpd',
'pred_class_10%',
]
for var in bin_var:
aqus[var] = np.where(aqus[var] == True, 1, 0)
# Rename
aqus.rename(columns={
'no_renew_nextpd': 'label_value',
'pred_class_10%': 'score'
},
inplace=True)
print('Define majority rule defined on relative proportion of the class',
'\n')
aqus['race'] = aqus[aqcol].idxmax(axis=1)
# Use quantile income distribution
aqus['income'] = pd.qcut(
aqus['Median household income (1999 dollars)_scale'],
3,
labels=["rich", "median", "poor"])
# Final form
aqus.drop(aqcol, axis=1, inplace=True)
aqus.drop(['Median household income (1999 dollars)_scale'],
axis=1,
inplace=True)
aq = aqus.reset_index()
aq.rename(columns={'index': 'entity_id'}, inplace=True)
aq['race'] = aq['race'].replace({
'Some other race alone_scale':
'Other',
'White alone_scale':
'White',
'Black/AfAmer alone_scale':
'African American',
'Asian alone_scale':
'Asian',
'HI alone_scale':
'Hispanic',
'AmInd/Alaskn alone_scale':
'American Indian'
})
# Consolidate types
aq['income'] = aq['income'].astype(object)
aq['entity_id'] = aq['entity_id'].astype(object)
aq['score'] = aq['score'].astype(object)
aq['label_value'] = aq['label_value'].astype(object)
# Distribuion of categories
aq_palette = sns.diverging_palette(225, 35, n=2)
by_race = sns.countplot(x="race", data=aq[aq.race.isin(aqc)])
by_race.set_xticklabels(by_race.get_xticklabels(), rotation=40, ha="right")
plt.savefig('charts/Racial distribution in data.png')
# Primary distribuion against score
aq_palette = sns.diverging_palette(225, 35, n=2)
by_race = sns.countplot(x="race",
hue="score",
data=aq[aq.race.isin(aqc)],
palette=aq_palette)
by_race.set_xticklabels(by_race.get_xticklabels(), rotation=40, ha="right")
# Race
plt.savefig('charts/race_score.png')
# Income
by_inc = sns.countplot(x="income",
hue="score",
data=aq,
palette=aq_palette)
plt.savefig('charts/income_score.png')
# Set Group
g = Group()
xtab, _ = g.get_crosstabs(aq)
# False Negative Rates
aqp = Plot()
fnr = aqp.plot_group_metric(xtab, 'fnr', min_group_size=0.05)
p = aqp.plot_group_metric_all(xtab,
metrics=['ppr', 'pprev', 'fnr', 'fpr'],
ncols=4)
p.savefig('charts/eth_metrics.png')
# Bias with respect to white rich category
b = Bias()
bdf = b.get_disparity_predefined_groups(xtab,
original_df=aq,
ref_groups_dict={
'race': 'White',
'income': 'rich'
},
alpha=0.05,
mask_significance=True)
bdf.style
calculated_disparities = b.list_disparities(bdf)
disparity_significance = b.list_significance(bdf)
aqp.plot_disparity(bdf,
group_metric='fpr_disparity',
attribute_name='race',
significance_alpha=0.05)
plt.savefig('charts/disparity.png')
# Fairness
hbdf = b.get_disparity_predefined_groups(xtab,
original_df=aq,
ref_groups_dict={
'race': 'African American',
'income': 'poor'
},
alpha=0.05,
mask_significance=False)
majority_bdf = b.get_disparity_major_group(xtab,
original_df=aq,
mask_significance=True)
min_metric_bdf = b.get_disparity_min_metric(df=xtab, original_df=aq)
f = Fairness()
fdf = f.get_group_value_fairness(bdf)
parity_detrminations = f.list_parities(fdf)
gaf = f.get_group_attribute_fairness(fdf)
gof = f.get_overall_fairness(fdf)
z = aqp.plot_fairness_group(fdf, group_metric='ppr')
plt.savefig('charts/fairness_overall.png')
# Checking for False Omission Rate and False Negative Rates
fg = aqp.plot_fairness_group_all(fdf, metrics=['for', 'fnr'], ncols=2)
fg.savefig('charts/fairness_metrics.png')
return None
def aq_analysis(arguments):
# Import result set from best model.
result_set = pd.read_csv(arguments.result_set) \
.drop_duplicates(subset="block_group") \
.rename(columns={"successful": "label_value"})
# Drop columns not needed for analysis.
features_to_drop = [
column for column in result_set.columns
if column in DROP_COLUMN_KEYWORDS and "count" not in column
]
result_set = result_set.drop(columns=features_to_drop)
# Initialize base comparison attributes dictionary.
base_comparison = {"pct_white": None, "pct_high_income": None}
base_comparison_label = "_".join(base_comparison.keys())
# Preprocess outside of Aequitas because preprocess_input_df() doesn't work.
for column in result_set.columns:
if column == "score":
result_set[column] = result_set[column].astype(float)
elif column == "label_value":
result_set[column] = result_set[column].astype(int)
else:
if result_set[column].nunique() > 1:
result_set[column], bins = pd.qcut(x=result_set[column],
q=4,
precision=2,
duplicates="drop",
retbins=True)
# Save label of highest quartile for base comparison attributes.
if column in base_comparison:
lb = str(round(bins[3], 2))
ub = str(round(bins[4], 2))
base_comparison[column] = "(" + lb + ", " + ub + "]"
result_set[column] = result_set[column].astype(str)
# Initialize Aequitas objects and export directory.
aqg, aqb, aqf, aqp = Group(), Bias(), Fairness(), Plot()
directory = "aequitas"
if not os.path.exists(directory):
os.makedirs(directory)
# Calculate crosstabs by distinct group.
crosstabs, _ = aqg.get_crosstabs(
df=result_set,
score_thresholds={"score": [float(arguments.threshold)]})
absolute_metrics = aqg.list_absolute_metrics(crosstabs)
crosstabs[["attribute_name", "attribute_value"] + absolute_metrics] \
.round(2) \
.to_csv(directory + "/aequitas_crosstabs.csv", index=False)
# Plot bias and fairness with respect to white, high income communities.
disparity_white_hiinc = aqb.get_disparity_predefined_groups(
crosstabs.loc[crosstabs["attribute_name"].isin(
base_comparison.keys())], result_set, base_comparison)
a = aqp.plot_disparity_all(disparity_white_hiinc,
metrics=METRICS,
show_figure=False)
a_filename = "bias_ref_" + base_comparison_label + ".png"
a.savefig(directory + "/" + a_filename)
b = aqp.plot_fairness_disparity_all(
aqf.get_group_value_fairness(disparity_white_hiinc),
metrics=METRICS,
show_figure=False)
b_filename = "fairness_ref_" + base_comparison_label + ".png"
b.savefig(directory + "/" + b_filename)
# Aequitas
from aequitas.preprocessing import preprocess_input_df
from aequitas.group import Group
from aequitas.plotting import Plot
from aequitas.bias import Bias
from aequitas.fairness import Fairness
ae_subset_df = pred_test_df[['race', 'gender', 'score', 'label_value']]
ae_df, _ = preprocess_input_df(ae_subset_df)
g = Group()
xtab, _ = g.get_crosstabs(ae_df)
absolute_metrics = g.list_absolute_metrics(xtab)
clean_xtab = xtab.fillna(-1)
aqp = Plot()
b = Bias()
# ## Reference Group Selection
# Below we have chosen the reference group for our analysis but feel free to select another one.
# In[166]:
# test reference group with Caucasian Male
bdf = b.get_disparity_predefined_groups(clean_xtab,
original_df=ae_df,
ref_groups_dict={'race':'Caucasian', 'gender':'Male'
},