def test_error_rate(self):
data = pd.read_csv('tests/validation.csv', index_col=0)
model_outcome = data['prediction'].values
truth_outcome = data['truth'].values
protected_class = data['gender'].values
assert_equals(metrics.rel_error_rate(protected_class,
model_outcome,
truth_outcome),
(0.125, 0.625, 0.75))
def do_audit(filename):
rawcsv = '{}/{}'.format(app.config['UPLOAD_FOLDER'], filename)
df = pd.read_csv(rawcsv)
audit = {}
audit['race_gf'] = metrics.group_fairness(df['race'].values, df['prediction'].values)
audit['race_num_majority'] = len(df[df['race'] == 1])
audit['race_num_minority'] = len(df[df['race'] == 0])
df_pos = df[df['prediction'] == 1]
audit['race_num_minority_pos'] = len(df_pos[df_pos['race'] == 0])
audit['race_num_majority_pos'] = len(df_pos[df_pos['race'] == 1])
audit['gender_num_majority'] = len(df[df['gender'] == 1])
audit['gender_num_minority'] = len(df[df['gender'] == 0])
audit['gender_num_minority_pos'] = len(df_pos[df_pos['gender'] == 0])
audit['gender_num_majority_pos'] = len(df_pos[df_pos['gender'] == 1])
audit['rel_race_error_rate'], audit['rel_pos_race_error_rate'], audit['rel_neg_race_error_rate'] = metrics.rel_error_rate(df['race'].values, df['prediction'].values, df['truth'].values)
audit['rel_gender_error_rate'], audit['rel_pos_gender_error_rate'], audit['rel_neg_gender_error_rate'] = metrics.rel_error_rate(df['gender'].values, df['prediction'].values, df['truth'].values)
audit['gender_gf'] = metrics.group_fairness(df['gender'].values, df['prediction'].values)
df_pos = df[df['truth'] == 1]
df['error'] = np.abs(df_pos['prediction'] - df_pos['truth'])
df_incorrect = df[df['error'] > 0]
audit['race_majority_wrong'] = float(len(df_incorrect['race'] == 1)) / float(len(df_pos))
audit['race_minority_wrong'] = float(len(df_incorrect['race'] == 0)) / float(len(df_pos))
audit['gender_majority_wrong'] = float(len(df_incorrect['gender'] == 1)) / float(len(df_pos))
audit['gender_minority_wrong'] = float(len(df_incorrect['gender'] == 0)) / float(len(df_pos))
return render_template('auditresult.html', df=df, filename=filename, audit=audit)