def fair_metrics(bst, data, column, thresh):
tr = list(data.get_label())
best_iteration = bst.best_ntree_limit
pred = bst.predict(data, ntree_limit=best_iteration)
pred = [1 if p > thresh else 0 for p in pred]
na0 = 0
na1 = 0
nd0 = 0
nd1 = 0
for p, c in zip(pred, column):
if (p == 1 and c == 0):
nd1 += 1
if (p == 1 and c == 1):
na1 += 1
if (p == 0 and c == 0):
nd0 += 1
if (p == 0 and c == 1):
na0 += 1
Pa1, Pd1, Pa0, Pd0 = na1 / (na1 + na0), nd1 / (nd1 + nd0), na0 / (
na1 + na0), nd0 / (nd1 + nd0)
dsp_metric = np.abs(Pd1 - Pa1)
#dsp_metric = np.abs((first-second)/(first+second))
sr_metric = selection_rate(tr, pred, pos_label=1)
dpd_metric = demographic_parity_difference(tr,
pred,
sensitive_features=column)
dpr_metric = demographic_parity_ratio(tr, pred, sensitive_features=column)
eod_metric = equalized_odds_difference(tr, pred, sensitive_features=column)
return dsp_metric, sr_metric, dpd_metric, dpr_metric, eod_metric
def equalized_odds(df_test_encoded, predictions, print_=False):
eod_sex = equalized_odds_difference(df_test_encoded.earnings, predictions, sensitive_features=df_test_encoded.sex)
eor_sex = equalized_odds_ratio(df_test_encoded.earnings, predictions, sensitive_features=df_test_encoded.sex)
if (print_):
print(f"equalised odds difference sex: {eod_sex:.3f}")
print(f"equalised odds ratio sex: {eor_sex:.3f}")
def __binary_group_fairness_measures(X,
prtc_attr,
y_true,
y_pred,
y_prob=None,
priv_grp=1):
"""[summary]
Args:
X (pandas DataFrame): Sample features
prtc_attr (named array-like): values for the protected attribute
(note: protected attribute may also be present in X)
y_true (pandas DataFrame): Sample targets
y_pred (pandas DataFrame): Sample target predictions
y_prob (pandas DataFrame, optional): Sample target probabilities. Defaults
to None.
Returns:
[type]: [description]
"""
pa_names = prtc_attr.columns.tolist()
gf_vals = {}
gf_key = 'Group Fairness'
gf_vals['Statistical Parity Difference'] = \
aif_mtrc.statistical_parity_difference(y_true, y_pred, prot_attr=pa_names)
gf_vals['Disparate Impact Ratio'] = \
aif_mtrc.disparate_impact_ratio(y_true, y_pred, prot_attr=pa_names)
if not helper.is_tutorial_running() and not len(pa_names) > 1:
gf_vals['Demographic Parity Difference'] = \
fl_mtrc.demographic_parity_difference(y_true, y_pred,
sensitive_features=prtc_attr)
gf_vals['Demographic Parity Ratio'] = \
fl_mtrc.demographic_parity_ratio(y_true, y_pred,
sensitive_features=prtc_attr)
gf_vals['Average Odds Difference'] = \
aif_mtrc.average_odds_difference(y_true, y_pred, prot_attr=pa_names)
gf_vals['Equal Opportunity Difference'] = \
aif_mtrc.equal_opportunity_difference(y_true, y_pred, prot_attr=pa_names)
if not helper.is_tutorial_running() and not len(pa_names) > 1:
gf_vals['Equalized Odds Difference'] = \
fl_mtrc.equalized_odds_difference(y_true, y_pred,
sensitive_features=prtc_attr)
gf_vals['Equalized Odds Ratio'] = \
fl_mtrc.equalized_odds_ratio(y_true, y_pred,
sensitive_features=prtc_attr)
gf_vals['Positive Predictive Parity Difference'] = \
aif_mtrc.difference(sk_metric.precision_score, y_true,
y_pred, prot_attr=pa_names, priv_group=priv_grp)
gf_vals['Balanced Accuracy Difference'] = \
aif_mtrc.difference(sk_metric.balanced_accuracy_score, y_true,
y_pred, prot_attr=pa_names, priv_group=priv_grp)
if y_prob is not None:
gf_vals['AUC Difference'] = \
aif_mtrc.difference(sk_metric.roc_auc_score, y_true, y_prob,
prot_attr=pa_names, priv_group=priv_grp)
return (gf_key, gf_vals)
def test_equalized_odds_difference(agg_method):
actual = equalized_odds_difference(y_t,
y_p,
sensitive_features=g_1,
method=agg_method)
metrics = {'tpr': true_positive_rate, 'fpr': false_positive_rate}
gm = MetricFrame(metrics, y_t, y_p, sensitive_features=g_1)
diffs = gm.difference(method=agg_method)
assert actual == diffs.max()
def test_equalized_odds_difference_weighted(agg_method):
actual = equalized_odds_difference(y_t,
y_p,
sensitive_features=g_1,
method=agg_method,
sample_weight=s_w)
metrics = {'tpr': true_positive_rate, 'fpr': false_positive_rate}
sw = {'sample_weight': s_w}
sp = {'tpr': sw, 'fpr': sw}
gm = MetricFrame(metrics,
y_t,
y_p,
sensitive_features=g_1,
sample_params=sp)
diffs = gm.difference(method=agg_method)
assert actual == diffs.max()
def evaluate_model(model, device, criterion, data_loader):
model.eval()
y_true = []
y_pred = []
y_out = []
sensitives = []
for i, data in enumerate(data_loader):
x, y, sensitive_features = data
x = x.to(device)
y = y.to(device)
sensitive_features = sensitive_features.to(device)
with torch.no_grad():
logit = model(x)
# logit : binary prediction size=(b, 1)
bina = (torch.sigmoid(logit) > 0.5).float()
y_true += y.cpu().tolist()
y_pred += bina.cpu().tolist()
y_out += torch.sigmoid(logit).tolist()
sensitives += sensitive_features.cpu().tolist()
result = {}
result["acc"] = skm.accuracy_score(y_true, y_pred)
result["f1score"] = skm.f1_score(y_true, y_pred)
result["AUC"] = skm.roc_auc_score(y_true, y_out)
result['DP'] = {
"diff":
flm.demographic_parity_difference(
y_true, y_pred, sensitive_features=sensitive_features),
"ratio":
flm.demographic_parity_ratio(y_true,
y_pred,
sensitive_features=sensitive_features),
}
result["EO"] = {
"diff":
flm.equalized_odds_difference(y_true,
y_pred,
sensitive_features=sensitive_features),
"ratio":
flm.equalized_odds_ratio(y_true,
y_pred,
sensitive_features=sensitive_features),
}
return result
def test(args, model, device, test_loader, test_size, sensitive_idx):
model.eval()
criterion = nn.BCELoss()
test_loss = 0
correct = 0
i = 0
avg_recall = 0
avg_precision = 0
overall_results = []
avg_eq_odds = 0
avg_dem_par = 0
avg_tpr = 0
avg_tp = 0
avg_tn = 0
avg_fp = 0
avg_fn = 0
with torch.no_grad():
for cats, conts, target in tqdm(test_loader):
print("*********")
#i += 1
cats, conts, target = cats.to(device), conts.to(device), target.to(device)
output = model(cats, conts)
test_loss += criterion(output, target).item() # sum up batch loss
pred = (output > 0.5).float()
correct += pred.eq(target.view_as(pred)).sum().item()
curr_datetime = datetime.now()
curr_hour = curr_datetime.hour
curr_min = curr_datetime.minute
pred_df = pd.DataFrame(pred.numpy())
pred_df.to_csv(f"pred_results/{args.run_name}_{curr_hour}-{curr_min}.csv")
# confusion matrixç
tn, fp, fn, tp = confusion_matrix(target, pred).ravel()
avg_tn+=tn
avg_fp+=fp
avg_fn+=fn
avg_tp+=tp
# position of col for sensitive values
sensitive = [i[sensitive_idx].item() for i in cats]
cat_len = max(sensitive)
print(cat_len)
#exit()
sub_cm = []
#print(cat_len)
for j in range(cat_len+1):
try:
idx = list(locate(sensitive, lambda x: x == j))
sub_tar = target[idx]
sub_pred = pred[idx]
sub_tn, sub_fp, sub_fn, sub_tp = confusion_matrix(sub_tar, sub_pred).ravel()
except:
# when only one value to predict
temp_tar = int(sub_tar.numpy()[0])
temp_pred = int(sub_pred.numpy()[0])
#print(tar, pred)
if temp_tar and temp_pred:
sub_tn, sub_fp, sub_fn, sub_tp = 0, 0, 0, 1
elif temp_tar and not temp_pred:
sub_tn, sub_fp, sub_fn, sub_tp = 0, 0, 1, 0
elif not temp_tar and not temp_pred:
sub_tn, sub_fp, sub_fn, sub_tp = 1, 0, 0, 0
elif not temp_tar and temp_pred:
sub_tn, sub_fp, sub_fn, sub_tp = 0, 1, 0, 0
else:
sub_tn, sub_fp, sub_fn, sub_tp = 0, 0, 0, 0
total = mysum(sub_tn, sub_fp, sub_fn, sub_tp)
sub_cm.append((sub_tn/total, sub_fp/total, sub_fn/total, sub_tp/total))
# Fairness metrics
group_metrics = MetricFrame({'precision': skm.precision_score, 'recall': skm.recall_score},
target, pred,
sensitive_features=sensitive)
demographic_parity = flm.demographic_parity_difference(target, pred,
sensitive_features=sensitive)
eq_odds = flm.equalized_odds_difference(target, pred,
sensitive_features=sensitive)
# metric_fns = {'true_positive_rate': true_positive_rate}
tpr = MetricFrame(true_positive_rate,
target, pred,
sensitive_features=sensitive)
# tpr = flm.true_positive_rate(target, pred,sample_weight=sensitive)
sub_results = group_metrics.overall.to_dict()
sub_results_by_group = group_metrics.by_group.to_dict()
#print("\n", group_metrics.by_group, "\n")
avg_precision += sub_results['precision']
avg_recall += sub_results['recall']
overall_results.append(sub_results_by_group)
avg_eq_odds += eq_odds
avg_dem_par += demographic_parity
avg_tpr += tpr.difference(method='between_groups')
print(i)
total = mysum(avg_tn, avg_fp, avg_fn, avg_tp)
cm = (avg_tn/total, avg_fp/total, avg_fn/total, avg_tp/total)
test_loss /= test_size
accuracy = correct / test_size
avg_loss = test_loss
return accuracy, avg_loss, avg_precision, avg_recall, avg_eq_odds, avg_tpr, avg_dem_par, cm, sub_cm, overall_results
def test_student(args, student_train_loader, student_labels,
student_test_loader, test_size, cat_emb_size, num_conts,
device, sensitive_idx):
student_model = RandomForestClassifier(random_state=42,
warm_start=True,
n_estimators=100)
print("========== Testing Student Model ==========")
for epoch in range(args.epochs):
train_loader = student_loader(student_train_loader, student_labels)
for (cats, conts), labels in train_loader:
X = torch.cat((cats, conts), 1)
student_model = student_model.fit(X, labels)
test_loss = 0
correct = 0
i = 0
avg_recall = 0
avg_precision = 0
overall_results = []
avg_eq_odds = 0
avg_dem_par = 0
avg_tpr = 0
avg_tp = 0
avg_tn = 0
avg_fp = 0
avg_fn = 0
with torch.no_grad():
for batch_idx, (cats, conts,
target) in enumerate(student_test_loader):
print("target\n", sum(target))
i += 1
X = torch.cat((cats, conts), 1)
output = student_model.predict(X)
output = torch.from_numpy(output)
pred = (output > 0.5).float()
print("pred\n", sum(pred))
correct += pred.eq(target.view_as(pred)).sum().item()
curr_datetime = datetime.now()
curr_hour = curr_datetime.hour
curr_min = curr_datetime.minute
pred_df = pd.DataFrame(pred.numpy())
pred_df.to_csv(
f"pred_results/{args.run_name}_{curr_hour}-{curr_min}.csv"
)
#print(pred, np.sum(np.squeeze(pred.eq(target.data.view_as(pred))).cpu().numpy()))
#correct += np.sum(np.squeeze(pred.eq(target.data.view_as(pred))).cpu().numpy())
#total += cats.size(0)
# confusion matrixç
tn, fp, fn, tp = confusion_matrix(target, pred).ravel()
avg_tn += tn
avg_fp += fp
avg_fn += fn
avg_tp += tp
# position of col for sensitive values
sensitive = [i[sensitive_idx].item() for i in cats]
cat_len = max(sensitive)
#exit()
sub_cm = []
# print(cat_len)
for j in range(cat_len + 1):
try:
idx = list(locate(sensitive, lambda x: x == j))
sub_tar = target[idx]
sub_pred = pred[idx]
sub_tn, sub_fp, sub_fn, sub_tp = confusion_matrix(
sub_tar, sub_pred).ravel()
except:
# when only one value to predict
temp_tar = int(sub_tar.numpy()[0])
temp_pred = int(sub_pred.numpy()[0])
# print(tar, pred)
if temp_tar and temp_pred:
sub_tn, sub_fp, sub_fn, sub_tp = 0, 0, 0, 1
elif temp_tar and not temp_pred:
sub_tn, sub_fp, sub_fn, sub_tp = 0, 0, 1, 0
elif not temp_tar and not temp_pred:
sub_tn, sub_fp, sub_fn, sub_tp = 1, 0, 0, 0
elif not temp_tar and temp_pred:
sub_tn, sub_fp, sub_fn, sub_tp = 0, 1, 0, 0
else:
sub_tn, sub_fp, sub_fn, sub_tp = 0, 0, 0, 0
total = mysum(sub_tn, sub_fp, sub_fn, sub_tp)
print("??", total)
sub_cm.append((sub_tn / total, sub_fp / total,
sub_fn / total, sub_tp / total))
# Fairness metrics
group_metrics = MetricFrame(
{
'precision': skm.precision_score,
'recall': skm.recall_score
},
target,
pred,
sensitive_features=sensitive)
print(target)
print(pred)
demographic_parity = flm.demographic_parity_difference(
target, pred, sensitive_features=sensitive)
eq_odds = flm.equalized_odds_difference(
target, pred, sensitive_features=sensitive)
# metric_fns = {'true_positive_rate': true_positive_rate}
tpr = MetricFrame(true_positive_rate,
target,
pred,
sensitive_features=sensitive)
# tpr = flm.true_positive_rate(target, pred,sample_weight=sensitive)
sub_results = group_metrics.overall.to_dict()
sub_results_by_group = group_metrics.by_group.to_dict()
# print("\n", group_metrics.by_group, "\n")
avg_precision += sub_results['precision']
avg_recall += sub_results['recall']
print("pre_rec", sub_results)
overall_results.append(sub_results_by_group)
avg_eq_odds += eq_odds
print("eqo", eq_odds)
avg_dem_par += demographic_parity
print("dempar", demographic_parity)
avg_tpr += tpr.difference(method='between_groups')
print("tpr", tpr.difference(method='between_groups'))
total = mysum(avg_tn, avg_fp, avg_fn, avg_tp)
print("!!", total)
cm = (avg_tn / total, avg_fp / total, avg_fn / total,
avg_tp / total)
test_loss /= test_size
accuracy = correct / test_size
avg_loss = test_loss
return accuracy, avg_loss, avg_precision, avg_recall, avg_eq_odds, avg_tpr, avg_dem_par, cm, sub_cm, overall_results
def test_student(args, student_train_loader, student_labels, student_test_loader, test_size, cat_emb_size, num_conts, device, sensitive_idx):
student_model = RegressionModel(emb_szs=cat_emb_size,
n_cont=num_conts,
emb_drop=0.04,
out_sz=1,
szs=[1000, 500, 250],
drops=[0.001, 0.01, 0.01],
y_range=(0, 1)).to(device)
criterion = nn.BCELoss()
optimizer = optim.SGD(student_model.parameters(), lr=args.lr, momentum=0)
steps = 0
running_loss = 0
correct = 0
print("========== Testing Student Model ==========")
for epoch in range(args.epochs):
student_model.train()
train_loader = student_loader(student_train_loader, student_labels)
for (cats, conts) , labels in train_loader:
#for _batch_idx, (data, target) in enumerate(tqdm(train_loader)):
#cats = data[0]
#conts = data[1]
steps += 1
optimizer.zero_grad()
output = student_model(cats, conts).view(-1)
labels = labels.to(torch.float32)
loss = criterion(output, labels)
loss.backward()
optimizer.step()
running_loss += loss.item()
# if steps % 50 == 0:
student_model.eval()
test_loss = 0
correct = 0
i = 0
avg_recall = 0
avg_precision = 0
overall_results = []
avg_eq_odds = 0
avg_dem_par = 0
avg_tpr = 0
avg_tp = 0
avg_tn = 0
avg_fp = 0
avg_fn = 0
with torch.no_grad():
for batch_idx, (cats, conts, target) in enumerate(student_test_loader):
print("target\n", sum(target))
i+=1
output = student_model(cats, conts)
loss += criterion(output, target).item()
test_loss = test_loss + ((1 / (batch_idx + 1)) * (loss.data - test_loss))
pred = (output > 0.5).float()
print("pred\n", sum(pred))
correct += pred.eq(target.view_as(pred)).sum().item()
curr_datetime = datetime.now()
curr_hour = curr_datetime.hour
curr_min = curr_datetime.minute
pred_df = pd.DataFrame(pred.numpy())
pred_df.to_csv(f"pred_results/{args.run_name}_{curr_hour}-{curr_min}.csv")
#print(pred, np.sum(np.squeeze(pred.eq(target.data.view_as(pred))).cpu().numpy()))
#correct += np.sum(np.squeeze(pred.eq(target.data.view_as(pred))).cpu().numpy())
#total += cats.size(0)
# confusion matrixç
tn, fp, fn, tp = confusion_matrix(target, pred).ravel()
avg_tn += tn
avg_fp += fp
avg_fn += fn
avg_tp += tp
# position of col for sensitive values
sensitive = [i[sensitive_idx].item() for i in cats]
cat_len = max(sensitive)
#exit()
sub_cm = []
# print(cat_len)
for j in range(cat_len+1):
try:
idx = list(locate(sensitive, lambda x: x == j))
sub_tar = target[idx]
sub_pred = pred[idx]
sub_tn, sub_fp, sub_fn, sub_tp = confusion_matrix(sub_tar, sub_pred).ravel()
except:
# when only one value to predict
print("----WHAT?")
temp_tar = int(sub_tar.numpy()[0])
temp_pred = int(sub_pred.numpy()[0])
# print(tar, pred)
if temp_tar and temp_pred:
sub_tn, sub_fp, sub_fn, sub_tp = 0, 0, 0, 1
elif temp_tar and not temp_pred:
sub_tn, sub_fp, sub_fn, sub_tp = 0, 0, 1, 0
elif not temp_tar and not temp_pred:
sub_tn, sub_fp, sub_fn, sub_tp = 1, 0, 0, 0
elif not temp_tar and temp_pred:
sub_tn, sub_fp, sub_fn, sub_tp = 0, 1, 0, 0
else:
sub_tn, sub_fp, sub_fn, sub_tp = 0, 0, 0, 0
total = mysum(sub_tn, sub_fp, sub_fn, sub_tp)
print("??", total)
sub_cm.append((sub_tn / total, sub_fp / total, sub_fn / total, sub_tp / total))
# Fairness metrics
group_metrics = MetricFrame({'precision': skm.precision_score, 'recall': skm.recall_score},
target, pred,
sensitive_features=sensitive)
demographic_parity = flm.demographic_parity_difference(target, pred,
sensitive_features=sensitive)
eq_odds = flm.equalized_odds_difference(target, pred,
sensitive_features=sensitive)
# metric_fns = {'true_positive_rate': true_positive_rate}
tpr = MetricFrame(true_positive_rate,
target, pred,
sensitive_features=sensitive)
# tpr = flm.true_positive_rate(target, pred,sample_weight=sensitive)
sub_results = group_metrics.overall.to_dict()
sub_results_by_group = group_metrics.by_group.to_dict()
# print("\n", group_metrics.by_group, "\n")
avg_precision += sub_results['precision']
avg_recall += sub_results['recall']
print("pre_rec", sub_results)
overall_results.append(sub_results_by_group)
avg_eq_odds += eq_odds
print("eqo", eq_odds)
avg_dem_par += demographic_parity
print("dempar", demographic_parity)
avg_tpr += tpr.difference(method='between_groups')
print("tpr", tpr.difference(method='between_groups'))
total = mysum(avg_tn, avg_fp, avg_fn, avg_tp)
print("!!", total)
cm = (avg_tn / total, avg_fp / total, avg_fn / total, avg_tp / total)
test_loss /= test_size
accuracy = correct / test_size
avg_loss = test_loss
return accuracy, avg_loss, avg_precision, avg_recall, avg_eq_odds, avg_tpr, avg_dem_par, cm, sub_cm, overall_results
def fair_metrics(gt, y, group):
metrics_dict = {
"DPd": demographic_parity_difference(gt, y, sensitive_features=group),
"EOd": equalized_odds_difference(gt, y, sensitive_features=group),
}
return metrics_dict