def morgan(result):
phi_l, phi_u, phi_m, fpr_morgan, adv_morgan, ppv_morgan = np.zeros(
B), np.zeros(B), np.zeros(B), np.zeros(B), np.zeros(B), np.zeros(B)
alpha_l = 0.22 # alpha value used to tune lower loss threshold
alpha_u = 0.3 # alpha value used to tune upper loss threshold
for run in RUNS:
if args.eps == None:
_, membership, per_instance_loss, _, _, _, proposed_mi_outputs = result[
'no_privacy'][run]
else:
_, membership, per_instance_loss, _, _, _, proposed_mi_outputs = result[
'gdp_'][args.eps][run]
true_y, v_true_y, v_membership, v_per_instance_loss, v_counts, counts = proposed_mi_outputs
low_thresh, _ = get_pred_mem_mi(per_instance_loss,
proposed_mi_outputs,
method='yeom',
fpr_threshold=alpha_l,
per_class_thresh=args.per_class_thresh,
fixed_thresh=args.fixed_thresh)
high_thresh, _ = get_pred_mem_mi(
per_instance_loss,
proposed_mi_outputs,
method='yeom',
fpr_threshold=alpha_u,
per_class_thresh=args.per_class_thresh,
fixed_thresh=args.fixed_thresh)
merlin_thresh, _ = get_pred_mem_mi(
per_instance_loss,
proposed_mi_outputs,
method='merlin',
fpr_threshold=alpha,
per_class_thresh=args.per_class_thresh,
fixed_thresh=args.fixed_thresh)
pred_1 = np.where(per_instance_loss >= low_thresh, 1, 0)
pred_2 = np.where(per_instance_loss <= high_thresh, 1, 0)
pred_3 = np.where(counts >= merlin_thresh, 1, 0)
pred = pred_1 & pred_2 & pred_3
fp, adv, ppv = get_fp(membership,
pred), get_adv(membership,
pred), get_ppv(membership, pred)
phi_l[run], phi_u[run], phi_m[run], fpr_morgan[run], adv_morgan[
run], ppv_morgan[
run] = low_thresh, high_thresh, merlin_thresh, fp / (
gamma * 10000), adv, ppv
print(
'\nMorgan:\nphi: (%f +/- %f, %f +/- %f, %f +/- %f)\nFPR: %.4f +/- %.4f\nTPR: %.4f +/- %.4f\nAdv: %.4f +/- %.4f\nPPV: %.4f +/- %.4f'
% (np.mean(phi_l), np.std(phi_l), np.mean(phi_u), np.std(phi_u),
np.mean(phi_m), np.std(phi_m), np.mean(fpr_morgan),
np.std(fpr_morgan), np.mean(adv_morgan + fpr_morgan),
np.std(adv_morgan + fpr_morgan), np.mean(adv_morgan),
np.std(adv_morgan), np.mean(ppv_morgan), np.std(ppv_morgan)))
def plot_privacy_leakage(result, eps=None, dp='gdp_'):
adv_yeom_vanilla_1, adv_yeom, adv_merlin = np.zeros(B), np.zeros(B), np.zeros(B)
ppv_yeom_vanilla_1, ppv_yeom, ppv_merlin = np.zeros(B), np.zeros(B), np.zeros(B)
fpr_yeom_vanilla_1, fpr_yeom, fpr_merlin = np.zeros(B), np.zeros(B), np.zeros(B)
thresh_yeom_vanilla_1, thresh_yeom, thresh_merlin = np.zeros(B), np.zeros(B), np.zeros(B)
yeom_zero_m, yeom_zero_nm, merlin_zero_m, merlin_zero_nm = [], [], [], []
for run in RUNS:
aux, membership, per_instance_loss, yeom_mi_outputs_1, yeom_mi_outputs_2, proposed_mi_outputs = result['no_privacy'][run] if not eps else result[dp][eps][run]
train_loss, train_acc, test_loss, test_acc = aux
true_y, v_true_y, v_membership, v_per_instance_loss, v_counts, counts = proposed_mi_outputs
m, nm = get_zeros(membership, per_instance_loss)
yeom_zero_m.append(m)
yeom_zero_nm.append(nm)
m, nm = get_zeros(membership, counts)
merlin_zero_m.append(m)
merlin_zero_nm.append(nm)
#plot_histogram(per_instance_loss)
#plot_distributions(per_instance_loss, membership, method='yeom')
#plot_sign_histogram(membership, counts, 100)
plot_distributions(counts, membership, method='merlin')
# As used below, method == 'yeom' runs a Yeom attack but finds a better threshold than is used in the original Yeom attack.
thresh, pred = get_pred_mem_mi(per_instance_loss, proposed_mi_outputs, method='yeom', fpr_threshold=alpha, per_class_thresh=args.per_class_thresh, fixed_thresh=args.fixed_thresh)
fp, adv, ppv = get_fp(membership, pred), get_adv(membership, pred), get_ppv(membership, pred)
thresh_yeom[run], fpr_yeom[run], adv_yeom[run], ppv_yeom[run] = thresh, fp / (gamma * 10000), adv, ppv
# As used below, method == 'merlin' runs a new threshold-based membership inference attack that uses the direction of the change in per-instance loss for the record.
thresh, pred = get_pred_mem_mi(per_instance_loss, proposed_mi_outputs, method='merlin', fpr_threshold=alpha, per_class_thresh=args.per_class_thresh, fixed_thresh=args.fixed_thresh)
fp, adv, ppv = get_fp(membership, pred), get_adv(membership, pred), get_ppv(membership, pred)
thresh_merlin[run], fpr_merlin[run], adv_merlin[run], ppv_merlin[run] = thresh, fp / (gamma * 10000), adv, ppv
# Original Yeom attack that uses expected training loss threshold
fp, adv, ppv = get_fp(membership, yeom_mi_outputs_1), get_adv(membership, yeom_mi_outputs_1), get_ppv(membership, yeom_mi_outputs_1)
thresh_yeom_vanilla_1[run], fpr_yeom_vanilla_1[run], adv_yeom_vanilla_1[run], ppv_yeom_vanilla_1[run] = train_loss, fp / (gamma * 10000), adv, ppv
print('\nYeom: \t %.2f +/- %.2f \t %.2f +/- %.2f' % (np.mean(yeom_zero_m), np.std(yeom_zero_m), np.mean(yeom_zero_nm), np.std(yeom_zero_nm)))
print('\nMerlin: \t %.2f +/- %.2f \t %.2f +/- %.2f' % (np.mean(merlin_zero_m), np.std(merlin_zero_m), np.mean(merlin_zero_nm), np.std(merlin_zero_nm)))
print('\nYeom Vanilla 1:\nphi: %f +/- %f\nFPR: %.4f +/- %.4f\nTPR: %.4f +/- %.4f\nAdv: %.4f +/- %.4f\nPPV: %.4f +/- %.4f' % (np.mean(thresh_yeom_vanilla_1), np.std(thresh_yeom_vanilla_1), np.mean(fpr_yeom_vanilla_1), np.std(fpr_yeom_vanilla_1), np.mean(adv_yeom_vanilla_1+fpr_yeom_vanilla_1), np.std(adv_yeom_vanilla_1+fpr_yeom_vanilla_1), np.mean(adv_yeom_vanilla_1), np.std(adv_yeom_vanilla_1), np.mean(ppv_yeom_vanilla_1), np.std(ppv_yeom_vanilla_1)))
print('\nYeom:\nphi: %f +/- %f\nFPR: %.4f +/- %.4f\nTPR: %.4f +/- %.4f\nAdv: %.4f +/- %.4f\nPPV: %.4f +/- %.4f' % (np.mean(thresh_yeom), np.std(thresh_yeom), np.mean(fpr_yeom), np.std(fpr_yeom), np.mean(adv_yeom+fpr_yeom), np.std(adv_yeom+fpr_yeom), np.mean(adv_yeom), np.std(adv_yeom), np.mean(ppv_yeom), np.std(ppv_yeom)))
print('\nMerlin:\nphi: %f +/- %f\nFPR: %.4f +/- %.4f\nTPR: %.4f +/- %.4f\nAdv: %.4f +/- %.4f\nPPV: %.4f +/- %.4f' % (np.mean(thresh_merlin), np.std(thresh_merlin), np.mean(fpr_merlin), np.std(fpr_merlin), np.mean(adv_merlin+fpr_merlin), np.std(adv_merlin+fpr_merlin), np.mean(adv_merlin), np.std(adv_merlin), np.mean(ppv_merlin), np.std(ppv_merlin)))
vals.append(
list(
pickle.load(
open(
DATA_PATH + MODEL + 'no_privacy_1e-08_' +
str(run + 1) + '.p', 'rb'))))
aux, membership, per_instance_loss, yeom_mi_outputs_1, yeom_mi_outputs_2, proposed_mi_outputs_arr = vals[
run]
train_loss, train_acc, test_loss, test_acc = aux
for i in range(5):
thresh, pred = get_pred_mem_mi(per_instance_loss,
proposed_mi_outputs_arr[i],
method=attack,
fpr_threshold=alpha)
fp, adv, ppv = get_fp(membership,
pred), get_adv(membership,
pred), get_ppv(membership, pred)
print(thresh, fp, adv * 10000 + fp)
thresh_attack[run][i], fpr_attack[run][i], adv_attack[run][
i], ppv_attack[run][i] = thresh, fp / 10000, adv, ppv
print(
'\nRun %d:\nphi: %f +/- %f\nFPR: %.4f +/- %.4f\nTPR: %.4f +/- %.4f\nAdv: %.4f +/- %.4f\nPPV: %.4f +/- %.4f'
% (run + 1, np.mean(thresh_attack[run]), np.std(thresh_attack[run]),
np.mean(fpr_attack[run]), np.std(
fpr_attack[run]), np.mean(adv_attack[run] + fpr_attack[run]),
np.std(adv_attack[run] + fpr_attack[run]), np.mean(adv_attack[run]),
np.std(adv_attack[run]), np.mean(
ppv_attack[run]), np.std(ppv_attack[run])))
print(
'\n\nAcross All Runs:\nphi: %f +/- %f\nFPR: %.4f +/- %.4f\nTPR: %.4f +/- %.4f\nAdv: %.4f +/- %.4f\nPPV: %.4f +/- %.4f\n'
% (np.mean(thresh_attack), np.std(thresh_attack), np.mean(fpr_attack),
def plot_privacy_leakage(result, eps=None, dp='gdp_'):
adv_y_mi_1, adv_y_mi_2, adv_p_mi_1, adv_p_mi_2 = np.zeros(B), np.zeros(
B), np.zeros(B), np.zeros(B)
ppv_y_mi_1, ppv_y_mi_2, ppv_p_mi_1, ppv_p_mi_2 = np.zeros(B), np.zeros(
B), np.zeros(B), np.zeros(B)
fpr_y_mi_1, fpr_y_mi_2, fpr_p_mi_1, fpr_p_mi_2 = np.zeros(B), np.zeros(
B), np.zeros(B), np.zeros(B)
thresh_y_mi_1, thresh_y_mi_2, thresh_p_mi_1, thresh_p_mi_2 = np.zeros(
B), np.zeros(B), np.zeros(B), np.zeros(B)
mi_1_zero_m, mi_1_zero_nm, mi_2_zero_m, mi_2_zero_nm = [], [], [], []
for run in RUNS:
aux, membership, per_instance_loss, yeom_mi_outputs_1, yeom_mi_outputs_2, proposed_mi_outputs = result[
'no_privacy'][run] if not eps else result[dp][eps][run]
train_loss, train_acc, test_loss, test_acc = aux
true_y, v_true_y, v_membership, v_per_instance_loss, v_counts, counts = proposed_mi_outputs
m, nm = get_zeros(membership, per_instance_loss)
mi_1_zero_m.append(m)
mi_1_zero_nm.append(nm)
m, nm = get_zeros(membership, counts)
mi_2_zero_m.append(m)
mi_2_zero_nm.append(nm)
plot_histogram(per_instance_loss)
plot_distributions(per_instance_loss, membership)
plot_sign_histogram(membership, counts, 100)
plot_distributions(counts, membership, 2)
thresh, pred = get_pred_mem_mi(per_instance_loss,
proposed_mi_outputs,
method=1,
fpr_threshold=alpha,
per_class_thresh=args.per_class_thresh,
fixed_thresh=args.fixed_thresh)
fp, adv, ppv = get_fp(membership,
pred), get_adv(membership,
pred), get_ppv(membership, pred)
thresh_p_mi_1[run], fpr_p_mi_1[run], adv_p_mi_1[run], ppv_p_mi_1[
run] = thresh, fp / (gamma * 10000), adv, ppv
thresh, pred = get_pred_mem_mi(per_instance_loss,
proposed_mi_outputs,
method=2,
fpr_threshold=alpha,
per_class_thresh=args.per_class_thresh,
fixed_thresh=args.fixed_thresh)
fp, adv, ppv = get_fp(membership,
pred), get_adv(membership,
pred), get_ppv(membership, pred)
thresh_p_mi_2[run], fpr_p_mi_2[run], adv_p_mi_2[run], ppv_p_mi_2[
run] = thresh, fp / (gamma * 10000), adv, ppv
fp, adv, ppv = get_fp(membership, yeom_mi_outputs_1), get_adv(
membership, yeom_mi_outputs_1), get_ppv(membership,
yeom_mi_outputs_1)
thresh_y_mi_1[run], fpr_y_mi_1[run], adv_y_mi_1[run], ppv_y_mi_1[
run] = train_loss, fp / (gamma * 10000), adv, ppv
print('\nMI 1: \t %.2f +/- %.2f \t %.2f +/- %.2f' %
(np.mean(mi_1_zero_m), np.std(mi_1_zero_m), np.mean(mi_1_zero_nm),
np.std(mi_1_zero_nm)))
print('\nMI 2: \t %.2f +/- %.2f \t %.2f +/- %.2f' %
(np.mean(mi_2_zero_m), np.std(mi_2_zero_m), np.mean(mi_2_zero_nm),
np.std(mi_2_zero_nm)))
print(
'\nYeom MI 1:\nphi: %f +/- %f\nFPR: %.4f +/- %.4f\nTPR: %.4f +/- %.4f\nAdv: %.4f +/- %.4f\nPPV: %.4f +/- %.4f'
% (np.mean(thresh_y_mi_1), np.std(thresh_y_mi_1), np.mean(fpr_y_mi_1),
np.std(fpr_y_mi_1), np.mean(adv_y_mi_1 + fpr_y_mi_1),
np.std(adv_y_mi_1 + fpr_y_mi_1), np.mean(adv_y_mi_1),
np.std(adv_y_mi_1), np.mean(ppv_y_mi_1), np.std(ppv_y_mi_1)))
print(
'\nProposed MI 1:\nphi: %f +/- %f\nFPR: %.4f +/- %.4f\nTPR: %.4f +/- %.4f\nAdv: %.4f +/- %.4f\nPPV: %.4f +/- %.4f'
% (np.mean(thresh_p_mi_1), np.std(thresh_p_mi_1), np.mean(fpr_p_mi_1),
np.std(fpr_p_mi_1), np.mean(adv_p_mi_1 + fpr_p_mi_1),
np.std(adv_p_mi_1 + fpr_p_mi_1), np.mean(adv_p_mi_1),
np.std(adv_p_mi_1), np.mean(ppv_p_mi_1), np.std(ppv_p_mi_1)))
print(
'\nProposed MI 2:\nphi: %f +/- %f\nFPR: %.4f +/- %.4f\nTPR: %.4f +/- %.4f\nAdv: %.4f +/- %.4f\nPPV: %.4f +/- %.4f'
% (np.mean(thresh_p_mi_2), np.std(thresh_p_mi_2), np.mean(fpr_p_mi_2),
np.std(fpr_p_mi_2), np.mean(adv_p_mi_2 + fpr_p_mi_2),
np.std(adv_p_mi_2 + fpr_p_mi_2), np.mean(adv_p_mi_2),
np.std(adv_p_mi_2), np.mean(ppv_p_mi_2), np.std(ppv_p_mi_2)))