def irfil_inversion(results_path, dataset, save_path):
noise_scale = "0.001"
its = 10
irfil_results = load_results(results_path, f"{dataset}_least_squares_irfil.json")
etas = np.array(irfil_results["etas"])
eta_means = np.array(irfil_results["eta_means"])[:its]
eta_stds = np.array(irfil_results["eta_stds"])[:its]
plotting.line_plot(
eta_means[None, :], np.arange(its),
xlabel="Steps of IRFIL",
ylabel="Mean $\\bar{\eta}$",
errors=eta_stds[None, :],
size=(4.85, 5.05),
filename=os.path.join(args.save_path, f"{dataset}_mean_fil"),
)
def compute_correct_ratio(etas, num_bins, predictions, target):
order = etas.argsort()
bin_size = len(target) // num_bins + 1
bin_accs = []
for prediction in predictions:
prediction = np.array(prediction)
correct = (prediction == target)
bin_accs.append([correct[order[lower:lower + bin_size]].mean()
for lower in range(0, len(correct), bin_size)])
return np.array(bin_accs)
inversion_results = load_results(
results_path,
f"{dataset}_least_squares_whitebox_private_inversion_irfil.json")
target = np.array(inversion_results[0]["target"])
num_bins = 10
ratio_means = []
ratio_stds = []
its = [0, 2, 10]
for it in its:
predictions = inversion_results[it][noise_scale]['predictions']
ratios = compute_correct_ratio(etas, num_bins, predictions, target)
ratio_means.append(ratios.mean(axis=0))
ratio_stds.append(ratios.std(axis=0))
ratio_means = np.array(ratio_means)
ratio_stds = np.array(ratio_stds)
plotting.line_plot(
ratio_means, np.arange(num_bins),
legend=["Iteration {}".format(it) for it in its],
xlabel="FIL ($\eta$) percentile",
ylabel="Attribute inversion accuracy",
errors=ratio_stds,
size=(5, 5),
filename=os.path.join(
args.save_path,
f"{dataset}_whitebox_eta_percentile"),
)
def plot_unit(unit_id, cc_filtered, cc_fits, out_dir):
"""
Plot cc unit with fits
"""
title = unit_id
try:
filtered_unit = cc_filtered[unit_id]
pos = filtered_unit['cluster'] >= 0
clusters = len(filtered_unit.loc[pos, 'cluster'].unique())
colors = sns.color_palette('colorblind', clusters)
cluster_units = []
cluster_fits = []
legend = []
for c, cluster_df in filtered_unit.loc[pos].groupby('cluster'):
cts = int(cluster_df.iloc[0]['cts'])
legend.append('CTs = {}'.format(cts))
cluster_units.append(cluster_df[['load', 'heat_rate']].values)
colors.append(lighten(colors[c], 0.5))
cluster_id = '{}-{}'.format(unit_id, c)
cluster_fits.append(get_hr_fit(cc_fits, cluster_id))
plt_data = cluster_units + cluster_fits
linestyles = ('', ) * len(legend) + ('--', ) * len(legend)
f_path = os.path.join(out_dir, '{}.png'.format(unit_id))
x = filtered_unit.loc[pos, 'load'].values
x_lim = np.nanmax(x[x != np.inf]) * 1.1
y = filtered_unit.loc[pos, 'heat_rate'].values
y_lim = np.nanmax(y[y != np.inf]) * 1.1
mplt.line_plot(*plt_data,
despine=True,
title=title,
linestyles=linestyles,
markers=('o', ),
colors=colors,
xlabel='Load (MWh)',
ylabel='Heat Rate (mmBTU/MWh)',
xlim=(0, x_lim),
ylim=(0, y_lim),
legend=legend,
legend_loc=0,
filename=f_path,
showplot=False)
except Exception as ex:
print("{} failed due to {}".format(unit_id, ex))
def iterated_reweighted_etas(results_path, save_path, prefix):
"""
Line plot of variance of Fisher information loss eta with iterated
reweighting.
"""
linear = load_results(results_path, f"{prefix}_linear_reweighted.json")
logistic = load_results(results_path, f"{prefix}_logistic_reweighted.json")
stds = np.array([linear["eta_stds"], logistic["eta_stds"]])
iterations = np.arange(stds.shape[1])
plotting.line_plot(
stds, iterations, legend=["Linear", "Logistic"],
xlabel="Iteration", ylabel="Standard deviation of $\eta$",
ylog=True,
size=(5, 5),
filename=os.path.join(save_path, f"{prefix}_eta_std_vs_iterations"))
for results, model in [(linear, "linear"), (logistic, "logistic")]:
em = results["eta_means"]
std = results["eta_stds"]
acc = results["test_accs"]
print("="*20)
print(f"{prefix} {model}")
print(f"Pre IRFP eta {em[0]:.3f}, std {std[0]:.3f}, test accuracy {acc[0]:.3f}")
print(f"Post IRFP eta {em[-1]:.3f}, std {std[-1]:.3f}, test accuracy {acc[-1]:.3f}")
def history_plot(history):
loss = history.__dict__['history']['loss']
line_plot(loss)
def private_inversion_accuracy(results_path, save_path):
L2s = ['1e-5', '1e-3', '1e-1', '1']
noise_scales = [
'1e-05', '2e-05', '5e-05', '0.0001', '0.0002',
'0.0005', '0.001', '0.002', '0.005', '0.01',
'0.02', '0.05', '0.1', '0.2', '0.5', '1.0',
]
inversion_results = load_results(
results_path, f"iwpc_least_squares_inversion_l2_1e-3.json")[0]
target = np.array(inversion_results["target"])
baseline = (target == np.array(inversion_results["baseline"])).mean()
# Load the max eta for each L2.
mean_etas = []
for l2 in L2s:
etas = load_results(
results_path, f"iwpc_least_squares_fil_l2_{l2}.json")["etas"]
mean_etas.append(np.mean(etas))
sigmas = np.array([float(ns) for ns in noise_scales])
mean_etas = np.array(mean_etas)[:, None] / sigmas
results = {"whitebox": {}, "fredrikson14": {}}
for inverter in ["whitebox", "fredrikson14"]:
results = []
for l2 in L2s:
# inversion results are in a list ordered by ieration or IRFIL,
# each dictionary is the results at a given noise scale along with
# the target values
inversion_results = load_results(
results_path,
f"iwpc_least_squares_{inverter}_private_inversion_l2_{l2}.json")
all_accs = []
for noise_scale in noise_scales:
accs = []
for prediction in inversion_results[0][noise_scale]['predictions']:
accs.append((np.array(prediction) == target).mean())
all_accs.append([np.mean(accs), np.std(accs)])
results.append(all_accs)
results = np.array(results) # [L2, noise scale, mean/std]
means = results[:, :, 0]
stds = results[:, :, 1]
legend = ["$\lambda=10^{%d}$"%int(math.log10(float(l2))) for l2 in L2s]
plotting.line_plot(
means, mean_etas, legend=legend,
xlabel="Mean $\\bar{\eta}$",
ylabel="Attribute inversion accuracy",
errors=stds,
ymax=1.02,
ymin=0.2,
xlog=True,
size=(5, 5))
plt.semilogx([0, 1e3], [baseline]*2, 'k--', label="Prior")
plt.legend()
plt.xlim(right=1e3)
plotting.savefig(os.path.join(
args.save_path,
f"iwpc_{inverter}_vs_eta_varying_l2"))
def private_mse_and_fil(results_path, save_path):
L2s = ['1e-5', '1e-3', '1e-1', '1']
noise_scales = [
'1e-05', '2e-05', '5e-05', '0.0001', '0.0002',
'0.0005', '0.001', '0.002', '0.005', '0.01',
'0.02', '0.05', '0.1', '0.2', '0.5', '1.0',
]
fils = []
mean_etas = []
mses = []
for l2 in L2s:
etas = load_results(
results_path, f"iwpc_least_squares_fil_l2_{l2}.json")["etas"]
fils.append(etas)
inversion_results = load_results(
results_path,
f"iwpc_least_squares_whitebox_private_inversion_l2_{l2}.json")
mses.append([inversion_results[0][noise_scale]['test_acc']
for noise_scale in noise_scales])
mean_etas.append(np.mean(etas))
sigmas = np.array([float(ns) for ns in noise_scales])
mean_etas = np.array(mean_etas)[:, None] / sigmas
l2s = np.array([float(l2) for l2 in L2s])
legend = ["$\lambda=10^{%d}$"%int(math.log10(float(l2))) for l2 in L2s]
# Plot FILs:
num_bins = 100
fil_counts = []
fil_centers = []
for fil in fils:
lower = math.log10(np.min(fil))
upper = math.log10(np.max(fil) + 1e-4)
bins = np.logspace(lower, upper, num_bins + 1)
counts, edges = np.histogram(fil, bins=bins)
centers = (edges[:-1] + edges[1:]) / 2
fil_counts.append(counts)
fil_centers.append(centers)
plotting.line_plot(
np.array(fil_counts),
np.array(fil_centers),
xlabel="FIL $\eta$ (at $\sigma=1$)",
ylabel="Number of examples",
legend=legend,
marker=None,
size=(5, 5),
xlog=True,
filename=os.path.join(
args.save_path,
f"iwpc_fil_counts_varying_l2"),
)
# PLot MSEs
mses = np.array(mses) # [L2, noise_scale, trials]
mse_means = mses.mean(axis=2)
mse_stds = mses.std(axis=2)
plotting.line_plot(
mse_means, mean_etas, legend=legend,
xlabel="Mean $\\bar{\eta}$",
ylabel="Test MSE",
ylog=True,
xlog=True,
size=(5, 5),
errors=mse_stds,
filename=os.path.join(args.save_path, f"iwpc_mse_vs_eta_varying_l2"),
)