def get_sigma(q, epoches, epsilon, delta): # sigma allows us to run 10 epoches
steps_per_epoch = int(1. / q)
steps = epoches * steps_per_epoch
sigma = 1
while (True): #get sigma in interge
loss = compute_rdp(q, sigma, steps, np.arange(2, 2 + 512))
if_delta, _ = _compute_delta(np.arange(2, 2 + 512), loss, epsilon)
if (if_delta > delta):
sigma += 1
else:
break
for i in range(10): #get sigma in .1
sigma -= .1
loss = compute_rdp(q, sigma, steps, np.arange(2, 2 + 512))
if_delta, _ = _compute_delta(np.arange(2, 2 + 512), loss, epsilon)
if (if_delta > delta):
sigma += .1
break
for i in range(10): #get sigma in .01
sigma -= .01
loss = compute_rdp(q, sigma, steps, np.arange(2, 2 + 512))
if_delta, _ = _compute_delta(np.arange(2, 2 + 512), loss, epsilon)
if (if_delta > delta):
sigma += .01
break
return sigma
def loop_for_sigma(q,
T,
eps,
delta,
cur_sigma,
interval,
rdp_orders=32,
rgp=True):
while True:
orders = np.arange(2, rdp_orders, 0.1)
steps = T
if (rgp):
rdp = compute_rdp(
q, cur_sigma, steps, orders
) * 2 ## when using residual gradients, the sensitivity is sqrt(2)
else:
rdp = compute_rdp(q, cur_sigma, steps, orders)
cur_eps, _, opt_order = get_privacy_spent(orders,
rdp,
target_delta=delta)
if (cur_eps < eps and cur_sigma > interval):
cur_sigma -= interval
previous_eps = cur_eps
else:
cur_sigma += interval
break
return cur_sigma, previous_eps
def test_compute_rdp(self):
rdp_scalar = rdp_accountant.compute_rdp(0.1, 2, 10, 5)
self.assertAlmostEqual(rdp_scalar, 0.07737, places=5)
rdp_vec = rdp_accountant.compute_rdp(0.01, 2.5, 50, [1.5, 2.5, 5, 50, 100])
correct = [0.00065, 0.001085, 0.00218075, 0.023846, 167.416307]
for i in range(len(rdp_vec)):
self.assertAlmostEqual(rdp_vec[i], correct[i], places=5)
def test_compute_rdp(self):
rdp_scalar = rdp_accountant.compute_rdp(0.1, 2, 10, 5)
self.assertAlmostEqual(rdp_scalar, 0.07737, places=5)
rdp_vec = rdp_accountant.compute_rdp(0.01, 2.5, 50, [1.5, 2.5, 5, 50, 100,
np.inf])
correct = [0.00065, 0.001085, 0.00218075, 0.023846, 167.416307, np.inf]
for i in range(len(rdp_vec)):
self.assertAlmostEqual(rdp_vec[i], correct[i], places=5)
def test_compute_rdp_sequence(self):
rdp_vec = rdp_accountant.compute_rdp(0.01, 2.5, 50,
[1.5, 2.5, 5, 50, 100, np.inf])
self.assertSequenceAlmostEqual(
rdp_vec,
[0.00065, 0.001085, 0.00218075, 0.023846, 167.416307, np.inf],
delta=1e-5)
def test_get_privacy_spent_check_target_eps(self):
orders = range(2, 33)
rdp = rdp_accountant.compute_rdp(0.01, 4, 10000, orders)
_, delta, opt_order = rdp_accountant.get_privacy_spent(
orders, rdp, target_eps=1.258575)
self.assertAlmostEqual(delta, 1e-5)
self.assertEqual(opt_order, 20)
def server_exec(self,mt):
i=1
while(True):
clear_output()
print('Comunication round: ', i)
self.eval_acc()
rdp = compute_rdp(float(mt/len(self.clients)), self.sigmat, i, self.orders)
_,delta_spent, opt_order = get_privacy_spent(self.orders, rdp, target_eps=self.epsilon)
print('Delta spent: ', delta_spent)
print('Delta budget: ', self.p_budget)
if self.p_budget < delta_spent:
break
Zt = np.random.choice(self.clients, mt)
deltas = []
norms = []
for client in Zt:
#print(client.number)
deltaW, normW = client.update(self.model.state_dict())
deltas.append(deltaW)
norms.append(normW)
#print('all updates')
self.model.to('cpu')
new_state_dict = self.sanitaze(mt, deltas, norms, self.sigmat, self.model.state_dict())
#print('sanitaze')
self.model.load_state_dict(new_state_dict)
i+=1
def test_get_privacy_spent_check_target_delta(self):
orders = range(2, 33)
rdp = rdp_accountant.compute_rdp(0.01, 4, 10000, orders)
eps, _, opt_order = rdp_accountant.get_privacy_spent(orders,
rdp,
target_delta=1e-5)
self.assertAlmostEqual(eps, 1.258575, places=5)
self.assertEqual(opt_order, 20)
def test_get_privacy_spent(self):
orders = range(2, 33)
rdp = rdp_accountant.compute_rdp(0.01, 4, 10000, orders)
eps, delta, opt_order = rdp_accountant.get_privacy_spent(orders, rdp,
target_delta=1e-5)
self.assertAlmostEqual(eps, 1.258575, places=5)
self.assertEqual(opt_order, 20)
eps, delta, _ = rdp_accountant.get_privacy_spent(orders, rdp,
target_eps=1.258575)
self.assertAlmostEqual(delta, 1e-5)
def test_compute_privacy_loss(self):
parameters = [(0.01, 4, 10000), (0.1, 2, 100)]
delta = 1e-5
orders = (1.25, 1.5, 1.75, 2., 2.5, 3., 4., 5., 6., 7., 8., 10., 12., 14.,
16., 20., 24., 28., 32., 64., 256.)
rdp = np.zeros_like(orders, dtype=float)
for q, sigma, steps in parameters:
rdp += rdp_accountant.compute_rdp(q, sigma, steps, orders)
eps, delta, opt_order = rdp_accountant.get_privacy_spent(orders, rdp,
target_delta=delta)
self.assertAlmostEqual(eps, 3.276237, places=5)
self.assertEqual(opt_order, 8)
def test_check_composition(self):
orders = (1.25, 1.5, 1.75, 2., 2.5, 3., 4., 5., 6., 7., 8., 10., 12.,
14., 16., 20., 24., 28., 32., 64., 256.)
rdp = rdp_accountant.compute_rdp(q=1e-4,
stddev_to_sensitivity_ratio=.4,
steps=40000,
orders=orders)
eps, _, opt_order = rdp_accountant.get_privacy_spent(orders,
rdp,
target_delta=1e-6)
rdp += rdp_accountant.compute_rdp(q=0.1,
stddev_to_sensitivity_ratio=2,
steps=100,
orders=orders)
eps, _, opt_order = rdp_accountant.get_privacy_spent(orders,
rdp,
target_delta=1e-5)
self.assertAlmostEqual(eps, 8.509656, places=5)
self.assertEqual(opt_order, 2.5)
def test_compute_privacy_loss(self):
parameters = [(0.01, 4, 10000), (0.1, 2, 100)]
delta = 1e-5
orders = (1.25, 1.5, 1.75, 2., 2.5, 3., 4., 5., 6., 7., 8., 10., 12., 14.,
16., 20., 24., 28., 32., 64., 256.)
rdp = np.zeros_like(orders, dtype=float)
for q, sigma, steps in parameters:
rdp += rdp_accountant.compute_rdp(q, sigma, steps, orders)
eps, delta, opt_order = rdp_accountant.get_privacy_spent(orders, rdp,
target_delta=delta)
self.assertAlmostEqual(eps, 3.276237, places=5)
self.assertEqual(opt_order, 8)
def compute_epsilon(steps):
print('in compute_epsilon, steps', steps)
"""Computes epsilon value for given hyperparameters."""
if FLAGS.noise_multiplier == 0.0:
return float('inf')
orders = [1 + x / 10. for x in range(1, 100)] + list(range(12, 64))
sampling_probability = (batch_size * 1.0 /
config['hist_len']) / num_examples
print('sampling_probability', sampling_probability)
rdp = compute_rdp(q=sampling_probability,
noise_multiplier=FLAGS.noise_multiplier,
steps=steps,
orders=orders)
# Delta is set to 1e-5 because Penn TreeBank has 60000 training points.
return get_privacy_spent(orders, rdp, target_delta=FLAGS.target_delta)[0]
def _apply_dp_sgd_analysis(q, sigma, iterations, orders, delta):
"""Calculates epsilon for stochastic gradient descent.
Args:
q (float): Sampling probability, generally batch_size / number_of_samples
sigma (float): Noise multiplier
iterations (float): Number of iterations mechanism is applied
orders (list(float)): Orders to try for finding optimal epsilon
delta (float): Target delta
Returns:
float: epsilon
Example::
>>> epsilon(10000, 256, 0.3, 100, 1e-5)
"""
rdp = compute_rdp(q, sigma, iterations, orders)
eps, _, opt_order = get_privacy_spent(orders, rdp, target_delta=delta)
return eps
def test_compute_rdp_scalar(self):
rdp_scalar = rdp_accountant.compute_rdp(0.1, 2, 10, 5)
self.assertAlmostEqual(rdp_scalar, 0.07737, places=5)
def test_compute_rdp_no_sampling(self):
# q = 1, RDP = alpha/2 * sigma^2
self.assertEqual(rdp_accountant.compute_rdp(1, 10, 1, 20), 0.1)
def test_compute_rdp_no_data(self):
# q = 0
self.assertEqual(rdp_accountant.compute_rdp(0, 10, 1, 20), 0)
def train_dp(net,
train_loader,
test_loader,
optimizer,
criterion,
epoch,
batch_size,
classes=10,
noise_multiplier=1.1):
best_ACC = 0.0
best_eps = 0.0
step = 0
best_model = None
#for epoch in range(n_epochs):
tf = time.time()
net.train()
loss_tot, avg_norm, cnt = 0.0, 0.0, 0
for i, batch in enumerate(train_loader):
imgs, labels = batch
imgs, labels = imgs.to(device), labels.to(device)
bs = imgs.size(0)
outputs = net(imgs)[0]
loss = criterion(outputs, labels)
grad = [torch.zeros_like(param) for param in net.parameters()]
num_microbatch = bs / microbatch_size
for j in range(0, bs, microbatch_size):
optimizer.zero_grad()
torch.autograd.backward(loss[j:j + microbatch_size],
retain_graph=True)
L2_norm = 0.0
for param in net.parameters():
L2_norm += (param.grad * param.grad).sum()
L2_norm = torch.sqrt(L2_norm)
avg_norm = avg_norm * 0.95 + L2_norm * 0.05
coef = float(g_clip_thres) / max(g_clip_thres, L2_norm.item())
grad = [
g + param.grad * coef
for param, g in zip(net.parameters(), grad)
]
for param, g in zip(net.parameters(), grad):
if noise_multiplier > 0:
param.grad.data = g + torch.cuda.FloatTensor(g.size()).normal_(
0, noise_multiplier * float(g_clip_thres))
else:
param.grad.data = g
param.grad.data /= num_microbatch
optimizer.step()
loss_tot += torch.sum(loss).item()
cnt += imgs.shape[0]
step += 1
train_loss = loss_tot / cnt
q = batch_size * 1.0 / cnt
orders = [1 + x / 10. for x in range(1, 100)] + list(range(12, 256))
if noise_multiplier > 0:
rdp = compute_rdp(q=q,
noise_multiplier=noise_multiplier,
steps=step,
orders=orders)
eps = get_privacy_spent(orders, rdp, target_delta=delta)[0]
else:
eps = 0
train_accuracy = eval_net(net, train_loader, classes)
test_accuracy = eval_net(net, test_loader, classes)
interval = time.time() - tf
if test_accuracy > best_ACC:
best_ACC = test_accuracy
best_model = deepcopy(net)
best_eps = eps
print(
"Epoch:{}\tTime:{:.2f}\tTrain Loss:{:.2f}\tEps:{:.2f}\tTrain Acc:{:.2f}\tTest Acc:{:.2f}"
.format(epoch, interval, train_loss, eps, train_accuracy,
test_accuracy))
#print("Best Acc:{:.2f}".format(best_ACC))
#print("Best Eps:{:.2f}".format(best_eps))
#return best_model, best_ACC, best_eps
return test_accuracy
