def get_sigma(q, epoches, epsilon, delta): # sigma allows us to run 10 epoches steps_per_epoch = int(1. / q) steps = epoches * steps_per_epoch sigma = 1 while (True): #get sigma in interge loss = compute_rdp(q, sigma, steps, np.arange(2, 2 + 512)) if_delta, _ = _compute_delta(np.arange(2, 2 + 512), loss, epsilon) if (if_delta > delta): sigma += 1 else: break for i in range(10): #get sigma in .1 sigma -= .1 loss = compute_rdp(q, sigma, steps, np.arange(2, 2 + 512)) if_delta, _ = _compute_delta(np.arange(2, 2 + 512), loss, epsilon) if (if_delta > delta): sigma += .1 break for i in range(10): #get sigma in .01 sigma -= .01 loss = compute_rdp(q, sigma, steps, np.arange(2, 2 + 512)) if_delta, _ = _compute_delta(np.arange(2, 2 + 512), loss, epsilon) if (if_delta > delta): sigma += .01 break return sigma
def loop_for_sigma(q, T, eps, delta, cur_sigma, interval, rdp_orders=32, rgp=True): while True: orders = np.arange(2, rdp_orders, 0.1) steps = T if (rgp): rdp = compute_rdp( q, cur_sigma, steps, orders ) * 2 ## when using residual gradients, the sensitivity is sqrt(2) else: rdp = compute_rdp(q, cur_sigma, steps, orders) cur_eps, _, opt_order = get_privacy_spent(orders, rdp, target_delta=delta) if (cur_eps < eps and cur_sigma > interval): cur_sigma -= interval previous_eps = cur_eps else: cur_sigma += interval break return cur_sigma, previous_eps
def test_compute_rdp(self): rdp_scalar = rdp_accountant.compute_rdp(0.1, 2, 10, 5) self.assertAlmostEqual(rdp_scalar, 0.07737, places=5) rdp_vec = rdp_accountant.compute_rdp(0.01, 2.5, 50, [1.5, 2.5, 5, 50, 100]) correct = [0.00065, 0.001085, 0.00218075, 0.023846, 167.416307] for i in range(len(rdp_vec)): self.assertAlmostEqual(rdp_vec[i], correct[i], places=5)
def test_compute_rdp(self): rdp_scalar = rdp_accountant.compute_rdp(0.1, 2, 10, 5) self.assertAlmostEqual(rdp_scalar, 0.07737, places=5) rdp_vec = rdp_accountant.compute_rdp(0.01, 2.5, 50, [1.5, 2.5, 5, 50, 100, np.inf]) correct = [0.00065, 0.001085, 0.00218075, 0.023846, 167.416307, np.inf] for i in range(len(rdp_vec)): self.assertAlmostEqual(rdp_vec[i], correct[i], places=5)
def test_compute_rdp_sequence(self): rdp_vec = rdp_accountant.compute_rdp(0.01, 2.5, 50, [1.5, 2.5, 5, 50, 100, np.inf]) self.assertSequenceAlmostEqual( rdp_vec, [0.00065, 0.001085, 0.00218075, 0.023846, 167.416307, np.inf], delta=1e-5)
def test_get_privacy_spent_check_target_eps(self): orders = range(2, 33) rdp = rdp_accountant.compute_rdp(0.01, 4, 10000, orders) _, delta, opt_order = rdp_accountant.get_privacy_spent( orders, rdp, target_eps=1.258575) self.assertAlmostEqual(delta, 1e-5) self.assertEqual(opt_order, 20)
def server_exec(self,mt): i=1 while(True): clear_output() print('Comunication round: ', i) self.eval_acc() rdp = compute_rdp(float(mt/len(self.clients)), self.sigmat, i, self.orders) _,delta_spent, opt_order = get_privacy_spent(self.orders, rdp, target_eps=self.epsilon) print('Delta spent: ', delta_spent) print('Delta budget: ', self.p_budget) if self.p_budget < delta_spent: break Zt = np.random.choice(self.clients, mt) deltas = [] norms = [] for client in Zt: #print(client.number) deltaW, normW = client.update(self.model.state_dict()) deltas.append(deltaW) norms.append(normW) #print('all updates') self.model.to('cpu') new_state_dict = self.sanitaze(mt, deltas, norms, self.sigmat, self.model.state_dict()) #print('sanitaze') self.model.load_state_dict(new_state_dict) i+=1
def test_get_privacy_spent_check_target_delta(self): orders = range(2, 33) rdp = rdp_accountant.compute_rdp(0.01, 4, 10000, orders) eps, _, opt_order = rdp_accountant.get_privacy_spent(orders, rdp, target_delta=1e-5) self.assertAlmostEqual(eps, 1.258575, places=5) self.assertEqual(opt_order, 20)
def test_get_privacy_spent(self): orders = range(2, 33) rdp = rdp_accountant.compute_rdp(0.01, 4, 10000, orders) eps, delta, opt_order = rdp_accountant.get_privacy_spent(orders, rdp, target_delta=1e-5) self.assertAlmostEqual(eps, 1.258575, places=5) self.assertEqual(opt_order, 20) eps, delta, _ = rdp_accountant.get_privacy_spent(orders, rdp, target_eps=1.258575) self.assertAlmostEqual(delta, 1e-5)
def test_compute_privacy_loss(self): parameters = [(0.01, 4, 10000), (0.1, 2, 100)] delta = 1e-5 orders = (1.25, 1.5, 1.75, 2., 2.5, 3., 4., 5., 6., 7., 8., 10., 12., 14., 16., 20., 24., 28., 32., 64., 256.) rdp = np.zeros_like(orders, dtype=float) for q, sigma, steps in parameters: rdp += rdp_accountant.compute_rdp(q, sigma, steps, orders) eps, delta, opt_order = rdp_accountant.get_privacy_spent(orders, rdp, target_delta=delta) self.assertAlmostEqual(eps, 3.276237, places=5) self.assertEqual(opt_order, 8)
def test_check_composition(self): orders = (1.25, 1.5, 1.75, 2., 2.5, 3., 4., 5., 6., 7., 8., 10., 12., 14., 16., 20., 24., 28., 32., 64., 256.) rdp = rdp_accountant.compute_rdp(q=1e-4, stddev_to_sensitivity_ratio=.4, steps=40000, orders=orders) eps, _, opt_order = rdp_accountant.get_privacy_spent(orders, rdp, target_delta=1e-6) rdp += rdp_accountant.compute_rdp(q=0.1, stddev_to_sensitivity_ratio=2, steps=100, orders=orders) eps, _, opt_order = rdp_accountant.get_privacy_spent(orders, rdp, target_delta=1e-5) self.assertAlmostEqual(eps, 8.509656, places=5) self.assertEqual(opt_order, 2.5)
def compute_epsilon(steps): print('in compute_epsilon, steps', steps) """Computes epsilon value for given hyperparameters.""" if FLAGS.noise_multiplier == 0.0: return float('inf') orders = [1 + x / 10. for x in range(1, 100)] + list(range(12, 64)) sampling_probability = (batch_size * 1.0 / config['hist_len']) / num_examples print('sampling_probability', sampling_probability) rdp = compute_rdp(q=sampling_probability, noise_multiplier=FLAGS.noise_multiplier, steps=steps, orders=orders) # Delta is set to 1e-5 because Penn TreeBank has 60000 training points. return get_privacy_spent(orders, rdp, target_delta=FLAGS.target_delta)[0]
def _apply_dp_sgd_analysis(q, sigma, iterations, orders, delta): """Calculates epsilon for stochastic gradient descent. Args: q (float): Sampling probability, generally batch_size / number_of_samples sigma (float): Noise multiplier iterations (float): Number of iterations mechanism is applied orders (list(float)): Orders to try for finding optimal epsilon delta (float): Target delta Returns: float: epsilon Example:: >>> epsilon(10000, 256, 0.3, 100, 1e-5) """ rdp = compute_rdp(q, sigma, iterations, orders) eps, _, opt_order = get_privacy_spent(orders, rdp, target_delta=delta) return eps
def test_compute_rdp_scalar(self): rdp_scalar = rdp_accountant.compute_rdp(0.1, 2, 10, 5) self.assertAlmostEqual(rdp_scalar, 0.07737, places=5)
def test_compute_rdp_no_sampling(self): # q = 1, RDP = alpha/2 * sigma^2 self.assertEqual(rdp_accountant.compute_rdp(1, 10, 1, 20), 0.1)
def test_compute_rdp_no_data(self): # q = 0 self.assertEqual(rdp_accountant.compute_rdp(0, 10, 1, 20), 0)
def train_dp(net, train_loader, test_loader, optimizer, criterion, epoch, batch_size, classes=10, noise_multiplier=1.1): best_ACC = 0.0 best_eps = 0.0 step = 0 best_model = None #for epoch in range(n_epochs): tf = time.time() net.train() loss_tot, avg_norm, cnt = 0.0, 0.0, 0 for i, batch in enumerate(train_loader): imgs, labels = batch imgs, labels = imgs.to(device), labels.to(device) bs = imgs.size(0) outputs = net(imgs)[0] loss = criterion(outputs, labels) grad = [torch.zeros_like(param) for param in net.parameters()] num_microbatch = bs / microbatch_size for j in range(0, bs, microbatch_size): optimizer.zero_grad() torch.autograd.backward(loss[j:j + microbatch_size], retain_graph=True) L2_norm = 0.0 for param in net.parameters(): L2_norm += (param.grad * param.grad).sum() L2_norm = torch.sqrt(L2_norm) avg_norm = avg_norm * 0.95 + L2_norm * 0.05 coef = float(g_clip_thres) / max(g_clip_thres, L2_norm.item()) grad = [ g + param.grad * coef for param, g in zip(net.parameters(), grad) ] for param, g in zip(net.parameters(), grad): if noise_multiplier > 0: param.grad.data = g + torch.cuda.FloatTensor(g.size()).normal_( 0, noise_multiplier * float(g_clip_thres)) else: param.grad.data = g param.grad.data /= num_microbatch optimizer.step() loss_tot += torch.sum(loss).item() cnt += imgs.shape[0] step += 1 train_loss = loss_tot / cnt q = batch_size * 1.0 / cnt orders = [1 + x / 10. for x in range(1, 100)] + list(range(12, 256)) if noise_multiplier > 0: rdp = compute_rdp(q=q, noise_multiplier=noise_multiplier, steps=step, orders=orders) eps = get_privacy_spent(orders, rdp, target_delta=delta)[0] else: eps = 0 train_accuracy = eval_net(net, train_loader, classes) test_accuracy = eval_net(net, test_loader, classes) interval = time.time() - tf if test_accuracy > best_ACC: best_ACC = test_accuracy best_model = deepcopy(net) best_eps = eps print( "Epoch:{}\tTime:{:.2f}\tTrain Loss:{:.2f}\tEps:{:.2f}\tTrain Acc:{:.2f}\tTest Acc:{:.2f}" .format(epoch, interval, train_loss, eps, train_accuracy, test_accuracy)) #print("Best Acc:{:.2f}".format(best_ACC)) #print("Best Eps:{:.2f}".format(best_eps)) #return best_model, best_ACC, best_eps return test_accuracy