def main(_): # Load training and test data data = ld_cifar10() # Instantiate model, loss, and optimizer for training net = CNN(in_channels=3) device = 'cuda' if torch.cuda.is_available() else 'cpu' if device == 'cuda': net = net.cuda() loss_fn = torch.nn.CrossEntropyLoss(reduction='mean') optimizer = torch.optim.Adam(net.parameters(), lr=1e-3) # Train vanilla model net.train() for epoch in range(1, FLAGS.nb_epochs + 1): train_loss = 0. for x, y in data.train: x, y = x.to(device), y.to(device) if FLAGS.adv_train: # Replace clean example with adversarial example for adversarial training x = projected_gradient_descent(net, x, FLAGS.eps, 0.01, 40, np.inf) # Stop backward from entering the graph that created the adv example x = x.clone().detach() optimizer.zero_grad() loss = loss_fn(net(x), y) loss.backward() optimizer.step() train_loss += loss.item() print('epoch: {}/{}, train loss: {:.3f}'.format( epoch, FLAGS.nb_epochs, train_loss)) # Evaluate on clean and adversarial data net.eval() report = EasyDict(nb_test=0, correct=0, correct_fgm=0, correct_pgd=0) for x, y in data.test: x, y = x.to(device), y.to(device) x_fgm = fast_gradient_method(net, x, FLAGS.eps, np.inf) x_pgd = projected_gradient_descent(net, x, FLAGS.eps, 0.01, 40, np.inf) _, y_pred = net(x).max(1) # model prediction on clean examples _, y_pred_fgm = net(x_fgm).max( 1) # model prediction on FGM adversarial examples _, y_pred_pgd = net(x_pgd).max( 1) # model prediction on PGD adversarial examples report.nb_test += y.size(0) report.correct += y_pred.eq(y).sum().item() report.correct_fgm += y_pred_fgm.eq(y).sum().item() report.correct_pgd += y_pred_pgd.eq(y).sum().item() print('test acc on clean examples (%): {:.3f}'.format( report.correct / report.nb_test * 100.)) print('test acc on FGM adversarial examples (%): {:.3f}'.format( report.correct_fgm / report.nb_test * 100.)) print('test acc on PGD adversarial examples (%): {:.3f}'.format( report.correct_pgd / report.nb_test * 100.))
def test(opt, net, loader): correct = 0 if opt.adversarial: correct_fgm, correct_pgd = 0, 0 total = 0 net.eval() logger.info('Starting testing...') with torch.set_grad_enabled(opt.adversarial): # adversarial attacks need grad computations n_batches = len(loader) for i, (input, target) in enumerate(loader): logger.info('batch: {}/{}'.format(i + 1, n_batches)) input, target = \ input.to(device, non_blocking=True), target.to( device, non_blocking=True) output = net(input, update_centers=False) _, predicted = torch.max(output.data, 1) total += target.size(0) correct += (predicted == target).sum().item() if opt.adversarial: def net_fn(input): return net(input, update_centers=False) input_fgm = fast_gradient_method(net_fn, input, opt.adversary_eps, opt.adversary_norm) input_pgd = projected_gradient_descent(net_fn, input, opt.adversary_eps, opt.pgd_step_eps, opt.pgd_n_steps, opt.adversary_norm) output_fgm = net(input_fgm, update_centers=False) output_pgd = net(input_pgd, update_centers=False) _, predicted_fgm = torch.max(output_fgm.data, 1) _, predicted_pgd = torch.max(output_pgd.data, 1) correct_fgm += (predicted_fgm == target).sum().item() correct_pgd += (predicted_pgd == target).sum().item() acc = 100 * correct / total results = {'accuracy (%)': acc} logger.info('Accuracy (%): {:.3f}'.format(acc)) if opt.adversarial: acc_fgm = 100 * correct_fgm / total logger.info('Accuracy under FGM (%): {:.3f}'.format(acc_fgm)) acc_pgd = 100 * correct_pgd / total logger.info('Accuracy under PGD (%): {:.3f}'.format(acc_pgd)) results['accuracy under FGM (%)'] = acc_fgm results['accuracy under PGD (%)'] = acc_pgd with open(os.path.join(opt.save_dir, 'test.json'), 'w') as out: json.dump(results, out, indent=2) logger.info('Testing finished!')
def test(opt, net, loader): correct = 0 if opt.adversarial: correct_fgm, correct_pgd = 0, 0 total = 0 net.eval() logger.info('Starting testing...') with torch.set_grad_enabled(opt.adversarial): # adversarial attacks need grad computations for i, (input, target) in enumerate(loader): logger.info('batch: {}/{}'.format(i, len(loader))) input, target = \ input.to(device, non_blocking=True), target.to(device, non_blocking=True) output = net(input, update_centers=False) # save tensors for visualization raw_data = input if i == 0 else torch.cat((raw_data, input)) labels = target if i == 0 else torch.cat((labels, target)) activations = net_head(input) all_activations = activations if i == 0 else torch.cat((all_activations, activations)) _, predicted = torch.max(output.data, 1) total += target.size(0) correct += (predicted == target).sum().item() if opt.adversarial: net_fn = lambda input: net(input, update_centers=False) input_fgm = fast_gradient_method(net_fn, input, opt.adversary_eps, opt.adversary_norm) input_pgd = projected_gradient_descent(net_fn, input, opt.adversary_eps, opt.pgd_step_eps, opt.pgd_n_steps, opt.adversary_norm) output_fgm = net(input_fgm, update_centers=False) output_pgd = net(input_pgd, update_centers=False) _, predicted_fgm = torch.max(output_fgm.data, 1) _, predicted_pgd = torch.max(output_pgd.data, 1) correct_fgm += (predicted_fgm == target).sum().item() correct_pgd += (predicted_pgd == target).sum().item() logger.info('Accuracy (%): {:.3f}'.format(100 * correct / total)) if opt.adversarial: logger.info('Accuracy under FGM (%): {:.3f}'.format(100 * correct_fgm / total)) logger.info('Accuracy under PGD (%): {:.3f}'.format(100 * correct_pgd / total)) logger.info('Testing finished!') return raw_data.cpu().numpy(), labels.cpu().numpy(), all_activations.cpu().numpy()
def run_attacks_cleverhans(res_path, ncl=False): if ncl: MODEL_DIR = '/mnt/md0/orville/Miriam/modular-loss-experiments-morph/results_ncl/CIFAR-10/densenet-82-8-8' rel_dirs = ['alpha_0.0_gamma_0.02_n_models_2_1583114412120', 'alpha_0.0_gamma_0.05_n_models_2_1583114439810'] alpha = ['0.02', '0.05'] res_path = res_path + '_ncl' else: MODEL_DIR = '/mnt/md0/orville/Miriam/modular-loss-experiments-morph/results/CIFAR-10/densenet-82-8-8' # UNCORR_MODEL_DIR = 'alpha_0.0_gamma_0.0_n_models_2_1581641733617' # CORR_MODEL_DIR = 'alpha_0.1_gamma_0.0_n_models_2_1581641746832' # CORR_MODEL_DIR_2 = 'alpha_0.2_gamma_0.0_n_models_2_1581641777871' # UNCORR_MODEL_DIR = 'alpha_0.0_gamma_0.0_n_models_3_1585505819121' # CORR_MODEL_DIR = 'alpha_0.1_gamma_0.0_n_models_3_1585505685528' # CORR_MODEL_DIR_2 = 'alpha_0.2_gamma_0.0_n_models_3_1585505042819' # rel_dirs = [UNCORR_MODEL_DIR, CORR_MODEL_DIR, CORR_MODEL_DIR_2] rel_dirs = ['alpha_0.0_gamma_0.0_n_models_3_1585505819121', 'alpha_0.1_gamma_0.0_n_models_3_1589795142450', 'alpha_0.2_gamma_0.0_n_models_3_1589794987034', 'alpha_0.3_gamma_0.0_n_models_3_1589795486214', 'alpha_0.4_gamma_0.0_n_models_3_1589796192038', 'alpha_0.5_gamma_0.0_n_models_3_1589796200262', 'alpha_0.6_gamma_0.0_n_models_3_1589796218204', 'alpha_0.7_gamma_0.0_n_models_3_1589796234665'] alpha = list(map(lambda x: format(x, '2.1f'), np.arange(0.0, 0.8, 0.1))) batch_size = 256 # 128 # 516 n_workers = 20 dataset = 'CIFAR-10' network = 'densenet-82-8-8' loaders, _ = get_dataloaders_(batch_size, 0, dataset, False, early_stop=False, n_workers=n_workers) n_models = 2 if ncl else 3 params = {} params['densenet-82-8-8'] = {'num_modules': n_models, 'bottleneck': True, 'reduction': 0.5, 'depth': 82, 'growth_rate': 8, 'input_shape': (3, 32, 32), 'output_dim': 10} network = 'densenet-82-8-8' model = DenseNet(input_shape=params[network]['input_shape'], output_dim=params[network]['output_dim'], growth_rate=params[network]['growth_rate'], depth=params[network]['depth'], reduction=params[network]['reduction'], bottleneck=params[network]['bottleneck'], num_modules=n_models) device = torch.device("cuda") reports = dict.fromkeys(alpha) for model_path, curr_alpha in tqdm(zip(rel_dirs, alpha), total=len(alpha)): if ncl: weight_path = path.join(MODEL_DIR, model_path, 'trial_0/' + curr_alpha + '/weights/final_weights.pt') else: weight_path = path.join(MODEL_DIR, model_path, 'trial_0/0.0/weights/final_weights.pt') model.reset_parameters() model.load_state_dict(torch.load(weight_path)) model.eval() # model.train(mode=False) net = ModelMeanEP(model).to(device) report = dict() for x, y in tqdm(loaders['test'], total=len(loaders['test'])): x, y = x.to(device), y.to(device) report['nb_test'] = report.get('nb_test', 0) + y.size(0) _, y_pred = net(x).max(1) # model prediction on clean examples report['acc'] = report.get('acc', 0) + y_pred.eq(y).sum().item() # model prediction on FGM adversarial examples x_adv = fast_gradient_method(net, x, 0.02, np.inf) _, y_pred = net(x_adv).max(1) # model prediction on FGM adversarial examples report['FGM_0.02'] = report.get('FGM_0.02', 0) + y_pred.eq(y).sum().item() x_adv = fast_gradient_method(net, x, 0.04, np.inf) _, y_pred = net(x_adv).max(1) # model prediction on FGM adversarial examples report['FGM_0.04'] = report.get('FGM_0.04', 0) + y_pred.eq(y).sum().item() # model prediction on BIM adversarial examples x_adv = projected_gradient_descent(net, x, eps=0.01, eps_iter=0.01 / 10, nb_iter=10, norm=np.inf, rand_init=0) _, y_pred = net(x_adv).max(1) report['BIM_0.01'] = report.get('BIM_0.01', 0) + y_pred.eq(y).sum().item() x_adv = projected_gradient_descent(net, x, eps=0.02, eps_iter=0.02 / 10, nb_iter=10, norm=np.inf, rand_init=0) _, y_pred = net(x_adv).max(1) report['BIM_0.02'] = report.get('BIM_0.02', 0) + y_pred.eq(y).sum().item() # model prediction on PGD adversarial examples x_adv = projected_gradient_descent(net, x, eps=0.01, eps_iter=0.01 / 10, nb_iter=10, norm=np.inf) _, y_pred = net(x_adv).max(1) report['PGD_0.01'] = report.get('PGD_0.01', 0) + y_pred.eq(y).sum().item() x_adv = projected_gradient_descent(net, x, eps=0.02, eps_iter=0.02 / 10, nb_iter=10, norm=np.inf) _, y_pred = net(x_adv).max(1) report['PGD_0.02'] = report.get('PGD_0.02', 0) + y_pred.eq(y).sum().item() for key in ['acc', 'FGM_0.02', 'FGM_0.04', 'BIM_0.01', 'BIM_0.02', 'PGD_0.01', 'PGD_0.02']: report[key] = (report[key] / report['nb_test']) * 100. reports[curr_alpha] = report pickle.dump(reports, open(res_path, 'wb')) pickle.dump(reports, open(res_path, 'wb'))
def main(_): # Load training and test data data = ld_cifar10() # Instantiate model, loss, and optimizer for training # net = CNN(in_channels=3) net = resnet.ResNet18() device = 'cuda' if torch.cuda.is_available() else 'cpu' logging.info('Using GPU' if device == 'cuda' else 'Using CPU') if device == 'cuda': net = net.cuda() loss_fn = torch.nn.CrossEntropyLoss(reduction='mean') optimizer = torch.optim.Adam(net.parameters(), lr=1e-3) # load checkpoint if exists if os.path.exists(FLAGS.checkpoint): ckpt = torch.load(FLAGS.checkpoint) net.load_state_dict(ckpt['net']) logging.info('Loaded model %s with accuracy %.3f', FLAGS.checkpoint, ckpt['acc']) else: # Train vanilla model if input('No checkpoint found, continue? y/[n]') != 'y': return -1 net.train() with trange(1, FLAGS.nb_epochs + 1, desc='Training', unit='Epoch') as t: for epoch in t: train_loss = 0. for x, y in data.train: x, y = x.to(device), y.to(device) if FLAGS.adv_train: # Replace clean example with adversarial example for adversarial training x = projected_gradient_descent(net, x, FLAGS.eps, 0.01, 40, np.inf) optimizer.zero_grad() loss = loss_fn(net(x), y) loss.backward() optimizer.step() train_loss += loss.item() t.set_description('Train Loss=%.3f' % train_loss) # Evaluate on clean and adversarial data net.eval() report = EasyDict(nb_test=0, correct=0, correct_fgm=0, correct_pgd=0) for x, y in tqdm(data.test, unit='Samples', desc='Testing'): x, y = x.to(device), y.to(device) x_fgm = fast_gradient_method(net, x, FLAGS.eps, np.inf) x_pgd = projected_gradient_descent(net, x, FLAGS.eps, 0.01, 40, np.inf) _, y_pred = net(x).max(1) # model prediction on clean examples _, y_pred_fgm = net(x_fgm).max( 1) # model prediction on FGM adversarial examples _, y_pred_pgd = net(x_pgd).max( 1) # model prediction on PGD adversarial examples report.nb_test += y.size(0) report.correct += y_pred.eq(y).sum().item() report.correct_fgm += y_pred_fgm.eq(y).sum().item() report.correct_pgd += y_pred_pgd.eq(y).sum().item() print('test acc on clean examples (%): {:.3f}'.format( report.correct / report.nb_test * 100.)) print('test acc on FGM adversarial examples (%): {:.3f}'.format( report.correct_fgm / report.nb_test * 100.)) print('test acc on PGD adversarial examples (%): {:.3f}'.format( report.correct_pgd / report.nb_test * 100.)) return 0