def evaluate(description): load_cfg_fom_args(description) assert cfg.CORRUPTION.DATASET == 'cifar10' base_model = load_model(cfg.MODEL.ARCH, cfg.CKPT_DIR, 'cifar10', ThreatModel.Linf).cuda() if cfg.MODEL.ADAPTATION == "dent": assert cfg.MODEL.EPISODIC dent_model = Dent(base_model, cfg.OPTIM) logger.info(dent_model.model) x_test, y_test = load_cifar10(cfg.CORRUPTION.NUM_EX, cfg.DATA_DIR) x_test, y_test = x_test.cuda(), y_test.cuda() adversary = AutoAttack( dent_model, norm='Linf', eps=8./255., version='standard', log_path=osp.join(cfg.SAVE_DIR, cfg.LOG_DEST)) adversary.run_standard_evaluation( x_test, y_test, bs=cfg.TEST.BATCH_SIZE)
def eval(model, args, dataset, logger=None): def out(s): if logger is not None: logger.info(s) else: print(s) model.eval() args.attack_iters = 10 attack = PGD(args, model=model) out("Test pgd10") pgd10_acc, nat_acc = pgd(model, dataset.test_loader, attack, need_nature_acc=True) out(f"Nature: {nat_acc}") out(f"PGD-10: {pgd10_acc}") args.attack_iters = 20 out("Test pgd20") attack = PGD(args, model=model) pgd20_acc, _ = pgd(model, dataset.test_loader, attack) out(f"PGD-20: {pgd20_acc}") args.attack_iters = 1 args.pgd_alpha = 10 out("Test fgsm") attack = PGD(args, model=model) fgsm_acc, _ = pgd(model, dataset.test_loader, attack) out("=" * 70) out(f"Nature: {nat_acc}") out(f"FGSM: {fgsm_acc}") out(f"PGD-10: {pgd10_acc}") out(f"PGD-20: {pgd20_acc}") ### Evaluate AutoAttack ### l = [x for (x, y) in dataset.test_loader] x_test = torch.cat(l, 0) l = [y for (x, y) in dataset.test_loader] y_test = torch.cat(l, 0) epsilon = 8 / 255.0 adversary = AutoAttack(model, norm="Linf", eps=epsilon, version="standard") adversary.run_standard_evaluation(x_test, y_test, bs=256)
def eval_batch(model, x_test, y_test, eps): n_correct = {} num_test = x_test.shape[0] eps = eps/255 adversary = AutoAttack(model, norm='dftinf', eps=eps, version='custom', attacks_to_run=['apgd-ce', 'apgd-dlr']) adversary.apgd.n_restarts = 1 # adversary.apgd.n_iter = 1 x_adv = adversary.run_standard_evaluation(x_test, y_test) n_correct['dftinf'] = adversary.clean_accuracy(x_adv, y_test)*num_test n_correct['clean'] = adversary.clean_accuracy(x_test, y_test)*num_test return n_correct
def evaluate_aa(args, model): test_transform_nonorm = transforms.Compose([transforms.ToTensor()]) num_workers = 2 test_dataset_nonorm = datasets.CIFAR10(args.data_dir, train=False, transform=test_transform_nonorm, download=True) test_loader_nonorm = torch.utils.data.DataLoader( dataset=test_dataset_nonorm, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2, ) model.eval() l = [x for (x, y) in test_loader_nonorm] x_test = torch.cat(l, 0) l = [y for (x, y) in test_loader_nonorm] y_test = torch.cat(l, 0) class normalize_model(): def __init__(self, model): self.model_test = model def __call__(self, x): x_norm = normalize(x) return self.model_test(x_norm) new_model = normalize_model(model) epsilon = 8 / 255. adversary = AutoAttack(new_model, norm='Linf', eps=epsilon, version='standard') X_adv = adversary.run_standard_evaluation(x_test, y_test, bs=128)
def auto(model, device, val_loader, criterion, args, epoch=0): """ Evaluate on atuo-attack adversarial validation set inputs. """ batch_time = AverageMeter("Time", ":6.3f") losses = AverageMeter("Loss", ":.4f") adv_losses = AverageMeter("Adv_Loss", ":.4f") top1 = AverageMeter("Acc_1", ":6.2f") top5 = AverageMeter("Acc_5", ":6.2f") top1_adv = AverageMeter("Adv-Acc_1", ":6.2f") top5_adv = AverageMeter("Adv-Acc_5", ":6.2f") progress = ProgressMeter( len(val_loader), [batch_time, losses, adv_losses, top1, top5, top1_adv, top5_adv], prefix="Test: ", ) # switch to evaluation mode model.eval() assert args.distance in ["linf", "l2"] print( "USING ONLY APGD_CE & APGD-T attacks. Rest of them don't change robust accuracy much but take super long to finish." ) adversary = AutoAttack(model, norm="Linf" if args.distance == "linf" else "L2", eps=args.epsilon) adversary.attacks_to_run = ['apgd-ce'] with torch.no_grad(): end = time.time() for i, data in enumerate(val_loader): images, target = data[0].to(device), data[1].to(device) # clean images output = model(images) loss = criterion(output, target) acc1, acc5 = accuracy(output, target, topk=(1, 2)) losses.update(loss.item(), images.size(0)) top1.update(acc1[0], images.size(0)) top5.update(acc5[0], images.size(0)) images = adversary.run_standard_evaluation(images, target, bs=len(images)) # compute output output = model(images) loss = criterion(output, target) # measure accuracy and record loss acc1, acc5 = accuracy(output, target, topk=(1, 2)) adv_losses.update(loss.item(), images.size(0)) top1_adv.update(acc1[0], images.size(0)) top5_adv.update(acc5[0], images.size(0)) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if (i + 1) % args.print_freq == 0 and args.local_rank == 0: progress.display(i) if args.local_rank == 0: progress.display(i) # print final results result = { "top1": top1.avg, "top5": top5.avg, "loss": losses.avg, "top1_adv": top1_adv.avg, "top5_adv": top5_adv.avg, "adv_loss": adv_losses.avg } return result
clean_y_test, batch_size=batch_size, device=device) print(f'Clean accuracy: {accuracy:.2%}') if threat_model_ in {ThreatModel.Linf, ThreatModel.L2}: if eps is None: raise ValueError( "If the threat model is L2 or Linf, `eps` must be specified.") adversary = AutoAttack(model, norm=threat_model_.value, eps=eps, version='standard', device=device) x_adv = adversary.run_standard_evaluation(clean_x_test, clean_y_test) adv_accuracy = clean_accuracy(model, x_adv, clean_y_test, batch_size=batch_size, device=device) elif threat_model_ == ThreatModel.corruptions: print(f"Evaluating over {len(CORRUPTIONS)} corruptions") # Save into a dict to make a Pandas DF with nested index adv_accuracy = corruptions_evaluation(batch_size, data_dir, dataset_, device, model, n_examples, to_disk, model_name) else: raise NotImplementedError print(f'Adversarial accuracy: {adv_accuracy:.2%}')
x_test = torch.cat(l, 0) l = [y for (x, y) in test_loader] y_test = torch.cat(l, 0) # cheap version if args.cheap: adversary.cheap() # plus version if args.plus: adversary.plus = True # run attack and save images with torch.no_grad(): if not args.individual: adv_complete = adversary.run_standard_evaluation( x_test[:args.n_ex], y_test[:args.n_ex], bs=args.batch_size) torch.save({'adv_complete': adv_complete}, '{}/{}_1_{}_eps_{:.5f}_plus_{}_cheap_{}.pth'.format( args.save_dir, 'aa', adv_complete.shape[0], args.epsilon, args.plus, args.cheap)) else: # individual version, each attack is run on all test points # specify attacks to run with # adversary.attacks_to_run = ['apgd-ce'] adv_complete = adversary.run_standard_evaluation_individual( x_test[:args.n_ex], y_test[:args.n_ex], bs=args.batch_size) torch.save( adv_complete,
testY = np.argmax(testY, axis=1) torch_testX = torch.from_numpy(np.transpose(testX, (0, 3, 1, 2))).float().cuda() torch_testY = torch.from_numpy(testY).float() # load model from saved weights print('[INFO] MODEL_PATH: {:s}'.format(MODEL_PATH)) tf_model = mnist_model() tf_model.load_weights(MODEL_PATH) # remove 'softmax layer' and put it into adapter atk_model = tf.keras.models.Model( inputs=tf_model.input, outputs=tf_model.get_layer(index=-2).output) atk_model.summary() y_input = tf.placeholder(tf.int64, shape=[None]) x_input = atk_model.input logits = atk_model.output model_adapted = utils_tf.ModelAdapter(logits, x_input, y_input, sess) # run attack adversary = AutoAttack(model_adapted, norm='Linf', eps=epsilon, plus=False, is_tf_model=True) x_adv = adversary.run_standard_evaluation(torch_testX, torch_testY, bs=batch_size) np_x_adv = np.moveaxis(x_adv.cpu().numpy(), 1, 3) np.save("./output/mnist_adv.npy", np_x_adv)
test_loader, 0, adversary=adversary_t, logger=None, ret='adv') elif P.mode == 'test_auto_attack': from autoattack import AutoAttack auto_adversary = AutoAttack(model, norm=P.distance, eps=P.epsilon, version='standard') x_test, y_test = extract_dataset(test_loader) x_adv = auto_adversary.run_standard_evaluation(x_test, y_test) elif P.mode == 'test_mce': from evals import test_classifier mean_corruption_error = 0. for name in corruption_list: error = test_classifier(P, model, corruption_loader[name], 0, logger=None) mean_corruption_error += error print(f'Error of {name}: {error}%\n') print(f'MCE: {mean_corruption_error/len(corruption_list)} %')
def robust_test(model, loss_fn, loader, args, att_dir, epoch=0, training_output_dir_name=None, figure_dir_name=None, n_batches=0, train_data=False, training_time=False): """ n_batches (int): Number of batches for evaluation. """ model.eval() num_correct, num_correct_adv, num_samples = 0, 0, 0 steps = 1 losses_adv = [] losses_ben = [] prob_dict = {} if args.track_hard: loss_dict = collections.OrderedDict() pred_dict = {} if training_time and args.track_hard: f_name = training_output_dir_name + 'losses.json' f = open(f_name, 'a+') loss_dict['epoch'] = epoch if 'PGD' in att_dir['attack']: for t, (x, y, idx, ez, m) in enumerate(loader): x = x.cuda() y = y.cuda() x_var = Variable(x, requires_grad=True) y_var = Variable(y, requires_grad=False) if att_dir['targeted']: y_target = generate_target_label_tensor(y_var.cpu(), args).cuda() else: y_target = y_var if 'PGD_linf' in att_dir['attack']: adv_x = pgd_attack(model, x, x_var, y_target, att_dir['attack_iter'], att_dir['epsilon'], att_dir['eps_step'], att_dir['clip_min'], att_dir['clip_max'], att_dir['targeted'], att_dir['rand_init']) elif 'PGD_l2' in att_dir['attack']: adv_x = pgd_l2_attack(model, x, x_var, y_target, att_dir['attack_iter'], att_dir['epsilon'], att_dir['eps_step'], att_dir['clip_min'], att_dir['clip_max'], att_dir['targeted'], att_dir['rand_init'], att_dir['num_restarts']) # Predictions scores = model(x.cuda()) _, preds = scores.data.max(1) scores_adv = model(adv_x) _, preds_adv = scores_adv.data.max(1) # Losses batch_loss_adv = loss_fn(scores_adv, y) loss_adv = torch.mean(batch_loss_adv) losses_adv.append(loss_adv.data.cpu().numpy()) batch_loss_ben = loss_fn(scores, y) loss_ben = torch.mean(batch_loss_ben) losses_ben.append(loss_ben.data.cpu().numpy()) # Correct count num_correct += (preds == y).sum() num_correct_adv += (preds_adv == y).sum() num_samples += len(preds) # Adding probs to dict count = 0 for i in idx.numpy(): score_curr = scores_adv[count].cpu().detach().numpy() prob_dict[str(i)] = softmax(score_curr) # print(count) count += 1 if args.track_hard: idx_matched = idx[~ez].numpy() m_matched = m[~ez].numpy() preds_adv_matched = preds_adv[~ez].cpu().numpy() loss_adv_matched = batch_loss_adv[~ez].cpu().detach().numpy() y_np_matched = y[~ez].cpu().numpy() ez_np = ez.data.cpu().numpy() for i in range(len(y_np_matched)): pred_dict[str(idx_matched[i])] = [ m_matched[i], y_np_matched[i], preds_adv_matched[i] ] loss_dict = track_hard_losses(ez_np, batch_loss_adv, batch_loss_ben, loss_dict, t) if not training_time: if args.viz and steps == 1: if not os.path.exists(figure_dir_name): os.makedirs(figure_dir_name) custom_save_image(adv_x, preds_adv, y, args, figure_dir_name, train_data) if n_batches > 0 and steps == n_batches: break steps += 1 elif 'AA' in att_dir['attack']: x_test = [] y_test = [] idx_all = [] for t, (x, y, idx, ez, m) in enumerate(loader): x_test.append(x) y_test.append(y) idx_all.extend(idx.numpy()) x_test = torch.cat(x_test, 0) y_test = torch.cat(y_test, 0) print(x_test.size()) print(y_test.size()) adversary = AutoAttack(model, norm='L2', eps=att_dir['epsilon'], version='standard') adversary.attacks_to_run = ['apgd-ce', 'fab'] adv_x = adversary.run_standard_evaluation(x_test, y_test) loss_adv, loss_ben, num_correct, num_correct_adv, num_samples, scores_adv = compute_scores_and_losses( model, loss_fn, x_test, y_test, adv_x) losses_adv.append(loss_adv.data.cpu().numpy()) losses_ben.append(loss_ben.data.cpu().numpy()) # Adding probs to dict count = 0 for i in idx_all: score_curr = scores_adv[count].cpu().detach().numpy() prob_dict[str(i)] = softmax(score_curr) # print(count) count += 1 acc = float(num_correct) / num_samples acc_adv = float(num_correct_adv) / num_samples print('Clean accuracy: {:.2f}% ({}/{})'.format( 100. * acc, num_correct, num_samples, )) print('Adversarial accuracy: {:.2f}% ({}/{})'.format( 100. * acc_adv, num_correct_adv, num_samples, )) if args.track_hard: if not training_time: print('Counting hard points') hard_point_class_count(pred_dict) if training_time: json.dump(loss_dict, f) f.write('\n') if len(loss_dict['batch_losses_hard']) > 0: print('Adv loss easy: %.8f' % np.mean(loss_dict['batch_losses_easy'])) print('Adv loss hard: %.8f' % np.mean(loss_dict['batch_losses_hard'])) print('Ben loss easy: %.8f' % np.mean(loss_dict['batch_losses_ben_easy'])) print('Ben loss hard: %.8f' % np.mean(loss_dict['batch_losses_ben_hard'])) return 100. * acc, 100. * acc_adv, np.mean(losses_ben), np.mean( losses_adv), prob_dict
def main(config): os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" os.environ["CUDA_VISIBLE_DEVICES"] = config['meta']['gpu_id'] model_name = config['meta']['model_name'] # Set all random seeds seed = config['meta']['seed'] np.random.seed(seed) torch.manual_seed(seed) device = 'cuda' if torch.cuda.is_available() else 'cpu' if device == 'cuda': cudnn.benchmark = True # Set up model directory save_dir = os.path.join(config['meta']['save_path'], './') if not os.path.isdir(save_dir): os.makedirs(save_dir) model_path = os.path.join(save_dir, model_name) timenow = datetime.datetime.now().strftime("%m-%d-%Y-%H:%M:%S") logfile = 'test_' + model_name + '_' + timenow log = get_logger(logfile, '') log.info('\n%s', yaml.dump(config)) log.info('Preparing data...') if config['test']['dataset'] == 'cifar10': _, _, (x_test, y_test) = load_cifar10_all(data_dir=config['meta']['data_path'], val_size=0.1, shuffle=False, seed=seed) num_classes = 10 elif config['test']['dataset'] == 'cifar100': _, _, (x_test, y_test) = load_cifar100_all( data_dir=config['meta']['data_path'], val_size=0.1, shuffle=False, seed=seed) num_classes = 100 elif config['test']['dataset'] == 'mnist': _, _, (x_test, y_test) = load_mnist_all(data_dir=config['meta']['data_path'], val_size=0.1, shuffle=True, seed=seed) num_classes = 10 else: raise NotImplementedError('invalid dataset.') log.info('Building model...') if config['test']['network'] == 'resnet': # use ResNetV2-20 net = PreActResNet(PreActBlock, [2, 2, 2, 2], num_classes=num_classes) elif config['test']['network'] == 'wideresnet': # use WideResNet-34-10 net = WideResNet(num_classes=num_classes) elif config['test']['network'] == 'basic': net = BasicModel(num_classes=num_classes) else: raise NotImplementedError('Specified network not implemented.') net.load_state_dict(torch.load(model_path + '.pt')) net = net.eval().to(device) num_test_samples = config['test']['num_test_samples'] log.info('Starting attack...') adversary = AutoAttack(net, norm='Linf', eps=config['test']['epsilon'], version='standard', log_path=logfile + '.log') x_adv = adversary.run_standard_evaluation(x_test[:num_test_samples], y_test[:num_test_samples], bs=config['test']['batch_size']) y_pred = classify(net, x_adv) adv_acc = get_acc(y_pred, y_test[:num_test_samples]) log.info('AA acc: %.4f.', adv_acc)
def main(): args = get_args() np.random.seed(args.seed) torch.manual_seed(args.seed) torch.cuda.manual_seed(args.seed) logger = logging.getLogger(__name__) logging.basicConfig( format='[%(asctime)s] - %(message)s', datefmt='%Y/%m/%d %H:%M:%S', level=logging.DEBUG, handlers=[ logging.StreamHandler() ]) logger.info(args) _, test_loader = get_loaders(args.data_dir, args.batch_size) best_state_dict = torch.load(os.path.join(args.out_dir, 'model_best.pth')) # Evaluation model_test = PreActResNet18().cuda() # model_test = WideResNet(34, 10, widen_factor=10, dropRate=0.0) model_test = nn.DataParallel(model_test).cuda() if 'state_dict' in best_state_dict.keys(): model_test.load_state_dict(best_state_dict['state_dict']) else: model_test.load_state_dict(best_state_dict) model_test.float() model_test.eval() ### Evaluate clean acc ### _, test_acc = evaluate_standard(test_loader, model_test) print('Clean acc: ', test_acc) ### Evaluate PGD (CE loss) acc ### _, pgd_acc_CE = evaluate_pgd(test_loader, model_test, attack_iters=10, restarts=1, eps=8, step=2, use_CWloss=False) print('PGD-10 (10 restarts, step 2, CE loss) acc: ', pgd_acc_CE) ### Evaluate PGD (CW loss) acc ### _, pgd_acc_CW = evaluate_pgd(test_loader, model_test, attack_iters=10, restarts=1, eps=8, step=2, use_CWloss=True) print('PGD-10 (10 restarts, step 2, CW loss) acc: ', pgd_acc_CW) ### Evaluate AutoAttack ### l = [x for (x, y) in test_loader] x_test = torch.cat(l, 0) l = [y for (x, y) in test_loader] y_test = torch.cat(l, 0) class normalize_model(): def __init__(self, model): self.model_test = model def __call__(self, x): return self.model_test(normalize(x)) new_model = normalize_model(model_test) epsilon = 8 / 255. adversary = AutoAttack(new_model, norm='Linf', eps=epsilon, version='standard') X_adv = adversary.run_standard_evaluation(x_test, y_test, bs=128)
shuffle=False, num_workers=0) from autoattack import AutoAttack log = 'store/' + log_folder + '/AA_eval-new.txt' model = model.cuda() adversary = AutoAttack(model, norm='Linf', eps=8 / 255, log_path=log, version='standard') adversary.attacks_to_run = ['apgd-t'] l = [x for (x, y) in test_loader] x_test = torch.cat(l, 0).cuda() l = [y for (x, y) in test_loader] y_test = torch.cat(l, 0).cuda() adv_complete = adversary.run_standard_evaluation(x_test, y_test, bs=128) save_dir = 'store/' + log_folder print(adv_complete) torch.save({'adv_complete': adv_complete}, '{}/{}_{}_1_{}_eps_{:.5f}.pth'.format(save_dir, 'aa', 'standard', adv_complete.shape[0], 8 / 255)) ## PGD20 EVAL ## out_store = cox.store.Store('store', exp_id=log_folder) eval_pgd20_args = Parameters({ 'constraint': 'inf', 'eps': 8 / 255, 'attack_lr': 2 / 255, 'epochs': 10,
dataloader = dl.get_CIFAR100(False, batch_size=ROBUSTNESS_DATAPOINTS, augm_type='none') else: raise NotImplementedError() data_iterator = iter(dataloader) ref_data, target = next(data_iterator) if L2: print('Eps: 0.5') attack = AutoAttack(model, device=device, norm='L2', eps=0.5, verbose=True) attack.run_standard_evaluation(ref_data, target, bs=bs) # print('Eps: 1.0') # attack = AutoAttack(model, device=device, norm='L2', eps=1.0, attacks_to_run=attacks_to_run,verbose=True) # attack.run_standard_evaluation(ref_data, target, bs=bs) if LINF: print('Eps: 8/255') attack = AutoAttack(model, device=device, norm='Linf', eps=8. / 255., verbose=True) attack.run_standard_evaluation(ref_data, target, bs=bs)
def get_all_cifar10_data(constants, is_test=True): ds = constants.get_dataset() if is_test: _, data_loader = ds.make_loaders(batch_size=16, only_val=True, workers=8, data_aug=False) else: data_loader, _ = ds.make_loaders(batch_size=16, workers=8, data_aug=False) X, Y = [], [] for (x, y) in data_loader: X.append(x) Y.append(y) X, Y = ch.cat(X), ch.cat(Y) return (X, Y) if __name__ == "__main__": epsilon = 8 / 255 constants = utils.CIFAR10() ds = constants.get_dataset() model = constants.get_model( "/p/adversarialml/as9rw/models_cifar10_vgg19/custom_adv_train_try_10.000000_100.000000_16_0.010000_3_fast_1/checkpoint.pt.best", "vgg19") wrapped_model = ModelWrapper(model) images, labels = get_all_cifar10_data(constants) adversary = AutoAttack(wrapped_model, norm='Linf', eps=epsilon, plus=False) x_adv = adversary.run_standard_evaluation(images, labels, bs=128)
model_adapted = utils_tf2.ModelAdapter(model_ensemble) adversary = AutoAttack(model_adapted, norm='Linf', eps=eps, version='standard', is_tf_model=True, log_path=logfile) # Convert to required format: NHWC --> NCHW _x = torch.from_numpy(np.moveaxis(x_test, 3, 1)) _y = torch.from_numpy(np.argmax(y_test, axis=-1)) _x.float().to("cuda") _y.to("cuda") x_adv = adversary.run_standard_evaluation(_x, _y, bs=bs) x_adv = x_adv.cpu().numpy() # Convert back to NHWC x_adv = np.moveaxis(x_adv, 1, 3) pred_adv = model_ensemble.predict(x=x_adv) pred_nat = model_ensemble.predict(x=x_test) acc_nat_test = np.mean( np.argmax(pred_nat, axis=-1) == np.argmax(y_test, axis=-1)) acc_adv_test = np.mean( np.argmax(pred_adv, axis=-1) == np.argmax(y_test, axis=-1)) h_nat, m_nat, s_nat = get_statistic(pred_nat) h_adv, m_adv, s_adv = get_statistic(pred_adv)