def eval_fn(model): advcorrect = 0 model.to(args.dev) model.eval() with ctx_noparamgrad_and_eval(model): if args.source_arch == 'googlenet': adv_complete_list = [] for batch_idx, (x_batch, y_batch) in enumerate(test_loader): if (batch_idx + 1) * args.test_batch_size > args.batch_size: break x_batch, y_batch = x_batch.to(args.dev), y_batch.to(args.dev) adv_complete_list.append(attacker.perturb(x_batch, target=y_batch)) adv_complete = torch.cat(adv_complete_list) else: adv_complete = attacker.perturb(x_test[:args.batch_size], target=y_test[:args.batch_size]) adv_complete = torch.clamp(adv_complete, min=0., max=1.0) output = model(adv_complete) pred = output.max(1, keepdim=True)[1] advcorrect += pred.eq(y_test[:args.batch_size].view_as(pred)).sum().item() fool_rate = 1 - advcorrect / float(args.batch_size) print('Test set base model fool rate: %f' %(fool_rate)) model.cpu() if args.transfer: adv_img_list = [] y_orig = y_test[:args.batch_size] for i in range(0, len(adv_complete)): adv_img_list.append([adv_complete[i].unsqueeze(0), y_orig[i]]) # Free memory del model torch.cuda.empty_cache() baseline_transfer(args, attacker, "AEG", model_type, adv_img_list, l_test_classif_paths, adv_models)
def attack_dataset_images(self, args, attacker, arch_name, target_model, codec, l_test_classif_paths, adv_models, result_dump_path='.'): success_list, query_list, adv_img_list = [], [], [] for batch_idx, data_tuple in enumerate(self.dataset_loader): print(batch_idx) if batch_idx > args.num_attack: break if args.dataset == "ImageNet": if args.input_size >= 299: images, true_labels = data_tuple[1], data_tuple[2] else: images, true_labels = data_tuple[0], data_tuple[2] else: images, true_labels = data_tuple[0], data_tuple[1] if images.size(-1) != args.input_size: images = F.interpolate(images, size=target_model.module.input_size[-1], mode='bilinear',align_corners=True) success, query, adv_images = self.make_adversarial_examples(batch_idx, images.cuda(), true_labels.cuda(), args, attacker, target_model, codec) success_list.append(success) adv_img_list.append([adv_images, true_labels]) avg_correct = sum(success_list) / float(len(success_list)) print('{} is attacked finished ({} images)'.format(arch_name, self.total_images)) print(' avg correct: {:.4f}'.format(avg_correct.item())) # print(' avg correct: {:.4f}'.format(self.correct_all.mean().item())) print(' avg not_done: {:.4f}'.format(self.not_done_all.mean().item())) # 有多少图没做完 if self.success_all.sum().item() > 0: print( ' avg mean_query: {:.4f}'.format(self.success_query_all[self.success_all.byte()].mean().item())) print( ' avg median_query: {:.4f}'.format(self.success_query_all[self.success_all.byte()].median().item())) print(' max query: {}'.format(self.success_query_all[self.success_all.byte()].max().item())) if self.not_done_all.sum().item() > 0: print( ' avg not_done_loss: {:.4f}'.format(self.not_done_loss_all[self.not_done_all.byte()].mean().item())) print( ' avg not_done_prob: {:.4f}'.format(self.not_done_prob_all[self.not_done_all.byte()].mean().item())) print('Saving results to {}'.format(result_dump_path)) # meta_info_dict = {"avg_correct": self.correct_all.mean().item(), # "avg_not_done": self.not_done_all.mean().item(), # "mean_query": self.success_query_all[self.success_all.byte()].mean().item(), # "median_query": self.success_query_all[self.success_all.byte()].median().item(), # "max_query": self.success_query_all[self.success_all.byte()].max().item(), # "not_done_loss": self.not_done_loss_all[self.not_done_all.byte()].mean().item(), # "not_done_prob": self.not_done_prob_all[self.not_done_all.byte()].mean().item()} # meta_info_dict['args'] = vars(args) # with open(result_dump_path, "w") as result_file_obj: # json.dump(meta_info_dict, result_file_obj, indent=4, sort_keys=True) print("done, write stats info to {}".format(result_dump_path)) if args.transfer: baseline_transfer(args, attacker, args.attack_method, arch_name, adv_img_list, l_test_classif_paths, adv_models)
def run_standard_evaluation_individual(self, args, x_orig, y_orig, bs=250, adv_models=None, transfer_eval=False, test_paths=None): # update attacks list if plus activated after initialization if self.plus: if not 'apgd-t' in self.attacks_to_run: self.attacks_to_run.extend(['apgd-t']) if not 'fab-t' in self.attacks_to_run: self.attacks_to_run.extend(['fab-t']) l_attacks = self.attacks_to_run adv = {} self.plus = False verbose_indiv = self.verbose attack_fool_rate = [] acc_indiv = self.clean_accuracy(x_orig, y_orig, bs=bs) for c in l_attacks: startt = time.time() self.attacks_to_run = [c] adv[c], robust_accuracy = self.run_standard_evaluation( x_orig, y_orig, bs=bs, adv_models=adv_models) if transfer_eval: model_type = args.source_arch adv_img_list = [] for i in range(0, len(adv[c])): adv_img_list.append([adv[c][i].unsqueeze(0), y_orig[i]]) baseline_transfer(args, c, c, model_type, adv_img_list, test_paths, adv_models) fool_rate = 1 - robust_accuracy attack_fool_rate.append([c, fool_rate]) space = '\t \t' if c == 'fab' else '\t' print('robust accuracy by {} {} {:.2%} \t (time attack: {:.1f} s)'. format(c.upper(), space, acc_indiv, time.time() - startt)) return adv, attack_fool_rate
def eval(args, attacker, attack_name, eval_helpers, num_eval_samples=None): model, model_type, adv_models, l_test_classif_paths, test_loader = eval_helpers[:] advcorrect = 0 with ctx_noparamgrad_and_eval(model): adv_complete_list, output_list, y_list = [], [], [] for batch_idx, (x_batch, y_batch) in enumerate(test_loader): x_batch, y_batch = x_batch.to(args.dev), y_batch.to(args.dev) adv_imgs = attacker.perturb(x_batch, y_batch).detach() adv_complete_list.append(adv_imgs.cpu()) output_list.append(model(adv_imgs).cpu()) y_list.append(y_batch.cpu()) output = torch.cat(output_list) adv_complete = torch.cat(adv_complete_list) y_test = torch.cat(y_list) pred = output.max(1, keepdim=True)[1] advcorrect += pred.eq(y_test.view_as(pred)).sum().item() fool_rate = 1 - advcorrect / float(len(test_loader.dataset)) print('Test set base model fool rate: %f' % (fool_rate)) if args.transfer: if num_eval_samples is not None: adv_complete = adv_complete[:num_eval_samples] y_test = y_test[:num_eval_samples] adv_img_list = torch.utils.data.TensorDataset(adv_complete, y_test) adv_img_list = torch.utils.data.DataLoader( adv_img_list, batch_size=args.test_batch_size) # Free memory del model torch.cuda.empty_cache() fool_rate = [] if args.target_arch is not None: model_type = args.target_arch if not isinstance(model_type, list): model_type = [model_type] for name in model_type: fool_rate.append( baseline_transfer(args, attacker, attack_name, name, adv_img_list, l_test_classif_paths, adv_models)) return np.mean(fool_rate) return fool_rate
def main(): parser = argparse.ArgumentParser() parser.add_argument('--dataset', type=str, default='cifar') parser.add_argument('--start', type=int, default=0) parser.add_argument('--end', type=int, default=100) parser.add_argument('--n_iter', type=int, default=500) parser.add_argument('--query_step', type=int, default=1) parser.add_argument('--transfer', action='store_true') parser.add_argument('--debug', action='store_true') parser.add_argument('--sweep', action='store_true') parser.add_argument("--wandb", action="store_true", default=False, help='Use wandb for logging') parser.add_argument('--noise', type=float, default=0.3) parser.add_argument('--eps', type=float, default=0.031) parser.add_argument('--batch_size', type=int, default=256, metavar='S') parser.add_argument('--test_batch_size', type=int, default=512, metavar='S') parser.add_argument('--train_set', default='test', choices=['train_and_test', 'test', 'train'], help='add the test set in the training set') # parser.add_argument('--modelIn', type=str, default='../../Nattack/all_models/robustnet/noise_0.3.pth') parser.add_argument( '--modelIn', type=str, default='../pretrained_classifiers/cifar/res18/model_0.pt') parser.add_argument( '--robust_model_path', type=str, default= "../madry_challenge_models/mnist/adv_trained/mnist_lenet5_advtrained.pt" ) parser.add_argument( '--dir_test_models', type=str, default="../", help= "The path to the directory containing the classifier models for evaluation." ) parser.add_argument( "--max_test_model", type=int, default=2, help="The maximum number of pretrained classifiers to use for testing." ) parser.add_argument('--train_on_madry', default=False, action='store_true', help='Train using Madry tf grad') parser.add_argument('--train_on_list', default=False, action='store_true', help='train on a list of classifiers') parser.add_argument('--attack_ball', type=str, default="Linf", choices=['L2', 'Linf']) parser.add_argument('--source_arch', default="res18", help="The architecture we want to attack on CIFAR.") parser.add_argument( '--target_arch', default=None, help="The architecture we want to blackbox transfer to on CIFAR.") parser.add_argument('--epsilon', type=float, default=0.1, metavar='M', help='Epsilon for Delta (default: 0.1)') parser.add_argument('--train_with_critic_path', type=str, default=None, help='Train generator with saved critic model') parser.add_argument('--namestr', type=str, default='NoBox', \ help='additional info in output filename to describe experiments') args = parser.parse_args() args.dev = torch.device('cuda' if torch.cuda.is_available() else 'cpu') train_loader, test_loader = create_loaders(args, root='../data') if os.path.isfile("../settings.json"): with open('../settings.json') as f: data = json.load(f) args.wandb_apikey = data.get("wandbapikey") if args.wandb: os.environ['WANDB_API_KEY'] = args.wandb_apikey wandb.init(project='NoBox-sweeps', name='AutoAttack-{}'.format(args.dataset)) adv_models = None if args.dataset == 'cifar': args.nc, args.h, args.w = 3, 32, 32 model, l_test_classif_paths = load_all_classifiers( args, load_archs=[args.source_arch]) model_type = args.source_arch if args.target_arch is not None: model_target, l_test_classif_paths = load_all_classifiers( args, load_archs=[args.target_arch]) model_type = args.target_arch del model_target torch.cuda.empty_cache() elif args.dataset == 'mnist': if args.source_arch == 'natural': model, l_test_classif_paths = load_all_classifiers( args, load_archs=["natural"]) model_type = 'natural' elif args.source_arch == 'ens_adv': adv_model_names = args.adv_models adv_models = [None] * len(adv_model_names) for i in range(len(adv_model_names)): type = get_model_type(adv_model_names[i]) adv_models[i] = load_model(args, adv_model_names[i], type=type).to(args.dev) path = os.path.join(args.dir_test_models, "pretrained_classifiers", args.dataset, "ensemble_adv_trained", args.model) model = load_model(args, args.model, type=args.type) l_test_classif_paths = [path] model_type = 'Ensemble Adversarial' model.to(args.dev) model.eval() print("Testing on %d Test Classifiers" % (len(l_test_classif_paths))) l = [x.unsqueeze(0) for (x, y) in test_loader.dataset] x_test = torch.cat(l, 0) l = [y for (x, y) in test_loader.dataset] y_test = torch.Tensor(l).long() device_count = torch.cuda.device_count() if device_count > 1: print( "CUDA Device Count is %d, Error might happen. Use export CUDA_VISIBLE_DEVICES=0" % (device_count)) if not args.sweep: attacker = AutoAttack(args, args.n_iter, model, norm=args.attack_ball, verbose=True, eps=args.epsilon) adv, attack_fool_rate = attacker.run_standard_evaluation_individual( args, x_test[:args.batch_size], y_test[:args.batch_size], bs=args.batch_size, transfer_eval=args.transfer, test_paths=l_test_classif_paths, adv_models=adv_models) adv_complete, success_rate = attacker.run_standard_evaluation( x_test[:args.batch_size], y_test[:args.batch_size], bs=args.batch_size, adv_models=adv_models) if args.transfer: adv_img_list = [] y_orig = y_test[:args.batch_size] for i in range(0, len(adv_complete)): adv_img_list.append([adv_complete[i].unsqueeze(0), y_orig[i]]) baseline_transfer(args, attacker, "AutoAttack", model_type, adv_img_list, l_test_classif_paths, adv_models) else: for n_iter in range(0, args.n_iter, args.query_step): attacker = AutoAttack(args, n_iter, model, verbose=True, norm=args.attack_ball, eps=args.epsilon) adv, attack_fool_rate = attacker.run_standard_evaluation_individual( args, x_test[:args.batch_size], y_test[:args.batch_size], bs=args.batch_size, transfer_eval=args.transfer, test_paths=l_test_classif_paths, adv_models=adv_models) adv_complete, success_rate = attacker.run_standard_evaluation( x_test[:args.batch_size], y_test[:args.batch_size], bs=args.batch_size, adv_models=adv_models) if args.transfer: adv_img_list = [] y_orig = y_test[:args.batch_size] for i in range(0, len(adv_complete)): adv_img_list.append( [adv_complete[i].unsqueeze(0), y_orig[i]]) baseline_transfer(args, attacker, "AutoAttack", model_type, adv_img_list, l_test_classif_paths, adv_models) if args.wandb: wandb.log({ "APGD-CE": attack_fool_rate[0][1], "APGD-DLR": attack_fool_rate[1][1], "FAB": attack_fool_rate[2][1], "Square": attack_fool_rate[3][1], "queries": n_iter })
def main(): parser = argparse.ArgumentParser(description='NoBox') # Hparams parser.add_argument('--gp_coeff', type=float, default=0., help='coeff for the gradient penalty') parser.add_argument('--latent_dim', type=int, default=20, metavar='N', help='Latent dim for VAE') parser.add_argument('--lr', type=float, default=0.01, metavar='LR', help='learning rate for the generator (default: 0.01)') parser.add_argument('--lr_model', type=float, default=None, metavar='LR', help='learning rate for the model (default: None -> default to args.lr)') parser.add_argument('--momentum', type=float, default=0.5, metavar='M', help='optimizer momentum (default: 0.5)') parser.add_argument('--extragradient', default=False, action='store_true', help='Use extragadient algorithm') parser.add_argument('--latent_size', type=int, default=50, metavar='N', help='Size of latent distribution (default: 50)') parser.add_argument('--flow_model', default=None, const='soft', nargs='?', choices=[None, 'RealNVP', 'planar', 'radial'], help='Type of Normalizing Flow (default: %(default)s)') parser.add_argument('--flow_layer_type', type=str, default='Linear', help='Which type of layer to use ---i.e. GRevNet or Linear') parser.add_argument('--flow_hidden_size', type=int, default=128, help='Hidden layer size for Flows.') parser.add_argument('--n_blocks', type=int, default=2, help='Number of blocks to stack in flow') parser.add_argument('--flow_hidden', type=int, default=1, help='Number of hidden layers in each Flow.') parser.add_argument('--eval_set', default="test", help="Evaluate model on test or validation set.") parser.add_argument('--train_with_critic_path', type=str, default=None, help='Train generator with saved critic model') parser.add_argument('--train_on_file', default=False, action='store_true', help='Train using Madry tf grad') # Training parser.add_argument('--lambda_on_clean', default=0.0, type=float, help='train the critic on clean examples of the train set') parser.add_argument('--not_use_labels', default=False, action='store_true', help='Use the labels for the conditional generator') parser.add_argument('--hinge_coeff', default=10., type=float, help='coeff for the hinge loss penalty') parser.add_argument('--anneal_eps', default=0., type=float, help='coeff for the epsilon annealing') parser.add_argument('--fixed_critic', default=False, action='store_true', help='Critic is not trained') parser.add_argument('--train_on_list', default=False, action='store_true', help='train on a list of classifiers') parser.add_argument('--train_set', default='train', choices=['train_and_test','test','train'], help='add the test set in the training set') parser.add_argument('--epochs', type=int, default=10, metavar='N', help='number of epochs to train (default: 10)') parser.add_argument('--n_iter', type=int, default=500, help='N iters for quere based attacks') parser.add_argument('--PGD_steps', type=int, default=40, metavar='N', help='max gradient steps (default: 30)') parser.add_argument('--max_iter', type=int, default=10, metavar='N', help='max gradient steps (default: 10)') parser.add_argument('--epsilon', type=float, default=0.1, metavar='M', help='Epsilon for Delta (default: 0.1)') parser.add_argument('--attack_ball', type=str, default="L2", choices= ['L2','Linf'], help='type of box attack') parser.add_argument('--bb_steps', type=int, default=2000, metavar='N', help='Max black box steps per sample(default: 1000)') parser.add_argument('--attack_epochs', type=int, default=100, metavar='N', help='Max numbe of epochs to train G') parser.add_argument('--num_flows', type=int, default=2, metavar='N', help='Number of Flows') parser.add_argument('--seed', type=int, metavar='S', help='random seed (default: None)') parser.add_argument('--input_size', type=int, default=784, metavar='S', help='Input size for MNIST is default') parser.add_argument('--batch_size', type=int, default=256, metavar='S', help='Batch size') parser.add_argument('--test_batch_size', type=int, default=512, metavar='S', help='Test Batch size') parser.add_argument('--pgd_on_critic', default=False, action='store_true', help='Train Critic on pgd samples') parser.add_argument('--train_with_robust', default=False, action='store_true', help='Train with Robust model + Critic') parser.add_argument('--test', default=False, action='store_true', help='just test model and print accuracy') parser.add_argument('--clip_grad', default=True, action='store_true', help='Clip grad norm') parser.add_argument('--train_vae', default=False, action='store_true', help='Train VAE') parser.add_argument('--train_ae', default=False, action='store_true', help='Train AE') parser.add_argument('--attack_type', type=str, default='nobox', help='Which attack to run') parser.add_argument('--attack_loss', type=str, default='cross_entropy', help='Which loss func. to use to optimize G') parser.add_argument('--perturb_loss', type=str, default='L2', choices= ['L2','Linf'], help='Which loss func. to use to optimize to compute constraint') parser.add_argument('--dataset', type=str, default='mnist') parser.add_argument('--model', type=str, default=None) parser.add_argument('--deterministic_G', default=False, action='store_true', help='Deterministic Latent State') parser.add_argument('--run_baseline', default=False, action='store_true', help='Run baseline PGD') parser.add_argument('--resample_test', default=False, action='store_true', help='Load model and test resampling capability') parser.add_argument('--resample_iterations', type=int, default=100, metavar='N', help='How many times to resample (default: 100)') parser.add_argument('--architecture', default="VGG16", help="The architecture we want to attack on CIFAR.") parser.add_argument('--eval_freq', default=5, type=int, help="Evaluate and save model every eval_freq epochs.") parser.add_argument('--num_test_samples', default=None, type=int, help="The number of samples used to train and test the attacker.") parser.add_argument('--num_eval_samples', default=None, type=int, help="The number of samples used to train and test the attacker.") # Bells parser.add_argument("--wandb", action="store_true", default=False, help='Use wandb for logging') parser.add_argument('--model_path', type=str, default="mnist_cnn.pt", help='where to save/load') parser.add_argument('--namestr', type=str, default='NoBox', \ help='additional info in output filename to describe experiments') parser.add_argument('--dir_test_models', type=str, default="./dir_test_models", help="The path to the directory containing the classifier models for evaluation.") parser.add_argument('--robust_model_path', type=str, default="./madry_challenge_models/mnist/adv_trained/mnist_lenet5_advtrained.pt", help="The path to our adv robust classifier") parser.add_argument('--robust_sample_prob', type=float, default=1e-1, metavar='N', help='1-P(robust)') #parser.add_argument('--madry_model_path', type=str, default="./madry_challenge_models", # help="The path to the directory containing madrys classifiers for testing") parser.add_argument("--max_test_model", type=int, default=1, help="The maximum number of pretrained classifiers to use for testing.") parser.add_argument("--perturb_magnitude", type=float, default=None, help="The amount of perturbation we want to enforce with lagrangian.") parser.add_argument("--log_path", type=str, default="./logs", help="Where to save logs if logger is specified.") parser.add_argument("--save_model", type=str, default=None, help="Where to save the models, if it is specified.") parser.add_argument("--fixed_testset", action="store_true", help="If used then makes sure that the same set of samples is always used for testing.") parser.add_argument('--normalize', default=None, choices=(None, "default", "meanstd")) ### parser.add_argument('--source_arch', default="res18", help="The architecture we want to attack on CIFAR.") parser.add_argument('--target_arch', nargs='*', help="The architecture we want to blackbox transfer to on CIFAR.") parser.add_argument('--ensemble_adv_trained', action='store_true') parser.add_argument('--adv_models', nargs='*', help='path to adv model(s)') parser.add_argument('--type', type=int, default=0, help='Model type (default: 0)') parser.add_argument('--model_name', help='path to model') parser.add_argument('--transfer', action='store_true') parser.add_argument('--command', choices=("eval", "train"), default="train") parser.add_argument('--split', type=int, default=None, help="Which subsplit to use.") parser.add_argument('--path_to_data', default="../data", type=str) args = parser.parse_args() args.dev = torch.device('cuda' if torch.cuda.is_available() else 'cpu') normalize = None if args.normalize == "meanstd": normalize = transforms.Normalize(cf.mean["cifar10"], cf.std["cifar10"]) elif args.normalize == "default": normalize = CIFAR_NORMALIZATION train_loader, test_loader, split_train_loader, split_test_loader = create_loaders(args, root=args.path_to_data, split=args.split, num_test_samples=args.num_test_samples, normalize=normalize) if args.split is not None: train_loader = split_train_loader test_loader = split_test_loader if os.path.isfile("../settings.json"): with open('../settings.json') as f: data = json.load(f) args.wandb_apikey = data.get("wandbapikey") if args.wandb: os.environ['WANDB_API_KEY'] = args.wandb_apikey wandb.init(project='NoBox-table2', name='NoBox-Attack-{}-{}'.format(args.dataset, args.namestr)) model, adv_models, l_test_classif_paths, args.model_type = data_and_model_setup(args, no_box_attack=True) model.to(args.dev) model.eval() print("Testing on %d Test Classifiers with Source Model %s" %(len(l_test_classif_paths), args.source_arch)) x_test, y_test = load_data(args, test_loader) if args.dataset == "mnist": critic = load_unk_model(args) elif args.dataset == "cifar": name = args.source_arch if args.source_arch == "adv": name = "res18" critic = load_unk_model(args, name=name) misclassify_loss_func = kwargs_attack_loss[args.attack_loss] attacker = NoBoxAttack(critic, misclassify_loss_func, args) print("Evaluating clean error rate:") list_model = [args.source_arch] if args.source_arch == "adv": list_model = [args.model_type] if args.target_arch is not None: list_model = args.target_arch for model_type in list_model: num_samples = args.num_eval_samples if num_samples is None: num_samples = len(test_loader.dataset) eval_loader = torch.utils.data.Subset(test_loader.dataset, np.random.randint(len(test_loader.dataset), size=(num_samples,))) eval_loader = torch.utils.data.DataLoader(eval_loader, batch_size=args.test_batch_size) baseline_transfer(args, None, "Clean", model_type, eval_loader, list_classifiers=l_test_classif_paths) def eval_fn(model): advcorrect = 0 model.to(args.dev) model.eval() with ctx_noparamgrad_and_eval(model): if args.source_arch == 'googlenet': adv_complete_list = [] for batch_idx, (x_batch, y_batch) in enumerate(test_loader): if (batch_idx + 1) * args.test_batch_size > args.batch_size: break x_batch, y_batch = x_batch.to(args.dev), y_batch.to(args.dev) adv_complete_list.append(attacker.perturb(x_batch, target=y_batch)) adv_complete = torch.cat(adv_complete_list) else: adv_complete = attacker.perturb(x_test[:args.batch_size], target=y_test[:args.batch_size]) adv_complete = torch.clamp(adv_complete, min=0., max=1.0) output = model(adv_complete) pred = output.max(1, keepdim=True)[1] advcorrect += pred.eq(y_test[:args.batch_size].view_as(pred)).sum().item() fool_rate = 1 - advcorrect / float(args.batch_size) print('Test set base model fool rate: %f' %(fool_rate)) model.cpu() if args.transfer: adv_img_list = [] y_orig = y_test[:args.batch_size] for i in range(0, len(adv_complete)): adv_img_list.append([adv_complete[i].unsqueeze(0), y_orig[i]]) # Free memory del model torch.cuda.empty_cache() baseline_transfer(args, attacker, "AEG", model_type, adv_img_list, l_test_classif_paths, adv_models) if args.command == "eval": attacker.load(args) elif args.command == "train": attacker.train(train_loader, test_loader, adv_models, l_test_classif_paths, l_train_classif={"source_model": model}, eval_fn=eval_fn)
def main(): parser = argparse.ArgumentParser() parser.add_argument( '--data-path', type=str, default='../../Nattack/cifar10_data/test_batch', help= 'path to the test_batch file from http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz' ) parser.add_argument('--dataset', type=str, default='cifar') parser.add_argument('--start', type=int, default=0) parser.add_argument('--end', type=int, default=100) parser.add_argument('--n_iter', type=int, default=500) parser.add_argument('--query_step', type=int, default=1) parser.add_argument('--transfer', action='store_true') parser.add_argument('--debug', action='store_true') parser.add_argument('--sweep', action='store_true') parser.add_argument("--wandb", action="store_true", default=False, help='Use wandb for logging') parser.add_argument('--noise', type=float, default=0.3) parser.add_argument('--epsilon', type=float, default=0.031) parser.add_argument('--batch_size', type=int, default=256, metavar='S') parser.add_argument('--test_batch_size', type=int, default=512, metavar='S') parser.add_argument('--train_set', default='test', choices=['train_and_test', 'test', 'train'], help='add the test set in the training set') parser.add_argument( '--modelIn', type=str, default='../../Nattack/all_models/robustnet/noise_0.3.pth') parser.add_argument( '--robust_model_path', type=str, default= "../madry_challenge_models/mnist/adv_trained/mnist_lenet5_advtrained.pt" ) parser.add_argument('--source_arch', default="res18", help="The architecture we want to attack on CIFAR.") parser.add_argument( '--target_arch', default=None, help="The architecture we want to blackbox transfer to on CIFAR.") parser.add_argument( '--dir_test_models', type=str, default="../", help= "The path to the directory containing the classifier models for evaluation." ) parser.add_argument( "--max_test_model", type=int, default=2, help="The maximum number of pretrained classifiers to use for testing." ) parser.add_argument('--train_on_madry', default=False, action='store_true', help='Train using Madry tf grad') parser.add_argument('--train_on_list', default=False, action='store_true', help='train on a list of classifiers') parser.add_argument('--train_with_critic_path', type=str, default=None, help='Train generator with saved critic model') parser.add_argument('--namestr', type=str, default='NoBox', \ help='additional info in output filename to describe experiments') args = parser.parse_args() args.dev = torch.device('cuda' if torch.cuda.is_available() else 'cpu') if os.path.isfile("../settings.json"): with open('../settings.json') as f: data = json.load(f) args.wandb_apikey = data.get("wandbapikey") if args.wandb: os.environ['WANDB_API_KEY'] = args.wandb_apikey wandb.init(project='NoBox-sweeps', name='NAttack-{}'.format(args.dataset)) adv_models = None if args.dataset == 'cifar': args.nc, args.h, args.w = 3, 32, 32 provider = robustml.provider.CIFAR10(args.data_path) model, l_test_classif_paths = load_all_classifiers( args, load_archs=[args.source_arch]) model_type = args.source_arch if args.target_arch is not None: model_target, l_test_classif_paths = load_all_classifiers( args, load_archs=[args.target_arch]) model_type = args.target_arch del model_target torch.cuda.empty_cache() elif args.dataset == 'mnist': mnist_data_path = '../data/MNIST/raw/t10k-images-idx3-ubyte.gz' mnist_label_path = '../data/MNIST/raw/t10k-labels-idx1-ubyte.gz' args.nc, args.h, args.w = 1, 28, 28 provider = robustml.provider.MNIST(mnist_data_path, mnist_label_path) if args.source_arch == 'natural': model, l_test_classif_paths = load_all_classifiers( args, load_archs=["natural"]) model_type = 'natural' elif args.source_arch == 'ens_adv': adv_model_names = args.adv_models adv_models = [None] * len(adv_model_names) for i in range(len(adv_model_names)): type = get_model_type(adv_model_names[i]) adv_models[i] = load_model(args, adv_model_names[i], type=type).to(args.dev) path = os.path.join(args.dir_test_models, "pretrained_classifiers", args.dataset, "ensemble_adv_trained", args.model) model = load_model(args, args.model, type=args.type) l_test_classif_paths = [path] model_type = 'Ensemble Adversarial' model.to(args.dev) model.eval() train_loader, test_loader = create_loaders(args, root='../data') test_classifier(args, model, args.dev, test_loader, 1, logger=None) print("Testing on %d Test Classifiers" % (len(l_test_classif_paths))) if not args.sweep: attacker = NAttack(args, model, dataset=args.dataset, n_iter=args.n_iter, eps=args.epsilon, end=args.end) fool_rate, adv_img_list = attacker.perturb(provider) if args.wandb: wandb.log({"Fool Rate": fool_rate, "queries": args.n_iter}) if args.transfer: baseline_transfer(args, attacker, "NAttack", model_type, adv_img_list, l_test_classif_paths, adv_models) else: for n_iter in range(0, args.n_iter, args.query_step): attacker = NAttack(args, model, dataset=args.dataset, n_iter=n_iter, eps=args.epsilon, end=args.end) fool_rate = attacker.perturb(provider) if args.wandb: wandb.log({"Fool Rate": fool_rate, "queries": n_iter}) if args.transfer: baseline_transfer(args, attacker, "NAttack", model_type, adv_img_list, l_test_classif_paths, adv_models)
def main(): parser = argparse.ArgumentParser() parser.add_argument('--dataset', type=str, default='cifar') parser.add_argument('--start', type=int, default=0) parser.add_argument('--end', type=int, default=100) parser.add_argument("--wandb", action="store_true", default=False, help='Use wandb for logging') parser.add_argument('--batch_size', type=int, default=256, metavar='S') parser.add_argument('--test_batch_size', type=int, default=512, metavar='S') parser.add_argument('--train_set', default='test', choices=['train_and_test','test','train'], help='add the test set in the training set') parser.add_argument('--modelIn', type=str, default='../pretrained_classifiers/cifar/res18/model_0.pt') parser.add_argument('--robust_model_path', type=str, default="../madry_challenge_models/mnist/adv_trained/mnist_lenet5_advtrained.pt") parser.add_argument('--dir_test_models', type=str, default="../", help="The path to the directory containing the classifier models for evaluation.") parser.add_argument("--max_test_model", type=int, default=2, help="The maximum number of pretrained classifiers to use for testing.") parser.add_argument('--train_on_madry', default=False, action='store_true', help='Train using Madry tf grad') parser.add_argument('--train_on_list', default=False, action='store_true', help='train on a list of classifiers') parser.add_argument('--attack_ball', type=str, default="Linf", choices= ['L2','Linf']) parser.add_argument('--source_arch', default="res18", help="The architecture we want to attack on CIFAR.") parser.add_argument('--train_with_critic_path', type=str, default=None, help='Train generator with saved critic model') parser.add_argument('--model', help='path to model') parser.add_argument('--namestr', type=str, default='NoBox', \ help='additional info in output filename to describe experiments') args = parser.parse_args() args.dev = torch.device('cuda' if torch.cuda.is_available() else 'cpu') train_loader, test_loader = create_loaders(args, root='../data') if os.path.isfile("../settings.json"): with open('../settings.json') as f: data = json.load(f) args.wandb_apikey = data.get("wandbapikey") if args.wandb: os.environ['WANDB_API_KEY'] = args.wandb_apikey wandb.init(project='NoBox-sweeps', name='MI-Attack-{}'.format(args.dataset)) adv_models = None attacker = None if args.dataset == 'cifar': args.nc, args.h, args.w = 3, 32, 32 model, l_test_classif_paths = load_all_classifiers(args, load_archs=[args.source_arch]) model_type = args.source_arch elif args.dataset == 'mnist': if args.source_arch == 'natural': model, l_test_classif_paths = load_all_classifiers(args, load_archs=["natural"]) model_type = 'natural' elif args.source_arch == 'ens_adv': adv_model_names = args.adv_models adv_models = [None] * len(adv_model_names) for i in range(len(adv_model_names)): type = get_model_type(adv_model_names[i]) adv_models[i] = load_model(args, adv_model_names[i], type=type).to(args.dev) path = os.path.join(args.dir_test_models, "pretrained_classifiers", args.dataset, "ensemble_adv_trained", args.model) model = load_model(args, args.model, type=args.type) l_test_classif_paths = [path] model_type = 'Ensemble Adversarial' model.to(args.dev) model.eval() print("Testing on %d Test Classifiers with Source Model %s" %(len(l_test_classif_paths), args.source_arch)) l = [x.unsqueeze(0) for (x, y) in test_loader.dataset] x_test = torch.cat(l, 0).to(args.dev) l = [y for (x, y) in test_loader.dataset] y_test = torch.Tensor(l).long().to(args.dev) device_count = torch.cuda.device_count() if device_count > 1: print("CUDA Device Count is %d, Error might happen. Use export CUDA_VISIBLE_DEVICES=0" %(device_count)) test_img_list = [] x_orig = x_test[:args.batch_size] y_orig = y_test[:args.batch_size] for i in range(0, len(x_orig)): test_img_list.append([x_orig[i].unsqueeze(0), y_orig[i]]) # Free memory del model torch.cuda.empty_cache() baseline_transfer(args, attacker, "Clean", model_type, test_img_list, l_test_classif_paths, adv_models)
def main(): parser = argparse.ArgumentParser() parser.add_argument('--max_epsilon', default=32.0, type=float, help='Maximum size of adversarial perturbation.') parser.add_argument('--num_iter', default=10, type=int, help="Number of iterations.") parser.add_argument("--batch_size", default=256, type=int, help="How many images process at one time.") parser.add_argument("--momentum", default=1.0, type=float, help="Momentum.") parser.add_argument('--dataset', type=str, default='cifar') parser.add_argument('--start', type=int, default=0) parser.add_argument('--end', type=int, default=100) parser.add_argument('--n_iter', type=int, default=1000) parser.add_argument('--transfer', action='store_true') parser.add_argument('--debug', action='store_true') parser.add_argument('--sweep', action='store_true') parser.add_argument("--wandb", action="store_true", default=False, help='Use wandb for logging') parser.add_argument('--ensemble_adv_trained', action='store_true') parser.add_argument('--test_batch_size', type=int, default=32, metavar='S') parser.add_argument('--train_set', default='test', choices=['train_and_test', 'test', 'train'], help='add the test set in the training set') parser.add_argument( '--modelIn', type=str, default='../pretrained_classifiers/cifar/res18/model_0.pt') parser.add_argument( '--robust_model_path', type=str, default= "../madry_challenge_models/mnist/adv_trained/mnist_lenet5_advtrained.pt" ) parser.add_argument( '--dir_test_models', type=str, default="../", help= "The path to the directory containing the classifier models for evaluation." ) parser.add_argument( "--max_test_model", type=int, default=2, help="The maximum number of pretrained classifiers to use for testing." ) parser.add_argument('--train_on_madry', default=False, action='store_true', help='Train using Madry tf grad') parser.add_argument('--train_on_list', default=False, action='store_true', help='train on a list of classifiers') parser.add_argument('--attack_ball', type=str, default="Linf", choices=['L2', 'Linf']) parser.add_argument('--source_arch', default="res18", help="The architecture we want to attack on CIFAR.") parser.add_argument( '--target_arch', default=None, help="The architecture we want to blackbox transfer to on CIFAR.") parser.add_argument('--transform_prob', type=float, default=0.5, metavar='M', help='Randomly apply input Transformation') parser.add_argument('--resize_factor', type=float, default=1.1, metavar='M', help='Resize Factor for Random Resizing') parser.add_argument('--epsilon', type=float, default=0.1, metavar='M', help='Epsilon for Delta (default: 0.1)') parser.add_argument('--train_with_critic_path', type=str, default=None, help='Train generator with saved critic model') parser.add_argument('--model', help='path to model') parser.add_argument('--adv_models', nargs='*', help='path to adv model(s)') parser.add_argument('--type', type=int, default=0, help='Model type (default: 0)') parser.add_argument('--namestr', type=str, default='NoBox', \ help='additional info in output filename to describe experiments') args = parser.parse_args() args.dev = torch.device('cuda' if torch.cuda.is_available() else 'cpu') train_loader, test_loader = create_loaders(args, root='../data') if os.path.isfile("../settings.json"): with open('../settings.json') as f: data = json.load(f) args.wandb_apikey = data.get("wandbapikey") if args.wandb: os.environ['WANDB_API_KEY'] = args.wandb_apikey wandb.init(project='NoBox-sweeps', name='AutoAttack-{}'.format(args.dataset)) adv_models = None if args.dataset == 'cifar': args.nc, args.h, args.w = 3, 32, 32 model, l_test_classif_paths = load_all_classifiers( args, load_archs=[args.source_arch]) model_type = args.source_arch if args.target_arch is not None: model_target, l_test_classif_paths = load_all_classifiers( args, load_archs=[args.target_arch]) model_type = args.target_arch del model_target torch.cuda.empty_cache() if args.ensemble_adv_trained: adv_model_names = args.adv_models l_test_classif_paths = [] adv_models = [None] * len(adv_model_names) for i in range(len(adv_model_names)): adv_path = os.path.join(args.dir_test_models, "pretrained_classifiers", args.dataset, "ensemble_adv_trained", adv_model_names[i] + '.pt') init_func, _ = ARCHITECTURES[adv_model_names[i]] temp_model = init_func().to(args.dev) adv_models[i] = nn.DataParallel(temp_model) adv_models[i].load_state_dict(torch.load(adv_path)) l_test_classif_paths.append([adv_path]) model_type = 'Ensemble Adversarial' elif args.dataset == 'mnist': args.nc, args.h, args.w = 1, 28, 28 if args.source_arch == 'natural': model, l_test_classif_paths = load_all_classifiers( args, load_archs=["natural"]) model_type = 'natural' elif args.source_arch == 'ens_adv' or args.ensemble_adv_trained: adv_model_names = args.adv_models adv_models = [None] * len(adv_model_names) for i in range(len(adv_model_names)): type = get_model_type(adv_model_names[i]) adv_models[i] = load_model(args, adv_model_names[i], type=type).to(args.dev) path = os.path.join(args.dir_test_models, "pretrained_classifiers", args.dataset, "ensemble_adv_trained", args.model) model, l_test_classif_paths = load_all_classifiers( args, load_archs=["natural"]) # model = load_model(args, args.model, type=args.type) l_test_classif_paths = [path] model_type = 'Ensemble Adversarial' model.to(args.dev) model.eval() print("Testing on %d Test Classifiers with Source Model %s" % (len(l_test_classif_paths), args.source_arch)) l = [x.unsqueeze(0) for (x, y) in test_loader.dataset] x_test = torch.cat(l, 0).to(args.dev) l = [y for (x, y) in test_loader.dataset] y_test = torch.Tensor(l).long().to(args.dev) device_count = torch.cuda.device_count() if device_count > 1: print( "CUDA Device Count is %d, Error might happen. Use export CUDA_VISIBLE_DEVICES=0" % (device_count)) stack_kernel_list = [] # This is needed for Depth-wise convolution for i in range(0, args.nc): kernel = gkern(15, 3).astype(np.float32) stack_kernel = np.stack([kernel, kernel, kernel]).swapaxes(2, 0) stack_kernel_list.append(np.expand_dims(stack_kernel, 3)) stack_kernel = np.stack(stack_kernel_list).squeeze() stack_kernel = torch.tensor(stack_kernel).permute(0, 3, 1, 2).to(args.dev) attacker = TIM(args, stack_kernel=stack_kernel, model=model, attack_ball=args.attack_ball, eps=args.epsilon, n_iter=args.n_iter, decay_factor=args.momentum, eps_iter=0.01) advcorrect = 0 with ctx_noparamgrad_and_eval(model): adv_complete_list = [] if args.dataset == 'cifar': for batch_idx, (x_batch, y_batch) in enumerate(test_loader): if (batch_idx + 1) * args.test_batch_size > args.batch_size: break x_batch, y_batch = x_batch.to(args.dev), y_batch.to(args.dev) adv_complete_list.append(attacker.perturb(x_batch, y_batch)) adv_complete = torch.cat(adv_complete_list) else: adv_complete = attacker.perturb(x_test[:args.batch_size], y_test[:args.batch_size]) if args.transfer: adv_img_list = [] y_orig = y_test[:args.batch_size] for i in range(0, len(adv_complete)): adv_img_list.append([adv_complete[i].unsqueeze(0), y_orig[i]]) # Free memory del model torch.cuda.empty_cache() baseline_transfer(args, attacker, "TI-DI-Attack", model_type, adv_img_list, l_test_classif_paths, adv_models)