def prepare_advclient_model(self): if self.args.aggr == "fedavg": adv_client = self.server.client_list[self.adv_client_idx] branch_w = self.server.branches[self.adv_branch_idx] adv_client.model_trainer.set_model_params(branch_w) self.advclient_model = copy.deepcopy(adv_client.model_trainer.model) ensemble_info = None elif self.args.aggr == "blockensemble": state_dict_pair, ensemble_info = self.server.prepare_branch_dict() self.adv_client = self.server.client_list[self.adv_client_idx] self.adv_client.model_trainer.set_model_params(state_dict_pair) # self.advclient_model = TwoModelWarpper( # self.adv_client.model_trainer.model1, # self.adv_client.model_trainer.model2, # ) self.advclient_model = self.adv_client.model_trainer.model2 elif self.args.aggr == "heteroensemble": model_pair, ensemble_info = self.server.prepare_branch_dict(client_idx=0) self.adv_client = self.server.client_list[self.adv_client_idx] self.advclient_model = TwoModelWarpper( model_pair[0], model_pair[1], ) # self.advclient_model = model_pair[0] else: raise NotImplementedError self.advclient_model.eval() self.advclient_model = PyTorchModel( self.advclient_model, bounds = self.bounds, preprocessing = self.preprocessing ) return ensemble_info
def main(): parser = argparse.ArgumentParser() parser.add_argument('--steps', type=int, default=20000, help='Iteration of BA') parser.add_argument('--targeted', action='store', default=False, help='For targeted attack') args = parser.parse_args() model = Net() model.load_state_dict(torch.load('mnist_cnn.pt')) model.eval() preprocessing = dict(mean=0.1307, std=0.3081) fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing) fmodel = fmodel.transform_bounds((0, 1)) assert fmodel.bounds == (0, 1) images, labels = ep.astensors( *samples(fmodel, dataset="mnist", batchsize=10)) print('Model accuracy on clean examples: {}'.format( accuracy(fmodel, images, labels))) if args.targeted: target_class = (labels + 7) % 10 criterion = fb.criteria.TargetedMisclassification(target_class) else: criterion = fb.criteria.Misclassification(labels) attack = fa.BoundaryAttack(steps=args.steps, tensorboard=None) epsilons = np.linspace(0.01, 10, 20) raw, clipped, success = attack(fmodel, images, labels, epsilons=epsilons) robust_accuracy = 1 - success.float32().mean(axis=-1) plt.plot(epsilons, robust_accuracy.numpy()) plt.xlabel("Epsilons") plt.ylabel("Robust Accuracy") plt.savefig('mnist_BA_robust_acc.jpg') plt.show() mean_distance = [] for i in range(len(clipped)): dist = np.mean(fb.distances.l2(clipped[i], images).numpy()) mean_distance.append(dist) plt.plot(epsilons, mean_distance) plt.xlabel('Epsilons') plt.ylabel('Mean L2 distance') plt.savefig("mnist_BA_mean_L2distance.jpg") plt.show()
def main(): parser = argparse.ArgumentParser() parser.add_argument('--steps', type=int, default=10000, help='Iteration of BA') parser.add_argument('--targeted', action='store', default=False, help='For targeted attack') args = parser.parse_args() model = Net() model.load_state_dict(torch.load('mnist_cnn.pt')) model.eval() preprocessing = dict(mean=0.1307, std=0.3081) fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing) fmodel = fmodel.transform_bounds((0, 1)) assert fmodel.bounds == (0, 1) images, labels = ep.astensors( *samples(fmodel, dataset="mnist", batchsize=10)) print('Model accuracy on clean examples: {}'.format( accuracy(fmodel, images, labels))) epsilons = np.linspace(0.01, 10, 20) boundary_attack = fa.BoundaryAttack(steps=args.steps, tensorboard=None) _, _, ba_success = boundary_attack(fmodel, images, labels, epsilons=epsilons) ba_robust_accuracy = 1 - ba_success.float32().mean(axis=-1) random_attack = fa.L2RepeatedAdditiveGaussianNoiseAttack( repeats=args.steps) _, _, ra_success = random_attack(fmodel, images, labels, epsilons=epsilons) ra_robust_accuracy = 1 - ra_success.float32().mean(axis=-1) legends = ["Boundary Attack", "Random Attack"] plt.plot(epsilons, ba_robust_accuracy.numpy()) plt.plot(epsilons, ra_robust_accuracy.numpy()) plt.legend(legends, loc='upper right') plt.xlabel("Perturbation Norm (L2)") plt.ylabel("Robust Accuracy") plt.title("{} Queries".format(args.steps)) plt.savefig('mnist_robust_acc.jpg') plt.show()
def main() -> None: model = models.resnet101(pretrained=True).eval() preprocessing = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], axis=-3) fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing) images, labels = fb.utils.samples(fmodel, dataset='imagenet', batchsize=16) clean_acc = fb.utils.accuracy(fmodel, images, labels) print(f"clean accuracy: {clean_acc * 100:.1f} %") attack = fb.attacks.LinfDeepFoolAttack() epsilons = np.linspace(0.0, 0.005, num=20) images = ep.astensor(images) labels = ep.astensor(labels) criterion = fb.criteria.Misclassification(labels) raw, clipped, is_adv = attack(fmodel, images, labels, epsilons=epsilons) robust_accuracy = 1 - is_adv.float32().mean(axis=-1) print("robust accuracy for perturbations with") for eps, acc in zip(epsilons, robust_accuracy): print(f" Linf norm ≤ {eps:<6}: {acc.item() * 100:4.1f} %") plt.plot(epsilons, robust_accuracy.numpy())
def exp_balckbox_attack(): model = model_name(pretrained=True).eval() preprocessing = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], axis=-3) fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing) total_success = 0 widgets = [ 'train :', Percentage(), ' ', Bar('#'), ' ', Timer(), ' ', ETA(), ' ', FileTransferSpeed() ] pbar = ProgressBar(widgets=widgets) length = len(train_loader) print("length train loader: ", length) total_imgs = 0 for batch_data in pbar(train_loader): images, labels = batch_data['image'].to( DEVICE), batch_data['label_idx'].to(DEVICE) acc = accuracy(fmodel, images, labels) print(images.shape[0]) total_imgs += images.shape[0] total_success += int(acc * images.shape[0]) print("success attack is ", str(total_imgs - total_success), "/" + str(total_imgs)) print("watermark attack rate is: ", str(1 - total_success / total_imgs))
def main() -> None: # instantiate a model (could also be a TensorFlow or JAX model) model = models.resnet18(pretrained=True).eval() preprocessing = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], axis=-3) fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing) # get data and test the model # wrapping the tensors with ep.astensors is optional, but it allows # us to work with EagerPy tensors in the following images, labels = ep.astensors( *samples(fmodel, dataset="imagenet", batchsize=16)) clean_acc = accuracy(fmodel, images, labels) * 100 print(f"clean accuracy: {clean_acc:.1f} %") # the attack trys a combination of specified rotations and translations to an image # stops early if adversarial shifts and translations for all images are found attack = fa.SpatialAttack( max_translation=6, # 6px so x in [x-6, x+6] and y in [y-6, y+6] num_translations=6, # number of translations in x, y. max_rotation=20, # +- rotation in degrees num_rotations=5, # number of rotations # max total iterations = num_rotations * num_translations**2 ) # report the success rate of the attack (percentage of samples that could # be adversarially perturbed) and the robust accuracy (the remaining accuracy # of the model when it is attacked) xp_, _, success = attack(fmodel, images, labels) suc = success.float32().mean().item() * 100 print(f"attack success: {suc:.1f} %" " (for the specified rotation and translation bounds)") print(f"robust accuracy: {100 - suc:.1f} %" " (for the specified rotation and translation bounds)")
def initialize_model(): model = download_model(model_name) # normalizacja do sredniej i odchylenia standardowego zbioru ImageNet preprocessing = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], axis=-3) return PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
def main(): parser = argparse.ArgumentParser() parser.add_argument('--steps', type=int, default=1000, help='Maximum number of steps to perform') parser.add_argument('--targeted', action='store', default=False, help='For targeted attack') args = parser.parse_args() model = Net() model.load_state_dict(torch.load('mnist_cnn.pt')) model.eval() preprocessing = dict(mean=0.1307, std=0.3081) fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing) fmodel = fmodel.transform_bounds((0, 1)) assert fmodel.bounds == (0, 1) images, labels = ep.astensors( *samples(fmodel, dataset="mnist", batchsize=10)) print('Model accuracy on clean examples: {}'.format( accuracy(fmodel, images, labels))) if args.targeted: target_class = (labels + 7) % 10 criterion = fb.criteria.TargetedMisclassification(target_class) else: criterion = fb.criteria.Misclassification(labels) attack = fa.L2DeepFoolAttack(steps=args.steps) epsilons = None raw, clipped, success = attack(fmodel, images, labels, epsilons=epsilons) robust_accuracy = 1 - success.float32().mean() print("Robust Accuracy", robust_accuracy.item()) dist = np.mean(fb.distances.l2(clipped, images).numpy()) print("Average perturbation norm", dist)
def get_model(): model = models.resnet18(pretrained=True).eval() mean = torch.Tensor([0.485, 0.456, 0.406]) std = torch.Tensor([0.229, 0.224, 0.225]) if torch.cuda.is_available(): mean = mean.cuda(0) std = std.cuda(0) preprocessing = dict(mean=mean, std=std, axis=-3) fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing) return fmodel
def run_attacks(MODEL_DIR, res_path): rel_dirs = [x for x in os.listdir(MODEL_DIR) if '2020' in x] alpha = [re.findall('a=([0-9, \.]*)_', d)[0] for d in rel_dirs] res = dict.fromkeys(alpha) learner = prep_learner() for model_path, curr_alpha in tqdm(zip(rel_dirs, alpha), total=len(alpha)): conf.save_path = Path(path.join(MODEL_DIR, model_path)) fix_str = [ x for x in os.listdir(path.join(MODEL_DIR, model_path)) if 'model' in x ][0][8:] learner.load_state(conf, fix_str, model_only=True, from_save_folder=True) # probs set_probes(learner) for model in learner.models: model = torch.nn.DataParallel(model.cuda(), device_ids=list(range(4))) model.eval() res[curr_alpha] = dict() for (attack, eps), attack_name in tqdm(zip(attack_list, attack_list_names), desc='attaking ' + str(curr_alpha), total=len(attack_list)): fmodel = JointModelEP( [PyTorchModel(m, bounds=(0, 1)) for m in learner.models], 'cuda') attack = attack() success_tot = [] for images, labels in tqdm(learner.eval_loader, total=len(learner.eval_loader), desc=attack_name): images, labels = ep.astensors(images.to('cuda'), labels.to('cuda')) _, _, success = attack(fmodel, images, labels, epsilons=eps) success_tot.append(success) success_tot = ep.concatenate(success_tot, -1) # calculate and report the robust accuracy robust_accuracy = 1 - success_tot.float32().mean(axis=-1) for epsilon, acc in zip(eps, robust_accuracy): res[curr_alpha][attack_name + '_' + str(epsilon)] = acc.item() pickle.dump(res, open(res_path, 'wb')) pickle.dump(res, open(res_path, 'wb'))
def test_attack(model, data_loader, attack_name, epsilon_values, args, device='cpu'): model.eval() attack_model = attacks[attack_name] # For adversarial testing, the pre-processing happens in the foolbox wrapper. if args['dataset'] in ('mnist', 'fmnist'): preprocessing = None elif args['dataset'] == 'cifar10': preprocessing = dict(mean=(0.4914, 0.4822, 0.4465), std=(0.2023, 0.1994, 0.2010), axis=-3) elif args['dataset'] == 'cifar100': preprocessing = dict(mean=(0.5071, 0.4867, 0.4408), std=(0.2675, 0.2565, 0.2761), axis=-3) elif args['dataset'] == 'svhn': preprocessing = dict(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5), axis=-3) else: raise NotImplementedError('Dataset not supported.') fbox_model = PyTorchModel(model, bounds=(0, 1), device=device, preprocessing=preprocessing) success_cum = [] for data, target in data_loader: data = data.to(device) target = target.to(device) if attack_name in ('FGSM', 'PGD', 'BIM', 'C&W'): advs, _, success = attack_model(fbox_model, data, target, epsilons=epsilon_values, mc=args['monte_carlo_runs']) elif attack_name in ('Few-Pixel', ): # Since the few-pixel attack is not supported by foolbox, we have a different testing pipeline. raise NotImplementedError() success_cum.append(success) success_cum = torch.cat(success_cum, dim=1) robust_accuracy = 1 - success_cum.float().mean(axis=-1) return robust_accuracy
def main() -> None: print(DEVICE) # instantiate a model (could also be a TensorFlow or JAX model) model = model_name(pretrained=True).eval() # preprocessing = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], axis=-3) # image_mean = torch.tensor([0.491, 0.482, 0.447]).view(1, 3, 1, 1) # image_std = torch.tensor([0.247, 0.243, 0.262]).view(1, 3, 1, 1) preprocessing = dict(mean=[0.491, 0.482, 0.447], std=[0.247, 0.243, 0.262], axis=-3) fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing) total_success = 0 # headers = ['img', 'advIndex', 'alpha ', 'angle', 'class', 'l2'] # with open('result_inception_v3_gen4_30.csv', 'w')as f: # f_csv = csv.writer(f) # f_csv.writerow(headers) attack = advDigitalMark() # SpatialAttack() epoch = 0 length = len(train_loader) print("length train loader: ", length) total_msg = '' for batch_data in train_loader: # if epoch == 1: # break images, labels, filenames = batch_data['image'].to( DEVICE), batch_data['label_idx'].to(DEVICE), batch_data['filename'] clean_acc = accuracy(fmodel, images, labels) logging.info("\nepoch " + str(epoch) + ":\nclean accuracy: " + str(clean_acc)) total_msg += "\nepoch " + str(epoch) + ":\nclean accuracy: " + str( clean_acc) count, msg = attack.run(fmodel, images, labels, filenames=filenames) total_success += count total_msg += msg logging.info("current success attack is " + str(total_success)) send_email(total_msg, title='fsy天池比赛V1模型3000-5000结果报告') epoch += 1 total_msg += "\n total attack success img is " + str(total_success) # logging.info("total succes is "+str(total_success)+"/100") send_email(total_msg, title='fsy天池比赛V1模型3000-5000结果报告')
def exp_otherAttack(): model = model_name(pretrained=True).eval() preprocessing = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], axis=-3) fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing) total_success = 0 attack = SpatialAttack() epoch = 0 widgets = [ 'train :', Percentage(), ' ', Bar('#'), ' ', Timer(), ' ', ETA(), ' ', FileTransferSpeed() ] pbar = ProgressBar(widgets=widgets) length = len(train_loader) print("length train loader: ", length) for batch_data in pbar(train_loader): if epoch == 25: break images, labels = batch_data['image'].to( DEVICE), batch_data['label_idx'].to(DEVICE) clean_acc = accuracy(fmodel, images, labels) print(f"\nepoch " + str(epoch) + ":\nclean accuracy: ", clean_acc) epsilons = [1] _, _, isSuceess = attack(fmodel, images, labels, epsilons=epsilons) isSuceess = isSuceess.cpu().numpy() total_success += np.sum(isSuceess == True) print("current success : ", total_success) # total_success += int(adv_acc*10) epoch += 1 print("success attack is ", str(total_success))
def prepare_server_model(self, ensemble_info): if self.args.aggr == "fedavg": server = self.server.client_list[self.adv_client_idx] branch_w = self.server.branches[self.adv_branch_idx] server.model_trainer.set_model_params(branch_w) self.server_model = copy.deepcopy(server.model_trainer.model) elif self.args.aggr == "blockensemble": self.server_model = self.server.server_model elif self.args.aggr == "heteroensemble": self.server_model = self.server.server_model # server_model = self.server.server_model # self.server_model = HeteroFeatAvgEnsembleDefense( # server_model, ensemble_info # ) else: raise NotImplementedError self.server_model.eval() self.server_model = PyTorchModel( self.server_model, bounds = self.bounds, preprocessing = self.preprocessing )
#!/usr/bin/env python3 import torchvision.models as models import eagerpy as ep from foolbox import PyTorchModel, accuracy, samples import foolbox.attacks as fa import numpy as np if __name__ == "__main__": # instantiate a model (could also be a TensorFlow or JAX model) model = models.resnet18(pretrained=True).eval() preprocessing = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], axis=-3) fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing) # get data and test the model # wrapping the tensors with ep.astensors is optional, but it allows # us to work with EagerPy tensors in the following images, labels = ep.astensors( *samples(fmodel, dataset="imagenet", batchsize=16)) clean_acc = accuracy(fmodel, images, labels) print(f"clean accuracy: {clean_acc * 100:.1f} %") print("") attacks = [ fa.FGSM(), fa.LinfPGD(), fa.LinfBasicIterativeAttack(), fa.LinfAdditiveUniformNoiseAttack(), fa.LinfDeepFoolAttack(), ]
def main_worker(gpu, ngpus_per_node, args): args.gpu = gpu attack = args.attack os.makedirs(args.output, exist_ok=True) acc_path = os.path.join(args.output, f'{args.attack}.txt') if args.gpu is not None: print("Use GPU: {} for training".format(args.gpu)) if args.distributed: if args.dist_url == "env://" and args.rank == -1: args.rank = int(os.environ["RANK"]) if args.multiprocessing_distributed: # For multiprocessing distributed training, rank needs to be the # global rank among all the processes args.rank = args.rank * ngpus_per_node + gpu dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, world_size=args.world_size, rank=args.rank) # create model if 'imagenet_100' in args.data: num_classes = 100 else: num_classes = 1000 if args.rank == 0: print("%s-way classification" % num_classes) if args.pretrained: print("=> using pre-trained model '{}'".format(args.arch)) model_name = str(args.arch) model = models_vgg.__dict__[args.arch](pretrained=True, num_classes=num_classes) else: print("=> creating model '{}'".format(args.arch)) model_name = str(args.arch) model = models_vgg.__dict__[model_name](pretrained=False, num_classes=num_classes) if not torch.cuda.is_available(): print('using CPU, this will be slow') elif args.distributed: # For multiprocessing distributed, DistributedDataParallel constructor # should always set the single device scope, otherwise, # DistributedDataParallel will use all available devices. if args.gpu is not None: torch.cuda.set_device(args.gpu) model.cuda(args.gpu) # When using a single GPU per process and per # DistributedDataParallel, we need to divide the batch size # ourselves based on the total number of GPUs we have args.batch_size = int(args.batch_size / ngpus_per_node) args.workers = int( (args.workers + ngpus_per_node - 1) / ngpus_per_node) model = torch.nn.parallel.DistributedDataParallel( model, device_ids=[args.gpu]) else: model.cuda() # DistributedDataParallel will divide and allocate batch_size to all # available GPUs if device_ids are not set model = torch.nn.parallel.DistributedDataParallel(model) elif args.gpu is not None: torch.cuda.set_device(args.gpu) model = model.cuda(args.gpu) else: # DataParallel will divide and allocate batch_size to all available GPUs if args.arch.startswith('alexnet') or args.arch.startswith('vgg'): model.features = torch.nn.DataParallel(model.features) model.cuda() else: model = torch.nn.DataParallel(model).cuda() print("=> loading checkpoint '{}'".format(args.checkpoint)) checkpoint = {'model': {}} if args.gpu is None: checkpoint_old = torch.load(args.checkpoint) else: # Map model to be loaded to specified single gpu. loc = 'cuda:{}'.format(args.gpu) checkpoint_old = torch.load(args.checkpoint, map_location=loc) for value in checkpoint_old['model']: if 'module' not in value: name = 'module.' + value checkpoint['model'][name] = checkpoint_old['model'][value] else: checkpoint['model'][value] = checkpoint_old['model'][value] model.load_state_dict(checkpoint['model'], strict=False) print("=> loaded checkpoint '{}' (epoch {})".format( args.checkpoint, checkpoint_old['epoch'])) model.eval() preprocessing = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], axis=-3) val_loader = torch.utils.data.DataLoader(datasets.ImageFolder( args.data, transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(160), transforms.ToTensor(), ])), batch_size=1, shuffle=False) fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing, device="cuda") # apply the attack attack = globals()[attack]() epsilons = [ 0.0000, 0.0002, 0.0004, 0.0006, 0.0008, 0.0010, 0.0012, 0.0014, 0.0016, 0.0018, 0.0020, ] clean_acc, robust_acc = validate(fmodel=fmodel, val_loader=val_loader, attack=attack, epsilons=epsilons) f = open(acc_path, 'a') print(f"clean accuracy: {clean_acc} %", file=f) print("robust accuracy for perturbations with", file=f) for eps, acc in zip(epsilons, robust_acc): print(f" {args.attack} norm ≤ {eps:<6}: {acc * 100} %", file=f) f.close()
elif params.get('metric') == 'mahalanobis': psd_matrix = np.loadtxt(params.get('psd_matrix_path'), ) psd_matrix = torch.tensor(psd_matrix, dtype=torch.float, device=params.get('device')) else: raise Exception('unsupported metric') knn_module = MahalanobisKnnModule(X_train, y_train, params.getint('k'), psd_matrix) knn_module.to(params.get('device')) knn_module.eval() fmodel = PyTorchModel(knn_module, bounds=(0, 1), device=params.get('device')) attack = BoundaryAttack() n_eval = params.getint('n_eval') perturbations_list = [] for i, (X_eval, y_eval) in enumerate( zip( torch.split(X_test[:n_eval], params.getint('attack_batch_size')), torch.split(y_test[:n_eval], params.getint('attack_batch_size')), )): print(i) _, advs, successful = attack( fmodel, X_eval, y_eval,
params.cp_latest_filename = cp_latest_filenames[model_index] train_loader, val_loader, test_loader, data_params = dataset_utils.load_dataset( params) for key, value in data_params.items(): setattr(params, key, value) model = loaders.load_model(params.model_type) model.setup(params, logger) model.params.analysis_out_dir = os.path.join( *[model.params.model_out_dir, 'analysis', model.params.version]) model.params.analysis_save_dir = os.path.join( model.params.analysis_out_dir, 'savefiles') if not os.path.exists(model.params.analysis_save_dir): os.makedirs(model.params.analysis_save_dir) model.to(params.device) model.load_checkpoint() fmodel = PyTorchModel(model.eval(), bounds=(0, 1)) print('\n', '~' * 79) num_batches = len(test_loader.dataset) // model.params.batch_size attack_success = np.zeros( (len(attacks), len(epsilons), num_batches, model.params.batch_size), dtype=np.bool) for batch_index, (data, target) in enumerate(test_loader): data = model.preprocess_data(data.to(model.params.device)) target = target.to(model.params.device) images, labels = ep.astensors(*(data, target)) del data del target print( f'Model type: {model.params.model_type} [{model_index+1} out of {len(log_files)}]' ) print(f'Batch {batch_index+1} out of {num_batches}')
def run_attacks(res_path): MORPH_MODEL_DIR = '/mnt/md0/orville/Miriam/modular-loss-experiments-morph/results_morph_correct/CIFAR-10/densenet-82-8-8' MODEL_DIR = '/mnt/md0/orville/Miriam/modular-loss-experiments-morph/results/CIFAR-10/densenet-82-8-8' # UNCORR_MODEL_DIR = 'alpha_0.0_gamma_0.0_n_models_2_1581641733617' # CORR_MODEL_DIR = 'alpha_0.1_gamma_0.0_n_models_2_1581641746832' # CORR_MODEL_DIR_2 = 'alpha_0.2_gamma_0.0_n_models_2_1581641777871' UNCORR_MODEL_DIR = 'alpha_0.0_gamma_0.0_n_models_3_1585505819121' CORR_MODEL_DIR = 'alpha_0.1_gamma_0.0_n_models_3_1585505685528' CORR_MODEL_DIR_2 = 'alpha_0.2_gamma_0.0_n_models_3_1585505042819' rel_dirs = [UNCORR_MODEL_DIR, CORR_MODEL_DIR, CORR_MODEL_DIR_2] alpha = ['0', '0.1', '0.2'] res = dict.fromkeys(alpha) batch_size = 516 n_workers = 20 dataset = 'CIFAR-10' network = 'densenet-82-8-8' loaders, _ = get_dataloaders_(batch_size, 0, dataset, False, early_stop=False, n_workers=n_workers) n_models = 3 params = {} params['densenet-82-8-8'] = {'num_modules': 2, 'bottleneck': True, 'reduction': 0.5, 'depth': 82, 'growth_rate': 8, 'input_shape': (3, 32, 32), 'output_dim': 10} network = 'densenet-82-8-8' model = DenseNet(input_shape=params[network]['input_shape'], output_dim=params[network]['output_dim'], growth_rate=params[network]['growth_rate'], depth=params[network]['depth'], reduction=params[network]['reduction'], bottleneck=params[network]['bottleneck'], num_modules=n_models) device = torch.device("cuda") for model_path, curr_alpha in tqdm(zip(rel_dirs, alpha), total=len(alpha)): weight_path = path.join(MODEL_DIR, model_path, 'trial_0/0.0/weights/final_weights.pt') model.reset_parameters() model.load_state_dict(torch.load(weight_path)) model.eval() # model.train(mode=False) fmodel = PyTorchModel(ModelMeanEP(model), bounds=(0, 1), device=device) res[curr_alpha] = dict() for (attack, eps), attack_name in tqdm(zip(attack_list, attack_list_names), desc='attaking ' + str(curr_alpha), total=len(attack_list)): attack = attack() success_tot = [] for images, labels in tqdm(loaders['test'], total=len(loaders['test']), desc=attack_name): images, labels = images.to(device), labels.to(device) _, _, success = attack(fmodel, images, labels, epsilons=eps) success_tot.append(success) success_tot = torch.cat(success_tot, -1) # calculate and report the robust accuracy robust_accuracy = 1 - success_tot.float().mean(axis=-1) for epsilon, acc in zip(eps, robust_accuracy): res[curr_alpha][attack_name + '_' + str(epsilon)] = acc.item() pickle.dump(res, open(res_path, 'wb')) pickle.dump(res, open(res_path, 'wb')) pickle.dump(res, open(res_path, 'wb'))
# for bn in bottlenecks: # results[vdepth][bn] = dict() results[n_bn] = dict() results[n_bn][rep] = dict() model = RetinalBottleneckModel(n_bn, 'resnet50', n_out=1000, n_inch=3, retina_kernel_size=7, transform=normalize) model.load_state_dict(torch.load(dir + model_file + '.pt')) # model = torch.hub.load('ecs-vlc/opponency:master', 'colour_full', n_bn=bn, d_vvs=vdepth, rep=run) model.eval() fmodel = PyTorchModel(model, bounds=(0, 1)) # images, labels = samples(fmodel, dataset="cifar10", batchsize=20) # images = images.contiguous() # results[vdepth][bn][run]["accuracy"] = accuracy(fmodel, images, labels) attack_success = np.zeros((len(attacks), len(epsilons), len(testset)), dtype=np.bool) # for i, attack in enumerate(attacks): print(attack) idx = 0 for images, labels in tqdm(testloader): images = images.to(fmodel.device) labels = labels.to(fmodel.device)
class AdvAttack(): def __init__(self, server, device, args, adv_client_idx=0, adv_branch_idx=0): self.server = server server.set_client_dataset() self.device = device self.args = args self.adv_client_idx = adv_client_idx self.adv_branch_idx = adv_branch_idx self.save_dir = osp.join(self.args.save_dir, f"{type(self).__name__}") if not os.path.exists(self.save_dir): os.makedirs(self.save_dir, exist_ok=True) # set adv client model params = self.set_attack_params() ensemble_info = self.prepare_advclient_model() self.prepare_server_model(ensemble_info) # test clean accuracys) # clean_acc = self.check_accuracy( # self.advclient_model, self.server.test_global, self.device # ) # logging.info(f"clean accuracy: {clean_acc * 100:.1f} %") # st() self.attack_fn = LinfPGD(**params) # self.attack_fn = L2CarliniWagnerAttack(**params) self.attack() def set_attack_params(self): if self.args.dataset in ["mnist", "fmnist", "emnist"]: range = 1 / 0.3081 params = { "steps": 10, "rel_stepsize": 0.1, } self.epsilon = range * 0.3 bound_min = (0-0.1307) / 0.3081 bound_max = (1-0.1307) / 0.3081 self.bounds = (bound_min, bound_max) self.preprocessing = None elif self.args.dataset in ['cifar10', 'cifar100']: CIFAR_MEAN = torch.Tensor([0.49139968, 0.48215827, 0.44653124]).view(1,3,1,1).to(self.device) CIFAR_STD = torch.Tensor([0.24703233, 0.24348505, 0.26158768]).view(1,3,1,1).to(self.device) range = 1 / 0.3081 params = { "steps": 20, "rel_stepsize": 0.1, } self.epsilon = 8/255 # bound_min = (0-0.1307) / 0.3081 # bound_max = (1-0.1307) / 0.3081 self.bounds = (0, 1) self.preprocessing = { "mean": CIFAR_MEAN, "std": CIFAR_STD } else: raise NotImplementedError return params def check_accuracy(self, fmodel, dataloader, device) -> float: total, correct = 0., 0. with torch.no_grad(): for batch_idx, (x, target) in enumerate(dataloader): x = x.to(device) target = target.to(device) if self.preprocessing is not None: x = x * self.preprocessing["std"] + self.preprocessing["mean"] predictions = fmodel(x).argmax(axis=-1) total += len(x) correct += (predictions == target).sum() accuracy = correct / total return accuracy.item() def prepare_advclient_model(self): if self.args.aggr == "fedavg": adv_client = self.server.client_list[self.adv_client_idx] branch_w = self.server.branches[self.adv_branch_idx] adv_client.model_trainer.set_model_params(branch_w) self.advclient_model = copy.deepcopy(adv_client.model_trainer.model) ensemble_info = None elif self.args.aggr == "blockensemble": state_dict_pair, ensemble_info = self.server.prepare_branch_dict() self.adv_client = self.server.client_list[self.adv_client_idx] self.adv_client.model_trainer.set_model_params(state_dict_pair) # self.advclient_model = TwoModelWarpper( # self.adv_client.model_trainer.model1, # self.adv_client.model_trainer.model2, # ) self.advclient_model = self.adv_client.model_trainer.model2 elif self.args.aggr == "heteroensemble": model_pair, ensemble_info = self.server.prepare_branch_dict(client_idx=0) self.adv_client = self.server.client_list[self.adv_client_idx] self.advclient_model = TwoModelWarpper( model_pair[0], model_pair[1], ) # self.advclient_model = model_pair[0] else: raise NotImplementedError self.advclient_model.eval() self.advclient_model = PyTorchModel( self.advclient_model, bounds = self.bounds, preprocessing = self.preprocessing ) return ensemble_info def prepare_server_model(self, ensemble_info): if self.args.aggr == "fedavg": server = self.server.client_list[self.adv_client_idx] branch_w = self.server.branches[self.adv_branch_idx] server.model_trainer.set_model_params(branch_w) self.server_model = copy.deepcopy(server.model_trainer.model) elif self.args.aggr == "blockensemble": self.server_model = self.server.server_model elif self.args.aggr == "heteroensemble": self.server_model = self.server.server_model # server_model = self.server.server_model # self.server_model = HeteroFeatAvgEnsembleDefense( # server_model, ensemble_info # ) else: raise NotImplementedError self.server_model.eval() self.server_model = PyTorchModel( self.server_model, bounds = self.bounds, preprocessing = self.preprocessing ) def attack(self): total, correct, success, server_success = 0, 0, 0, 0 client_total, server_total = 0, 0 fmodel = self.advclient_model server_model = self.server_model dataloader = self.server.test_global for batch_idx, (x, target) in enumerate(dataloader): x = x.to(self.device) target = target.to(self.device) if self.preprocessing is not None: x = x * self.preprocessing["std"] + self.preprocessing["mean"] # x = x * 255 clean_pred = fmodel(x).argmax(axis=-1) pred_correct = clean_pred == target correct += (pred_correct).sum().float() raw_advs, clipped_advs, _success = self.attack_fn(fmodel, x, clean_pred, epsilons=self.epsilon) # total += len(x) # success += (_success).sum().float() # server_pred = server_model(clipped_advs).argmax(axis=-1) # server_wrong = (server_pred != target) # server_success += server_wrong.sum().float() # success += (_success*pred_correct).sum().float() server_clean_pred = server_model(x).argmax(axis=-1) client_valid = (clean_pred == target) client_total += client_valid.sum().float() success += (_success * client_valid).sum().float() server_adv_pred = server_model(clipped_advs).argmax(axis=-1) server_valid = (server_clean_pred == target) server_total += server_valid.sum().float() server_wrong = (server_adv_pred != target) * server_valid server_success += server_wrong.sum().float() # print(f"Total {total}, success {success}, server success {server_success}") # st() # diff = (x-clipped_advs).max() # for MNIST # x = x * 0.3081 + 0.1307 # clipped_advs = clipped_advs * 0.3081 + 0.1307 # for idx in range(5): # img, adv_img = x[idx], clipped_advs[idx] # img = (img*255).cpu().numpy().astype(np.uint8) # adv_img = (adv_img*255).cpu().numpy().astype(np.uint8) # img = np.concatenate([img, adv_img], axis=2) # img = np.transpose(img, (1,2,0)) # path = osp.join("debug", f"{idx}.png") # from PIL import Image # for MNIST # img = img.repeat(3, axis=2) # im = Image.fromarray(img,) # im.save(path) # diff = (x - clipped_advs).max()*255 # st() success_rate = (success / client_total).item() server_success_rate = (server_success / server_total).item() clean_acc = (correct / total).item() logging.info(f"clean acc: {clean_acc*100:.1f}%, client success rate: {success_rate * 100:.1f}%, server success rate: {server_success_rate*100:.1f}%" ) return clean_acc, success_rate, server_success_rate
def foolbox_attack(filter=None, filter_preserve='low', free_parm='eps', plot_num=None): # get model. model = get_model() model = nn.DataParallel(model).to(device) model = model.eval() preprocessing = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], axis=-3) fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing) if plot_num: free_parm = '' val_loader = get_val_loader(plot_num) else: # Load images. val_loader = get_val_loader(args.attack_batch_size) if 'eps' in free_parm: epsilons = [0.001, 0.003, 0.005, 0.008, 0.01, 0.1] else: epsilons = [0.01] if 'step' in free_parm: steps = [1, 5, 10, 30, 40, 50] else: steps = [args.iteration] for step in steps: # Adversarial attack. if args.attack_type == 'LinfPGD': attack = LinfPGD(steps=step) elif args.attack_type == 'FGSM': attack = FGSM() clean_acc = 0.0 for i, data in enumerate(val_loader, 0): # Samples (attack_batch_size * attack_epochs) images for adversarial attack. if i >= args.attack_epochs: break images, labels = data[0].to(device), data[1].to(device) if step == steps[0]: clean_acc += (get_acc( fmodel, images, labels )) / args.attack_epochs # accumulate for attack epochs. _images, _labels = ep.astensors(images, labels) raw_advs, clipped_advs, success = attack(fmodel, _images, _labels, epsilons=epsilons) if plot_num: grad = torch.from_numpy( raw_advs[0].numpy()).to(device) - images grad = grad.clone().detach_() return grad if filter: robust_accuracy = torch.empty(len(epsilons)) for eps_id in range(len(epsilons)): grad = torch.from_numpy( raw_advs[eps_id].numpy()).to(device) - images grad = grad.clone().detach_() freq = dct.dct_2d(grad) if filter_preserve == 'low': mask = torch.zeros(freq.size()).to(device) mask[:, :, :filter, :filter] = 1 elif filter_preserve == 'high': mask = torch.zeros(freq.size()).to(device) mask[:, :, filter:, filter:] = 1 masked_freq = torch.mul(freq, mask) new_grad = dct.idct_2d(masked_freq) x_adv = torch.clamp(images + new_grad, 0, 1).detach_() robust_accuracy[eps_id] = (get_acc(fmodel, x_adv, labels)) else: robust_accuracy = 1 - success.float32().mean(axis=-1) if i == 0: robust_acc = robust_accuracy / args.attack_epochs else: robust_acc += robust_accuracy / args.attack_epochs if step == steps[0]: print("sample size is : ", args.attack_batch_size * args.attack_epochs) print(f"clean accuracy: {clean_acc * 100:.1f} %") print( f"Model {args.model} robust accuracy for {args.attack_type} perturbations with" ) for eps, acc in zip(epsilons, robust_acc): print( f" Step {step}, Linf norm ≤ {eps:<6}: {acc.item() * 100:4.1f} %" ) print(' -------------------')
for epoch in range(1, epochs + 1): print("Epoch #", epoch) y_pred = model.forward(x_train) loss = loss_fn(y_pred.squeeze(), y_train) print_(loss.item()) optimizer.zero_grad() loss.backward() optimizer.step() print("Training error:", mean_squared_error(model(x_train).detach().numpy(), y_train)) y_pred = model(x_test).detach().numpy() print("Test error:", mean_squared_error(y_pred, y_test)) fmodel = PyTorchModel(model.eval(), bounds=(-1, 2)) attack = L2AdditiveGaussianNoiseAttack() epsilons = np.linspace(0.0, 6.0, num=6, endpoint=False) advs, _, success = attack(fmodel, x_final, y_final, epsilons=epsilons) perturbed = np.zeros((len(X), 7)) added = np.zeros(len(X)) print("Finding adversarial inputs") for eps, adv in zip(epsilons, advs): if 0 not in added: break pred = model(adv) pred = pred.detach().numpy().ravel() pred_binned = np.digitize(pred, bins)
def exp1(): print(DEVICE) parser = argparse.ArgumentParser( description='CIFAR10 Training against DDN Attack') parser.add_argument('--data', default='../defenses/data/cifar10', help='path to dataset') parser.add_argument('--workers', default=2, type=int, help='number of data loading workers') parser.add_argument('--cpu', action='store_true', help='force training on cpu') parser.add_argument('--save-folder', '--sf', default='weights/cifar10_inception_v3/', help='folder to save state dicts') parser.add_argument('--save-freq', '--sfr', default=10, type=int, help='save frequency') parser.add_argument('--save-name', '--sn', default='cifar10', help='name for saving the final state dict') parser.add_argument('--batch-size', '-b', default=20, type=int, help='mini-batch size') parser.add_argument('--epochs', '-e', default=100, type=int, help='number of total epochs to run') parser.add_argument('--lr', '--learning-rate', default=0.001, type=float, help='initial learning rate') parser.add_argument('--lr-decay', '--lrd', default=0.2, type=float, help='decay for learning rate') parser.add_argument('--lr-step', '--lrs', default=10, type=int, help='step size for learning rate decay') parser.add_argument('--momentum', default=0.9, type=float, help='momentum') parser.add_argument('--weight-decay', '--wd', default=5e-4, type=float, help='weight decay') parser.add_argument('--drop', default=0.3, type=float, help='dropout rate of the classifier') parser.add_argument('--adv', type=int, default=None, help='epoch to start training with adversarial images') parser.add_argument('--max-norm', type=float, default=1, help='max norm for the adversarial perturbations') parser.add_argument('--steps', default=10, type=int, help='number of steps for the attack') parser.add_argument( '--visdom-port', '--vp', type=int, default=8097, help='For visualization, which port visdom is running.') parser.add_argument('--print-freq', '--pf', default=10, type=int, help='print frequency') args = parser.parse_args() print(args) train_transform = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), ]) test_transform = transforms.Compose([ transforms.ToTensor(), ]) train_set = data.Subset( CIFAR10(args.data, train=True, transform=train_transform, download=True), list(range(30000, 50000))) val_set = data.Subset( CIFAR10(args.data, train=True, transform=test_transform, download=True), list(range(48000, 50000))) test_set = CIFAR10(args.data, train=False, transform=test_transform, download=True) train_loader = data.DataLoader(train_set, batch_size=args.batch_size, shuffle=True, num_workers=args.workers, drop_last=True, pin_memory=True) val_loader = data.DataLoader(val_set, batch_size=100, shuffle=False, num_workers=args.workers, pin_memory=True) test_loader = data.DataLoader(test_set, batch_size=100, shuffle=False, num_workers=args.workers, pin_memory=True) image_mean = torch.tensor([0.491, 0.482, 0.447]).view(1, 3, 1, 1) image_std = torch.tensor([0.247, 0.243, 0.262]).view(1, 3, 1, 1) m = AlexNet().eval() model = NormalizedModel(model=m, mean=image_mean, std=image_std).to( DEVICE) # keep images in the [0, 1] range model_file = '/home/frankfeng/researchData/code/adversarial_training_code/PLP/fast_adv/defenses/weights/cifar10_AlexNet/cifar10_valacc0.8019999772310257.pth' model_dict = torch.load(model_file) model.load_state_dict(model_dict) # preprocessing = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], axis=-3) # image_mean = torch.tensor([0.491, 0.482, 0.447]).view(1, 3, 1, 1) # image_std = torch.tensor([0.247, 0.243, 0.262]).view(1, 3, 1, 1) preprocessing = dict(mean=[0.491, 0.482, 0.447], std=[0.247, 0.243, 0.262], axis=-3) fmodel = PyTorchModel(model, bounds=(0, 1)) total_success = 0 # headers = ['img', 'advIndex', 'alpha ', 'angle', 'class', 'l2'] # with open('result_inception_v3_gen4_30.csv', 'w')as f: # f_csv = csv.writer(f) # f_csv.writerow(headers) attack = advDigitalMark() # SpatialAttack() epoch = 0 widgets = [ 'train :', Percentage(), ' ', Bar('#'), ' ', Timer(), ' ', ETA(), ' ', FileTransferSpeed() ] pbar = ProgressBar(widgets=widgets) length = len(train_loader) print("length train loader: ", length) for epoch in range(args.epochs): if epoch == 25: break for i, (images, labels) in enumerate(tqdm.tqdm(train_loader, ncols=80)): images, labels = images.to(DEVICE), labels.to(DEVICE) clean_acc = accuracy(fmodel, images, labels) print(f"\nepoch " + str(epoch) + ":\nclean accuracy: ", clean_acc) count = attack.run(fmodel, images, labels) total_success += count epoch += 1 print("current success attack is ", str(total_success)) print( str(model_name) + "_" + str(max_gen) + "_" + str(max_gen) + ": total attack success img is ", str(total_success))
labels.append(label) return image_paths, labels num_classes = 2 batchsize = 16 data_dir = "../cat_dog/val" classifier_path = "../ckpts1/best_classifier.pth" image_suffix = ".jpg" classfier = resnet18(pretrained=False, num_classes=num_classes) classfier.load_state_dict(torch.load(classifier_path, map_location="cpu")) classfier.eval() device = torch.device("cuda" if torch.cuda.is_available() else "cpu") accuracys = [] fmodel = PyTorchModel(classfier, bounds=(0, 1), device=device) image_paths, labels = get_datas(data_dir, image_suffix) eval_dataset = PALMClassifyDataset(image_paths, labels) eval_loader = torch.utils.data.DataLoader(eval_dataset, batch_size=batchsize, num_workers=0, shuffle=False) preds = [] for x, _ in eval_loader: x = x.to(device) pred = fmodel(x).argmax(axis=-1).tolist() preds += pred acc_clean = accuracy_score(labels, preds) print(f"clean accuracy: {acc_clean * 100:.2f} %")
def main() -> None: # instantiate a model (could also be a TensorFlow or JAX model) #model = models.resnet18(pretrained=True).eval() #model=torch.load('/data1/zyh/copycat/Framework/cifar_model.pth') model = AlexNet() path = "./cifar_net.pth" #path = '/data1/zyh/copycat/Framework/cifar_model.pth' #model.load_state_dict(torch.load('/data1/zyh/copycat/Framework/cifar_model.pth')) #pretrained_dict = {k: v for k, v in model_pretrained.items() if k in model_dict} #model_dict.update(pretrained_dict) #model.load_state_dict(state_dict) model.load_state_dict(torch.load(path), strict=True) model = model.to(device) model.eval() print(type(model)) #preprocessing = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], axis=-3) preprocessing = dict(mean=[0.5] * 3, std=[0.5] * 3, axis=-3) fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing) # get data and test the model # wrapping the tensors with ep.astensors is optional, but it allows # us to work with EagerPy tensors in the following test_dataset = torchvision.datasets.CIFAR10( root='~/.torch/', train=False, #transform = transforms.Compose([transforms.Resize((256,256)),transforms.ToTensor()]), transform=transforms.Compose([transforms.ToTensor()]), download=True) # test_dataset .data = test_dataset.data[:128*5] test_loader = torch.utils.data.DataLoader( dataset=test_dataset, batch_size=128, #该参数表示每次读取的批样本个数 shuffle=False) #该参数表示读取时是否打乱样本顺序 # 创建迭代器 #data_iter = iter(test_loader) #images, labels = next(data_iter) # 当迭代开始时, 队列和线程开始读取数据 #images, labels = data_iter.next() #im=images #images=im.resize(100,3,128,128) with torch.no_grad(): all_clean_acc_foolbox = [] ## native predict predict_func(test_loader, model) for ii, (imgs, lbls) in tqdm.tqdm(enumerate(test_loader), total=len(test_loader)): imgs = imgs.to(device) lbls = lbls.to(device) images, labels = ep.astensors(imgs, lbls) ## calc with foolbox pred_lbl_foolbox = fmodel(images) clean_acc_one = accuracy(fmodel, imgs, lbls) all_clean_acc_foolbox.append(clean_acc_one) clean_acc = sum(all_clean_acc_foolbox) / len(all_clean_acc_foolbox) print(f"clean accuracy: {clean_acc * 100:.1f} %") # apply the attack attack = LinfPGD() '''epsilons = [ 0.0, 0.0002, 0.0005, 0.0008, 0.001, 0.0015, 0.002, 0.003, 0.01, 0.1, 0.3, 0.5, 1.0, ]''' epsilons = [ 0.0005, 0.001, 0.002, 0.01, 0.1, ] def attack_one_batch(fmodel, images, labels, iter=0, verbose=True): images, labels = ep.astensors(images, labels) raw_advs, clipped_advs, success = attack(fmodel, images, labels, epsilons=epsilons) if verbose: print("===" * 8, iter, "===" * 8) if verbose: robust_accuracy = 1 - success.float32().mean(axis=-1) print("robust accuracy for perturbations with") for eps, acc in zip(epsilons, robust_accuracy): print(f" Linf norm ≤ {eps:<6}: {acc.item() * 100:4.1f} %") if verbose: fig = plt.gcf() os.makedirs("./image/", exist_ok=True) for i in range(len(raw_advs)): img_v = raw_advs[i].raw torchvision.utils.save_image( img_v, f'./image/{str(iter).zfill(4)}_{str(i).zfill(3)}_.png') return [x.raw for x in raw_advs] # print("====" * 8, "start attack", "====" * 8) collection_adv = [] collection_gt = [] for ii, (imgs, lbls) in tqdm.tqdm(enumerate(test_loader), total=len(test_loader)): imgs = imgs.to(device) lbls = lbls.to(device) # images, labels = ep.astensors(images,labels) adv_ret = attack_one_batch(fmodel=fmodel, images=imgs, labels=lbls, iter=ii, verbose=True) collection_adv.append(torch.stack(adv_ret)) collection_gt.append(lbls.cpu()) print("====" * 8, "start evaluation", "====" * 8) with torch.no_grad(): adv_total_dataset = torch.cat(collection_adv, dim=1) lbl_total_dataset = torch.cat(collection_gt).to(device) # print (adv_total_dataset.mean(dim=(1,2,3,4)),"the mean if each eps") for (eps, ep_adv_dataset) in zip(epsilons, adv_total_dataset): # print ("eps:",eps,"===>"*8) # print (ep_adv_dataset.mean(),"each...") advs_ = ep_adv_dataset.to(device) acc2 = accuracy(fmodel, advs_, lbl_total_dataset) print(f" Linf norm ≤ {eps:<6}: {acc2 * 100:4.1f} %") dataset = torch.utils.data.TensorDataset(ep_adv_dataset, lbl_total_dataset) dl = torch.utils.data.DataLoader(dataset, batch_size=128) predict_func(dl, model)
def main() -> None: # instantiate a model (could also be a TensorFlow or JAX model) model = models.resnet18(pretrained=True).eval() preprocessing = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], axis=-3) fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing) # get data and test the model # wrapping the tensors with ep.astensors is optional, but it allows # us to work with EagerPy tensors in the following images, labels = ep.astensors( *samples(fmodel, dataset="imagenet", batchsize=16)) clean_acc = accuracy(fmodel, images, labels) print(f"clean accuracy: {clean_acc * 100:.1f} %") # replace the gradient with the gradient from another model model2 = fmodel # demo, we just use the same model # TODO: this is still a bit annoying because we need # to overwrite run to get the labels class Attack(LinfPGD): def value_and_grad(self, loss_fn, x): val1 = loss_fn(x) loss_fn2 = self.get_loss_fn(model2, self.labels) _, grad2 = ep.value_and_grad(loss_fn2, x) return val1, grad2 def run(self, model, inputs, criterion, *, epsilon, **kwargs): criterion_ = get_criterion(criterion) self.labels = criterion_.labels return super().run(model, inputs, criterion_, epsilon=epsilon, **kwargs) # apply the attack attack = Attack() epsilons = [ 0.0, 0.0002, 0.0005, 0.0008, 0.001, 0.0015, 0.002, 0.003, 0.01, 0.1, 0.3, 0.5, 1.0, ] raw_advs, clipped_advs, success = attack(fmodel, images, labels, epsilons=epsilons) # calculate and report the robust accuracy (the accuracy of the model when # it is attacked) robust_accuracy = 1 - success.float32().mean(axis=-1) print("robust accuracy for perturbations with") for eps, acc in zip(epsilons, robust_accuracy): print(f" Linf norm ≤ {eps:<6}: {acc.item() * 100:4.1f} %") # we can also manually check this # we will use the clipped advs instead of the raw advs, otherwise # we would need to check if the perturbation sizes are actually # within the specified epsilon bound print() print("we can also manually check this:") print() print("robust accuracy for perturbations with") for eps, advs_ in zip(epsilons, clipped_advs): acc2 = accuracy(fmodel, advs_, labels) print(f" Linf norm ≤ {eps:<6}: {acc2 * 100:4.1f} %") print(" perturbation sizes:") perturbation_sizes = (advs_ - images).norms.linf(axis=(1, 2, 3)).numpy() print(" ", str(perturbation_sizes).replace("\n", "\n" + " ")) if acc2 == 0: break
def main() -> None: # instantiate a model (could also be a TensorFlow or JAX model) #model = models.resnet18(pretrained=True).eval() #model=torch.load('/data1/zyh/copycat/Framework/cifar_model.pth') model =AlexNet() path = "./cifar_net.pth" #path = '/data1/zyh/copycat/Framework/cifar_model.pth' #model.load_state_dict(torch.load('/data1/zyh/copycat/Framework/cifar_model.pth')) #pretrained_dict = {k: v for k, v in model_pretrained.items() if k in model_dict} #model_dict.update(pretrained_dict) #model.load_state_dict(state_dict) model.load_state_dict(torch.load(path),strict=True) model.eval() print(type(model)) #preprocessing = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], axis=-3) preprocessing = dict(mean=[0.5]*3, std=[0.5]*3, axis=-3) fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing) # get data and test the model # wrapping the tensors with ep.astensors is optional, but it allows # us to work with EagerPy tensors in the following #test_dataset = torchvision.datasets.CIFAR10(root='~/.torch/', # train=True, # #transform = transforms.Compose([transforms.Resize((256,256)),transforms.ToTensor()]), # transform = transforms.Compose([transforms.ToTensor()]), # download=True) #test_loader = torch.utils.data.DataLoader(dataset=test_dataset, # batch_size=128, #该参数表示每次读取的批样本个数 # shuffle=False) #该参数表示读取时是否打乱样本顺序 # # 创建迭代器 #data_iter = iter(test_loader) #images, labels = next(data_iter) # 当迭代开始时, 队列和线程开始读取数据 #images, labels = data_iter.next() #images=images.to(device) #labels=labels.to(device) #im=images #images=im.resize(100,3,128,128) images, labels = ep.astensors(*samples(fmodel, dataset="cifar10", batchsize=16)) #images, labels = ep.astensors(*samples(fmodel, dataset="imagenet", batchsize=16)) #print(images.shape) clean_acc = accuracy(fmodel, images, labels) print(f"clean accuracy: {clean_acc * 100:.1f} %") # apply the attack attack = LinfPGD() '''epsilons = [ 0.0, 0.0002, 0.0005, 0.0008, 0.001, 0.0015, 0.002, 0.003, 0.01, 0.1, 0.3, 0.5, 1.0, ]''' epsilons = [ 0.0005, 0.001, 0.002, 0.01, 0.1, ] raw_advs, clipped_advs, success = attack(fmodel, images, labels, epsilons=epsilons) print(type(raw_advs)) print("atest") # calculate and report the robust accuracy (the accuracy of the model when # it is attacked) robust_accuracy = 1 - success.float32().mean(axis=-1) print("robust accuracy for perturbations with") for eps, acc in zip(epsilons, robust_accuracy): print(f" Linf norm ≤ {eps:<6}: {acc.item() * 100:4.1f} %") # we can also manually check this # we will use the clipped advs instead of the raw advs, otherwise # we would need to check if the perturbation sizes are actually # within the specified epsilon bound print() print("we can also manually check this:") print() print("robust accuracy for perturbations with") for eps, advs_ in zip(epsilons, clipped_advs): acc2 = accuracy(fmodel, advs_, labels) print(f" Linf norm ≤ {eps:<6}: {acc2 * 100:4.1f} %") print(" perturbation sizes:") perturbation_sizes = (advs_ - images).norms.linf(axis=(1, 2, 3)).numpy() print(" ", str(perturbation_sizes).replace("\n", "\n" + " ")) if acc2 == 0: break fig = plt.gcf() os.makedirs("./image/",exist_ok=True) for i in range(len(raw_advs)): img_v = raw_advs[i].raw torchvision.utils.save_image(img_v, './image/'+str(i) +'.png')
def main(): parser = argparse.ArgumentParser() parser.add_argument('--min_norm', type=float, default=0.01, help='Minimum perturbation norm') parser.add_argument('--max_norm', type=float, default=15, help='Maximum perturbation norm') parser.add_argument('--num', type=int, default=12, help='Number of norms to evaluate on') args = parser.parse_args() device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # For Colab # class args: # # repeats = 1000 # min_norm = 0.01 # max_norm = 15 # num = 12 # List of max query count queries = [10, 100, 1000, 5000] # Load the pretrained model model = Net() model.load_state_dict(torch.load('mnist_cnn.pt')) model.eval() # preprocess the model inputs and set pixel bound in [0, 1] preprocessing = dict(mean=0.1307, std=0.3081) fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing) fmodel = fmodel.transform_bounds((0, 1)) assert fmodel.bounds == (0, 1) # Set the perturbation norm space epsilons = np.linspace(args.min_norm, args.max_norm, args.num) test_loader = torch.utils.data.DataLoader( datasets.MNIST('../data', train=False, download=True, transform=transforms.ToTensor()), batch_size=10000, shuffle=True) total = 0 # total input count successful = torch.zeros(args.num, device=device) # success for each norm legends = [] start = time() for query in queries: for images, labels in test_loader: images = images.to(device) labels = labels.to(device) ep.astensor_(images) ep.astensor_(labels) # Additive Gaussian noise attack with L2 norm attack = fa.L2RepeatedAdditiveGaussianNoiseAttack(repeats=query) raw, clipped, success = attack(fmodel, images, labels, epsilons=epsilons) # Add the total number of successful attacks for each norm value successful += success.sum(axis=1) total += len(labels) robust_accuracy = (1 - 1.0 * successful / total).cpu() plt.plot(epsilons, robust_accuracy.numpy()) legends.append("{} Queries".format(query)) plt.xlabel("Perturbation Norm (L2)") plt.ylabel("Robust Accuracy") plt.title("Gaussian Noise") plt.legend(legends, loc='upper right') plt.ylim([0, 1]) plt.savefig('mnist_RA_robust_acc.jpg') plt.show() end = time() print("Time taken: {:.1f} minutes".format((end - start) / 60))
def main() -> None: # instantiate a model (could also be a TensorFlow or JAX model) model = models.resnet18(pretrained=True).eval() preprocessing = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], axis=-3) fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing) # get data and test the model # wrapping the tensors with ep.astensors is optional, but it allows # us to work with EagerPy tensors in the following images, labels = ep.astensors( *samples(fmodel, dataset="imagenet", batchsize=16)) clean_acc = accuracy(fmodel, images, labels) print(f"clean accuracy: {clean_acc * 100:.1f} %") # apply the attack attack = LinfPGD() epsilons = [ 0.0, 0.0002, 0.0005, 0.0008, 0.001, 0.0015, 0.002, 0.003, 0.01, 0.1, 0.3, 0.5, 1.0, ] raw_advs, clipped_advs, success = attack(fmodel, images, labels, epsilons=epsilons) # calculate and report the robust accuracy (the accuracy of the model when # it is attacked) robust_accuracy = 1 - success.float32().mean(axis=-1) print("robust accuracy for perturbations with") for eps, acc in zip(epsilons, robust_accuracy): print(f" Linf norm ≤ {eps:<6}: {acc.item() * 100:4.1f} %") # we can also manually check this # we will use the clipped advs instead of the raw advs, otherwise # we would need to check if the perturbation sizes are actually # within the specified epsilon bound print() print("we can also manually check this:") print() print("robust accuracy for perturbations with") for eps, advs_ in zip(epsilons, clipped_advs): acc2 = accuracy(fmodel, advs_, labels) print(f" Linf norm ≤ {eps:<6}: {acc2 * 100:4.1f} %") print(" perturbation sizes:") perturbation_sizes = (advs_ - images).norms.linf(axis=(1, 2, 3)).numpy() print(" ", str(perturbation_sizes).replace("\n", "\n" + " ")) if acc2 == 0: break