def prepare_advclient_model(self):
if self.args.aggr == "fedavg":
adv_client = self.server.client_list[self.adv_client_idx]
branch_w = self.server.branches[self.adv_branch_idx]
adv_client.model_trainer.set_model_params(branch_w)
self.advclient_model = copy.deepcopy(adv_client.model_trainer.model)
ensemble_info = None
elif self.args.aggr == "blockensemble":
state_dict_pair, ensemble_info = self.server.prepare_branch_dict()
self.adv_client = self.server.client_list[self.adv_client_idx]
self.adv_client.model_trainer.set_model_params(state_dict_pair)
# self.advclient_model = TwoModelWarpper(
# self.adv_client.model_trainer.model1,
# self.adv_client.model_trainer.model2,
# )
self.advclient_model = self.adv_client.model_trainer.model2
elif self.args.aggr == "heteroensemble":
model_pair, ensemble_info = self.server.prepare_branch_dict(client_idx=0)
self.adv_client = self.server.client_list[self.adv_client_idx]
self.advclient_model = TwoModelWarpper(
model_pair[0],
model_pair[1],
)
# self.advclient_model = model_pair[0]
else:
raise NotImplementedError
self.advclient_model.eval()
self.advclient_model = PyTorchModel(
self.advclient_model, bounds = self.bounds,
preprocessing = self.preprocessing
)
return ensemble_info
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--steps',
type=int,
default=20000,
help='Iteration of BA')
parser.add_argument('--targeted',
action='store',
default=False,
help='For targeted attack')
args = parser.parse_args()
model = Net()
model.load_state_dict(torch.load('mnist_cnn.pt'))
model.eval()
preprocessing = dict(mean=0.1307, std=0.3081)
fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
fmodel = fmodel.transform_bounds((0, 1))
assert fmodel.bounds == (0, 1)
images, labels = ep.astensors(
*samples(fmodel, dataset="mnist", batchsize=10))
print('Model accuracy on clean examples: {}'.format(
accuracy(fmodel, images, labels)))
if args.targeted:
target_class = (labels + 7) % 10
criterion = fb.criteria.TargetedMisclassification(target_class)
else:
criterion = fb.criteria.Misclassification(labels)
attack = fa.BoundaryAttack(steps=args.steps, tensorboard=None)
epsilons = np.linspace(0.01, 10, 20)
raw, clipped, success = attack(fmodel, images, labels, epsilons=epsilons)
robust_accuracy = 1 - success.float32().mean(axis=-1)
plt.plot(epsilons, robust_accuracy.numpy())
plt.xlabel("Epsilons")
plt.ylabel("Robust Accuracy")
plt.savefig('mnist_BA_robust_acc.jpg')
plt.show()
mean_distance = []
for i in range(len(clipped)):
dist = np.mean(fb.distances.l2(clipped[i], images).numpy())
mean_distance.append(dist)
plt.plot(epsilons, mean_distance)
plt.xlabel('Epsilons')
plt.ylabel('Mean L2 distance')
plt.savefig("mnist_BA_mean_L2distance.jpg")
plt.show()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--steps',
type=int,
default=10000,
help='Iteration of BA')
parser.add_argument('--targeted',
action='store',
default=False,
help='For targeted attack')
args = parser.parse_args()
model = Net()
model.load_state_dict(torch.load('mnist_cnn.pt'))
model.eval()
preprocessing = dict(mean=0.1307, std=0.3081)
fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
fmodel = fmodel.transform_bounds((0, 1))
assert fmodel.bounds == (0, 1)
images, labels = ep.astensors(
*samples(fmodel, dataset="mnist", batchsize=10))
print('Model accuracy on clean examples: {}'.format(
accuracy(fmodel, images, labels)))
epsilons = np.linspace(0.01, 10, 20)
boundary_attack = fa.BoundaryAttack(steps=args.steps, tensorboard=None)
_, _, ba_success = boundary_attack(fmodel,
images,
labels,
epsilons=epsilons)
ba_robust_accuracy = 1 - ba_success.float32().mean(axis=-1)
random_attack = fa.L2RepeatedAdditiveGaussianNoiseAttack(
repeats=args.steps)
_, _, ra_success = random_attack(fmodel, images, labels, epsilons=epsilons)
ra_robust_accuracy = 1 - ra_success.float32().mean(axis=-1)
legends = ["Boundary Attack", "Random Attack"]
plt.plot(epsilons, ba_robust_accuracy.numpy())
plt.plot(epsilons, ra_robust_accuracy.numpy())
plt.legend(legends, loc='upper right')
plt.xlabel("Perturbation Norm (L2)")
plt.ylabel("Robust Accuracy")
plt.title("{} Queries".format(args.steps))
plt.savefig('mnist_robust_acc.jpg')
plt.show()
def main() -> None:
model = models.resnet101(pretrained=True).eval()
preprocessing = dict(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
axis=-3)
fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
images, labels = fb.utils.samples(fmodel, dataset='imagenet', batchsize=16)
clean_acc = fb.utils.accuracy(fmodel, images, labels)
print(f"clean accuracy: {clean_acc * 100:.1f} %")
attack = fb.attacks.LinfDeepFoolAttack()
epsilons = np.linspace(0.0, 0.005, num=20)
images = ep.astensor(images)
labels = ep.astensor(labels)
criterion = fb.criteria.Misclassification(labels)
raw, clipped, is_adv = attack(fmodel, images, labels, epsilons=epsilons)
robust_accuracy = 1 - is_adv.float32().mean(axis=-1)
print("robust accuracy for perturbations with")
for eps, acc in zip(epsilons, robust_accuracy):
print(f" Linf norm ≤ {eps:<6}: {acc.item() * 100:4.1f} %")
plt.plot(epsilons, robust_accuracy.numpy())
def exp_balckbox_attack():
model = model_name(pretrained=True).eval()
preprocessing = dict(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
axis=-3)
fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
total_success = 0
widgets = [
'train :',
Percentage(), ' ',
Bar('#'), ' ',
Timer(), ' ',
ETA(), ' ',
FileTransferSpeed()
]
pbar = ProgressBar(widgets=widgets)
length = len(train_loader)
print("length train loader: ", length)
total_imgs = 0
for batch_data in pbar(train_loader):
images, labels = batch_data['image'].to(
DEVICE), batch_data['label_idx'].to(DEVICE)
acc = accuracy(fmodel, images, labels)
print(images.shape[0])
total_imgs += images.shape[0]
total_success += int(acc * images.shape[0])
print("success attack is ", str(total_imgs - total_success),
"/" + str(total_imgs))
print("watermark attack rate is: ", str(1 - total_success / total_imgs))
def main() -> None:
# instantiate a model (could also be a TensorFlow or JAX model)
model = models.resnet18(pretrained=True).eval()
preprocessing = dict(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
axis=-3)
fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
# get data and test the model
# wrapping the tensors with ep.astensors is optional, but it allows
# us to work with EagerPy tensors in the following
images, labels = ep.astensors(
*samples(fmodel, dataset="imagenet", batchsize=16))
clean_acc = accuracy(fmodel, images, labels) * 100
print(f"clean accuracy: {clean_acc:.1f} %")
# the attack trys a combination of specified rotations and translations to an image
# stops early if adversarial shifts and translations for all images are found
attack = fa.SpatialAttack(
max_translation=6, # 6px so x in [x-6, x+6] and y in [y-6, y+6]
num_translations=6, # number of translations in x, y.
max_rotation=20, # +- rotation in degrees
num_rotations=5, # number of rotations
# max total iterations = num_rotations * num_translations**2
)
# report the success rate of the attack (percentage of samples that could
# be adversarially perturbed) and the robust accuracy (the remaining accuracy
# of the model when it is attacked)
xp_, _, success = attack(fmodel, images, labels)
suc = success.float32().mean().item() * 100
print(f"attack success: {suc:.1f} %"
" (for the specified rotation and translation bounds)")
print(f"robust accuracy: {100 - suc:.1f} %"
" (for the specified rotation and translation bounds)")
def initialize_model():
model = download_model(model_name)
# normalizacja do sredniej i odchylenia standardowego zbioru ImageNet
preprocessing = dict(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
axis=-3)
return PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--steps',
type=int,
default=1000,
help='Maximum number of steps to perform')
parser.add_argument('--targeted',
action='store',
default=False,
help='For targeted attack')
args = parser.parse_args()
model = Net()
model.load_state_dict(torch.load('mnist_cnn.pt'))
model.eval()
preprocessing = dict(mean=0.1307, std=0.3081)
fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
fmodel = fmodel.transform_bounds((0, 1))
assert fmodel.bounds == (0, 1)
images, labels = ep.astensors(
*samples(fmodel, dataset="mnist", batchsize=10))
print('Model accuracy on clean examples: {}'.format(
accuracy(fmodel, images, labels)))
if args.targeted:
target_class = (labels + 7) % 10
criterion = fb.criteria.TargetedMisclassification(target_class)
else:
criterion = fb.criteria.Misclassification(labels)
attack = fa.L2DeepFoolAttack(steps=args.steps)
epsilons = None
raw, clipped, success = attack(fmodel, images, labels, epsilons=epsilons)
robust_accuracy = 1 - success.float32().mean()
print("Robust Accuracy", robust_accuracy.item())
dist = np.mean(fb.distances.l2(clipped, images).numpy())
print("Average perturbation norm", dist)
def get_model():
model = models.resnet18(pretrained=True).eval()
mean = torch.Tensor([0.485, 0.456, 0.406])
std = torch.Tensor([0.229, 0.224, 0.225])
if torch.cuda.is_available():
mean = mean.cuda(0)
std = std.cuda(0)
preprocessing = dict(mean=mean, std=std, axis=-3)
fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
return fmodel
def run_attacks(MODEL_DIR, res_path):
rel_dirs = [x for x in os.listdir(MODEL_DIR) if '2020' in x]
alpha = [re.findall('a=([0-9, \.]*)_', d)[0] for d in rel_dirs]
res = dict.fromkeys(alpha)
learner = prep_learner()
for model_path, curr_alpha in tqdm(zip(rel_dirs, alpha), total=len(alpha)):
conf.save_path = Path(path.join(MODEL_DIR, model_path))
fix_str = [
x for x in os.listdir(path.join(MODEL_DIR, model_path))
if 'model' in x
][0][8:]
learner.load_state(conf,
fix_str,
model_only=True,
from_save_folder=True)
# probs
set_probes(learner)
for model in learner.models:
model = torch.nn.DataParallel(model.cuda(),
device_ids=list(range(4)))
model.eval()
res[curr_alpha] = dict()
for (attack,
eps), attack_name in tqdm(zip(attack_list, attack_list_names),
desc='attaking ' + str(curr_alpha),
total=len(attack_list)):
fmodel = JointModelEP(
[PyTorchModel(m, bounds=(0, 1)) for m in learner.models],
'cuda')
attack = attack()
success_tot = []
for images, labels in tqdm(learner.eval_loader,
total=len(learner.eval_loader),
desc=attack_name):
images, labels = ep.astensors(images.to('cuda'),
labels.to('cuda'))
_, _, success = attack(fmodel, images, labels, epsilons=eps)
success_tot.append(success)
success_tot = ep.concatenate(success_tot, -1)
# calculate and report the robust accuracy
robust_accuracy = 1 - success_tot.float32().mean(axis=-1)
for epsilon, acc in zip(eps, robust_accuracy):
res[curr_alpha][attack_name + '_' + str(epsilon)] = acc.item()
pickle.dump(res, open(res_path, 'wb'))
pickle.dump(res, open(res_path, 'wb'))
def test_attack(model,
data_loader,
attack_name,
epsilon_values,
args,
device='cpu'):
model.eval()
attack_model = attacks[attack_name]
# For adversarial testing, the pre-processing happens in the foolbox wrapper.
if args['dataset'] in ('mnist', 'fmnist'):
preprocessing = None
elif args['dataset'] == 'cifar10':
preprocessing = dict(mean=(0.4914, 0.4822, 0.4465),
std=(0.2023, 0.1994, 0.2010),
axis=-3)
elif args['dataset'] == 'cifar100':
preprocessing = dict(mean=(0.5071, 0.4867, 0.4408),
std=(0.2675, 0.2565, 0.2761),
axis=-3)
elif args['dataset'] == 'svhn':
preprocessing = dict(mean=(0.5, 0.5, 0.5),
std=(0.5, 0.5, 0.5),
axis=-3)
else:
raise NotImplementedError('Dataset not supported.')
fbox_model = PyTorchModel(model,
bounds=(0, 1),
device=device,
preprocessing=preprocessing)
success_cum = []
for data, target in data_loader:
data = data.to(device)
target = target.to(device)
if attack_name in ('FGSM', 'PGD', 'BIM', 'C&W'):
advs, _, success = attack_model(fbox_model,
data,
target,
epsilons=epsilon_values,
mc=args['monte_carlo_runs'])
elif attack_name in ('Few-Pixel', ):
# Since the few-pixel attack is not supported by foolbox, we have a different testing pipeline.
raise NotImplementedError()
success_cum.append(success)
success_cum = torch.cat(success_cum, dim=1)
robust_accuracy = 1 - success_cum.float().mean(axis=-1)
return robust_accuracy
def main() -> None:
print(DEVICE)
# instantiate a model (could also be a TensorFlow or JAX model)
model = model_name(pretrained=True).eval()
# preprocessing = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], axis=-3)
# image_mean = torch.tensor([0.491, 0.482, 0.447]).view(1, 3, 1, 1)
# image_std = torch.tensor([0.247, 0.243, 0.262]).view(1, 3, 1, 1)
preprocessing = dict(mean=[0.491, 0.482, 0.447],
std=[0.247, 0.243, 0.262],
axis=-3)
fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
total_success = 0
# headers = ['img', 'advIndex', 'alpha ', 'angle', 'class', 'l2']
# with open('result_inception_v3_gen4_30.csv', 'w')as f:
# f_csv = csv.writer(f)
# f_csv.writerow(headers)
attack = advDigitalMark()
# SpatialAttack()
epoch = 0
length = len(train_loader)
print("length train loader: ", length)
total_msg = ''
for batch_data in train_loader:
# if epoch == 1:
# break
images, labels, filenames = batch_data['image'].to(
DEVICE), batch_data['label_idx'].to(DEVICE), batch_data['filename']
clean_acc = accuracy(fmodel, images, labels)
logging.info("\nepoch " + str(epoch) + ":\nclean accuracy: " +
str(clean_acc))
total_msg += "\nepoch " + str(epoch) + ":\nclean accuracy: " + str(
clean_acc)
count, msg = attack.run(fmodel, images, labels, filenames=filenames)
total_success += count
total_msg += msg
logging.info("current success attack is " + str(total_success))
send_email(total_msg, title='fsy天池比赛V1模型3000-5000结果报告')
epoch += 1
total_msg += "\n total attack success img is " + str(total_success)
# logging.info("total succes is "+str(total_success)+"/100")
send_email(total_msg, title='fsy天池比赛V1模型3000-5000结果报告')
def exp_otherAttack():
model = model_name(pretrained=True).eval()
preprocessing = dict(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
axis=-3)
fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
total_success = 0
attack = SpatialAttack()
epoch = 0
widgets = [
'train :',
Percentage(), ' ',
Bar('#'), ' ',
Timer(), ' ',
ETA(), ' ',
FileTransferSpeed()
]
pbar = ProgressBar(widgets=widgets)
length = len(train_loader)
print("length train loader: ", length)
for batch_data in pbar(train_loader):
if epoch == 25:
break
images, labels = batch_data['image'].to(
DEVICE), batch_data['label_idx'].to(DEVICE)
clean_acc = accuracy(fmodel, images, labels)
print(f"\nepoch " + str(epoch) + ":\nclean accuracy: ", clean_acc)
epsilons = [1]
_, _, isSuceess = attack(fmodel, images, labels, epsilons=epsilons)
isSuceess = isSuceess.cpu().numpy()
total_success += np.sum(isSuceess == True)
print("current success : ", total_success)
# total_success += int(adv_acc*10)
epoch += 1
print("success attack is ", str(total_success))
def prepare_server_model(self, ensemble_info):
if self.args.aggr == "fedavg":
server = self.server.client_list[self.adv_client_idx]
branch_w = self.server.branches[self.adv_branch_idx]
server.model_trainer.set_model_params(branch_w)
self.server_model = copy.deepcopy(server.model_trainer.model)
elif self.args.aggr == "blockensemble":
self.server_model = self.server.server_model
elif self.args.aggr == "heteroensemble":
self.server_model = self.server.server_model
# server_model = self.server.server_model
# self.server_model = HeteroFeatAvgEnsembleDefense(
# server_model, ensemble_info
# )
else:
raise NotImplementedError
self.server_model.eval()
self.server_model = PyTorchModel(
self.server_model, bounds = self.bounds,
preprocessing = self.preprocessing
)
#!/usr/bin/env python3
import torchvision.models as models
import eagerpy as ep
from foolbox import PyTorchModel, accuracy, samples
import foolbox.attacks as fa
import numpy as np
if __name__ == "__main__":
# instantiate a model (could also be a TensorFlow or JAX model)
model = models.resnet18(pretrained=True).eval()
preprocessing = dict(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
axis=-3)
fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
# get data and test the model
# wrapping the tensors with ep.astensors is optional, but it allows
# us to work with EagerPy tensors in the following
images, labels = ep.astensors(
*samples(fmodel, dataset="imagenet", batchsize=16))
clean_acc = accuracy(fmodel, images, labels)
print(f"clean accuracy: {clean_acc * 100:.1f} %")
print("")
attacks = [
fa.FGSM(),
fa.LinfPGD(),
fa.LinfBasicIterativeAttack(),
fa.LinfAdditiveUniformNoiseAttack(),
fa.LinfDeepFoolAttack(),
]
def main_worker(gpu, ngpus_per_node, args):
args.gpu = gpu
attack = args.attack
os.makedirs(args.output, exist_ok=True)
acc_path = os.path.join(args.output, f'{args.attack}.txt')
if args.gpu is not None:
print("Use GPU: {} for training".format(args.gpu))
if args.distributed:
if args.dist_url == "env://" and args.rank == -1:
args.rank = int(os.environ["RANK"])
if args.multiprocessing_distributed:
# For multiprocessing distributed training, rank needs to be the
# global rank among all the processes
args.rank = args.rank * ngpus_per_node + gpu
dist.init_process_group(backend=args.dist_backend,
init_method=args.dist_url,
world_size=args.world_size,
rank=args.rank)
# create model
if 'imagenet_100' in args.data:
num_classes = 100
else:
num_classes = 1000
if args.rank == 0:
print("%s-way classification" % num_classes)
if args.pretrained:
print("=> using pre-trained model '{}'".format(args.arch))
model_name = str(args.arch)
model = models_vgg.__dict__[args.arch](pretrained=True,
num_classes=num_classes)
else:
print("=> creating model '{}'".format(args.arch))
model_name = str(args.arch)
model = models_vgg.__dict__[model_name](pretrained=False,
num_classes=num_classes)
if not torch.cuda.is_available():
print('using CPU, this will be slow')
elif args.distributed:
# For multiprocessing distributed, DistributedDataParallel constructor
# should always set the single device scope, otherwise,
# DistributedDataParallel will use all available devices.
if args.gpu is not None:
torch.cuda.set_device(args.gpu)
model.cuda(args.gpu)
# When using a single GPU per process and per
# DistributedDataParallel, we need to divide the batch size
# ourselves based on the total number of GPUs we have
args.batch_size = int(args.batch_size / ngpus_per_node)
args.workers = int(
(args.workers + ngpus_per_node - 1) / ngpus_per_node)
model = torch.nn.parallel.DistributedDataParallel(
model, device_ids=[args.gpu])
else:
model.cuda()
# DistributedDataParallel will divide and allocate batch_size to all
# available GPUs if device_ids are not set
model = torch.nn.parallel.DistributedDataParallel(model)
elif args.gpu is not None:
torch.cuda.set_device(args.gpu)
model = model.cuda(args.gpu)
else:
# DataParallel will divide and allocate batch_size to all available GPUs
if args.arch.startswith('alexnet') or args.arch.startswith('vgg'):
model.features = torch.nn.DataParallel(model.features)
model.cuda()
else:
model = torch.nn.DataParallel(model).cuda()
print("=> loading checkpoint '{}'".format(args.checkpoint))
checkpoint = {'model': {}}
if args.gpu is None:
checkpoint_old = torch.load(args.checkpoint)
else:
# Map model to be loaded to specified single gpu.
loc = 'cuda:{}'.format(args.gpu)
checkpoint_old = torch.load(args.checkpoint, map_location=loc)
for value in checkpoint_old['model']:
if 'module' not in value:
name = 'module.' + value
checkpoint['model'][name] = checkpoint_old['model'][value]
else:
checkpoint['model'][value] = checkpoint_old['model'][value]
model.load_state_dict(checkpoint['model'], strict=False)
print("=> loaded checkpoint '{}' (epoch {})".format(
args.checkpoint, checkpoint_old['epoch']))
model.eval()
preprocessing = dict(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
axis=-3)
val_loader = torch.utils.data.DataLoader(datasets.ImageFolder(
args.data,
transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(160),
transforms.ToTensor(),
])),
batch_size=1,
shuffle=False)
fmodel = PyTorchModel(model,
bounds=(0, 1),
preprocessing=preprocessing,
device="cuda")
# apply the attack
attack = globals()[attack]()
epsilons = [
0.0000,
0.0002,
0.0004,
0.0006,
0.0008,
0.0010,
0.0012,
0.0014,
0.0016,
0.0018,
0.0020,
]
clean_acc, robust_acc = validate(fmodel=fmodel,
val_loader=val_loader,
attack=attack,
epsilons=epsilons)
f = open(acc_path, 'a')
print(f"clean accuracy: {clean_acc} %", file=f)
print("robust accuracy for perturbations with", file=f)
for eps, acc in zip(epsilons, robust_acc):
print(f" {args.attack} norm ≤ {eps:<6}: {acc * 100} %", file=f)
f.close()
elif params.get('metric') == 'mahalanobis':
psd_matrix = np.loadtxt(params.get('psd_matrix_path'), )
psd_matrix = torch.tensor(psd_matrix,
dtype=torch.float,
device=params.get('device'))
else:
raise Exception('unsupported metric')
knn_module = MahalanobisKnnModule(X_train, y_train, params.getint('k'),
psd_matrix)
knn_module.to(params.get('device'))
knn_module.eval()
fmodel = PyTorchModel(knn_module, bounds=(0, 1), device=params.get('device'))
attack = BoundaryAttack()
n_eval = params.getint('n_eval')
perturbations_list = []
for i, (X_eval, y_eval) in enumerate(
zip(
torch.split(X_test[:n_eval], params.getint('attack_batch_size')),
torch.split(y_test[:n_eval], params.getint('attack_batch_size')),
)):
print(i)
_, advs, successful = attack(
fmodel,
X_eval,
y_eval,
params.cp_latest_filename = cp_latest_filenames[model_index]
train_loader, val_loader, test_loader, data_params = dataset_utils.load_dataset(
params)
for key, value in data_params.items():
setattr(params, key, value)
model = loaders.load_model(params.model_type)
model.setup(params, logger)
model.params.analysis_out_dir = os.path.join(
*[model.params.model_out_dir, 'analysis', model.params.version])
model.params.analysis_save_dir = os.path.join(
model.params.analysis_out_dir, 'savefiles')
if not os.path.exists(model.params.analysis_save_dir):
os.makedirs(model.params.analysis_save_dir)
model.to(params.device)
model.load_checkpoint()
fmodel = PyTorchModel(model.eval(), bounds=(0, 1))
print('\n', '~' * 79)
num_batches = len(test_loader.dataset) // model.params.batch_size
attack_success = np.zeros(
(len(attacks), len(epsilons), num_batches, model.params.batch_size),
dtype=np.bool)
for batch_index, (data, target) in enumerate(test_loader):
data = model.preprocess_data(data.to(model.params.device))
target = target.to(model.params.device)
images, labels = ep.astensors(*(data, target))
del data
del target
print(
f'Model type: {model.params.model_type} [{model_index+1} out of {len(log_files)}]'
)
print(f'Batch {batch_index+1} out of {num_batches}')
def run_attacks(res_path):
MORPH_MODEL_DIR = '/mnt/md0/orville/Miriam/modular-loss-experiments-morph/results_morph_correct/CIFAR-10/densenet-82-8-8'
MODEL_DIR = '/mnt/md0/orville/Miriam/modular-loss-experiments-morph/results/CIFAR-10/densenet-82-8-8'
# UNCORR_MODEL_DIR = 'alpha_0.0_gamma_0.0_n_models_2_1581641733617'
# CORR_MODEL_DIR = 'alpha_0.1_gamma_0.0_n_models_2_1581641746832'
# CORR_MODEL_DIR_2 = 'alpha_0.2_gamma_0.0_n_models_2_1581641777871'
UNCORR_MODEL_DIR = 'alpha_0.0_gamma_0.0_n_models_3_1585505819121'
CORR_MODEL_DIR = 'alpha_0.1_gamma_0.0_n_models_3_1585505685528'
CORR_MODEL_DIR_2 = 'alpha_0.2_gamma_0.0_n_models_3_1585505042819'
rel_dirs = [UNCORR_MODEL_DIR, CORR_MODEL_DIR, CORR_MODEL_DIR_2]
alpha = ['0', '0.1', '0.2']
res = dict.fromkeys(alpha)
batch_size = 516
n_workers = 20
dataset = 'CIFAR-10'
network = 'densenet-82-8-8'
loaders, _ = get_dataloaders_(batch_size, 0, dataset, False, early_stop=False, n_workers=n_workers)
n_models = 3
params = {}
params['densenet-82-8-8'] = {'num_modules': 2, 'bottleneck': True, 'reduction': 0.5, 'depth': 82, 'growth_rate': 8,
'input_shape': (3, 32, 32), 'output_dim': 10}
network = 'densenet-82-8-8'
model = DenseNet(input_shape=params[network]['input_shape'],
output_dim=params[network]['output_dim'],
growth_rate=params[network]['growth_rate'],
depth=params[network]['depth'],
reduction=params[network]['reduction'],
bottleneck=params[network]['bottleneck'],
num_modules=n_models)
device = torch.device("cuda")
for model_path, curr_alpha in tqdm(zip(rel_dirs, alpha), total=len(alpha)):
weight_path = path.join(MODEL_DIR, model_path, 'trial_0/0.0/weights/final_weights.pt')
model.reset_parameters()
model.load_state_dict(torch.load(weight_path))
model.eval() # model.train(mode=False)
fmodel = PyTorchModel(ModelMeanEP(model), bounds=(0, 1), device=device)
res[curr_alpha] = dict()
for (attack, eps), attack_name in tqdm(zip(attack_list, attack_list_names),
desc='attaking ' + str(curr_alpha), total=len(attack_list)):
attack = attack()
success_tot = []
for images, labels in tqdm(loaders['test'], total=len(loaders['test']), desc=attack_name):
images, labels = images.to(device), labels.to(device)
_, _, success = attack(fmodel, images, labels, epsilons=eps)
success_tot.append(success)
success_tot = torch.cat(success_tot, -1)
# calculate and report the robust accuracy
robust_accuracy = 1 - success_tot.float().mean(axis=-1)
for epsilon, acc in zip(eps, robust_accuracy):
res[curr_alpha][attack_name + '_' + str(epsilon)] = acc.item()
pickle.dump(res, open(res_path, 'wb'))
pickle.dump(res, open(res_path, 'wb'))
pickle.dump(res, open(res_path, 'wb'))
# for bn in bottlenecks:
# results[vdepth][bn] = dict()
results[n_bn] = dict()
results[n_bn][rep] = dict()
model = RetinalBottleneckModel(n_bn,
'resnet50',
n_out=1000,
n_inch=3,
retina_kernel_size=7,
transform=normalize)
model.load_state_dict(torch.load(dir + model_file + '.pt'))
# model = torch.hub.load('ecs-vlc/opponency:master', 'colour_full', n_bn=bn, d_vvs=vdepth, rep=run)
model.eval()
fmodel = PyTorchModel(model, bounds=(0, 1))
# images, labels = samples(fmodel, dataset="cifar10", batchsize=20)
# images = images.contiguous()
# results[vdepth][bn][run]["accuracy"] = accuracy(fmodel, images, labels)
attack_success = np.zeros((len(attacks), len(epsilons), len(testset)),
dtype=np.bool)
# for i, attack in enumerate(attacks):
print(attack)
idx = 0
for images, labels in tqdm(testloader):
images = images.to(fmodel.device)
labels = labels.to(fmodel.device)
class AdvAttack():
def __init__(self, server, device, args, adv_client_idx=0, adv_branch_idx=0):
self.server = server
server.set_client_dataset()
self.device = device
self.args = args
self.adv_client_idx = adv_client_idx
self.adv_branch_idx = adv_branch_idx
self.save_dir = osp.join(self.args.save_dir, f"{type(self).__name__}")
if not os.path.exists(self.save_dir):
os.makedirs(self.save_dir, exist_ok=True)
# set adv client model
params = self.set_attack_params()
ensemble_info = self.prepare_advclient_model()
self.prepare_server_model(ensemble_info)
# test clean accuracys)
# clean_acc = self.check_accuracy(
# self.advclient_model, self.server.test_global, self.device
# )
# logging.info(f"clean accuracy: {clean_acc * 100:.1f} %")
# st()
self.attack_fn = LinfPGD(**params)
# self.attack_fn = L2CarliniWagnerAttack(**params)
self.attack()
def set_attack_params(self):
if self.args.dataset in ["mnist", "fmnist", "emnist"]:
range = 1 / 0.3081
params = {
"steps": 10, "rel_stepsize": 0.1,
}
self.epsilon = range * 0.3
bound_min = (0-0.1307) / 0.3081
bound_max = (1-0.1307) / 0.3081
self.bounds = (bound_min, bound_max)
self.preprocessing = None
elif self.args.dataset in ['cifar10', 'cifar100']:
CIFAR_MEAN = torch.Tensor([0.49139968, 0.48215827, 0.44653124]).view(1,3,1,1).to(self.device)
CIFAR_STD = torch.Tensor([0.24703233, 0.24348505, 0.26158768]).view(1,3,1,1).to(self.device)
range = 1 / 0.3081
params = {
"steps": 20, "rel_stepsize": 0.1,
}
self.epsilon = 8/255
# bound_min = (0-0.1307) / 0.3081
# bound_max = (1-0.1307) / 0.3081
self.bounds = (0, 1)
self.preprocessing = {
"mean": CIFAR_MEAN, "std": CIFAR_STD
}
else:
raise NotImplementedError
return params
def check_accuracy(self, fmodel, dataloader, device) -> float:
total, correct = 0., 0.
with torch.no_grad():
for batch_idx, (x, target) in enumerate(dataloader):
x = x.to(device)
target = target.to(device)
if self.preprocessing is not None:
x = x * self.preprocessing["std"] + self.preprocessing["mean"]
predictions = fmodel(x).argmax(axis=-1)
total += len(x)
correct += (predictions == target).sum()
accuracy = correct / total
return accuracy.item()
def prepare_advclient_model(self):
if self.args.aggr == "fedavg":
adv_client = self.server.client_list[self.adv_client_idx]
branch_w = self.server.branches[self.adv_branch_idx]
adv_client.model_trainer.set_model_params(branch_w)
self.advclient_model = copy.deepcopy(adv_client.model_trainer.model)
ensemble_info = None
elif self.args.aggr == "blockensemble":
state_dict_pair, ensemble_info = self.server.prepare_branch_dict()
self.adv_client = self.server.client_list[self.adv_client_idx]
self.adv_client.model_trainer.set_model_params(state_dict_pair)
# self.advclient_model = TwoModelWarpper(
# self.adv_client.model_trainer.model1,
# self.adv_client.model_trainer.model2,
# )
self.advclient_model = self.adv_client.model_trainer.model2
elif self.args.aggr == "heteroensemble":
model_pair, ensemble_info = self.server.prepare_branch_dict(client_idx=0)
self.adv_client = self.server.client_list[self.adv_client_idx]
self.advclient_model = TwoModelWarpper(
model_pair[0],
model_pair[1],
)
# self.advclient_model = model_pair[0]
else:
raise NotImplementedError
self.advclient_model.eval()
self.advclient_model = PyTorchModel(
self.advclient_model, bounds = self.bounds,
preprocessing = self.preprocessing
)
return ensemble_info
def prepare_server_model(self, ensemble_info):
if self.args.aggr == "fedavg":
server = self.server.client_list[self.adv_client_idx]
branch_w = self.server.branches[self.adv_branch_idx]
server.model_trainer.set_model_params(branch_w)
self.server_model = copy.deepcopy(server.model_trainer.model)
elif self.args.aggr == "blockensemble":
self.server_model = self.server.server_model
elif self.args.aggr == "heteroensemble":
self.server_model = self.server.server_model
# server_model = self.server.server_model
# self.server_model = HeteroFeatAvgEnsembleDefense(
# server_model, ensemble_info
# )
else:
raise NotImplementedError
self.server_model.eval()
self.server_model = PyTorchModel(
self.server_model, bounds = self.bounds,
preprocessing = self.preprocessing
)
def attack(self):
total, correct, success, server_success = 0, 0, 0, 0
client_total, server_total = 0, 0
fmodel = self.advclient_model
server_model = self.server_model
dataloader = self.server.test_global
for batch_idx, (x, target) in enumerate(dataloader):
x = x.to(self.device)
target = target.to(self.device)
if self.preprocessing is not None:
x = x * self.preprocessing["std"] + self.preprocessing["mean"]
# x = x * 255
clean_pred = fmodel(x).argmax(axis=-1)
pred_correct = clean_pred == target
correct += (pred_correct).sum().float()
raw_advs, clipped_advs, _success = self.attack_fn(fmodel, x, clean_pred, epsilons=self.epsilon)
# total += len(x)
# success += (_success).sum().float()
# server_pred = server_model(clipped_advs).argmax(axis=-1)
# server_wrong = (server_pred != target)
# server_success += server_wrong.sum().float()
# success += (_success*pred_correct).sum().float()
server_clean_pred = server_model(x).argmax(axis=-1)
client_valid = (clean_pred == target)
client_total += client_valid.sum().float()
success += (_success * client_valid).sum().float()
server_adv_pred = server_model(clipped_advs).argmax(axis=-1)
server_valid = (server_clean_pred == target)
server_total += server_valid.sum().float()
server_wrong = (server_adv_pred != target) * server_valid
server_success += server_wrong.sum().float()
# print(f"Total {total}, success {success}, server success {server_success}")
# st()
# diff = (x-clipped_advs).max()
# for MNIST
# x = x * 0.3081 + 0.1307
# clipped_advs = clipped_advs * 0.3081 + 0.1307
# for idx in range(5):
# img, adv_img = x[idx], clipped_advs[idx]
# img = (img*255).cpu().numpy().astype(np.uint8)
# adv_img = (adv_img*255).cpu().numpy().astype(np.uint8)
# img = np.concatenate([img, adv_img], axis=2)
# img = np.transpose(img, (1,2,0))
# path = osp.join("debug", f"{idx}.png")
# from PIL import Image
# for MNIST
# img = img.repeat(3, axis=2)
# im = Image.fromarray(img,)
# im.save(path)
# diff = (x - clipped_advs).max()*255
# st()
success_rate = (success / client_total).item()
server_success_rate = (server_success / server_total).item()
clean_acc = (correct / total).item()
logging.info(f"clean acc: {clean_acc*100:.1f}%, client success rate: {success_rate * 100:.1f}%, server success rate: {server_success_rate*100:.1f}%" )
return clean_acc, success_rate, server_success_rate
def foolbox_attack(filter=None,
filter_preserve='low',
free_parm='eps',
plot_num=None):
# get model.
model = get_model()
model = nn.DataParallel(model).to(device)
model = model.eval()
preprocessing = dict(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
axis=-3)
fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
if plot_num:
free_parm = ''
val_loader = get_val_loader(plot_num)
else:
# Load images.
val_loader = get_val_loader(args.attack_batch_size)
if 'eps' in free_parm:
epsilons = [0.001, 0.003, 0.005, 0.008, 0.01, 0.1]
else:
epsilons = [0.01]
if 'step' in free_parm:
steps = [1, 5, 10, 30, 40, 50]
else:
steps = [args.iteration]
for step in steps:
# Adversarial attack.
if args.attack_type == 'LinfPGD':
attack = LinfPGD(steps=step)
elif args.attack_type == 'FGSM':
attack = FGSM()
clean_acc = 0.0
for i, data in enumerate(val_loader, 0):
# Samples (attack_batch_size * attack_epochs) images for adversarial attack.
if i >= args.attack_epochs:
break
images, labels = data[0].to(device), data[1].to(device)
if step == steps[0]:
clean_acc += (get_acc(
fmodel, images, labels
)) / args.attack_epochs # accumulate for attack epochs.
_images, _labels = ep.astensors(images, labels)
raw_advs, clipped_advs, success = attack(fmodel,
_images,
_labels,
epsilons=epsilons)
if plot_num:
grad = torch.from_numpy(
raw_advs[0].numpy()).to(device) - images
grad = grad.clone().detach_()
return grad
if filter:
robust_accuracy = torch.empty(len(epsilons))
for eps_id in range(len(epsilons)):
grad = torch.from_numpy(
raw_advs[eps_id].numpy()).to(device) - images
grad = grad.clone().detach_()
freq = dct.dct_2d(grad)
if filter_preserve == 'low':
mask = torch.zeros(freq.size()).to(device)
mask[:, :, :filter, :filter] = 1
elif filter_preserve == 'high':
mask = torch.zeros(freq.size()).to(device)
mask[:, :, filter:, filter:] = 1
masked_freq = torch.mul(freq, mask)
new_grad = dct.idct_2d(masked_freq)
x_adv = torch.clamp(images + new_grad, 0, 1).detach_()
robust_accuracy[eps_id] = (get_acc(fmodel, x_adv, labels))
else:
robust_accuracy = 1 - success.float32().mean(axis=-1)
if i == 0:
robust_acc = robust_accuracy / args.attack_epochs
else:
robust_acc += robust_accuracy / args.attack_epochs
if step == steps[0]:
print("sample size is : ",
args.attack_batch_size * args.attack_epochs)
print(f"clean accuracy: {clean_acc * 100:.1f} %")
print(
f"Model {args.model} robust accuracy for {args.attack_type} perturbations with"
)
for eps, acc in zip(epsilons, robust_acc):
print(
f" Step {step}, Linf norm ≤ {eps:<6}: {acc.item() * 100:4.1f} %"
)
print(' -------------------')
for epoch in range(1, epochs + 1):
print("Epoch #", epoch)
y_pred = model.forward(x_train)
loss = loss_fn(y_pred.squeeze(), y_train)
print_(loss.item())
optimizer.zero_grad()
loss.backward()
optimizer.step()
print("Training error:",
mean_squared_error(model(x_train).detach().numpy(), y_train))
y_pred = model(x_test).detach().numpy()
print("Test error:", mean_squared_error(y_pred, y_test))
fmodel = PyTorchModel(model.eval(), bounds=(-1, 2))
attack = L2AdditiveGaussianNoiseAttack()
epsilons = np.linspace(0.0, 6.0, num=6, endpoint=False)
advs, _, success = attack(fmodel, x_final, y_final, epsilons=epsilons)
perturbed = np.zeros((len(X), 7))
added = np.zeros(len(X))
print("Finding adversarial inputs")
for eps, adv in zip(epsilons, advs):
if 0 not in added:
break
pred = model(adv)
pred = pred.detach().numpy().ravel()
pred_binned = np.digitize(pred, bins)
def exp1():
print(DEVICE)
parser = argparse.ArgumentParser(
description='CIFAR10 Training against DDN Attack')
parser.add_argument('--data',
default='../defenses/data/cifar10',
help='path to dataset')
parser.add_argument('--workers',
default=2,
type=int,
help='number of data loading workers')
parser.add_argument('--cpu',
action='store_true',
help='force training on cpu')
parser.add_argument('--save-folder',
'--sf',
default='weights/cifar10_inception_v3/',
help='folder to save state dicts')
parser.add_argument('--save-freq',
'--sfr',
default=10,
type=int,
help='save frequency')
parser.add_argument('--save-name',
'--sn',
default='cifar10',
help='name for saving the final state dict')
parser.add_argument('--batch-size',
'-b',
default=20,
type=int,
help='mini-batch size')
parser.add_argument('--epochs',
'-e',
default=100,
type=int,
help='number of total epochs to run')
parser.add_argument('--lr',
'--learning-rate',
default=0.001,
type=float,
help='initial learning rate')
parser.add_argument('--lr-decay',
'--lrd',
default=0.2,
type=float,
help='decay for learning rate')
parser.add_argument('--lr-step',
'--lrs',
default=10,
type=int,
help='step size for learning rate decay')
parser.add_argument('--momentum', default=0.9, type=float, help='momentum')
parser.add_argument('--weight-decay',
'--wd',
default=5e-4,
type=float,
help='weight decay')
parser.add_argument('--drop',
default=0.3,
type=float,
help='dropout rate of the classifier')
parser.add_argument('--adv',
type=int,
default=None,
help='epoch to start training with adversarial images')
parser.add_argument('--max-norm',
type=float,
default=1,
help='max norm for the adversarial perturbations')
parser.add_argument('--steps',
default=10,
type=int,
help='number of steps for the attack')
parser.add_argument(
'--visdom-port',
'--vp',
type=int,
default=8097,
help='For visualization, which port visdom is running.')
parser.add_argument('--print-freq',
'--pf',
default=10,
type=int,
help='print frequency')
args = parser.parse_args()
print(args)
train_transform = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
])
test_transform = transforms.Compose([
transforms.ToTensor(),
])
train_set = data.Subset(
CIFAR10(args.data,
train=True,
transform=train_transform,
download=True), list(range(30000, 50000)))
val_set = data.Subset(
CIFAR10(args.data, train=True, transform=test_transform,
download=True), list(range(48000, 50000)))
test_set = CIFAR10(args.data,
train=False,
transform=test_transform,
download=True)
train_loader = data.DataLoader(train_set,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
drop_last=True,
pin_memory=True)
val_loader = data.DataLoader(val_set,
batch_size=100,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
test_loader = data.DataLoader(test_set,
batch_size=100,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
image_mean = torch.tensor([0.491, 0.482, 0.447]).view(1, 3, 1, 1)
image_std = torch.tensor([0.247, 0.243, 0.262]).view(1, 3, 1, 1)
m = AlexNet().eval()
model = NormalizedModel(model=m, mean=image_mean, std=image_std).to(
DEVICE) # keep images in the [0, 1] range
model_file = '/home/frankfeng/researchData/code/adversarial_training_code/PLP/fast_adv/defenses/weights/cifar10_AlexNet/cifar10_valacc0.8019999772310257.pth'
model_dict = torch.load(model_file)
model.load_state_dict(model_dict)
# preprocessing = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], axis=-3)
# image_mean = torch.tensor([0.491, 0.482, 0.447]).view(1, 3, 1, 1)
# image_std = torch.tensor([0.247, 0.243, 0.262]).view(1, 3, 1, 1)
preprocessing = dict(mean=[0.491, 0.482, 0.447],
std=[0.247, 0.243, 0.262],
axis=-3)
fmodel = PyTorchModel(model, bounds=(0, 1))
total_success = 0
# headers = ['img', 'advIndex', 'alpha ', 'angle', 'class', 'l2']
# with open('result_inception_v3_gen4_30.csv', 'w')as f:
# f_csv = csv.writer(f)
# f_csv.writerow(headers)
attack = advDigitalMark()
# SpatialAttack()
epoch = 0
widgets = [
'train :',
Percentage(), ' ',
Bar('#'), ' ',
Timer(), ' ',
ETA(), ' ',
FileTransferSpeed()
]
pbar = ProgressBar(widgets=widgets)
length = len(train_loader)
print("length train loader: ", length)
for epoch in range(args.epochs):
if epoch == 25:
break
for i, (images, labels) in enumerate(tqdm.tqdm(train_loader,
ncols=80)):
images, labels = images.to(DEVICE), labels.to(DEVICE)
clean_acc = accuracy(fmodel, images, labels)
print(f"\nepoch " + str(epoch) + ":\nclean accuracy: ", clean_acc)
count = attack.run(fmodel, images, labels)
total_success += count
epoch += 1
print("current success attack is ", str(total_success))
print(
str(model_name) + "_" + str(max_gen) + "_" + str(max_gen) +
": total attack success img is ", str(total_success))
labels.append(label)
return image_paths, labels
num_classes = 2
batchsize = 16
data_dir = "../cat_dog/val"
classifier_path = "../ckpts1/best_classifier.pth"
image_suffix = ".jpg"
classfier = resnet18(pretrained=False, num_classes=num_classes)
classfier.load_state_dict(torch.load(classifier_path, map_location="cpu"))
classfier.eval()
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
accuracys = []
fmodel = PyTorchModel(classfier, bounds=(0, 1), device=device)
image_paths, labels = get_datas(data_dir, image_suffix)
eval_dataset = PALMClassifyDataset(image_paths, labels)
eval_loader = torch.utils.data.DataLoader(eval_dataset,
batch_size=batchsize,
num_workers=0,
shuffle=False)
preds = []
for x, _ in eval_loader:
x = x.to(device)
pred = fmodel(x).argmax(axis=-1).tolist()
preds += pred
acc_clean = accuracy_score(labels, preds)
print(f"clean accuracy: {acc_clean * 100:.2f} %")
def main() -> None:
# instantiate a model (could also be a TensorFlow or JAX model)
#model = models.resnet18(pretrained=True).eval()
#model=torch.load('/data1/zyh/copycat/Framework/cifar_model.pth')
model = AlexNet()
path = "./cifar_net.pth"
#path = '/data1/zyh/copycat/Framework/cifar_model.pth'
#model.load_state_dict(torch.load('/data1/zyh/copycat/Framework/cifar_model.pth'))
#pretrained_dict = {k: v for k, v in model_pretrained.items() if k in model_dict}
#model_dict.update(pretrained_dict)
#model.load_state_dict(state_dict)
model.load_state_dict(torch.load(path), strict=True)
model = model.to(device)
model.eval()
print(type(model))
#preprocessing = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], axis=-3)
preprocessing = dict(mean=[0.5] * 3, std=[0.5] * 3, axis=-3)
fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
# get data and test the model
# wrapping the tensors with ep.astensors is optional, but it allows
# us to work with EagerPy tensors in the following
test_dataset = torchvision.datasets.CIFAR10(
root='~/.torch/',
train=False,
#transform = transforms.Compose([transforms.Resize((256,256)),transforms.ToTensor()]),
transform=transforms.Compose([transforms.ToTensor()]),
download=True)
# test_dataset .data = test_dataset.data[:128*5]
test_loader = torch.utils.data.DataLoader(
dataset=test_dataset,
batch_size=128, #该参数表示每次读取的批样本个数
shuffle=False) #该参数表示读取时是否打乱样本顺序
# 创建迭代器
#data_iter = iter(test_loader)
#images, labels = next(data_iter)
# 当迭代开始时, 队列和线程开始读取数据
#images, labels = data_iter.next()
#im=images
#images=im.resize(100,3,128,128)
with torch.no_grad():
all_clean_acc_foolbox = []
## native predict
predict_func(test_loader, model)
for ii, (imgs, lbls) in tqdm.tqdm(enumerate(test_loader),
total=len(test_loader)):
imgs = imgs.to(device)
lbls = lbls.to(device)
images, labels = ep.astensors(imgs, lbls)
## calc with foolbox
pred_lbl_foolbox = fmodel(images)
clean_acc_one = accuracy(fmodel, imgs, lbls)
all_clean_acc_foolbox.append(clean_acc_one)
clean_acc = sum(all_clean_acc_foolbox) / len(all_clean_acc_foolbox)
print(f"clean accuracy: {clean_acc * 100:.1f} %")
# apply the attack
attack = LinfPGD()
'''epsilons = [
0.0,
0.0002,
0.0005,
0.0008,
0.001,
0.0015,
0.002,
0.003,
0.01,
0.1,
0.3,
0.5,
1.0,
]'''
epsilons = [
0.0005,
0.001,
0.002,
0.01,
0.1,
]
def attack_one_batch(fmodel, images, labels, iter=0, verbose=True):
images, labels = ep.astensors(images, labels)
raw_advs, clipped_advs, success = attack(fmodel,
images,
labels,
epsilons=epsilons)
if verbose: print("===" * 8, iter, "===" * 8)
if verbose:
robust_accuracy = 1 - success.float32().mean(axis=-1)
print("robust accuracy for perturbations with")
for eps, acc in zip(epsilons, robust_accuracy):
print(f" Linf norm ≤ {eps:<6}: {acc.item() * 100:4.1f} %")
if verbose:
fig = plt.gcf()
os.makedirs("./image/", exist_ok=True)
for i in range(len(raw_advs)):
img_v = raw_advs[i].raw
torchvision.utils.save_image(
img_v,
f'./image/{str(iter).zfill(4)}_{str(i).zfill(3)}_.png')
return [x.raw for x in raw_advs] #
print("====" * 8, "start attack", "====" * 8)
collection_adv = []
collection_gt = []
for ii, (imgs, lbls) in tqdm.tqdm(enumerate(test_loader),
total=len(test_loader)):
imgs = imgs.to(device)
lbls = lbls.to(device)
# images, labels = ep.astensors(images,labels)
adv_ret = attack_one_batch(fmodel=fmodel,
images=imgs,
labels=lbls,
iter=ii,
verbose=True)
collection_adv.append(torch.stack(adv_ret))
collection_gt.append(lbls.cpu())
print("====" * 8, "start evaluation", "====" * 8)
with torch.no_grad():
adv_total_dataset = torch.cat(collection_adv, dim=1)
lbl_total_dataset = torch.cat(collection_gt).to(device)
# print (adv_total_dataset.mean(dim=(1,2,3,4)),"the mean if each eps")
for (eps, ep_adv_dataset) in zip(epsilons, adv_total_dataset):
# print ("eps:",eps,"===>"*8)
# print (ep_adv_dataset.mean(),"each...")
advs_ = ep_adv_dataset.to(device)
acc2 = accuracy(fmodel, advs_, lbl_total_dataset)
print(f" Linf norm ≤ {eps:<6}: {acc2 * 100:4.1f} %")
dataset = torch.utils.data.TensorDataset(ep_adv_dataset,
lbl_total_dataset)
dl = torch.utils.data.DataLoader(dataset, batch_size=128)
predict_func(dl, model)
def main() -> None:
# instantiate a model (could also be a TensorFlow or JAX model)
model = models.resnet18(pretrained=True).eval()
preprocessing = dict(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
axis=-3)
fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
# get data and test the model
# wrapping the tensors with ep.astensors is optional, but it allows
# us to work with EagerPy tensors in the following
images, labels = ep.astensors(
*samples(fmodel, dataset="imagenet", batchsize=16))
clean_acc = accuracy(fmodel, images, labels)
print(f"clean accuracy: {clean_acc * 100:.1f} %")
# replace the gradient with the gradient from another model
model2 = fmodel # demo, we just use the same model
# TODO: this is still a bit annoying because we need
# to overwrite run to get the labels
class Attack(LinfPGD):
def value_and_grad(self, loss_fn, x):
val1 = loss_fn(x)
loss_fn2 = self.get_loss_fn(model2, self.labels)
_, grad2 = ep.value_and_grad(loss_fn2, x)
return val1, grad2
def run(self, model, inputs, criterion, *, epsilon, **kwargs):
criterion_ = get_criterion(criterion)
self.labels = criterion_.labels
return super().run(model,
inputs,
criterion_,
epsilon=epsilon,
**kwargs)
# apply the attack
attack = Attack()
epsilons = [
0.0,
0.0002,
0.0005,
0.0008,
0.001,
0.0015,
0.002,
0.003,
0.01,
0.1,
0.3,
0.5,
1.0,
]
raw_advs, clipped_advs, success = attack(fmodel,
images,
labels,
epsilons=epsilons)
# calculate and report the robust accuracy (the accuracy of the model when
# it is attacked)
robust_accuracy = 1 - success.float32().mean(axis=-1)
print("robust accuracy for perturbations with")
for eps, acc in zip(epsilons, robust_accuracy):
print(f" Linf norm ≤ {eps:<6}: {acc.item() * 100:4.1f} %")
# we can also manually check this
# we will use the clipped advs instead of the raw advs, otherwise
# we would need to check if the perturbation sizes are actually
# within the specified epsilon bound
print()
print("we can also manually check this:")
print()
print("robust accuracy for perturbations with")
for eps, advs_ in zip(epsilons, clipped_advs):
acc2 = accuracy(fmodel, advs_, labels)
print(f" Linf norm ≤ {eps:<6}: {acc2 * 100:4.1f} %")
print(" perturbation sizes:")
perturbation_sizes = (advs_ - images).norms.linf(axis=(1, 2,
3)).numpy()
print(" ", str(perturbation_sizes).replace("\n", "\n" + " "))
if acc2 == 0:
break
def main() -> None:
# instantiate a model (could also be a TensorFlow or JAX model)
#model = models.resnet18(pretrained=True).eval()
#model=torch.load('/data1/zyh/copycat/Framework/cifar_model.pth')
model =AlexNet()
path = "./cifar_net.pth"
#path = '/data1/zyh/copycat/Framework/cifar_model.pth'
#model.load_state_dict(torch.load('/data1/zyh/copycat/Framework/cifar_model.pth'))
#pretrained_dict = {k: v for k, v in model_pretrained.items() if k in model_dict}
#model_dict.update(pretrained_dict)
#model.load_state_dict(state_dict)
model.load_state_dict(torch.load(path),strict=True)
model.eval()
print(type(model))
#preprocessing = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], axis=-3)
preprocessing = dict(mean=[0.5]*3, std=[0.5]*3, axis=-3)
fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
# get data and test the model
# wrapping the tensors with ep.astensors is optional, but it allows
# us to work with EagerPy tensors in the following
#test_dataset = torchvision.datasets.CIFAR10(root='~/.torch/',
# train=True,
# #transform = transforms.Compose([transforms.Resize((256,256)),transforms.ToTensor()]),
# transform = transforms.Compose([transforms.ToTensor()]),
# download=True)
#test_loader = torch.utils.data.DataLoader(dataset=test_dataset,
# batch_size=128, #该参数表示每次读取的批样本个数
# shuffle=False) #该参数表示读取时是否打乱样本顺序
# # 创建迭代器
#data_iter = iter(test_loader)
#images, labels = next(data_iter)
# 当迭代开始时, 队列和线程开始读取数据
#images, labels = data_iter.next()
#images=images.to(device)
#labels=labels.to(device)
#im=images
#images=im.resize(100,3,128,128)
images, labels = ep.astensors(*samples(fmodel, dataset="cifar10", batchsize=16))
#images, labels = ep.astensors(*samples(fmodel, dataset="imagenet", batchsize=16))
#print(images.shape)
clean_acc = accuracy(fmodel, images, labels)
print(f"clean accuracy: {clean_acc * 100:.1f} %")
# apply the attack
attack = LinfPGD()
'''epsilons = [
0.0,
0.0002,
0.0005,
0.0008,
0.001,
0.0015,
0.002,
0.003,
0.01,
0.1,
0.3,
0.5,
1.0,
]'''
epsilons = [
0.0005,
0.001,
0.002,
0.01,
0.1,
]
raw_advs, clipped_advs, success = attack(fmodel, images, labels, epsilons=epsilons)
print(type(raw_advs))
print("atest")
# calculate and report the robust accuracy (the accuracy of the model when
# it is attacked)
robust_accuracy = 1 - success.float32().mean(axis=-1)
print("robust accuracy for perturbations with")
for eps, acc in zip(epsilons, robust_accuracy):
print(f" Linf norm ≤ {eps:<6}: {acc.item() * 100:4.1f} %")
# we can also manually check this
# we will use the clipped advs instead of the raw advs, otherwise
# we would need to check if the perturbation sizes are actually
# within the specified epsilon bound
print()
print("we can also manually check this:")
print()
print("robust accuracy for perturbations with")
for eps, advs_ in zip(epsilons, clipped_advs):
acc2 = accuracy(fmodel, advs_, labels)
print(f" Linf norm ≤ {eps:<6}: {acc2 * 100:4.1f} %")
print(" perturbation sizes:")
perturbation_sizes = (advs_ - images).norms.linf(axis=(1, 2, 3)).numpy()
print(" ", str(perturbation_sizes).replace("\n", "\n" + " "))
if acc2 == 0:
break
fig = plt.gcf()
os.makedirs("./image/",exist_ok=True)
for i in range(len(raw_advs)):
img_v = raw_advs[i].raw
torchvision.utils.save_image(img_v, './image/'+str(i) +'.png')
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--min_norm', type=float, default=0.01,
help='Minimum perturbation norm')
parser.add_argument('--max_norm', type=float, default=15,
help='Maximum perturbation norm')
parser.add_argument('--num', type=int, default=12,
help='Number of norms to evaluate on')
args = parser.parse_args()
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# For Colab
# class args:
# # repeats = 1000
# min_norm = 0.01
# max_norm = 15
# num = 12
# List of max query count
queries = [10, 100, 1000, 5000]
# Load the pretrained model
model = Net()
model.load_state_dict(torch.load('mnist_cnn.pt'))
model.eval()
# preprocess the model inputs and set pixel bound in [0, 1]
preprocessing = dict(mean=0.1307, std=0.3081)
fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
fmodel = fmodel.transform_bounds((0, 1))
assert fmodel.bounds == (0, 1)
# Set the perturbation norm space
epsilons = np.linspace(args.min_norm, args.max_norm, args.num)
test_loader = torch.utils.data.DataLoader(
datasets.MNIST('../data', train=False, download=True,
transform=transforms.ToTensor()),
batch_size=10000, shuffle=True)
total = 0 # total input count
successful = torch.zeros(args.num, device=device) # success for each norm
legends = []
start = time()
for query in queries:
for images, labels in test_loader:
images = images.to(device)
labels = labels.to(device)
ep.astensor_(images)
ep.astensor_(labels)
# Additive Gaussian noise attack with L2 norm
attack = fa.L2RepeatedAdditiveGaussianNoiseAttack(repeats=query)
raw, clipped, success = attack(fmodel, images, labels,
epsilons=epsilons)
# Add the total number of successful attacks for each norm value
successful += success.sum(axis=1)
total += len(labels)
robust_accuracy = (1 - 1.0 * successful / total).cpu()
plt.plot(epsilons, robust_accuracy.numpy())
legends.append("{} Queries".format(query))
plt.xlabel("Perturbation Norm (L2)")
plt.ylabel("Robust Accuracy")
plt.title("Gaussian Noise")
plt.legend(legends, loc='upper right')
plt.ylim([0, 1])
plt.savefig('mnist_RA_robust_acc.jpg')
plt.show()
end = time()
print("Time taken: {:.1f} minutes".format((end - start) / 60))
def main() -> None:
# instantiate a model (could also be a TensorFlow or JAX model)
model = models.resnet18(pretrained=True).eval()
preprocessing = dict(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
axis=-3)
fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
# get data and test the model
# wrapping the tensors with ep.astensors is optional, but it allows
# us to work with EagerPy tensors in the following
images, labels = ep.astensors(
*samples(fmodel, dataset="imagenet", batchsize=16))
clean_acc = accuracy(fmodel, images, labels)
print(f"clean accuracy: {clean_acc * 100:.1f} %")
# apply the attack
attack = LinfPGD()
epsilons = [
0.0,
0.0002,
0.0005,
0.0008,
0.001,
0.0015,
0.002,
0.003,
0.01,
0.1,
0.3,
0.5,
1.0,
]
raw_advs, clipped_advs, success = attack(fmodel,
images,
labels,
epsilons=epsilons)
# calculate and report the robust accuracy (the accuracy of the model when
# it is attacked)
robust_accuracy = 1 - success.float32().mean(axis=-1)
print("robust accuracy for perturbations with")
for eps, acc in zip(epsilons, robust_accuracy):
print(f" Linf norm ≤ {eps:<6}: {acc.item() * 100:4.1f} %")
# we can also manually check this
# we will use the clipped advs instead of the raw advs, otherwise
# we would need to check if the perturbation sizes are actually
# within the specified epsilon bound
print()
print("we can also manually check this:")
print()
print("robust accuracy for perturbations with")
for eps, advs_ in zip(epsilons, clipped_advs):
acc2 = accuracy(fmodel, advs_, labels)
print(f" Linf norm ≤ {eps:<6}: {acc2 * 100:4.1f} %")
print(" perturbation sizes:")
perturbation_sizes = (advs_ - images).norms.linf(axis=(1, 2,
3)).numpy()
print(" ", str(perturbation_sizes).replace("\n", "\n" + " "))
if acc2 == 0:
break
