def eval_fn(model):
advcorrect = 0
model.to(args.dev)
model.eval()
with ctx_noparamgrad_and_eval(model):
if args.source_arch == 'googlenet':
adv_complete_list = []
for batch_idx, (x_batch, y_batch) in enumerate(test_loader):
if (batch_idx + 1) * args.test_batch_size > args.batch_size:
break
x_batch, y_batch = x_batch.to(args.dev), y_batch.to(args.dev)
adv_complete_list.append(attacker.perturb(x_batch, target=y_batch))
adv_complete = torch.cat(adv_complete_list)
else:
adv_complete = attacker.perturb(x_test[:args.batch_size],
target=y_test[:args.batch_size])
adv_complete = torch.clamp(adv_complete, min=0., max=1.0)
output = model(adv_complete)
pred = output.max(1, keepdim=True)[1]
advcorrect += pred.eq(y_test[:args.batch_size].view_as(pred)).sum().item()
fool_rate = 1 - advcorrect / float(args.batch_size)
print('Test set base model fool rate: %f' %(fool_rate))
model.cpu()
if args.transfer:
adv_img_list = []
y_orig = y_test[:args.batch_size]
for i in range(0, len(adv_complete)):
adv_img_list.append([adv_complete[i].unsqueeze(0), y_orig[i]])
# Free memory
del model
torch.cuda.empty_cache()
baseline_transfer(args, attacker, "AEG", model_type,
adv_img_list, l_test_classif_paths, adv_models)
def attack_dataset_images(self, args, attacker, arch_name, target_model,
codec, l_test_classif_paths, adv_models, result_dump_path='.'):
success_list, query_list, adv_img_list = [], [], []
for batch_idx, data_tuple in enumerate(self.dataset_loader):
print(batch_idx)
if batch_idx > args.num_attack:
break
if args.dataset == "ImageNet":
if args.input_size >= 299:
images, true_labels = data_tuple[1], data_tuple[2]
else:
images, true_labels = data_tuple[0], data_tuple[2]
else:
images, true_labels = data_tuple[0], data_tuple[1]
if images.size(-1) != args.input_size:
images = F.interpolate(images, size=target_model.module.input_size[-1], mode='bilinear',align_corners=True)
success, query, adv_images = self.make_adversarial_examples(batch_idx, images.cuda(), true_labels.cuda(),
args, attacker, target_model, codec)
success_list.append(success)
adv_img_list.append([adv_images, true_labels])
avg_correct = sum(success_list) / float(len(success_list))
print('{} is attacked finished ({} images)'.format(arch_name, self.total_images))
print(' avg correct: {:.4f}'.format(avg_correct.item()))
# print(' avg correct: {:.4f}'.format(self.correct_all.mean().item()))
print(' avg not_done: {:.4f}'.format(self.not_done_all.mean().item())) # 有多少图没做完
if self.success_all.sum().item() > 0:
print(
' avg mean_query: {:.4f}'.format(self.success_query_all[self.success_all.byte()].mean().item()))
print(
' avg median_query: {:.4f}'.format(self.success_query_all[self.success_all.byte()].median().item()))
print(' max query: {}'.format(self.success_query_all[self.success_all.byte()].max().item()))
if self.not_done_all.sum().item() > 0:
print(
' avg not_done_loss: {:.4f}'.format(self.not_done_loss_all[self.not_done_all.byte()].mean().item()))
print(
' avg not_done_prob: {:.4f}'.format(self.not_done_prob_all[self.not_done_all.byte()].mean().item()))
print('Saving results to {}'.format(result_dump_path))
# meta_info_dict = {"avg_correct": self.correct_all.mean().item(),
# "avg_not_done": self.not_done_all.mean().item(),
# "mean_query": self.success_query_all[self.success_all.byte()].mean().item(),
# "median_query": self.success_query_all[self.success_all.byte()].median().item(),
# "max_query": self.success_query_all[self.success_all.byte()].max().item(),
# "not_done_loss": self.not_done_loss_all[self.not_done_all.byte()].mean().item(),
# "not_done_prob": self.not_done_prob_all[self.not_done_all.byte()].mean().item()}
# meta_info_dict['args'] = vars(args)
# with open(result_dump_path, "w") as result_file_obj:
# json.dump(meta_info_dict, result_file_obj, indent=4, sort_keys=True)
print("done, write stats info to {}".format(result_dump_path))
if args.transfer:
baseline_transfer(args, attacker, args.attack_method, arch_name,
adv_img_list, l_test_classif_paths, adv_models)
def run_standard_evaluation_individual(self,
args,
x_orig,
y_orig,
bs=250,
adv_models=None,
transfer_eval=False,
test_paths=None):
# update attacks list if plus activated after initialization
if self.plus:
if not 'apgd-t' in self.attacks_to_run:
self.attacks_to_run.extend(['apgd-t'])
if not 'fab-t' in self.attacks_to_run:
self.attacks_to_run.extend(['fab-t'])
l_attacks = self.attacks_to_run
adv = {}
self.plus = False
verbose_indiv = self.verbose
attack_fool_rate = []
acc_indiv = self.clean_accuracy(x_orig, y_orig, bs=bs)
for c in l_attacks:
startt = time.time()
self.attacks_to_run = [c]
adv[c], robust_accuracy = self.run_standard_evaluation(
x_orig, y_orig, bs=bs, adv_models=adv_models)
if transfer_eval:
model_type = args.source_arch
adv_img_list = []
for i in range(0, len(adv[c])):
adv_img_list.append([adv[c][i].unsqueeze(0), y_orig[i]])
baseline_transfer(args, c, c, model_type, adv_img_list,
test_paths, adv_models)
fool_rate = 1 - robust_accuracy
attack_fool_rate.append([c, fool_rate])
space = '\t \t' if c == 'fab' else '\t'
print('robust accuracy by {} {} {:.2%} \t (time attack: {:.1f} s)'.
format(c.upper(), space, acc_indiv,
time.time() - startt))
return adv, attack_fool_rate
def eval(args, attacker, attack_name, eval_helpers, num_eval_samples=None):
model, model_type, adv_models, l_test_classif_paths, test_loader = eval_helpers[:]
advcorrect = 0
with ctx_noparamgrad_and_eval(model):
adv_complete_list, output_list, y_list = [], [], []
for batch_idx, (x_batch, y_batch) in enumerate(test_loader):
x_batch, y_batch = x_batch.to(args.dev), y_batch.to(args.dev)
adv_imgs = attacker.perturb(x_batch, y_batch).detach()
adv_complete_list.append(adv_imgs.cpu())
output_list.append(model(adv_imgs).cpu())
y_list.append(y_batch.cpu())
output = torch.cat(output_list)
adv_complete = torch.cat(adv_complete_list)
y_test = torch.cat(y_list)
pred = output.max(1, keepdim=True)[1]
advcorrect += pred.eq(y_test.view_as(pred)).sum().item()
fool_rate = 1 - advcorrect / float(len(test_loader.dataset))
print('Test set base model fool rate: %f' % (fool_rate))
if args.transfer:
if num_eval_samples is not None:
adv_complete = adv_complete[:num_eval_samples]
y_test = y_test[:num_eval_samples]
adv_img_list = torch.utils.data.TensorDataset(adv_complete, y_test)
adv_img_list = torch.utils.data.DataLoader(
adv_img_list, batch_size=args.test_batch_size)
# Free memory
del model
torch.cuda.empty_cache()
fool_rate = []
if args.target_arch is not None:
model_type = args.target_arch
if not isinstance(model_type, list):
model_type = [model_type]
for name in model_type:
fool_rate.append(
baseline_transfer(args, attacker, attack_name, name,
adv_img_list, l_test_classif_paths,
adv_models))
return np.mean(fool_rate)
return fool_rate
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--dataset', type=str, default='cifar')
parser.add_argument('--start', type=int, default=0)
parser.add_argument('--end', type=int, default=100)
parser.add_argument('--n_iter', type=int, default=500)
parser.add_argument('--query_step', type=int, default=1)
parser.add_argument('--transfer', action='store_true')
parser.add_argument('--debug', action='store_true')
parser.add_argument('--sweep', action='store_true')
parser.add_argument("--wandb",
action="store_true",
default=False,
help='Use wandb for logging')
parser.add_argument('--noise', type=float, default=0.3)
parser.add_argument('--eps', type=float, default=0.031)
parser.add_argument('--batch_size', type=int, default=256, metavar='S')
parser.add_argument('--test_batch_size',
type=int,
default=512,
metavar='S')
parser.add_argument('--train_set',
default='test',
choices=['train_and_test', 'test', 'train'],
help='add the test set in the training set')
# parser.add_argument('--modelIn', type=str, default='../../Nattack/all_models/robustnet/noise_0.3.pth')
parser.add_argument(
'--modelIn',
type=str,
default='../pretrained_classifiers/cifar/res18/model_0.pt')
parser.add_argument(
'--robust_model_path',
type=str,
default=
"../madry_challenge_models/mnist/adv_trained/mnist_lenet5_advtrained.pt"
)
parser.add_argument(
'--dir_test_models',
type=str,
default="../",
help=
"The path to the directory containing the classifier models for evaluation."
)
parser.add_argument(
"--max_test_model",
type=int,
default=2,
help="The maximum number of pretrained classifiers to use for testing."
)
parser.add_argument('--train_on_madry',
default=False,
action='store_true',
help='Train using Madry tf grad')
parser.add_argument('--train_on_list',
default=False,
action='store_true',
help='train on a list of classifiers')
parser.add_argument('--attack_ball',
type=str,
default="Linf",
choices=['L2', 'Linf'])
parser.add_argument('--source_arch',
default="res18",
help="The architecture we want to attack on CIFAR.")
parser.add_argument(
'--target_arch',
default=None,
help="The architecture we want to blackbox transfer to on CIFAR.")
parser.add_argument('--epsilon',
type=float,
default=0.1,
metavar='M',
help='Epsilon for Delta (default: 0.1)')
parser.add_argument('--train_with_critic_path',
type=str,
default=None,
help='Train generator with saved critic model')
parser.add_argument('--namestr', type=str, default='NoBox', \
help='additional info in output filename to describe experiments')
args = parser.parse_args()
args.dev = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
train_loader, test_loader = create_loaders(args, root='../data')
if os.path.isfile("../settings.json"):
with open('../settings.json') as f:
data = json.load(f)
args.wandb_apikey = data.get("wandbapikey")
if args.wandb:
os.environ['WANDB_API_KEY'] = args.wandb_apikey
wandb.init(project='NoBox-sweeps',
name='AutoAttack-{}'.format(args.dataset))
adv_models = None
if args.dataset == 'cifar':
args.nc, args.h, args.w = 3, 32, 32
model, l_test_classif_paths = load_all_classifiers(
args, load_archs=[args.source_arch])
model_type = args.source_arch
if args.target_arch is not None:
model_target, l_test_classif_paths = load_all_classifiers(
args, load_archs=[args.target_arch])
model_type = args.target_arch
del model_target
torch.cuda.empty_cache()
elif args.dataset == 'mnist':
if args.source_arch == 'natural':
model, l_test_classif_paths = load_all_classifiers(
args, load_archs=["natural"])
model_type = 'natural'
elif args.source_arch == 'ens_adv':
adv_model_names = args.adv_models
adv_models = [None] * len(adv_model_names)
for i in range(len(adv_model_names)):
type = get_model_type(adv_model_names[i])
adv_models[i] = load_model(args, adv_model_names[i],
type=type).to(args.dev)
path = os.path.join(args.dir_test_models, "pretrained_classifiers",
args.dataset, "ensemble_adv_trained",
args.model)
model = load_model(args, args.model, type=args.type)
l_test_classif_paths = [path]
model_type = 'Ensemble Adversarial'
model.to(args.dev)
model.eval()
print("Testing on %d Test Classifiers" % (len(l_test_classif_paths)))
l = [x.unsqueeze(0) for (x, y) in test_loader.dataset]
x_test = torch.cat(l, 0)
l = [y for (x, y) in test_loader.dataset]
y_test = torch.Tensor(l).long()
device_count = torch.cuda.device_count()
if device_count > 1:
print(
"CUDA Device Count is %d, Error might happen. Use export CUDA_VISIBLE_DEVICES=0"
% (device_count))
if not args.sweep:
attacker = AutoAttack(args,
args.n_iter,
model,
norm=args.attack_ball,
verbose=True,
eps=args.epsilon)
adv, attack_fool_rate = attacker.run_standard_evaluation_individual(
args,
x_test[:args.batch_size],
y_test[:args.batch_size],
bs=args.batch_size,
transfer_eval=args.transfer,
test_paths=l_test_classif_paths,
adv_models=adv_models)
adv_complete, success_rate = attacker.run_standard_evaluation(
x_test[:args.batch_size],
y_test[:args.batch_size],
bs=args.batch_size,
adv_models=adv_models)
if args.transfer:
adv_img_list = []
y_orig = y_test[:args.batch_size]
for i in range(0, len(adv_complete)):
adv_img_list.append([adv_complete[i].unsqueeze(0), y_orig[i]])
baseline_transfer(args, attacker, "AutoAttack", model_type,
adv_img_list, l_test_classif_paths, adv_models)
else:
for n_iter in range(0, args.n_iter, args.query_step):
attacker = AutoAttack(args,
n_iter,
model,
verbose=True,
norm=args.attack_ball,
eps=args.epsilon)
adv, attack_fool_rate = attacker.run_standard_evaluation_individual(
args,
x_test[:args.batch_size],
y_test[:args.batch_size],
bs=args.batch_size,
transfer_eval=args.transfer,
test_paths=l_test_classif_paths,
adv_models=adv_models)
adv_complete, success_rate = attacker.run_standard_evaluation(
x_test[:args.batch_size],
y_test[:args.batch_size],
bs=args.batch_size,
adv_models=adv_models)
if args.transfer:
adv_img_list = []
y_orig = y_test[:args.batch_size]
for i in range(0, len(adv_complete)):
adv_img_list.append(
[adv_complete[i].unsqueeze(0), y_orig[i]])
baseline_transfer(args, attacker, "AutoAttack", model_type,
adv_img_list, l_test_classif_paths,
adv_models)
if args.wandb:
wandb.log({
"APGD-CE": attack_fool_rate[0][1],
"APGD-DLR": attack_fool_rate[1][1],
"FAB": attack_fool_rate[2][1],
"Square": attack_fool_rate[3][1],
"queries": n_iter
})
def main():
parser = argparse.ArgumentParser(description='NoBox')
# Hparams
parser.add_argument('--gp_coeff', type=float, default=0.,
help='coeff for the gradient penalty')
parser.add_argument('--latent_dim', type=int, default=20, metavar='N',
help='Latent dim for VAE')
parser.add_argument('--lr', type=float, default=0.01, metavar='LR',
help='learning rate for the generator (default: 0.01)')
parser.add_argument('--lr_model', type=float, default=None, metavar='LR',
help='learning rate for the model (default: None -> default to args.lr)')
parser.add_argument('--momentum', type=float, default=0.5, metavar='M',
help='optimizer momentum (default: 0.5)')
parser.add_argument('--extragradient', default=False, action='store_true',
help='Use extragadient algorithm')
parser.add_argument('--latent_size', type=int, default=50, metavar='N',
help='Size of latent distribution (default: 50)')
parser.add_argument('--flow_model', default=None, const='soft',
nargs='?', choices=[None, 'RealNVP', 'planar', 'radial'],
help='Type of Normalizing Flow (default: %(default)s)')
parser.add_argument('--flow_layer_type', type=str, default='Linear',
help='Which type of layer to use ---i.e. GRevNet or Linear')
parser.add_argument('--flow_hidden_size', type=int, default=128,
help='Hidden layer size for Flows.')
parser.add_argument('--n_blocks', type=int, default=2,
help='Number of blocks to stack in flow')
parser.add_argument('--flow_hidden', type=int, default=1, help='Number of hidden layers in each Flow.')
parser.add_argument('--eval_set', default="test",
help="Evaluate model on test or validation set.")
parser.add_argument('--train_with_critic_path', type=str, default=None,
help='Train generator with saved critic model')
parser.add_argument('--train_on_file', default=False, action='store_true',
help='Train using Madry tf grad')
# Training
parser.add_argument('--lambda_on_clean', default=0.0, type=float,
help='train the critic on clean examples of the train set')
parser.add_argument('--not_use_labels', default=False, action='store_true',
help='Use the labels for the conditional generator')
parser.add_argument('--hinge_coeff', default=10., type=float,
help='coeff for the hinge loss penalty')
parser.add_argument('--anneal_eps', default=0., type=float,
help='coeff for the epsilon annealing')
parser.add_argument('--fixed_critic', default=False, action='store_true',
help='Critic is not trained')
parser.add_argument('--train_on_list', default=False, action='store_true',
help='train on a list of classifiers')
parser.add_argument('--train_set', default='train',
choices=['train_and_test','test','train'],
help='add the test set in the training set')
parser.add_argument('--epochs', type=int, default=10, metavar='N',
help='number of epochs to train (default: 10)')
parser.add_argument('--n_iter', type=int, default=500,
help='N iters for quere based attacks')
parser.add_argument('--PGD_steps', type=int, default=40, metavar='N',
help='max gradient steps (default: 30)')
parser.add_argument('--max_iter', type=int, default=10, metavar='N',
help='max gradient steps (default: 10)')
parser.add_argument('--epsilon', type=float, default=0.1, metavar='M',
help='Epsilon for Delta (default: 0.1)')
parser.add_argument('--attack_ball', type=str, default="L2",
choices= ['L2','Linf'],
help='type of box attack')
parser.add_argument('--bb_steps', type=int, default=2000, metavar='N',
help='Max black box steps per sample(default: 1000)')
parser.add_argument('--attack_epochs', type=int, default=100, metavar='N',
help='Max numbe of epochs to train G')
parser.add_argument('--num_flows', type=int, default=2, metavar='N',
help='Number of Flows')
parser.add_argument('--seed', type=int, metavar='S',
help='random seed (default: None)')
parser.add_argument('--input_size', type=int, default=784, metavar='S',
help='Input size for MNIST is default')
parser.add_argument('--batch_size', type=int, default=256, metavar='S',
help='Batch size')
parser.add_argument('--test_batch_size', type=int, default=512, metavar='S',
help='Test Batch size')
parser.add_argument('--pgd_on_critic', default=False, action='store_true',
help='Train Critic on pgd samples')
parser.add_argument('--train_with_robust', default=False, action='store_true',
help='Train with Robust model + Critic')
parser.add_argument('--test', default=False, action='store_true',
help='just test model and print accuracy')
parser.add_argument('--clip_grad', default=True, action='store_true',
help='Clip grad norm')
parser.add_argument('--train_vae', default=False, action='store_true',
help='Train VAE')
parser.add_argument('--train_ae', default=False, action='store_true',
help='Train AE')
parser.add_argument('--attack_type', type=str, default='nobox',
help='Which attack to run')
parser.add_argument('--attack_loss', type=str, default='cross_entropy',
help='Which loss func. to use to optimize G')
parser.add_argument('--perturb_loss', type=str, default='L2', choices= ['L2','Linf'],
help='Which loss func. to use to optimize to compute constraint')
parser.add_argument('--dataset', type=str, default='mnist')
parser.add_argument('--model', type=str, default=None)
parser.add_argument('--deterministic_G', default=False, action='store_true',
help='Deterministic Latent State')
parser.add_argument('--run_baseline', default=False, action='store_true',
help='Run baseline PGD')
parser.add_argument('--resample_test', default=False, action='store_true',
help='Load model and test resampling capability')
parser.add_argument('--resample_iterations', type=int, default=100, metavar='N',
help='How many times to resample (default: 100)')
parser.add_argument('--architecture', default="VGG16",
help="The architecture we want to attack on CIFAR.")
parser.add_argument('--eval_freq', default=5, type=int,
help="Evaluate and save model every eval_freq epochs.")
parser.add_argument('--num_test_samples', default=None, type=int,
help="The number of samples used to train and test the attacker.")
parser.add_argument('--num_eval_samples', default=None, type=int,
help="The number of samples used to train and test the attacker.")
# Bells
parser.add_argument("--wandb", action="store_true", default=False, help='Use wandb for logging')
parser.add_argument('--model_path', type=str, default="mnist_cnn.pt",
help='where to save/load')
parser.add_argument('--namestr', type=str, default='NoBox', \
help='additional info in output filename to describe experiments')
parser.add_argument('--dir_test_models', type=str, default="./dir_test_models",
help="The path to the directory containing the classifier models for evaluation.")
parser.add_argument('--robust_model_path', type=str,
default="./madry_challenge_models/mnist/adv_trained/mnist_lenet5_advtrained.pt",
help="The path to our adv robust classifier")
parser.add_argument('--robust_sample_prob', type=float, default=1e-1, metavar='N',
help='1-P(robust)')
#parser.add_argument('--madry_model_path', type=str, default="./madry_challenge_models",
# help="The path to the directory containing madrys classifiers for testing")
parser.add_argument("--max_test_model", type=int, default=1,
help="The maximum number of pretrained classifiers to use for testing.")
parser.add_argument("--perturb_magnitude", type=float, default=None,
help="The amount of perturbation we want to enforce with lagrangian.")
parser.add_argument("--log_path", type=str, default="./logs",
help="Where to save logs if logger is specified.")
parser.add_argument("--save_model", type=str, default=None,
help="Where to save the models, if it is specified.")
parser.add_argument("--fixed_testset", action="store_true",
help="If used then makes sure that the same set of samples is always used for testing.")
parser.add_argument('--normalize', default=None, choices=(None, "default", "meanstd"))
###
parser.add_argument('--source_arch', default="res18",
help="The architecture we want to attack on CIFAR.")
parser.add_argument('--target_arch', nargs='*',
help="The architecture we want to blackbox transfer to on CIFAR.")
parser.add_argument('--ensemble_adv_trained', action='store_true')
parser.add_argument('--adv_models', nargs='*', help='path to adv model(s)')
parser.add_argument('--type', type=int, default=0, help='Model type (default: 0)')
parser.add_argument('--model_name', help='path to model')
parser.add_argument('--transfer', action='store_true')
parser.add_argument('--command', choices=("eval", "train"), default="train")
parser.add_argument('--split', type=int, default=None,
help="Which subsplit to use.")
parser.add_argument('--path_to_data', default="../data", type=str)
args = parser.parse_args()
args.dev = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
normalize = None
if args.normalize == "meanstd":
normalize = transforms.Normalize(cf.mean["cifar10"], cf.std["cifar10"])
elif args.normalize == "default":
normalize = CIFAR_NORMALIZATION
train_loader, test_loader, split_train_loader, split_test_loader = create_loaders(args,
root=args.path_to_data, split=args.split, num_test_samples=args.num_test_samples, normalize=normalize)
if args.split is not None:
train_loader = split_train_loader
test_loader = split_test_loader
if os.path.isfile("../settings.json"):
with open('../settings.json') as f:
data = json.load(f)
args.wandb_apikey = data.get("wandbapikey")
if args.wandb:
os.environ['WANDB_API_KEY'] = args.wandb_apikey
wandb.init(project='NoBox-table2',
name='NoBox-Attack-{}-{}'.format(args.dataset, args.namestr))
model, adv_models, l_test_classif_paths, args.model_type = data_and_model_setup(args, no_box_attack=True)
model.to(args.dev)
model.eval()
print("Testing on %d Test Classifiers with Source Model %s" %(len(l_test_classif_paths), args.source_arch))
x_test, y_test = load_data(args, test_loader)
if args.dataset == "mnist":
critic = load_unk_model(args)
elif args.dataset == "cifar":
name = args.source_arch
if args.source_arch == "adv":
name = "res18"
critic = load_unk_model(args, name=name)
misclassify_loss_func = kwargs_attack_loss[args.attack_loss]
attacker = NoBoxAttack(critic, misclassify_loss_func, args)
print("Evaluating clean error rate:")
list_model = [args.source_arch]
if args.source_arch == "adv":
list_model = [args.model_type]
if args.target_arch is not None:
list_model = args.target_arch
for model_type in list_model:
num_samples = args.num_eval_samples
if num_samples is None:
num_samples = len(test_loader.dataset)
eval_loader = torch.utils.data.Subset(test_loader.dataset,
np.random.randint(len(test_loader.dataset), size=(num_samples,)))
eval_loader = torch.utils.data.DataLoader(eval_loader, batch_size=args.test_batch_size)
baseline_transfer(args, None, "Clean", model_type, eval_loader,
list_classifiers=l_test_classif_paths)
def eval_fn(model):
advcorrect = 0
model.to(args.dev)
model.eval()
with ctx_noparamgrad_and_eval(model):
if args.source_arch == 'googlenet':
adv_complete_list = []
for batch_idx, (x_batch, y_batch) in enumerate(test_loader):
if (batch_idx + 1) * args.test_batch_size > args.batch_size:
break
x_batch, y_batch = x_batch.to(args.dev), y_batch.to(args.dev)
adv_complete_list.append(attacker.perturb(x_batch, target=y_batch))
adv_complete = torch.cat(adv_complete_list)
else:
adv_complete = attacker.perturb(x_test[:args.batch_size],
target=y_test[:args.batch_size])
adv_complete = torch.clamp(adv_complete, min=0., max=1.0)
output = model(adv_complete)
pred = output.max(1, keepdim=True)[1]
advcorrect += pred.eq(y_test[:args.batch_size].view_as(pred)).sum().item()
fool_rate = 1 - advcorrect / float(args.batch_size)
print('Test set base model fool rate: %f' %(fool_rate))
model.cpu()
if args.transfer:
adv_img_list = []
y_orig = y_test[:args.batch_size]
for i in range(0, len(adv_complete)):
adv_img_list.append([adv_complete[i].unsqueeze(0), y_orig[i]])
# Free memory
del model
torch.cuda.empty_cache()
baseline_transfer(args, attacker, "AEG", model_type,
adv_img_list, l_test_classif_paths, adv_models)
if args.command == "eval":
attacker.load(args)
elif args.command == "train":
attacker.train(train_loader, test_loader, adv_models,
l_test_classif_paths, l_train_classif={"source_model": model},
eval_fn=eval_fn)
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'--data-path',
type=str,
default='../../Nattack/cifar10_data/test_batch',
help=
'path to the test_batch file from http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz'
)
parser.add_argument('--dataset', type=str, default='cifar')
parser.add_argument('--start', type=int, default=0)
parser.add_argument('--end', type=int, default=100)
parser.add_argument('--n_iter', type=int, default=500)
parser.add_argument('--query_step', type=int, default=1)
parser.add_argument('--transfer', action='store_true')
parser.add_argument('--debug', action='store_true')
parser.add_argument('--sweep', action='store_true')
parser.add_argument("--wandb",
action="store_true",
default=False,
help='Use wandb for logging')
parser.add_argument('--noise', type=float, default=0.3)
parser.add_argument('--epsilon', type=float, default=0.031)
parser.add_argument('--batch_size', type=int, default=256, metavar='S')
parser.add_argument('--test_batch_size',
type=int,
default=512,
metavar='S')
parser.add_argument('--train_set',
default='test',
choices=['train_and_test', 'test', 'train'],
help='add the test set in the training set')
parser.add_argument(
'--modelIn',
type=str,
default='../../Nattack/all_models/robustnet/noise_0.3.pth')
parser.add_argument(
'--robust_model_path',
type=str,
default=
"../madry_challenge_models/mnist/adv_trained/mnist_lenet5_advtrained.pt"
)
parser.add_argument('--source_arch',
default="res18",
help="The architecture we want to attack on CIFAR.")
parser.add_argument(
'--target_arch',
default=None,
help="The architecture we want to blackbox transfer to on CIFAR.")
parser.add_argument(
'--dir_test_models',
type=str,
default="../",
help=
"The path to the directory containing the classifier models for evaluation."
)
parser.add_argument(
"--max_test_model",
type=int,
default=2,
help="The maximum number of pretrained classifiers to use for testing."
)
parser.add_argument('--train_on_madry',
default=False,
action='store_true',
help='Train using Madry tf grad')
parser.add_argument('--train_on_list',
default=False,
action='store_true',
help='train on a list of classifiers')
parser.add_argument('--train_with_critic_path',
type=str,
default=None,
help='Train generator with saved critic model')
parser.add_argument('--namestr', type=str, default='NoBox', \
help='additional info in output filename to describe experiments')
args = parser.parse_args()
args.dev = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
if os.path.isfile("../settings.json"):
with open('../settings.json') as f:
data = json.load(f)
args.wandb_apikey = data.get("wandbapikey")
if args.wandb:
os.environ['WANDB_API_KEY'] = args.wandb_apikey
wandb.init(project='NoBox-sweeps',
name='NAttack-{}'.format(args.dataset))
adv_models = None
if args.dataset == 'cifar':
args.nc, args.h, args.w = 3, 32, 32
provider = robustml.provider.CIFAR10(args.data_path)
model, l_test_classif_paths = load_all_classifiers(
args, load_archs=[args.source_arch])
model_type = args.source_arch
if args.target_arch is not None:
model_target, l_test_classif_paths = load_all_classifiers(
args, load_archs=[args.target_arch])
model_type = args.target_arch
del model_target
torch.cuda.empty_cache()
elif args.dataset == 'mnist':
mnist_data_path = '../data/MNIST/raw/t10k-images-idx3-ubyte.gz'
mnist_label_path = '../data/MNIST/raw/t10k-labels-idx1-ubyte.gz'
args.nc, args.h, args.w = 1, 28, 28
provider = robustml.provider.MNIST(mnist_data_path, mnist_label_path)
if args.source_arch == 'natural':
model, l_test_classif_paths = load_all_classifiers(
args, load_archs=["natural"])
model_type = 'natural'
elif args.source_arch == 'ens_adv':
adv_model_names = args.adv_models
adv_models = [None] * len(adv_model_names)
for i in range(len(adv_model_names)):
type = get_model_type(adv_model_names[i])
adv_models[i] = load_model(args, adv_model_names[i],
type=type).to(args.dev)
path = os.path.join(args.dir_test_models, "pretrained_classifiers",
args.dataset, "ensemble_adv_trained",
args.model)
model = load_model(args, args.model, type=args.type)
l_test_classif_paths = [path]
model_type = 'Ensemble Adversarial'
model.to(args.dev)
model.eval()
train_loader, test_loader = create_loaders(args, root='../data')
test_classifier(args, model, args.dev, test_loader, 1, logger=None)
print("Testing on %d Test Classifiers" % (len(l_test_classif_paths)))
if not args.sweep:
attacker = NAttack(args,
model,
dataset=args.dataset,
n_iter=args.n_iter,
eps=args.epsilon,
end=args.end)
fool_rate, adv_img_list = attacker.perturb(provider)
if args.wandb:
wandb.log({"Fool Rate": fool_rate, "queries": args.n_iter})
if args.transfer:
baseline_transfer(args, attacker, "NAttack", model_type,
adv_img_list, l_test_classif_paths, adv_models)
else:
for n_iter in range(0, args.n_iter, args.query_step):
attacker = NAttack(args,
model,
dataset=args.dataset,
n_iter=n_iter,
eps=args.epsilon,
end=args.end)
fool_rate = attacker.perturb(provider)
if args.wandb:
wandb.log({"Fool Rate": fool_rate, "queries": n_iter})
if args.transfer:
baseline_transfer(args, attacker, "NAttack", model_type,
adv_img_list, l_test_classif_paths,
adv_models)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--dataset', type=str, default='cifar')
parser.add_argument('--start', type=int, default=0)
parser.add_argument('--end', type=int, default=100)
parser.add_argument("--wandb", action="store_true", default=False, help='Use wandb for logging')
parser.add_argument('--batch_size', type=int, default=256, metavar='S')
parser.add_argument('--test_batch_size', type=int, default=512, metavar='S')
parser.add_argument('--train_set', default='test',
choices=['train_and_test','test','train'],
help='add the test set in the training set')
parser.add_argument('--modelIn', type=str,
default='../pretrained_classifiers/cifar/res18/model_0.pt')
parser.add_argument('--robust_model_path', type=str,
default="../madry_challenge_models/mnist/adv_trained/mnist_lenet5_advtrained.pt")
parser.add_argument('--dir_test_models', type=str,
default="../",
help="The path to the directory containing the classifier models for evaluation.")
parser.add_argument("--max_test_model", type=int, default=2,
help="The maximum number of pretrained classifiers to use for testing.")
parser.add_argument('--train_on_madry', default=False, action='store_true',
help='Train using Madry tf grad')
parser.add_argument('--train_on_list', default=False, action='store_true',
help='train on a list of classifiers')
parser.add_argument('--attack_ball', type=str, default="Linf",
choices= ['L2','Linf'])
parser.add_argument('--source_arch', default="res18",
help="The architecture we want to attack on CIFAR.")
parser.add_argument('--train_with_critic_path', type=str, default=None,
help='Train generator with saved critic model')
parser.add_argument('--model', help='path to model')
parser.add_argument('--namestr', type=str, default='NoBox', \
help='additional info in output filename to describe experiments')
args = parser.parse_args()
args.dev = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
train_loader, test_loader = create_loaders(args, root='../data')
if os.path.isfile("../settings.json"):
with open('../settings.json') as f:
data = json.load(f)
args.wandb_apikey = data.get("wandbapikey")
if args.wandb:
os.environ['WANDB_API_KEY'] = args.wandb_apikey
wandb.init(project='NoBox-sweeps', name='MI-Attack-{}'.format(args.dataset))
adv_models = None
attacker = None
if args.dataset == 'cifar':
args.nc, args.h, args.w = 3, 32, 32
model, l_test_classif_paths = load_all_classifiers(args, load_archs=[args.source_arch])
model_type = args.source_arch
elif args.dataset == 'mnist':
if args.source_arch == 'natural':
model, l_test_classif_paths = load_all_classifiers(args, load_archs=["natural"])
model_type = 'natural'
elif args.source_arch == 'ens_adv':
adv_model_names = args.adv_models
adv_models = [None] * len(adv_model_names)
for i in range(len(adv_model_names)):
type = get_model_type(adv_model_names[i])
adv_models[i] = load_model(args, adv_model_names[i], type=type).to(args.dev)
path = os.path.join(args.dir_test_models, "pretrained_classifiers",
args.dataset, "ensemble_adv_trained", args.model)
model = load_model(args, args.model, type=args.type)
l_test_classif_paths = [path]
model_type = 'Ensemble Adversarial'
model.to(args.dev)
model.eval()
print("Testing on %d Test Classifiers with Source Model %s" %(len(l_test_classif_paths), args.source_arch))
l = [x.unsqueeze(0) for (x, y) in test_loader.dataset]
x_test = torch.cat(l, 0).to(args.dev)
l = [y for (x, y) in test_loader.dataset]
y_test = torch.Tensor(l).long().to(args.dev)
device_count = torch.cuda.device_count()
if device_count > 1:
print("CUDA Device Count is %d, Error might happen. Use export CUDA_VISIBLE_DEVICES=0" %(device_count))
test_img_list = []
x_orig = x_test[:args.batch_size]
y_orig = y_test[:args.batch_size]
for i in range(0, len(x_orig)):
test_img_list.append([x_orig[i].unsqueeze(0), y_orig[i]])
# Free memory
del model
torch.cuda.empty_cache()
baseline_transfer(args, attacker, "Clean", model_type,
test_img_list, l_test_classif_paths, adv_models)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--max_epsilon',
default=32.0,
type=float,
help='Maximum size of adversarial perturbation.')
parser.add_argument('--num_iter',
default=10,
type=int,
help="Number of iterations.")
parser.add_argument("--batch_size",
default=256,
type=int,
help="How many images process at one time.")
parser.add_argument("--momentum",
default=1.0,
type=float,
help="Momentum.")
parser.add_argument('--dataset', type=str, default='cifar')
parser.add_argument('--start', type=int, default=0)
parser.add_argument('--end', type=int, default=100)
parser.add_argument('--n_iter', type=int, default=1000)
parser.add_argument('--transfer', action='store_true')
parser.add_argument('--debug', action='store_true')
parser.add_argument('--sweep', action='store_true')
parser.add_argument("--wandb",
action="store_true",
default=False,
help='Use wandb for logging')
parser.add_argument('--ensemble_adv_trained', action='store_true')
parser.add_argument('--test_batch_size', type=int, default=32, metavar='S')
parser.add_argument('--train_set',
default='test',
choices=['train_and_test', 'test', 'train'],
help='add the test set in the training set')
parser.add_argument(
'--modelIn',
type=str,
default='../pretrained_classifiers/cifar/res18/model_0.pt')
parser.add_argument(
'--robust_model_path',
type=str,
default=
"../madry_challenge_models/mnist/adv_trained/mnist_lenet5_advtrained.pt"
)
parser.add_argument(
'--dir_test_models',
type=str,
default="../",
help=
"The path to the directory containing the classifier models for evaluation."
)
parser.add_argument(
"--max_test_model",
type=int,
default=2,
help="The maximum number of pretrained classifiers to use for testing."
)
parser.add_argument('--train_on_madry',
default=False,
action='store_true',
help='Train using Madry tf grad')
parser.add_argument('--train_on_list',
default=False,
action='store_true',
help='train on a list of classifiers')
parser.add_argument('--attack_ball',
type=str,
default="Linf",
choices=['L2', 'Linf'])
parser.add_argument('--source_arch',
default="res18",
help="The architecture we want to attack on CIFAR.")
parser.add_argument(
'--target_arch',
default=None,
help="The architecture we want to blackbox transfer to on CIFAR.")
parser.add_argument('--transform_prob',
type=float,
default=0.5,
metavar='M',
help='Randomly apply input Transformation')
parser.add_argument('--resize_factor',
type=float,
default=1.1,
metavar='M',
help='Resize Factor for Random Resizing')
parser.add_argument('--epsilon',
type=float,
default=0.1,
metavar='M',
help='Epsilon for Delta (default: 0.1)')
parser.add_argument('--train_with_critic_path',
type=str,
default=None,
help='Train generator with saved critic model')
parser.add_argument('--model', help='path to model')
parser.add_argument('--adv_models', nargs='*', help='path to adv model(s)')
parser.add_argument('--type',
type=int,
default=0,
help='Model type (default: 0)')
parser.add_argument('--namestr', type=str, default='NoBox', \
help='additional info in output filename to describe experiments')
args = parser.parse_args()
args.dev = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
train_loader, test_loader = create_loaders(args, root='../data')
if os.path.isfile("../settings.json"):
with open('../settings.json') as f:
data = json.load(f)
args.wandb_apikey = data.get("wandbapikey")
if args.wandb:
os.environ['WANDB_API_KEY'] = args.wandb_apikey
wandb.init(project='NoBox-sweeps',
name='AutoAttack-{}'.format(args.dataset))
adv_models = None
if args.dataset == 'cifar':
args.nc, args.h, args.w = 3, 32, 32
model, l_test_classif_paths = load_all_classifiers(
args, load_archs=[args.source_arch])
model_type = args.source_arch
if args.target_arch is not None:
model_target, l_test_classif_paths = load_all_classifiers(
args, load_archs=[args.target_arch])
model_type = args.target_arch
del model_target
torch.cuda.empty_cache()
if args.ensemble_adv_trained:
adv_model_names = args.adv_models
l_test_classif_paths = []
adv_models = [None] * len(adv_model_names)
for i in range(len(adv_model_names)):
adv_path = os.path.join(args.dir_test_models,
"pretrained_classifiers", args.dataset,
"ensemble_adv_trained",
adv_model_names[i] + '.pt')
init_func, _ = ARCHITECTURES[adv_model_names[i]]
temp_model = init_func().to(args.dev)
adv_models[i] = nn.DataParallel(temp_model)
adv_models[i].load_state_dict(torch.load(adv_path))
l_test_classif_paths.append([adv_path])
model_type = 'Ensemble Adversarial'
elif args.dataset == 'mnist':
args.nc, args.h, args.w = 1, 28, 28
if args.source_arch == 'natural':
model, l_test_classif_paths = load_all_classifiers(
args, load_archs=["natural"])
model_type = 'natural'
elif args.source_arch == 'ens_adv' or args.ensemble_adv_trained:
adv_model_names = args.adv_models
adv_models = [None] * len(adv_model_names)
for i in range(len(adv_model_names)):
type = get_model_type(adv_model_names[i])
adv_models[i] = load_model(args, adv_model_names[i],
type=type).to(args.dev)
path = os.path.join(args.dir_test_models, "pretrained_classifiers",
args.dataset, "ensemble_adv_trained",
args.model)
model, l_test_classif_paths = load_all_classifiers(
args, load_archs=["natural"])
# model = load_model(args, args.model, type=args.type)
l_test_classif_paths = [path]
model_type = 'Ensemble Adversarial'
model.to(args.dev)
model.eval()
print("Testing on %d Test Classifiers with Source Model %s" %
(len(l_test_classif_paths), args.source_arch))
l = [x.unsqueeze(0) for (x, y) in test_loader.dataset]
x_test = torch.cat(l, 0).to(args.dev)
l = [y for (x, y) in test_loader.dataset]
y_test = torch.Tensor(l).long().to(args.dev)
device_count = torch.cuda.device_count()
if device_count > 1:
print(
"CUDA Device Count is %d, Error might happen. Use export CUDA_VISIBLE_DEVICES=0"
% (device_count))
stack_kernel_list = []
# This is needed for Depth-wise convolution
for i in range(0, args.nc):
kernel = gkern(15, 3).astype(np.float32)
stack_kernel = np.stack([kernel, kernel, kernel]).swapaxes(2, 0)
stack_kernel_list.append(np.expand_dims(stack_kernel, 3))
stack_kernel = np.stack(stack_kernel_list).squeeze()
stack_kernel = torch.tensor(stack_kernel).permute(0, 3, 1, 2).to(args.dev)
attacker = TIM(args,
stack_kernel=stack_kernel,
model=model,
attack_ball=args.attack_ball,
eps=args.epsilon,
n_iter=args.n_iter,
decay_factor=args.momentum,
eps_iter=0.01)
advcorrect = 0
with ctx_noparamgrad_and_eval(model):
adv_complete_list = []
if args.dataset == 'cifar':
for batch_idx, (x_batch, y_batch) in enumerate(test_loader):
if (batch_idx + 1) * args.test_batch_size > args.batch_size:
break
x_batch, y_batch = x_batch.to(args.dev), y_batch.to(args.dev)
adv_complete_list.append(attacker.perturb(x_batch, y_batch))
adv_complete = torch.cat(adv_complete_list)
else:
adv_complete = attacker.perturb(x_test[:args.batch_size],
y_test[:args.batch_size])
if args.transfer:
adv_img_list = []
y_orig = y_test[:args.batch_size]
for i in range(0, len(adv_complete)):
adv_img_list.append([adv_complete[i].unsqueeze(0), y_orig[i]])
# Free memory
del model
torch.cuda.empty_cache()
baseline_transfer(args, attacker, "TI-DI-Attack", model_type,
adv_img_list, l_test_classif_paths, adv_models)
