def main(args):
args.dev = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
if os.path.isfile("../settings.json"):
with open('../settings.json') as f:
data = json.load(f)
args.wandb_apikey = data.get("wandbapikey")
if args.wandb:
os.environ['WANDB_API_KEY'] = args.wandb_apikey
wandb.init(project='NoBox-sweeps',
name='AutoAttack-{}'.format(args.dataset))
train_loader, test_loader, split_train_loader, split_test_loader = create_loaders(
args, root='../data', split=args.split)
if args.split is not None:
train_loader = split_train_loader
test_loader = split_test_loader
model, adv_models, l_test_classif_paths, model_type = data_and_model_setup(
args, di_attack=True)
model.to(args.dev)
model.eval()
print("Testing on %d Test Classifiers with Source Model %s" %
(len(l_test_classif_paths), args.source_arch))
stack_kernel_list = []
# This is needed for Depth-wise convolution
for i in range(0, args.nc):
kernel = gkern(15, 3).astype(np.float32)
stack_list = []
for j in range(0, args.nc):
stack_list.append(kernel)
stack_kernel = np.stack(stack_list)
stack_kernel_list.append(np.expand_dims(stack_kernel, 3))
stack_kernel = np.stack(stack_kernel_list)[:, :, :, :, 0]
stack_kernel = torch.tensor(stack_kernel).to(args.dev)
attacker = TIM(args,
stack_kernel=stack_kernel,
model=model,
attack_ball=args.attack_ball,
eps=args.epsilon,
n_iter=args.n_iter,
decay_factor=args.momentum,
eps_iter=0.01)
eval_helpers = [
model, model_type, adv_models, l_test_classif_paths, test_loader
]
total_fool_rate = eval(args, attacker, "TID-Attack", eval_helpers)
return total_fool_rate
def eval(args,
attacker,
test_loader=None,
list_classifiers=[],
logger=None,
epoch=0):
if test_loader is None:
_, test_loader = create_loaders(args)
all_acc = []
for name, classifier in list_classifiers.items():
acc = eval_classifier(args, attacker, test_loader, classifier, name)
all_acc.append(acc)
if logger is not None:
logger.write({"%s/acc" % name: acc}, epoch)
mean_acc = np.mean(all_acc)
std_acc = np.std(all_acc)
if logger is not None:
logger.write({"mean_acc": mean_acc, "loss": acc}, epoch)
return mean_acc, std_acc, all_acc
def main(args):
args.dev = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
if os.path.isfile("../settings.json"):
with open('../settings.json') as f:
data = json.load(f)
args.wandb_apikey = data.get("wandbapikey")
if args.wandb:
os.environ['WANDB_API_KEY'] = args.wandb_apikey
wandb.init(project='NoBox-sweeps',
name='AutoAttack-{}'.format(args.dataset))
train_loader, test_loader, split_train_loader, split_test_loader = create_loaders(
args, root='../data', split=args.split)
if args.split is not None:
train_loader = split_train_loader
test_loader = split_test_loader
model, adv_models, l_test_classif_paths, model_type = data_and_model_setup(
args, di_attack=True)
model.to(args.dev)
model.eval()
print("Testing on %d Test Classifiers with Source Model %s" %
(len(l_test_classif_paths), args.source_arch))
attacker = DIM(args,
model,
attack_ball=args.attack_ball,
eps=args.epsilon,
n_iter=args.n_iter,
decay_factor=args.momentum)
eval_helpers = [
model, model_type, adv_models, l_test_classif_paths, test_loader
]
total_fool_rate = eval(args, attacker, "DI-Attack", eval_helpers)
return total_fool_rate
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--dataset', type=str, default='cifar')
parser.add_argument('--start', type=int, default=0)
parser.add_argument('--end', type=int, default=100)
parser.add_argument('--n_iter', type=int, default=500)
parser.add_argument('--query_step', type=int, default=1)
parser.add_argument('--transfer', action='store_true')
parser.add_argument('--debug', action='store_true')
parser.add_argument('--sweep', action='store_true')
parser.add_argument("--wandb",
action="store_true",
default=False,
help='Use wandb for logging')
parser.add_argument('--noise', type=float, default=0.3)
parser.add_argument('--eps', type=float, default=0.031)
parser.add_argument('--batch_size', type=int, default=256, metavar='S')
parser.add_argument('--test_batch_size',
type=int,
default=512,
metavar='S')
parser.add_argument('--train_set',
default='test',
choices=['train_and_test', 'test', 'train'],
help='add the test set in the training set')
# parser.add_argument('--modelIn', type=str, default='../../Nattack/all_models/robustnet/noise_0.3.pth')
parser.add_argument(
'--modelIn',
type=str,
default='../pretrained_classifiers/cifar/res18/model_0.pt')
parser.add_argument(
'--robust_model_path',
type=str,
default=
"../madry_challenge_models/mnist/adv_trained/mnist_lenet5_advtrained.pt"
)
parser.add_argument(
'--dir_test_models',
type=str,
default="../",
help=
"The path to the directory containing the classifier models for evaluation."
)
parser.add_argument(
"--max_test_model",
type=int,
default=2,
help="The maximum number of pretrained classifiers to use for testing."
)
parser.add_argument('--train_on_madry',
default=False,
action='store_true',
help='Train using Madry tf grad')
parser.add_argument('--train_on_list',
default=False,
action='store_true',
help='train on a list of classifiers')
parser.add_argument('--attack_ball',
type=str,
default="Linf",
choices=['L2', 'Linf'])
parser.add_argument('--source_arch',
default="res18",
help="The architecture we want to attack on CIFAR.")
parser.add_argument(
'--target_arch',
default=None,
help="The architecture we want to blackbox transfer to on CIFAR.")
parser.add_argument('--epsilon',
type=float,
default=0.1,
metavar='M',
help='Epsilon for Delta (default: 0.1)')
parser.add_argument('--train_with_critic_path',
type=str,
default=None,
help='Train generator with saved critic model')
parser.add_argument('--namestr', type=str, default='NoBox', \
help='additional info in output filename to describe experiments')
args = parser.parse_args()
args.dev = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
train_loader, test_loader = create_loaders(args, root='../data')
if os.path.isfile("../settings.json"):
with open('../settings.json') as f:
data = json.load(f)
args.wandb_apikey = data.get("wandbapikey")
if args.wandb:
os.environ['WANDB_API_KEY'] = args.wandb_apikey
wandb.init(project='NoBox-sweeps',
name='AutoAttack-{}'.format(args.dataset))
adv_models = None
if args.dataset == 'cifar':
args.nc, args.h, args.w = 3, 32, 32
model, l_test_classif_paths = load_all_classifiers(
args, load_archs=[args.source_arch])
model_type = args.source_arch
if args.target_arch is not None:
model_target, l_test_classif_paths = load_all_classifiers(
args, load_archs=[args.target_arch])
model_type = args.target_arch
del model_target
torch.cuda.empty_cache()
elif args.dataset == 'mnist':
if args.source_arch == 'natural':
model, l_test_classif_paths = load_all_classifiers(
args, load_archs=["natural"])
model_type = 'natural'
elif args.source_arch == 'ens_adv':
adv_model_names = args.adv_models
adv_models = [None] * len(adv_model_names)
for i in range(len(adv_model_names)):
type = get_model_type(adv_model_names[i])
adv_models[i] = load_model(args, adv_model_names[i],
type=type).to(args.dev)
path = os.path.join(args.dir_test_models, "pretrained_classifiers",
args.dataset, "ensemble_adv_trained",
args.model)
model = load_model(args, args.model, type=args.type)
l_test_classif_paths = [path]
model_type = 'Ensemble Adversarial'
model.to(args.dev)
model.eval()
print("Testing on %d Test Classifiers" % (len(l_test_classif_paths)))
l = [x.unsqueeze(0) for (x, y) in test_loader.dataset]
x_test = torch.cat(l, 0)
l = [y for (x, y) in test_loader.dataset]
y_test = torch.Tensor(l).long()
device_count = torch.cuda.device_count()
if device_count > 1:
print(
"CUDA Device Count is %d, Error might happen. Use export CUDA_VISIBLE_DEVICES=0"
% (device_count))
if not args.sweep:
attacker = AutoAttack(args,
args.n_iter,
model,
norm=args.attack_ball,
verbose=True,
eps=args.epsilon)
adv, attack_fool_rate = attacker.run_standard_evaluation_individual(
args,
x_test[:args.batch_size],
y_test[:args.batch_size],
bs=args.batch_size,
transfer_eval=args.transfer,
test_paths=l_test_classif_paths,
adv_models=adv_models)
adv_complete, success_rate = attacker.run_standard_evaluation(
x_test[:args.batch_size],
y_test[:args.batch_size],
bs=args.batch_size,
adv_models=adv_models)
if args.transfer:
adv_img_list = []
y_orig = y_test[:args.batch_size]
for i in range(0, len(adv_complete)):
adv_img_list.append([adv_complete[i].unsqueeze(0), y_orig[i]])
baseline_transfer(args, attacker, "AutoAttack", model_type,
adv_img_list, l_test_classif_paths, adv_models)
else:
for n_iter in range(0, args.n_iter, args.query_step):
attacker = AutoAttack(args,
n_iter,
model,
verbose=True,
norm=args.attack_ball,
eps=args.epsilon)
adv, attack_fool_rate = attacker.run_standard_evaluation_individual(
args,
x_test[:args.batch_size],
y_test[:args.batch_size],
bs=args.batch_size,
transfer_eval=args.transfer,
test_paths=l_test_classif_paths,
adv_models=adv_models)
adv_complete, success_rate = attacker.run_standard_evaluation(
x_test[:args.batch_size],
y_test[:args.batch_size],
bs=args.batch_size,
adv_models=adv_models)
if args.transfer:
adv_img_list = []
y_orig = y_test[:args.batch_size]
for i in range(0, len(adv_complete)):
adv_img_list.append(
[adv_complete[i].unsqueeze(0), y_orig[i]])
baseline_transfer(args, attacker, "AutoAttack", model_type,
adv_img_list, l_test_classif_paths,
adv_models)
if args.wandb:
wandb.log({
"APGD-CE": attack_fool_rate[0][1],
"APGD-DLR": attack_fool_rate[1][1],
"FAB": attack_fool_rate[2][1],
"Square": attack_fool_rate[3][1],
"queries": n_iter
})
def train(args, logger=None):
from utils.utils import create_loaders, seed_everything, CIFAR_NORMALIZATION
import utils.config as cf
import os
import torch.backends.cudnn as cudnn
import time
seed_everything(args.seed)
normalize = None
if args.normalize == "meanstd":
from torchvision import transforms
normalize = transforms.Normalize(cf.mean["cifar10"], cf.std["cifar10"])
elif args.normalize == "default":
normalize = CIFAR_NORMALIZATION
# Hyper Parameter settings
use_cuda = torch.cuda.is_available()
best_acc = 0
start_epoch, num_epochs = cf.start_epoch, cf.num_epochs
# Data Uplaod
trainloader, testloader = create_loaders(args, augment=not args.no_augment, normalize=normalize)
# Model
print('\n[Phase 2] : Model setup')
net = Wide_ResNet(**vars(args))
file_name = os.path.join(args.output, "%s/%s/model_%i.pt" % (args.dataset, "wide_resnet", args.seed))
net.apply(conv_init)
if use_cuda:
net.cuda()
net = torch.nn.DataParallel(net, device_ids=range(torch.cuda.device_count()))
cudnn.benchmark = True
criterion = nn.CrossEntropyLoss()
if args.optimizer == "adam":
from torch.optim import Adam
optimizer = Adam(net.parameters(), lr=args.lr)
elif args.optimizer == "sgd":
from torch.optim import SGD
optimizer = None
elif args.optimizer == "sls":
from utils.sls import Sls
n_batches_per_epoch = len(trainloader)
print(n_batches_per_epoch)
optimizer = Sls(net.parameters(), n_batches_per_epoch=n_batches_per_epoch)
else:
raise ValueError("Only supports adam or sgd for optimizer.")
# Training
def train(epoch, optimizer=None):
net.train()
net.training = True
train_loss = 0
correct = 0
total = 0
if args.optimizer == "sgd":
optimizer = SGD(net.parameters(), lr=cf.learning_rate(args.lr, epoch), momentum=0.9, weight_decay=5e-4)
print('\n=> Training Epoch #%d, LR=%.4f' %(epoch, cf.learning_rate(args.lr, epoch)))
for batch_idx, (inputs, targets) in enumerate(trainloader):
if use_cuda:
inputs, targets = inputs.cuda(), targets.cuda() # GPU settings
optimizer.zero_grad()
inputs, targets = Variable(inputs), Variable(targets)
outputs = net(inputs) # Forward Propagation
loss = criterion(outputs, targets) # Loss
if args.optimizer == "sls":
def closure():
output = net(inputs)
loss = criterion(output, targets)
return loss
optimizer.step(closure)
else:
loss.backward()
optimizer.step()
train_loss += loss.item()
_, predicted = torch.max(outputs.data, 1)
total += targets.size(0)
correct += predicted.eq(targets.data).cpu().sum()
sys.stdout.write('\r')
sys.stdout.write('| Epoch [%3d/%3d] Iter[%3d/%3d]\t\tLoss: %.4f Acc@1: %.3f%%'
%(epoch, num_epochs, batch_idx+1,
len(trainloader), loss.item(), 100.*correct/total))
sys.stdout.flush()
if logger is not None:
logger.write(dict(train_accuracy=100. * correct / total, loss=loss.item()), epoch)
def test(epoch, best_acc=0):
net.eval()
net.training = False
test_loss = 0
correct = 0
total = 0
with torch.no_grad():
for batch_idx, (inputs, targets) in enumerate(testloader):
if use_cuda:
inputs, targets = inputs.cuda(), targets.cuda()
inputs, targets = Variable(inputs), Variable(targets)
outputs = net(inputs)
loss = criterion(outputs, targets)
test_loss += loss.item()
_, predicted = torch.max(outputs.data, 1)
total += targets.size(0)
correct += predicted.eq(targets.data).cpu().sum()
# Save checkpoint when best model
acc = 100.*correct/total
if logger is None:
print("\n| Validation Epoch #%d\t\t\tLoss: %.4f Acc@1: %.2f%%" %(epoch, loss.item(), acc))
else:
logger.write(dict(test_loss=loss.item(), test_accuracy=acc), epoch)
if acc > best_acc:
print('| Saving Best model...\t\t\tTop1 = %.2f%%' %(acc))
state = {
'net':net.module if use_cuda else net,
'acc':acc,
'epoch':epoch,
}
dirname = os.path.dirname(file_name)
if not os.path.exists(dirname):
os.makedirs(dirname)
torch.save(net.state_dict(), file_name)
best_acc = acc
return best_acc
print('\n[Phase 3] : Training model')
print('| Training Epochs = ' + str(num_epochs))
print('| Initial Learning Rate = ' + str(args.lr))
elapsed_time = 0
for epoch in range(start_epoch, start_epoch+num_epochs):
start_time = time.time()
train(epoch, optimizer)
best_acc = test(epoch, best_acc)
epoch_time = time.time() - start_time
elapsed_time += epoch_time
print('| Elapsed time : %d:%02d:%02d' %(cf.get_hms(elapsed_time)))
print('\n[Phase 4] : Testing model')
print('* Test results : Acc@1 = %.2f%%' %(best_acc))
def main():
parser = argparse.ArgumentParser(description='NoBox')
# Hparams
parser.add_argument('--gp_coeff', type=float, default=0.,
help='coeff for the gradient penalty')
parser.add_argument('--latent_dim', type=int, default=20, metavar='N',
help='Latent dim for VAE')
parser.add_argument('--lr', type=float, default=0.01, metavar='LR',
help='learning rate for the generator (default: 0.01)')
parser.add_argument('--lr_model', type=float, default=None, metavar='LR',
help='learning rate for the model (default: None -> default to args.lr)')
parser.add_argument('--momentum', type=float, default=0.5, metavar='M',
help='optimizer momentum (default: 0.5)')
parser.add_argument('--extragradient', default=False, action='store_true',
help='Use extragadient algorithm')
parser.add_argument('--latent_size', type=int, default=50, metavar='N',
help='Size of latent distribution (default: 50)')
parser.add_argument('--flow_model', default=None, const='soft',
nargs='?', choices=[None, 'RealNVP', 'planar', 'radial'],
help='Type of Normalizing Flow (default: %(default)s)')
parser.add_argument('--flow_layer_type', type=str, default='Linear',
help='Which type of layer to use ---i.e. GRevNet or Linear')
parser.add_argument('--flow_hidden_size', type=int, default=128,
help='Hidden layer size for Flows.')
parser.add_argument('--n_blocks', type=int, default=2,
help='Number of blocks to stack in flow')
parser.add_argument('--flow_hidden', type=int, default=1, help='Number of hidden layers in each Flow.')
parser.add_argument('--eval_set', default="test",
help="Evaluate model on test or validation set.")
parser.add_argument('--train_with_critic_path', type=str, default=None,
help='Train generator with saved critic model')
parser.add_argument('--train_on_file', default=False, action='store_true',
help='Train using Madry tf grad')
# Training
parser.add_argument('--lambda_on_clean', default=0.0, type=float,
help='train the critic on clean examples of the train set')
parser.add_argument('--not_use_labels', default=False, action='store_true',
help='Use the labels for the conditional generator')
parser.add_argument('--hinge_coeff', default=10., type=float,
help='coeff for the hinge loss penalty')
parser.add_argument('--anneal_eps', default=0., type=float,
help='coeff for the epsilon annealing')
parser.add_argument('--fixed_critic', default=False, action='store_true',
help='Critic is not trained')
parser.add_argument('--train_on_list', default=False, action='store_true',
help='train on a list of classifiers')
parser.add_argument('--train_set', default='train',
choices=['train_and_test','test','train'],
help='add the test set in the training set')
parser.add_argument('--epochs', type=int, default=10, metavar='N',
help='number of epochs to train (default: 10)')
parser.add_argument('--n_iter', type=int, default=500,
help='N iters for quere based attacks')
parser.add_argument('--PGD_steps', type=int, default=40, metavar='N',
help='max gradient steps (default: 30)')
parser.add_argument('--max_iter', type=int, default=10, metavar='N',
help='max gradient steps (default: 10)')
parser.add_argument('--epsilon', type=float, default=0.1, metavar='M',
help='Epsilon for Delta (default: 0.1)')
parser.add_argument('--attack_ball', type=str, default="L2",
choices= ['L2','Linf'],
help='type of box attack')
parser.add_argument('--bb_steps', type=int, default=2000, metavar='N',
help='Max black box steps per sample(default: 1000)')
parser.add_argument('--attack_epochs', type=int, default=100, metavar='N',
help='Max numbe of epochs to train G')
parser.add_argument('--num_flows', type=int, default=2, metavar='N',
help='Number of Flows')
parser.add_argument('--seed', type=int, metavar='S',
help='random seed (default: None)')
parser.add_argument('--input_size', type=int, default=784, metavar='S',
help='Input size for MNIST is default')
parser.add_argument('--batch_size', type=int, default=256, metavar='S',
help='Batch size')
parser.add_argument('--test_batch_size', type=int, default=512, metavar='S',
help='Test Batch size')
parser.add_argument('--pgd_on_critic', default=False, action='store_true',
help='Train Critic on pgd samples')
parser.add_argument('--train_with_robust', default=False, action='store_true',
help='Train with Robust model + Critic')
parser.add_argument('--test', default=False, action='store_true',
help='just test model and print accuracy')
parser.add_argument('--clip_grad', default=True, action='store_true',
help='Clip grad norm')
parser.add_argument('--train_vae', default=False, action='store_true',
help='Train VAE')
parser.add_argument('--train_ae', default=False, action='store_true',
help='Train AE')
parser.add_argument('--attack_type', type=str, default='nobox',
help='Which attack to run')
parser.add_argument('--attack_loss', type=str, default='cross_entropy',
help='Which loss func. to use to optimize G')
parser.add_argument('--perturb_loss', type=str, default='L2', choices= ['L2','Linf'],
help='Which loss func. to use to optimize to compute constraint')
parser.add_argument('--dataset', type=str, default='mnist')
parser.add_argument('--model', type=str, default=None)
parser.add_argument('--deterministic_G', default=False, action='store_true',
help='Deterministic Latent State')
parser.add_argument('--run_baseline', default=False, action='store_true',
help='Run baseline PGD')
parser.add_argument('--resample_test', default=False, action='store_true',
help='Load model and test resampling capability')
parser.add_argument('--resample_iterations', type=int, default=100, metavar='N',
help='How many times to resample (default: 100)')
parser.add_argument('--architecture', default="VGG16",
help="The architecture we want to attack on CIFAR.")
parser.add_argument('--eval_freq', default=5, type=int,
help="Evaluate and save model every eval_freq epochs.")
parser.add_argument('--num_test_samples', default=None, type=int,
help="The number of samples used to train and test the attacker.")
parser.add_argument('--num_eval_samples', default=None, type=int,
help="The number of samples used to train and test the attacker.")
# Bells
parser.add_argument("--wandb", action="store_true", default=False, help='Use wandb for logging')
parser.add_argument('--model_path', type=str, default="mnist_cnn.pt",
help='where to save/load')
parser.add_argument('--namestr', type=str, default='NoBox', \
help='additional info in output filename to describe experiments')
parser.add_argument('--dir_test_models', type=str, default="./dir_test_models",
help="The path to the directory containing the classifier models for evaluation.")
parser.add_argument('--robust_model_path', type=str,
default="./madry_challenge_models/mnist/adv_trained/mnist_lenet5_advtrained.pt",
help="The path to our adv robust classifier")
parser.add_argument('--robust_sample_prob', type=float, default=1e-1, metavar='N',
help='1-P(robust)')
#parser.add_argument('--madry_model_path', type=str, default="./madry_challenge_models",
# help="The path to the directory containing madrys classifiers for testing")
parser.add_argument("--max_test_model", type=int, default=1,
help="The maximum number of pretrained classifiers to use for testing.")
parser.add_argument("--perturb_magnitude", type=float, default=None,
help="The amount of perturbation we want to enforce with lagrangian.")
parser.add_argument("--log_path", type=str, default="./logs",
help="Where to save logs if logger is specified.")
parser.add_argument("--save_model", type=str, default=None,
help="Where to save the models, if it is specified.")
parser.add_argument("--fixed_testset", action="store_true",
help="If used then makes sure that the same set of samples is always used for testing.")
parser.add_argument('--normalize', default=None, choices=(None, "default", "meanstd"))
###
parser.add_argument('--source_arch', default="res18",
help="The architecture we want to attack on CIFAR.")
parser.add_argument('--target_arch', nargs='*',
help="The architecture we want to blackbox transfer to on CIFAR.")
parser.add_argument('--ensemble_adv_trained', action='store_true')
parser.add_argument('--adv_models', nargs='*', help='path to adv model(s)')
parser.add_argument('--type', type=int, default=0, help='Model type (default: 0)')
parser.add_argument('--model_name', help='path to model')
parser.add_argument('--transfer', action='store_true')
parser.add_argument('--command', choices=("eval", "train"), default="train")
parser.add_argument('--split', type=int, default=None,
help="Which subsplit to use.")
parser.add_argument('--path_to_data', default="../data", type=str)
args = parser.parse_args()
args.dev = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
normalize = None
if args.normalize == "meanstd":
normalize = transforms.Normalize(cf.mean["cifar10"], cf.std["cifar10"])
elif args.normalize == "default":
normalize = CIFAR_NORMALIZATION
train_loader, test_loader, split_train_loader, split_test_loader = create_loaders(args,
root=args.path_to_data, split=args.split, num_test_samples=args.num_test_samples, normalize=normalize)
if args.split is not None:
train_loader = split_train_loader
test_loader = split_test_loader
if os.path.isfile("../settings.json"):
with open('../settings.json') as f:
data = json.load(f)
args.wandb_apikey = data.get("wandbapikey")
if args.wandb:
os.environ['WANDB_API_KEY'] = args.wandb_apikey
wandb.init(project='NoBox-table2',
name='NoBox-Attack-{}-{}'.format(args.dataset, args.namestr))
model, adv_models, l_test_classif_paths, args.model_type = data_and_model_setup(args, no_box_attack=True)
model.to(args.dev)
model.eval()
print("Testing on %d Test Classifiers with Source Model %s" %(len(l_test_classif_paths), args.source_arch))
x_test, y_test = load_data(args, test_loader)
if args.dataset == "mnist":
critic = load_unk_model(args)
elif args.dataset == "cifar":
name = args.source_arch
if args.source_arch == "adv":
name = "res18"
critic = load_unk_model(args, name=name)
misclassify_loss_func = kwargs_attack_loss[args.attack_loss]
attacker = NoBoxAttack(critic, misclassify_loss_func, args)
print("Evaluating clean error rate:")
list_model = [args.source_arch]
if args.source_arch == "adv":
list_model = [args.model_type]
if args.target_arch is not None:
list_model = args.target_arch
for model_type in list_model:
num_samples = args.num_eval_samples
if num_samples is None:
num_samples = len(test_loader.dataset)
eval_loader = torch.utils.data.Subset(test_loader.dataset,
np.random.randint(len(test_loader.dataset), size=(num_samples,)))
eval_loader = torch.utils.data.DataLoader(eval_loader, batch_size=args.test_batch_size)
baseline_transfer(args, None, "Clean", model_type, eval_loader,
list_classifiers=l_test_classif_paths)
def eval_fn(model):
advcorrect = 0
model.to(args.dev)
model.eval()
with ctx_noparamgrad_and_eval(model):
if args.source_arch == 'googlenet':
adv_complete_list = []
for batch_idx, (x_batch, y_batch) in enumerate(test_loader):
if (batch_idx + 1) * args.test_batch_size > args.batch_size:
break
x_batch, y_batch = x_batch.to(args.dev), y_batch.to(args.dev)
adv_complete_list.append(attacker.perturb(x_batch, target=y_batch))
adv_complete = torch.cat(adv_complete_list)
else:
adv_complete = attacker.perturb(x_test[:args.batch_size],
target=y_test[:args.batch_size])
adv_complete = torch.clamp(adv_complete, min=0., max=1.0)
output = model(adv_complete)
pred = output.max(1, keepdim=True)[1]
advcorrect += pred.eq(y_test[:args.batch_size].view_as(pred)).sum().item()
fool_rate = 1 - advcorrect / float(args.batch_size)
print('Test set base model fool rate: %f' %(fool_rate))
model.cpu()
if args.transfer:
adv_img_list = []
y_orig = y_test[:args.batch_size]
for i in range(0, len(adv_complete)):
adv_img_list.append([adv_complete[i].unsqueeze(0), y_orig[i]])
# Free memory
del model
torch.cuda.empty_cache()
baseline_transfer(args, attacker, "AEG", model_type,
adv_img_list, l_test_classif_paths, adv_models)
if args.command == "eval":
attacker.load(args)
elif args.command == "train":
attacker.train(train_loader, test_loader, adv_models,
l_test_classif_paths, l_train_classif={"source_model": model},
eval_fn=eval_fn)
def main(args):
torch.manual_seed(args.seed)
device = torch.device('cuda' if args.cuda else 'cpu')
train_loader, test_loader = create_loaders(args, root='../data')
eps = args.epsilon
# if src_models is not None, we train on adversarial examples that come
# from multiple models
if args.train_adv:
adv_model_names = args.adv_models
adv_models = [None] * len(adv_model_names)
for i in range(len(adv_model_names)):
type = get_model_type(adv_model_names[i])
if args.dataset == 'cifar':
adv_models[i] = load_one_classifier(args,
load_archs=[adv_model_names[i]]).to(device)
acc = test_classifier(args, adv_models[i], args.dev, test_loader, epoch=0, logger=None)
print("Dataset: %s Model: %s Test set acc: %f" %(args.dataset,
adv_model_names[i], acc))
adv_models[i] = nn.DataParallel(adv_models[i])
else:
adv_models[i] = load_model(args, adv_model_names[i], type=type).to(device)
if args.dataset == 'cifar':
init_func, _ = ARCHITECTURES[args.model]
model = init_func().to(args.dev)
if "wide_resnet" in args.model:
model.apply(wide_resnet.conv_init)
model = nn.DataParallel(model)
else:
model = model_mnist(type=args.type).to(device)
optimizer = optim.Adam(model.parameters())
# Train model
if args.train_adv:
x_advs = [None] * (len(adv_models) + 1)
for epoch in range(args.epochs):
for batch_idx, (data, labels) in enumerate(train_loader):
data, labels = data.to(device), labels.to(device)
for i, m in enumerate(adv_models + [model]):
grad = gen_grad(data, m, labels, loss='training')
x_advs[i] = symbolic_fgs(data, grad, eps=eps)
loss_model = train_ens(epoch, batch_idx, model, data, labels, optimizer, x_advs=x_advs)
else:
for epoch in range(int(args.epochs / 2)):
for batch_idx, (data, labels) in enumerate(train_loader):
data, labels = data.to(device), labels.to(device)
loss_model = train_ens(epoch, batch_idx, model, data, labels, optimizer)
# Finally print the result
correct = 0
with torch.no_grad():
for (data, labels) in test_loader:
data, labels = data.to(device), labels.to(device)
correct += test(model, data, labels)
test_error = 100. - 100. * correct / len(test_loader.dataset)
print('Test Set Error Rate: {:.2f}%'.format(test_error))
path = os.path.join(args.dir_test_models, "pretrained_classifiers",
args.dataset, "ensemble_adv_trained", args.model)
dirname = os.path.dirname(path)
if not os.path.exists(dirname):
os.makedirs(dirname)
torch.save(model.state_dict(), path + args.namestr + '.pt')
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'--data-path',
type=str,
default='../../Nattack/cifar10_data/test_batch',
help=
'path to the test_batch file from http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz'
)
parser.add_argument('--dataset', type=str, default='cifar')
parser.add_argument('--start', type=int, default=0)
parser.add_argument('--end', type=int, default=100)
parser.add_argument('--n_iter', type=int, default=500)
parser.add_argument('--query_step', type=int, default=1)
parser.add_argument('--transfer', action='store_true')
parser.add_argument('--debug', action='store_true')
parser.add_argument('--sweep', action='store_true')
parser.add_argument("--wandb",
action="store_true",
default=False,
help='Use wandb for logging')
parser.add_argument('--noise', type=float, default=0.3)
parser.add_argument('--epsilon', type=float, default=0.031)
parser.add_argument('--batch_size', type=int, default=256, metavar='S')
parser.add_argument('--test_batch_size',
type=int,
default=512,
metavar='S')
parser.add_argument('--train_set',
default='test',
choices=['train_and_test', 'test', 'train'],
help='add the test set in the training set')
parser.add_argument(
'--modelIn',
type=str,
default='../../Nattack/all_models/robustnet/noise_0.3.pth')
parser.add_argument(
'--robust_model_path',
type=str,
default=
"../madry_challenge_models/mnist/adv_trained/mnist_lenet5_advtrained.pt"
)
parser.add_argument('--source_arch',
default="res18",
help="The architecture we want to attack on CIFAR.")
parser.add_argument(
'--target_arch',
default=None,
help="The architecture we want to blackbox transfer to on CIFAR.")
parser.add_argument(
'--dir_test_models',
type=str,
default="../",
help=
"The path to the directory containing the classifier models for evaluation."
)
parser.add_argument(
"--max_test_model",
type=int,
default=2,
help="The maximum number of pretrained classifiers to use for testing."
)
parser.add_argument('--train_on_madry',
default=False,
action='store_true',
help='Train using Madry tf grad')
parser.add_argument('--train_on_list',
default=False,
action='store_true',
help='train on a list of classifiers')
parser.add_argument('--train_with_critic_path',
type=str,
default=None,
help='Train generator with saved critic model')
parser.add_argument('--namestr', type=str, default='NoBox', \
help='additional info in output filename to describe experiments')
args = parser.parse_args()
args.dev = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
if os.path.isfile("../settings.json"):
with open('../settings.json') as f:
data = json.load(f)
args.wandb_apikey = data.get("wandbapikey")
if args.wandb:
os.environ['WANDB_API_KEY'] = args.wandb_apikey
wandb.init(project='NoBox-sweeps',
name='NAttack-{}'.format(args.dataset))
adv_models = None
if args.dataset == 'cifar':
args.nc, args.h, args.w = 3, 32, 32
provider = robustml.provider.CIFAR10(args.data_path)
model, l_test_classif_paths = load_all_classifiers(
args, load_archs=[args.source_arch])
model_type = args.source_arch
if args.target_arch is not None:
model_target, l_test_classif_paths = load_all_classifiers(
args, load_archs=[args.target_arch])
model_type = args.target_arch
del model_target
torch.cuda.empty_cache()
elif args.dataset == 'mnist':
mnist_data_path = '../data/MNIST/raw/t10k-images-idx3-ubyte.gz'
mnist_label_path = '../data/MNIST/raw/t10k-labels-idx1-ubyte.gz'
args.nc, args.h, args.w = 1, 28, 28
provider = robustml.provider.MNIST(mnist_data_path, mnist_label_path)
if args.source_arch == 'natural':
model, l_test_classif_paths = load_all_classifiers(
args, load_archs=["natural"])
model_type = 'natural'
elif args.source_arch == 'ens_adv':
adv_model_names = args.adv_models
adv_models = [None] * len(adv_model_names)
for i in range(len(adv_model_names)):
type = get_model_type(adv_model_names[i])
adv_models[i] = load_model(args, adv_model_names[i],
type=type).to(args.dev)
path = os.path.join(args.dir_test_models, "pretrained_classifiers",
args.dataset, "ensemble_adv_trained",
args.model)
model = load_model(args, args.model, type=args.type)
l_test_classif_paths = [path]
model_type = 'Ensemble Adversarial'
model.to(args.dev)
model.eval()
train_loader, test_loader = create_loaders(args, root='../data')
test_classifier(args, model, args.dev, test_loader, 1, logger=None)
print("Testing on %d Test Classifiers" % (len(l_test_classif_paths)))
if not args.sweep:
attacker = NAttack(args,
model,
dataset=args.dataset,
n_iter=args.n_iter,
eps=args.epsilon,
end=args.end)
fool_rate, adv_img_list = attacker.perturb(provider)
if args.wandb:
wandb.log({"Fool Rate": fool_rate, "queries": args.n_iter})
if args.transfer:
baseline_transfer(args, attacker, "NAttack", model_type,
adv_img_list, l_test_classif_paths, adv_models)
else:
for n_iter in range(0, args.n_iter, args.query_step):
attacker = NAttack(args,
model,
dataset=args.dataset,
n_iter=n_iter,
eps=args.epsilon,
end=args.end)
fool_rate = attacker.perturb(provider)
if args.wandb:
wandb.log({"Fool Rate": fool_rate, "queries": n_iter})
if args.transfer:
baseline_transfer(args, attacker, "NAttack", model_type,
adv_img_list, l_test_classif_paths,
adv_models)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--dataset', type=str, default='cifar')
parser.add_argument('--start', type=int, default=0)
parser.add_argument('--end', type=int, default=100)
parser.add_argument('--n_iter', type=int, default=1000)
parser.add_argument('--query_step', type=int, default=1)
parser.add_argument('--transfer', action='store_true')
parser.add_argument('--debug', action='store_true')
parser.add_argument('--sweep', action='store_true')
parser.add_argument("--wandb",
action="store_true",
default=False,
help='Use wandb for logging')
parser.add_argument('--ensemble_adv_trained', action='store_true')
parser.add_argument('--noise', type=float, default=0.3)
parser.add_argument('--batch_size', type=int, default=256, metavar='S')
parser.add_argument('--test_batch_size', type=int, default=32, metavar='S')
parser.add_argument('--train_set',
default='test',
choices=['train_and_test', 'test', 'train'],
help='add the test set in the training set')
parser.add_argument(
'--modelIn',
type=str,
default='../pretrained_classifiers/cifar/res18/model_0.pt')
parser.add_argument(
'--robust_model_path',
type=str,
default=
"../madry_challenge_models/mnist/adv_trained/mnist_lenet5_advtrained.pt"
)
parser.add_argument(
'--dir_test_models',
type=str,
default="../",
help=
"The path to the directory containing the classifier models for evaluation."
)
parser.add_argument(
"--max_test_model",
type=int,
default=2,
help="The maximum number of pretrained classifiers to use for testing."
)
parser.add_argument('--train_on_madry',
default=False,
action='store_true',
help='Train using Madry tf grad')
parser.add_argument('--train_on_list',
default=False,
action='store_true',
help='train on a list of classifiers')
parser.add_argument('--attack_ball',
type=str,
default="Linf",
choices=['L2', 'Linf'])
parser.add_argument('--source_arch',
default="res18",
help="The architecture we want to attack on CIFAR.")
parser.add_argument(
'--target_arch',
default=None,
help="The architecture we want to blackbox transfer to on CIFAR.")
parser.add_argument('--split',
type=int,
default=None,
help="Which subsplit to use.")
parser.add_argument('--epsilon',
type=float,
default=0.1,
metavar='M',
help='Epsilon for Delta (default: 0.1)')
parser.add_argument(
'--num_test_samples',
default=None,
type=int,
help="The number of samples used to train and test the attacker.")
parser.add_argument('--train_with_critic_path',
type=str,
default=None,
help='Train generator with saved critic model')
parser.add_argument('--model', help='path to model')
parser.add_argument('--adv_models', nargs='*', help='path to adv model(s)')
parser.add_argument('--type',
type=int,
default=0,
help='Model type (default: 0)')
parser.add_argument('--namestr', type=str, default='NoBox', \
help='additional info in output filename to describe experiments')
args = parser.parse_args()
args.dev = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
train_loader, test_loader, split_train_loader, split_test_loader = create_loaders(
args, root='../data', split=args.split)
if args.split is not None:
train_loader = split_train_loader
test_loader = split_test_loader
if os.path.isfile("../settings.json"):
with open('../settings.json') as f:
data = json.load(f)
args.wandb_apikey = data.get("wandbapikey")
if args.wandb:
os.environ['WANDB_API_KEY'] = args.wandb_apikey
wandb.init(project='NoBox-sweeps',
name='MI-Attack-{}'.format(args.dataset))
model, adv_models, l_test_classif_paths, model_type = data_and_model_setup(
args)
model.to(args.dev)
model.eval()
print("Testing on %d Test Classifiers with Source Model %s" %
(len(l_test_classif_paths), args.source_arch))
if args.attack_ball == 'Linf':
attacker = LinfMomentumIterativeAttack(
model,
loss_fn=nn.CrossEntropyLoss(reduction="sum"),
eps=args.epsilon,
nb_iter=args.n_iter,
decay_factor=1.,
eps_iter=0.01,
clip_min=0.,
clip_max=1.,
targeted=False)
elif args.attack_ball == 'L2':
attacker = L2MomentumIterativeAttack(
model,
loss_fn=nn.CrossEntropyLoss(reduction="sum"),
eps=args.epsilon,
nb_iter=args.n_iter,
decay_factor=1.,
eps_iter=0.01,
clip_min=0.,
clip_max=1.,
targeted=False)
else:
raise NotImplementedError
eval_helpers = [
model, model_type, adv_models, l_test_classif_paths, test_loader
]
total_fool_rate = eval(args, attacker, "MI-Attack", eval_helpers)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--dataset', type=str, default='cifar')
parser.add_argument('--start', type=int, default=0)
parser.add_argument('--end', type=int, default=100)
parser.add_argument("--wandb", action="store_true", default=False, help='Use wandb for logging')
parser.add_argument('--batch_size', type=int, default=256, metavar='S')
parser.add_argument('--test_batch_size', type=int, default=512, metavar='S')
parser.add_argument('--train_set', default='test',
choices=['train_and_test','test','train'],
help='add the test set in the training set')
parser.add_argument('--modelIn', type=str,
default='../pretrained_classifiers/cifar/res18/model_0.pt')
parser.add_argument('--robust_model_path', type=str,
default="../madry_challenge_models/mnist/adv_trained/mnist_lenet5_advtrained.pt")
parser.add_argument('--dir_test_models', type=str,
default="../",
help="The path to the directory containing the classifier models for evaluation.")
parser.add_argument("--max_test_model", type=int, default=2,
help="The maximum number of pretrained classifiers to use for testing.")
parser.add_argument('--train_on_madry', default=False, action='store_true',
help='Train using Madry tf grad')
parser.add_argument('--train_on_list', default=False, action='store_true',
help='train on a list of classifiers')
parser.add_argument('--attack_ball', type=str, default="Linf",
choices= ['L2','Linf'])
parser.add_argument('--source_arch', default="res18",
help="The architecture we want to attack on CIFAR.")
parser.add_argument('--train_with_critic_path', type=str, default=None,
help='Train generator with saved critic model')
parser.add_argument('--model', help='path to model')
parser.add_argument('--namestr', type=str, default='NoBox', \
help='additional info in output filename to describe experiments')
args = parser.parse_args()
args.dev = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
train_loader, test_loader = create_loaders(args, root='../data')
if os.path.isfile("../settings.json"):
with open('../settings.json') as f:
data = json.load(f)
args.wandb_apikey = data.get("wandbapikey")
if args.wandb:
os.environ['WANDB_API_KEY'] = args.wandb_apikey
wandb.init(project='NoBox-sweeps', name='MI-Attack-{}'.format(args.dataset))
adv_models = None
attacker = None
if args.dataset == 'cifar':
args.nc, args.h, args.w = 3, 32, 32
model, l_test_classif_paths = load_all_classifiers(args, load_archs=[args.source_arch])
model_type = args.source_arch
elif args.dataset == 'mnist':
if args.source_arch == 'natural':
model, l_test_classif_paths = load_all_classifiers(args, load_archs=["natural"])
model_type = 'natural'
elif args.source_arch == 'ens_adv':
adv_model_names = args.adv_models
adv_models = [None] * len(adv_model_names)
for i in range(len(adv_model_names)):
type = get_model_type(adv_model_names[i])
adv_models[i] = load_model(args, adv_model_names[i], type=type).to(args.dev)
path = os.path.join(args.dir_test_models, "pretrained_classifiers",
args.dataset, "ensemble_adv_trained", args.model)
model = load_model(args, args.model, type=args.type)
l_test_classif_paths = [path]
model_type = 'Ensemble Adversarial'
model.to(args.dev)
model.eval()
print("Testing on %d Test Classifiers with Source Model %s" %(len(l_test_classif_paths), args.source_arch))
l = [x.unsqueeze(0) for (x, y) in test_loader.dataset]
x_test = torch.cat(l, 0).to(args.dev)
l = [y for (x, y) in test_loader.dataset]
y_test = torch.Tensor(l).long().to(args.dev)
device_count = torch.cuda.device_count()
if device_count > 1:
print("CUDA Device Count is %d, Error might happen. Use export CUDA_VISIBLE_DEVICES=0" %(device_count))
test_img_list = []
x_orig = x_test[:args.batch_size]
y_orig = y_test[:args.batch_size]
for i in range(0, len(x_orig)):
test_img_list.append([x_orig[i].unsqueeze(0), y_orig[i]])
# Free memory
del model
torch.cuda.empty_cache()
baseline_transfer(args, attacker, "Clean", model_type,
test_img_list, l_test_classif_paths, adv_models)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--dataset', type=str, default='cifar')
parser.add_argument('--start', type=int, default=0)
parser.add_argument('--end', type=int, default=100)
parser.add_argument('--n_iter', type=int, default=1000)
parser.add_argument('--query_step', type=int, default=1)
parser.add_argument('--transfer', action='store_true')
parser.add_argument('--debug', action='store_true')
parser.add_argument('--sweep', action='store_true')
parser.add_argument("--wandb", action="store_true", default=False, help='Use wandb for logging')
parser.add_argument('--ensemble_adv_trained', action='store_true')
parser.add_argument('--gamma', type=float, default=0.5)
parser.add_argument('--batch_size', type=int, default=256, metavar='S')
parser.add_argument('--test_batch_size', type=int, default=128, metavar='S')
parser.add_argument('--train_set', default='test',
choices=['train_and_test','test','train'],
help='add the test set in the training set')
parser.add_argument('--modelIn', type=str,
default='../pretrained_classifiers/cifar/res18/model_0.pt')
parser.add_argument('--robust_model_path', type=str,
default="../madry_challenge_models/mnist/adv_trained/mnist_lenet5_advtrained.pt")
parser.add_argument('--dir_test_models', type=str,
default="../",
help="The path to the directory containing the classifier models for evaluation.")
parser.add_argument("--max_test_model", type=int, default=2,
help="The maximum number of pretrained classifiers to use for testing.")
parser.add_argument('--train_on_madry', default=False, action='store_true',
help='Train using Madry tf grad')
parser.add_argument('--momentum', default=0.0, type=float)
parser.add_argument('--train_on_list', default=False, action='store_true',
help='train on a list of classifiers')
parser.add_argument('--attack_ball', type=str, default="Linf",
choices= ['L2','Linf'])
parser.add_argument('--source_arch', default="res18",
help="The architecture we want to attack on CIFAR.")
parser.add_argument('--adv_models', nargs='*', help='path to adv model(s)')
parser.add_argument('--target_arch', default=None,
help="The architecture we want to blackbox transfer to on CIFAR.")
parser.add_argument('--epsilon', type=float, default=0.03125, metavar='M',
help='Epsilon for Delta (default: 0.1)')
parser.add_argument('--num_test_samples', default=None, type=int,
help="The number of samples used to train and test the attacker.")
parser.add_argument('--split', type=int, default=None,
help="Which subsplit to use.")
parser.add_argument('--step-size', default=2, type=float, help='perturb step size')
parser.add_argument('--train_with_critic_path', type=str, default=None,
help='Train generator with saved critic model')
parser.add_argument('--namestr', type=str, default='SGM', \
help='additional info in output filename to describe experiments')
args = parser.parse_args()
args.dev = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
train_loader, test_loader, split_train_loader, split_test_loader = create_loaders(args,
root='../data', split=args.split)
if os.path.isfile("../settings.json"):
with open('../settings.json') as f:
data = json.load(f)
args.wandb_apikey = data.get("wandbapikey")
if args.wandb:
os.environ['WANDB_API_KEY'] = args.wandb_apikey
wandb.init(project='NoBox-sweeps', name='AutoAttack-{}'.format(args.dataset))
model, adv_models, l_test_classif_paths, model_type = data_and_model_setup(args)
model.to(args.dev)
model.eval()
if args.step_size < 0:
step_size = args.epsilon / args.n_iter
else:
step_size = args.step_size / 255.0
# using our method - Skip Gradient Method (SGM)
if args.gamma < 1.0:
if args.source_arch in ['res18', 'res34', 'res50', 'res101', 'res152',
'wide_resnet']:
register_hook_for_resnet(model, arch=args.source_arch, gamma=args.gamma)
elif args.source_arch in ['dense121', 'dens169', 'dense201']:
register_hook_for_densenet(model, arch=args.source_arch, gamma=args.gamma)
else:
raise ValueError('Current code only supports resnet/densenet. '
'You can extend this code to other architectures.')
if args.momentum > 0.0:
print('using PGD attack with momentum = {}'.format(args.momentum))
attacker = MomentumIterativeAttack(predict=model,
loss_fn=nn.CrossEntropyLoss(reduction="sum"),
eps=args.epsilon,
nb_iter=args.n_iter,
eps_iter=step_size,
decay_factor=args.momentum,
clip_min=0.0, clip_max=1.0,
targeted=False)
else:
print('using Linf PGD attack')
attacker = LinfPGDAttack(predict=model,
loss_fn=nn.CrossEntropyLoss(reduction="sum"),
eps=args.epsilon, nb_iter=args.n_iter,
eps_iter=step_size, rand_init=False,
clip_min=0.0, clip_max=1.0, targeted=False)
eval_helpers = [model, model_type, adv_models, l_test_classif_paths, test_loader]
total_fool_rate = eval(args, attacker, "SGM-Attack", eval_helpers)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--max_epsilon',
default=32.0,
type=float,
help='Maximum size of adversarial perturbation.')
parser.add_argument('--num_iter',
default=10,
type=int,
help="Number of iterations.")
parser.add_argument("--batch_size",
default=256,
type=int,
help="How many images process at one time.")
parser.add_argument("--momentum",
default=1.0,
type=float,
help="Momentum.")
parser.add_argument('--dataset', type=str, default='cifar')
parser.add_argument('--start', type=int, default=0)
parser.add_argument('--end', type=int, default=100)
parser.add_argument('--n_iter', type=int, default=1000)
parser.add_argument('--transfer', action='store_true')
parser.add_argument('--debug', action='store_true')
parser.add_argument('--sweep', action='store_true')
parser.add_argument("--wandb",
action="store_true",
default=False,
help='Use wandb for logging')
parser.add_argument('--ensemble_adv_trained', action='store_true')
parser.add_argument('--test_batch_size', type=int, default=32, metavar='S')
parser.add_argument('--train_set',
default='test',
choices=['train_and_test', 'test', 'train'],
help='add the test set in the training set')
parser.add_argument(
'--modelIn',
type=str,
default='../pretrained_classifiers/cifar/res18/model_0.pt')
parser.add_argument(
'--robust_model_path',
type=str,
default=
"../madry_challenge_models/mnist/adv_trained/mnist_lenet5_advtrained.pt"
)
parser.add_argument(
'--dir_test_models',
type=str,
default="../",
help=
"The path to the directory containing the classifier models for evaluation."
)
parser.add_argument(
"--max_test_model",
type=int,
default=2,
help="The maximum number of pretrained classifiers to use for testing."
)
parser.add_argument('--train_on_madry',
default=False,
action='store_true',
help='Train using Madry tf grad')
parser.add_argument('--train_on_list',
default=False,
action='store_true',
help='train on a list of classifiers')
parser.add_argument('--attack_ball',
type=str,
default="Linf",
choices=['L2', 'Linf'])
parser.add_argument('--source_arch',
default="res18",
help="The architecture we want to attack on CIFAR.")
parser.add_argument(
'--target_arch',
default=None,
help="The architecture we want to blackbox transfer to on CIFAR.")
parser.add_argument('--transform_prob',
type=float,
default=0.5,
metavar='M',
help='Randomly apply input Transformation')
parser.add_argument('--resize_factor',
type=float,
default=1.1,
metavar='M',
help='Resize Factor for Random Resizing')
parser.add_argument('--epsilon',
type=float,
default=0.1,
metavar='M',
help='Epsilon for Delta (default: 0.1)')
parser.add_argument('--train_with_critic_path',
type=str,
default=None,
help='Train generator with saved critic model')
parser.add_argument('--model', help='path to model')
parser.add_argument('--adv_models', nargs='*', help='path to adv model(s)')
parser.add_argument('--type',
type=int,
default=0,
help='Model type (default: 0)')
parser.add_argument('--namestr', type=str, default='NoBox', \
help='additional info in output filename to describe experiments')
args = parser.parse_args()
args.dev = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
train_loader, test_loader = create_loaders(args, root='../data')
if os.path.isfile("../settings.json"):
with open('../settings.json') as f:
data = json.load(f)
args.wandb_apikey = data.get("wandbapikey")
if args.wandb:
os.environ['WANDB_API_KEY'] = args.wandb_apikey
wandb.init(project='NoBox-sweeps',
name='AutoAttack-{}'.format(args.dataset))
adv_models = None
if args.dataset == 'cifar':
args.nc, args.h, args.w = 3, 32, 32
model, l_test_classif_paths = load_all_classifiers(
args, load_archs=[args.source_arch])
model_type = args.source_arch
if args.target_arch is not None:
model_target, l_test_classif_paths = load_all_classifiers(
args, load_archs=[args.target_arch])
model_type = args.target_arch
del model_target
torch.cuda.empty_cache()
if args.ensemble_adv_trained:
adv_model_names = args.adv_models
l_test_classif_paths = []
adv_models = [None] * len(adv_model_names)
for i in range(len(adv_model_names)):
adv_path = os.path.join(args.dir_test_models,
"pretrained_classifiers", args.dataset,
"ensemble_adv_trained",
adv_model_names[i] + '.pt')
init_func, _ = ARCHITECTURES[adv_model_names[i]]
temp_model = init_func().to(args.dev)
adv_models[i] = nn.DataParallel(temp_model)
adv_models[i].load_state_dict(torch.load(adv_path))
l_test_classif_paths.append([adv_path])
model_type = 'Ensemble Adversarial'
elif args.dataset == 'mnist':
args.nc, args.h, args.w = 1, 28, 28
if args.source_arch == 'natural':
model, l_test_classif_paths = load_all_classifiers(
args, load_archs=["natural"])
model_type = 'natural'
elif args.source_arch == 'ens_adv' or args.ensemble_adv_trained:
adv_model_names = args.adv_models
adv_models = [None] * len(adv_model_names)
for i in range(len(adv_model_names)):
type = get_model_type(adv_model_names[i])
adv_models[i] = load_model(args, adv_model_names[i],
type=type).to(args.dev)
path = os.path.join(args.dir_test_models, "pretrained_classifiers",
args.dataset, "ensemble_adv_trained",
args.model)
model, l_test_classif_paths = load_all_classifiers(
args, load_archs=["natural"])
# model = load_model(args, args.model, type=args.type)
l_test_classif_paths = [path]
model_type = 'Ensemble Adversarial'
model.to(args.dev)
model.eval()
print("Testing on %d Test Classifiers with Source Model %s" %
(len(l_test_classif_paths), args.source_arch))
l = [x.unsqueeze(0) for (x, y) in test_loader.dataset]
x_test = torch.cat(l, 0).to(args.dev)
l = [y for (x, y) in test_loader.dataset]
y_test = torch.Tensor(l).long().to(args.dev)
device_count = torch.cuda.device_count()
if device_count > 1:
print(
"CUDA Device Count is %d, Error might happen. Use export CUDA_VISIBLE_DEVICES=0"
% (device_count))
stack_kernel_list = []
# This is needed for Depth-wise convolution
for i in range(0, args.nc):
kernel = gkern(15, 3).astype(np.float32)
stack_kernel = np.stack([kernel, kernel, kernel]).swapaxes(2, 0)
stack_kernel_list.append(np.expand_dims(stack_kernel, 3))
stack_kernel = np.stack(stack_kernel_list).squeeze()
stack_kernel = torch.tensor(stack_kernel).permute(0, 3, 1, 2).to(args.dev)
attacker = TIM(args,
stack_kernel=stack_kernel,
model=model,
attack_ball=args.attack_ball,
eps=args.epsilon,
n_iter=args.n_iter,
decay_factor=args.momentum,
eps_iter=0.01)
advcorrect = 0
with ctx_noparamgrad_and_eval(model):
adv_complete_list = []
if args.dataset == 'cifar':
for batch_idx, (x_batch, y_batch) in enumerate(test_loader):
if (batch_idx + 1) * args.test_batch_size > args.batch_size:
break
x_batch, y_batch = x_batch.to(args.dev), y_batch.to(args.dev)
adv_complete_list.append(attacker.perturb(x_batch, y_batch))
adv_complete = torch.cat(adv_complete_list)
else:
adv_complete = attacker.perturb(x_test[:args.batch_size],
y_test[:args.batch_size])
if args.transfer:
adv_img_list = []
y_orig = y_test[:args.batch_size]
for i in range(0, len(adv_complete)):
adv_img_list.append([adv_complete[i].unsqueeze(0), y_orig[i]])
# Free memory
del model
torch.cuda.empty_cache()
baseline_transfer(args, attacker, "TI-DI-Attack", model_type,
adv_img_list, l_test_classif_paths, adv_models)