def _generate(self, x, targets=None):
targeted = targets is not None
# handle the situation where targets are more than test set
if targets is not None:
assert len(targets) >= len(x)
targets = targets[:len(x)] # trancate targets
self._params['targeted'] = targeted
attack = BasicIterativeMethod(classifier=self.classifier,
**self._params)
# predict the outcomes
if targets is not None:
adv = attack.generate(x, targets)
else:
adv = attack.generate(x)
return adv
def create_attack(attack_type, classifier):
if attack_type == 'fgsm':
# Create a Fast Gradient Sign Method instance, specifying the classifier model, eps : attack step size
attacker = FastGradientMethod(classifier, eps=epsilon)
elif attack_type == 'pgd':
# Create a Projected Gradient Descent instance, specifying the classifier model, eps : Maximum perturbation that
# attacker can introduce, eps_step : Attack step size/input variation at each iteration,
# max_iter : maximum number of iterations, num_random_init : number of random initializations
attacker = ProjectedGradientDescent(classifier,
eps=epsilon,
eps_step=eps_step,
max_iter=max_iter,
num_random_init=num_random_init)
elif attack_type == 'bim':
# Create a Basic Iterative Method instance, specifying the classifier model, eps : Maximum perturbation,
# eps_step : attack step size, max_iter : maximum number of iterations
attacker = BasicIterativeMethod(classifier,
eps=epsilon,
eps_step=epsilon / max_iter,
max_iter=max_iter)
else:
print('No supported attack specified')
exit(0)
return attacker
def main():
with open('data.json') as data_json:
data_params = json.load(data_json)
parser = argparse.ArgumentParser()
parser.add_argument('--data', type=str)
parser.add_argument('--data_path', type=str, default='data')
parser.add_argument('--output_path', type=str, default='results')
parser.add_argument('--pretrained', type=str, required=True)
parser.add_argument('--batch_size', type=int, default=128)
parser.add_argument('--attack', type=str, required=True, choices=data_params['attacks'])
parser.add_argument('--eps', type=float, default=0.3)
# NOTE: In CW_L2 attack, eps is the upper bound of c.
parser.add_argument('--n_samples', type=int, default=2000)
parser.add_argument('--random_state', type=int, default=1234)
args = parser.parse_args()
print(args)
set_seeds(args.random_state)
if not os.path.exists(args.output_path):
print('Output folder does not exist. Create:', args.output_path)
os.mkdir(args.output_path)
print('Dataset:', args.data)
print('Pretrained model:', args.pretrained)
print('Running attack: {}'.format(args.attack))
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print('Device: {}'.format(device))
# Prepare data
transforms = tv.transforms.Compose([tv.transforms.ToTensor()])
if args.data == 'mnist':
dataset_train = datasets.MNIST(args.data_path, train=True, download=True, transform=transforms)
dataset_test = datasets.MNIST(args.data_path, train=False, download=True, transform=transforms)
elif args.data == 'cifar10':
dataset_train = datasets.CIFAR10(args.data_path, train=True, download=True, transform=transforms)
dataset_test = datasets.CIFAR10(args.data_path, train=False, download=True, transform=transforms)
else:
data_path = os.path.join(args.data_path, data_params['data'][args.data]['file_name'])
print('Read file:', data_path)
X, y = load_csv(data_path)
X_train, X_test, y_train, y_test = train_test_split(
X, y,
test_size=data_params['data'][args.data]['n_test'],
random_state=args.random_state)
scaler = MinMaxScaler().fit(X_train)
X_train = scaler.transform(X_train)
X_test = scaler.transform(X_test)
dataset_train = TensorDataset(torch.from_numpy(X_train).type(torch.float32), torch.from_numpy(y_train).type(torch.long))
dataset_test = TensorDataset(torch.from_numpy(X_test).type(torch.float32), torch.from_numpy(y_test).type(torch.long))
dataloader_train = DataLoader(dataset_train, 256, shuffle=False)
dataloader_test = DataLoader(dataset_test, 256, shuffle=False)
shape_train = get_shape(dataloader_train.dataset)
shape_test = get_shape(dataloader_test.dataset)
print('Train set:', shape_train)
print('Test set:', shape_test)
# Load model
use_prob = args.attack not in ['apgd', 'apgd1', 'apgd2', 'cw2', 'cwinf']
print('Attack:', args.attack)
print('Using softmax layer:', use_prob)
if args.data == 'mnist':
model = BaseModel(use_prob=use_prob).to(device)
model_name = 'basic'
elif args.data == 'cifar10':
model_name = args.pretrained.split('_')[1]
if model_name == 'resnet':
model = Resnet(use_prob=use_prob).to(device)
elif model_name == 'vgg':
model = Vgg(use_prob=use_prob).to(device)
else:
raise ValueError('Unknown model: {}'.format(model_name))
else:
n_features = data_params['data'][args.data]['n_features']
n_classes = data_params['data'][args.data]['n_classes']
model = NumericModel(
n_features,
n_hidden=n_features * 4,
n_classes=n_classes,
use_prob=use_prob).to(device)
model_name = 'basic' + str(n_features * 4)
optimizer = optim.SGD(model.parameters(), lr=0.01,
momentum=0.9, weight_decay=5e-4)
loss = nn.CrossEntropyLoss()
pretrained_path = os.path.join(args.output_path, args.pretrained)
model.load_state_dict(torch.load(pretrained_path, map_location=device))
_, acc_train = validate(model, dataloader_train, loss, device)
_, acc_test = validate(model, dataloader_test, loss, device)
print('Accuracy on train set: {:.4f}%'.format(acc_train * 100))
print('Accuracy on test set: {:.4f}%'.format(acc_test * 100))
# Create a subset which only contains recognisable samples.
tensor_test_X, tensor_test_y = get_correct_examples(
model, dataset_test, device=device, return_tensor=True)
dataset_perfect = TensorDataset(tensor_test_X, tensor_test_y)
loader_perfect = DataLoader(dataset_perfect, batch_size=512, shuffle=False)
_, acc_perfect = validate(model, loader_perfect, loss, device)
print('Accuracy on {} filtered test examples: {:.4f}%'.format(
len(dataset_perfect), acc_perfect * 100))
# Generate adversarial examples
n_features = data_params['data'][args.data]['n_features']
n_classes = data_params['data'][args.data]['n_classes']
if isinstance(n_features, int):
n_features = (n_features,)
classifier = PyTorchClassifier(
model=model,
loss=loss,
input_shape=n_features,
optimizer=optimizer,
nb_classes=n_classes,
clip_values=(0.0, 1.0),
device_type='gpu')
if args.attack == 'apgd':
eps_step = args.eps / 10.0 if args.eps <= 0.1 else 0.1
attack = AutoProjectedGradientDescent(
estimator=classifier,
eps=args.eps,
eps_step=eps_step,
max_iter=1000,
batch_size=args.batch_size,
targeted=False)
elif args.attack == 'apgd1':
attack = AutoProjectedGradientDescent(
estimator=classifier,
norm=1,
eps=args.eps,
eps_step=0.1,
max_iter=1000,
batch_size=args.batch_size,
targeted=False)
elif args.attack == 'apgd2':
attack = AutoProjectedGradientDescent(
estimator=classifier,
norm=2,
eps=args.eps,
eps_step=0.1,
max_iter=1000,
batch_size=args.batch_size,
targeted=False)
elif args.attack == 'bim':
eps_step = args.eps / 10.0
attack = BasicIterativeMethod(
estimator=classifier,
eps=args.eps,
eps_step=eps_step,
max_iter=1000,
batch_size=args.batch_size,
targeted=False)
elif args.attack == 'boundary':
attack = BoundaryAttack(
estimator=classifier,
max_iter=1000,
sample_size=args.batch_size,
targeted=False)
elif args.attack == 'cw2':
# NOTE: Do NOT increase the batch size!
attack = CarliniWagnerAttackL2(
model=model,
n_classes=n_classes,
confidence=args.eps,
verbose=True,
check_prob=False,
batch_size=args.batch_size,
targeted=False)
elif args.attack == 'cwinf':
attack = CarliniLInfMethod(
classifier=classifier,
confidence=args.eps,
max_iter=1000,
batch_size=args.batch_size,
targeted=False)
elif args.attack == 'deepfool':
attack = DeepFool(
classifier=classifier,
epsilon=args.eps,
batch_size=args.batch_size)
elif args.attack == 'fgsm':
attack = FastGradientMethod(
estimator=classifier,
eps=args.eps,
batch_size=args.batch_size)
elif args.attack == 'jsma':
attack = SaliencyMapMethod(
classifier=classifier,
gamma=args.eps,
batch_size=args.batch_size)
elif args.attack == 'line':
if args.data == 'mnist':
color = args.eps
elif args.data == 'cifar10':
color = (args.eps, args.eps, args.eps)
else:
raise NotImplementedError
attack = LineAttack(color=color, thickness=1)
elif args.attack == 'shadow':
attack = ShadowAttack(
estimator=classifier,
batch_size=args.batch_size,
targeted=False,
verbose=False)
elif args.attack == 'watermark':
attack = WaterMarkAttack(
eps=args.eps,
n_classes=data_params['data'][args.data]['n_classes'],
x_min=0.0,
x_max=1.0,
targeted=False)
X_train, y_train = get_correct_examples(model, dataset_train, device=device, return_tensor=True)
X_train = X_train.cpu().detach().numpy()
y_train = y_train.cpu().detach().numpy()
attack.fit(X_train, y_train)
else:
raise NotImplementedError
if len(dataset_perfect) > args.n_samples:
n = args.n_samples
else:
n = len(dataset_perfect)
X_benign = tensor_test_X[:n].cpu().detach().numpy()
y = tensor_test_y[:n].cpu().detach().numpy()
print('Creating {} adversarial examples with eps={} (Not all attacks use eps)'.format(n, args.eps))
time_start = time.time()
# Shadow attack only takes single sample!
if args.attack == 'shadow':
adv = np.zeros_like(X_benign)
for i in trange(len(X_benign)):
adv[i] = attack.generate(x=np.expand_dims(X_benign[i], axis=0))
elif args.attack == 'watermark':
# This is untargeted.
adv = attack.generate(X_benign, y)
else:
adv = attack.generate(x=X_benign)
time_elapsed = time.time() - time_start
print('Total time spend: {}'.format(str(datetime.timedelta(seconds=time_elapsed))))
pred_benign = np.argmax(classifier.predict(X_benign), axis=1)
acc_benign = np.sum(pred_benign == y) / n
pred_adv = np.argmax(classifier.predict(adv), axis=1)
acc_adv = np.sum(pred_adv == y) / n
print("Accuracy on benign samples: {:.4f}%".format(acc_benign * 100))
print("Accuracy on adversarial examples: {:.4f}%".format(acc_adv * 100))
# Save results
if args.n_samples < 2000:
output_file = '{}_{}_{}_{}_size{}'.format(args.data, model_name, args.attack, str(args.eps), args.n_samples)
else:
output_file = '{}_{}_{}_{}'.format(args.data, model_name, args.attack, str(args.eps))
path_x = os.path.join(args.output_path, '{}_x.npy'.format(output_file))
path_y = os.path.join(args.output_path, '{}_y.npy'.format(output_file))
path_adv = os.path.join(args.output_path, '{}_adv.npy'.format(output_file))
np.save(path_x, X_benign)
np.save(path_y, y)
np.save(path_adv, adv)
print('Saved to:', '{}_adv.npy'.format(output_file))
print()
def main():
with open('data.json') as data_json:
data_params = json.load(data_json)
parser = argparse.ArgumentParser()
parser.add_argument('--data', type=str, required=True, choices=data_params['datasets'])
parser.add_argument('--model', type=str, default='svm', choices=['svm', 'tree'])
parser.add_argument('--data_path', type=str, default='data')
parser.add_argument('--output_path', type=str, default='results')
parser.add_argument('--attack', type=str, required=True, choices=ATTACKS)
parser.add_argument('--eps', type=float, default=0.3)
# NOTE: In CW_L2 attack, eps is the upper bound of c.
parser.add_argument('--batch_size', type=int, default=128)
parser.add_argument('--n_samples', type=int, default=2000)
parser.add_argument('--random_state', type=int, default=1234)
args = parser.parse_args()
print(args)
set_seeds(args.random_state)
if not os.path.exists(args.output_path):
print('Output folder does not exist. Create:', args.output_path)
os.mkdir(args.output_path)
print('Dataset:', args.data)
print('Running attack:', args.attack)
# Prepare data
data_path = os.path.join(args.data_path, data_params['data'][args.data]['file_name'])
print('Read file: {}'.format(data_path))
X, y = load_csv(data_path)
# Apply scaling
scaler = MinMaxScaler().fit(X)
X = scaler.transform(X)
n_test = data_params['data'][args.data]['n_test']
random_state = args.random_state
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=n_test, random_state=random_state)
# Train model
if args.model == 'svm':
model = SVC(kernel="linear", C=1.0, gamma="scale", random_state=random_state)
elif args.model == 'tree':
model = ExtraTreeClassifier(random_state=random_state)
else:
raise NotImplementedError
model.fit(X_train, y_train)
acc_train = model.score(X_train, y_train)
acc_test = model.score(X_test, y_test)
print(('Train Acc: {:.4f}, ' + 'Test Acc: {:.4f}').format(acc_train, acc_test))
# Get perfect subset
pred_test = model.predict(X_test)
idx_correct = np.where(pred_test == y_test)[0]
X_test = X_test[idx_correct]
y_test = y_test[idx_correct]
classifier = SklearnClassifier(model=model, clip_values=(0.0, 1.0))
if args.attack == 'bim':
eps_step = args.eps / 10.0
attack = BasicIterativeMethod(
estimator=classifier,
eps=args.eps,
eps_step=eps_step,
max_iter=100,
targeted=False,
batch_size=args.batch_size)
elif args.attack == 'boundary':
attack = BoundaryAttack(
estimator=classifier,
max_iter=1000,
sample_size=20,
targeted=False)
elif args.attack == 'fgsm':
attack = FastGradientMethod(
estimator=classifier,
eps=args.eps,
batch_size=args.batch_size)
elif args.attack == 'tree':
attack = DecisionTreeAttack(
classifier=classifier)
else:
raise NotImplementedError
# How many examples do we have?
if len(X_test) > args.n_samples:
n = args.n_samples
else:
n = len(X_test)
X_benign = X_test[:n]
y_true = y_test[:n]
adv = attack.generate(X_benign)
acc = model.score(adv, y_true)
print('Acc on adv:', acc)
output_file = '{}_{}_{}_{}'.format(args.data, args.model, args.attack, str(args.eps))
path_x = os.path.join(args.output_path, '{}_x.npy'.format(output_file))
path_y = os.path.join(args.output_path, '{}_y.npy'.format(output_file))
path_adv = os.path.join(args.output_path, '{}_adv.npy'.format(output_file))
np.save(path_x, X_benign)
np.save(path_y, y_true)
np.save(path_adv, adv)
print('Saved to:', path_adv)
print()
def main(args):
assert args.dataset in ['mnist', 'cifar', 'svhn', 'tiny', 'tiny_gray'], \
"dataset parameter must be either 'mnist', 'cifar', 'svhn', or 'tiny'"
print('Dataset: %s' % args.dataset)
adv_path = '/home/aaldahdo/detectors/adv_data/'
if args.dataset == 'mnist':
from baselineCNN.cnn.cnn_mnist import MNISTCNN as model
model_mnist = model(mode='load', filename='cnn_{}.h5'.format(args.dataset))
classifier=model_mnist.model
sgd = optimizers.SGD(lr=0.05, decay=1e-6, momentum=0.9, nesterov=True)
classifier.compile(loss=categorical_crossentropy, optimizer=sgd, metrics=['accuracy'])
kclassifier = KerasClassifier(model=classifier, clip_values=(0, 1))
epsilons=[8/256, 16/256, 32/256, 64/256, 80/256, 128/256]
epsilons1=[5, 10, 15, 20, 25, 30, 40]
epsilons2=[0.125, 0.25, 0.3125, 0.5, 1, 1.5, 2]
eps_sa=0.3
pa_th=78
# random_restart = 20
# x_train = model_mnist.x_train
x_test = model_mnist.x_test
# y_train = model_mnist.y_train
y_test = model_mnist.y_test
y_test_labels = model_mnist.y_test_labels
translation = 10
rotation = 60
elif args.dataset == 'mnist_gray':
from baselineCNN.cnn.cnn_mnist_gray import MNISTCNN as model
model_mnist = model(mode='load', filename='cnn_{}.h5'.format(args.dataset))
classifier=model_mnist.model
sgd = optimizers.SGD(lr=0.05, decay=1e-6, momentum=0.9, nesterov=True)
classifier.compile(loss=categorical_crossentropy, optimizer=sgd, metrics=['accuracy'])
kclassifier = KerasClassifier(model=classifier, clip_values=(0, 1))
epsilons=[8/256, 16/256, 32/256, 64/256, 80/256, 128/256]
epsilons1=[5, 10, 15, 20, 25, 30, 40]
epsilons2=[0.125, 0.25, 0.3125, 0.5, 1, 1.5, 2]
eps_sa=0.3
pa_th=78
# random_restart = 20
# x_train = model_mnist.x_train
x_test = model_mnist.x_test
# y_train = model_mnist.y_train
y_test = model_mnist.y_test
y_test_labels = model_mnist.y_test_labels
translation = 10
rotation = 60
elif args.dataset == 'cifar':
from baselineCNN.cnn.cnn_cifar10 import CIFAR10CNN as model
model_cifar = model(mode='load', filename='cnn_{}.h5'.format(args.dataset))
classifier=model_cifar.model
sgd = optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
classifier.compile(loss=categorical_crossentropy, optimizer=sgd, metrics=['accuracy'])
kclassifier = KerasClassifier(model=classifier, clip_values=(0, 1))
epsilons=[8/256, 16/256, 32/256, 64/256, 80/256, 128/256]
epsilons1=[5, 10, 15, 20, 25, 30, 40]
epsilons2=[0.125, 0.25, 0.3125, 0.5, 1, 1.5, 2]
eps_sa=0.125
pa_th=100
# x_train = model_cifar.x_train
x_test = model_cifar.x_test
# y_train = model_cifar.y_train
y_test = model_cifar.y_test
y_test_labels = model_cifar.y_test_labels
translation = 8
rotation = 30
elif args.dataset == 'cifar_gray':
from baselineCNN.cnn.cnn_cifar10_gray import CIFAR10CNN as model
model_cifar = model(mode='load', filename='cnn_{}.h5'.format(args.dataset))
classifier=model_cifar.model
sgd = optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
classifier.compile(loss=categorical_crossentropy, optimizer=sgd, metrics=['accuracy'])
kclassifier = KerasClassifier(model=classifier, clip_values=(0, 1))
epsilons=[8/256, 16/256, 32/256, 64/256, 80/256, 128/256]
epsilons1=[5, 10, 15, 20, 25, 30, 40]
epsilons2=[0.125, 0.25, 0.3125, 0.5, 1, 1.5, 2]
eps_sa=0.125
pa_th=100
# x_train = model_cifar.x_train
x_test = model_cifar.x_test
# y_train = model_cifar.y_train
y_test = model_cifar.y_test
y_test_labels = model_cifar.y_test_labels
translation = 8
rotation = 30
elif args.dataset == 'svhn':
from baselineCNN.cnn.cnn_svhn import SVHNCNN as model
model_svhn = model(mode='load', filename='cnn_{}.h5'.format(args.dataset))
classifier=model_svhn.model
sgd = optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
classifier.compile(loss=categorical_crossentropy, optimizer=sgd, metrics=['accuracy'])
kclassifier = KerasClassifier(model=classifier, clip_values=(0, 1))
epsilons=[8/256, 16/256, 32/256, 64/256, 80/256, 128/256]
epsilons1=[5, 10, 15, 20, 25, 30, 40]
epsilons2=[0.125, 0.25, 0.3125, 0.5, 1, 1.5, 2]
eps_sa=0.125
pa_th=100
# x_train = model_svhn.x_train
x_test = model_svhn.x_test
# y_train = model_svhn.y_train
y_test = model_svhn.y_test
y_test_labels = model_svhn.y_test_labels
translation = 10
rotation = 60
elif args.dataset == 'svhn_gray':
from baselineCNN.cnn.cnn_svhn_gray import SVHNCNN as model
model_svhn = model(mode='load', filename='cnn_{}.h5'.format(args.dataset))
classifier=model_svhn.model
sgd = optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
classifier.compile(loss=categorical_crossentropy, optimizer=sgd, metrics=['accuracy'])
kclassifier = KerasClassifier(model=classifier, clip_values=(0, 1))
epsilons=[8/256, 16/256, 32/256, 64/256, 80/256, 128/256]
epsilons1=[5, 10, 15, 20, 25, 30, 40]
epsilons2=[0.125, 0.25, 0.3125, 0.5, 1, 1.5, 2]
eps_sa=0.125
pa_th=100
# x_train = model_svhn.x_train
x_test = model_svhn.x_test
# y_train = model_svhn.y_train
y_test = model_svhn.y_test
y_test_labels = model_svhn.y_test_labels
translation = 10
rotation = 60
elif args.dataset == 'tiny':
from baselineCNN.cnn.cnn_tiny import TINYCNN as model
model_tiny = model(mode='load', filename='cnn_{}.h5'.format(args.dataset))
classifier=model_tiny.model
sgd = optimizers.SGD(lr=0.001, decay=1e-6, momentum=0.9, nesterov=True)
classifier.compile(loss=categorical_crossentropy, optimizer=sgd, metrics=['accuracy'])
kclassifier = KerasClassifier(model=classifier, clip_values=(0, 1))
epsilons=[8/256, 16/256, 32/256, 64/256, 80/256, 128/256]
epsilons1=[5, 10, 15, 20, 25, 30, 40]
epsilons2=[0.125, 0.25, 0.3125, 0.5, 1, 1.5, 2]
eps_sa=0.125
pa_th=100
# x_train = model_tiny.x_train
x_test = model_tiny.x_test
# y_train = model_tiny.y_train
y_test = model_tiny.y_test
y_test_labels = model_tiny.y_test_labels
translation = 8
rotation = 30
del model_tiny
elif args.dataset == 'tiny_gray':
from baselineCNN.cnn.cnn_tiny_gray import TINYCNN as model
model_tiny = model(mode='load', filename='cnn_{}.h5'.format(args.dataset))
classifier=model_tiny.model
sgd = optimizers.SGD(lr=0.001, decay=1e-6, momentum=0.9, nesterov=True)
classifier.compile(loss=categorical_crossentropy, optimizer=sgd, metrics=['accuracy'])
kclassifier = KerasClassifier(model=classifier, clip_values=(0, 1))
epsilons=[8/256, 16/256, 32/256, 64/256, 80/256, 128/256]
epsilons1=[5, 10, 15, 20, 25, 30, 40]
epsilons2=[0.125, 0.25, 0.3125, 0.5, 1, 1.5, 2]
eps_sa=0.125
# x_train = model_tiny.x_train
x_test = model_tiny.x_test
# y_train = model_tiny.y_train
y_test = model_tiny.y_test
y_test_labels = model_tiny.y_test_labels
translation = 8
rotation = 30
del model_tiny
# batch_count_start = args.batch_indx
# bsize = args.batch_size
# batch_count_end = batch_count_start + 1
#FGSM
for e in epsilons:
attack = FastGradientMethod(estimator=kclassifier, eps=e, eps_step=0.01, batch_size=256)
adv_data = attack.generate(x=x_test)
adv_file_path = adv_path + args.dataset + '_fgsm_' + str(e) + '.npy'
np.save(adv_file_path, adv_data)
print('Done - {}'.format(adv_file_path))
#BIM
for e in epsilons:
attack = BasicIterativeMethod(estimator=kclassifier, eps=e, eps_step=0.01, batch_size=32, max_iter=int(e*256*1.25))
adv_data = attack.generate(x=x_test)
adv_file_path = adv_path + args.dataset + '_bim_' + str(e) + '.npy'
np.save(adv_file_path, adv_data)
print('Done - {}'.format(adv_file_path))
#PGD1
for e in epsilons1:
attack = ProjectedGradientDescent(estimator=kclassifier, norm=1, eps=e, eps_step=4, batch_size=32)
adv_data = attack.generate(x=x_test)
adv_file_path = adv_path + args.dataset + '_pgd1_' + str(e) + '.npy'
np.save(adv_file_path, adv_data)
print('Done - {}'.format(adv_file_path))
#PGD2
for e in epsilons2:
attack = ProjectedGradientDescent(estimator=kclassifier, norm=2, eps=e, eps_step=0.1, batch_size=32)
adv_data = attack.generate(x=x_test)
adv_file_path = adv_path + args.dataset + '_pgd2_' + str(e) + '.npy'
np.save(adv_file_path, adv_data)
print('Done - {}'.format(adv_file_path))
#PGDInf
for e in epsilons:
attack = ProjectedGradientDescent(estimator=kclassifier, norm=np.inf, eps=e, eps_step=0.01, batch_size=32)
adv_data = attack.generate(x=x_test)
adv_file_path = adv_path + args.dataset + '_pgdi_' + str(e) + '.npy'
np.save(adv_file_path, adv_data)
print('Done - {}'.format(adv_file_path))
#CWi
attack = CarliniLInfMethod(classifier=kclassifier, max_iter=200)
adv_data = attack.generate(x=x_test)
adv_file_path = adv_path + args.dataset + '_cwi.npy'
np.save(adv_file_path, adv_data)
print('Done - {}'.format(adv_file_path))
# #CWi
# if args.dataset=='tiny':
# for n, x, y in batch(x_test, y_test, batch_size=bsize):
# if n>=batch_count_start*bsize and n<batch_count_end*bsize:
# adv_file_path = adv_path + args.dataset + '_cwi_' + str(batch_count_start) + '.npy'
# if not os.path.isfile(adv_file_path):
# attack = CarliniLInfMethod(classifier=kclassifier, max_iter=100, batch_size=bsize)
# adv_data = attack.generate(x=x)
# np.save(adv_file_path, adv_data)
# print('Done - {}'.format(adv_file_path))
#CW2 - SLOW
attack = CarliniL2Method(classifier=kclassifier, max_iter=100, batch_size=1, confidence=10)
adv_data = attack.generate(x=x_test)
adv_file_path = adv_path + args.dataset + '_cw2.npy'
np.save(adv_file_path, adv_data)
print('Done - {}'.format(adv_file_path))
#DF
attack = DeepFool(classifier=kclassifier)
adv_data = attack.generate(x=x_test)
adv_file_path = adv_path + args.dataset + '_df.npy'
np.save(adv_file_path, adv_data)
print('Done - {}'.format(adv_file_path))
# #DF
# if args.dataset=='tiny':
# for n, x, y in batch(x_test, y_test, batch_size=bsize):
# if n>=batch_count_start*bsize and n<batch_count_end*bsize:
# attack = DeepFool(classifier=kclassifier, epsilon=9, max_iter=100)
# adv_data = attack.generate(x=x)
# adv_file_path = adv_path + args.dataset + '_df_'+ str(batch_count_start) + '.npy'
# np.save(adv_file_path, adv_data)
# print('Done - {}'.format(adv_file_path))
#Spatial transofrmation attack
attack = SpatialTransformation(classifier=kclassifier, max_translation=translation, max_rotation=rotation)
adv_data = attack.generate(x=x_test)
adv_file_path = adv_path + args.dataset + '_sta.npy'
np.save(adv_file_path, adv_data)
print('Done - {}'.format(adv_file_path))
#Square Attack
attack = SquareAttack(estimator=kclassifier, max_iter=200, eps=eps_sa)
adv_data = attack.generate(x=x_test, y=y_test)
adv_file_path = adv_path + args.dataset + '_sa.npy'
np.save(adv_file_path, adv_data)
print('Done - {}'.format(adv_file_path))
#HopSkipJump Attack
y_test_next= get_next_class(y_test)
attack = HopSkipJump(classifier=kclassifier, targeted=False, max_iter=0, max_eval=100, init_eval=10)
iter_step = 10
adv_data = np.zeros(x_test.shape)
# adv_data = adv_data[0:25]
for i in range(4):
adv_data = attack.generate(x=x_test, x_adv_init=adv_data, resume=True)
attack.max_iter = iter_step
# _, acc_normal = classifier.evaluate(x_test[0:25], y_test[0:25])
# _, acc_adv = classifier.evaluate(adv_data, y_test[0:25])
# print('Normal accuracy - {}\nAttack accuracy - {}'.format(acc_normal, acc_adv))
# subcount=1
# for i in range(0, 25):
# plt.subplot(5,5,subcount)
# if args.dataset=='mnist':
# plt.imshow(adv_data[i][:,:,0])
# else:
# plt.imshow(adv_data[i][:,:,:])
# plt.suptitle(args.dataset+ " sb")
# subcount = subcount + 1
# plt.show()
adv_file_path = adv_path + args.dataset + '_hop.npy'
np.save(adv_file_path, adv_data)
print('Done - {}'.format(adv_file_path))
#ZOO attack
attack = ZooAttack(classifier=kclassifier, batch_size=32)
adv_data = attack.generate(x=x_test, y=y_test)
adv_file_path = adv_path + args.dataset + '_zoo.npy'
np.save(adv_file_path, adv_data)
print('Done - {}'.format(adv_file_path))
def load_bid(classifier):
return BasicIterativeMethod(classifier=classifier, max_iter=20, eps=0.2,batch_size=1 )
def plot_attacks_acc(classifier, x, y, path_fig, dataset, title):
'''
Description:
This function takes in a classifier model and a list of images with labels and creates
a plot showing how the accuracy of model on the dataset decreases as attack strength (perturbation size)
increases for 3 different attacks (FGSM, PGD, BIM).
:param classifier: model to be evaluated
:param x: list of images to be predicted on
:param y: labels of images
:param path_fig: path to save the plot figure
:param dataset: name of dataset (e.g. mnist, cifar, ddsm, brain_mri, lidc)
:param title: title to define plot figure
:return: Figure will be saved with title
'''
if dataset == 'ddsm':
eps_range = [0.00001, 0.00005, 0.0001, 0.00025, 0.0005, 0.00075, 0.001, 0.00125, 0.0015, 0.00175, 0.002, 0.0025, 0.003, 0.0035, 0.004, 0.0045, 0.005, 0.0055, 0.006, 0.007, 0.008]
step_size = 0.001
elif dataset == 'brain_mri':
eps_range = [0.0001, 0.0005, 0.001, 0.0013, 0.0016, 0.002, 0.00225, 0.0025, 0.00275, 0.003, 0.00325, 0.0035, 0.00375, 0.004, 0.0045, 0.005, 0.006, 0.007, 0.008, 0.009, 0.01, 0.012]
step_size = 0.001
elif dataset == 'mnist':
eps_range = [0.0001, 0.01, 0.02, 0.05, 0.075, 0.1, 0.125, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.5]
step_size = 0.1
elif dataset == 'cifar':
eps_range = [0.0001, 0.001, 0.002, 0.003, 0.004, 0.005, 0.007, 0.009, 0.01, 0.015, 0.02, 0.03, 0.04, 0.05]
step_size = 0.01
elif dataset == 'lidc':
eps_range = [0.0001, 0.0003, 0.0006, 0.0008, 0.001, 0.00125, 0.0015, 0.00175, 0.002, 0.0023, 0.0026, 0.0028, 0.003, 0.004, 0.005, 0.006, 0.007, 0.008, 0.009, 0.01, 0.011, 0.012, 0.013, 0.014, 0.015, 0.016, 0.017, 0.018, 0.019, 0.02]
step_size = 0.001
nb_correct_fgsm = []
nb_correct_pgd = []
nb_correct_bim = []
for eps in eps_range:
attacker_fgsm = FastGradientMethod(classifier, eps=eps)
attacker_pgd = ProjectedGradientDescent(classifier, eps=eps, eps_step=eps/4, max_iter=10,
num_random_init=5)
attacker_bim = BasicIterativeMethod(classifier, eps=eps, eps_step=eps/10, max_iter=10)
x_fgsm = attacker_fgsm.generate(x)
x_pgd = attacker_pgd.generate(x)
x_bim = attacker_bim.generate(x)
x_pred_fgsm = np.argmax(classifier.predict(x_fgsm), axis=1)
nb_correct_fgsm += [np.sum(x_pred_fgsm == np.argmax(y, axis=1))]
x_pred_pgd = np.argmax(classifier.predict(x_pgd), axis=1)
nb_correct_pgd += [np.sum(x_pred_pgd == np.argmax(y, axis=1))]
x_pred_bim = np.argmax(classifier.predict(x_bim), axis=1)
nb_correct_bim += [np.sum(x_pred_bim == np.argmax(y, axis=1))]
fig, ax = plt.subplots()
ax.plot(np.array(eps_range) / step_size, 100 * np.array(nb_correct_fgsm) / y.shape[0], 'b--', label='FGSM')
ax.plot(np.array(eps_range) / step_size, 100 * np.array(nb_correct_pgd) / y.shape[0], 'r--', label='PGD')
ax.plot(np.array(eps_range) / step_size, 100 * np.array(nb_correct_bim) / y.shape[0], 'g--', label='BIM')
legend = ax.legend(loc='upper right', shadow=True, fontsize='large')
legend.get_frame().set_facecolor('#FFFFFF')
if dataset == 'mnist':
plt.xlabel('Perturbation (x ' + '$10^{-1}$' + ')')
elif dataset == 'cifar':
plt.xlabel('Perturbation (x ' + '$10^{-2}$' + ')')
else:
plt.xlabel('Perturbation (x ' + '$10^{-3}$' + ')')
plt.ylabel('Accuracy (%)')
plt.savefig(path_fig + dataset + '/' + title + '.png')
plt.clf()
data = [np.array(eps_range), np.array(nb_correct_fgsm) / y.shape[0], np.array(nb_correct_pgd) / y.shape[0], np.array(nb_correct_bim) / y.shape[0]]
out = csv.writer(open(path_csv + dataset + '/' + title + '.csv', "w"), delimiter=',', quoting=csv.QUOTE_ALL)
out.writerows(zip(*data))
return 0
def load_bim(classifier):
return BasicIterativeMethod(classifier=classifier, max_iter=20, eps=0.2)