batch_size = 512
epsilon = 0.3
if __name__ == '__main__':
# print(use_cuda)
ckpt_path = args.ckpt_path
# Obtain the model object
model = SmallCNN().to(device)
# Define the loss function and the optimizer
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.5)
# Initialize the classifier
mnist_classifier = PyTorchClassifier(clip_values=(0, 1), model=model, loss=criterion, optimizer=optimizer,
input_shape=(1, 28, 28), nb_classes=10)
# Load the classifier
model.load_state_dict(torch.load(ckpt_path))
# Test the classifier
predictions = mnist_classifier.predict(test_dataset_array)
accuracy = np.sum(np.argmax(predictions, axis=1) == test_label_dataset_array) / len(test_label_dataset_array)
print('Accuracy before attack: {}%'.format(accuracy * 100))
# Craft the adversarial examples
# PGD-20
# adv_crafter_pgd_40 = ProjectedGradientDescent(mnist_classifier, eps=epsilon, max_iter=40, batch_size=batch_size)
#
fp = gzip.open('../../data/cifar/testing_labels.npy', 'rb')
y_test = pickle.load(fp)
# Obtain the model object
model = ResNet18()
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=0.9,
weight_decay=5e-4)
# Initialize the classifier
cifar_classifier = PyTorchClassifier(clip_values=(0, 1),
model=model,
loss=criterion,
optimizer=optimizer,
input_shape=(3, 32, 32),
nb_classes=10)
# Train the classifier
#cifar_classifier.fit(x_train, y_train, batch_size=128, nb_epochs=10)
state = load_data('../../data/cifar/cifar_target_classifier.npy')
cifar_classifier.__setstate__(state)
# Test the classifier
predictions = cifar_classifier.predict(x_test)
accuracy = np.sum(
np.argmax(predictions, axis=1) == np.argmax(y_test, axis=1)) / len(y_test)
print('Accuracy before attack: {}%'.format(accuracy * 100))
#x_test_adv = load_data('../../data/cifar/fgsm_adversarial.npy')
def test_ptclassifier(self):
"""
Third test with the PyTorchClassifier.
:return:
"""
# Get MNIST
(x_train, y_train), (x_test, y_test) = self.mnist
x_train = np.swapaxes(x_train, 1, 3)
x_test = np.swapaxes(x_test, 1, 3)
# Define the network
model = Model()
# Define a loss function and optimizer
loss_fn = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.01)
# Get classifier
ptc = PyTorchClassifier((0, 1), model, loss_fn, optimizer, (1, 28, 28),
10)
ptc.fit(x_train, y_train, batch_size=BATCH_SIZE, nb_epochs=10)
# First attack
cl2m = CarliniL2Method(classifier=ptc,
targeted=True,
max_iter=100,
binary_search_steps=1,
learning_rate=1,
initial_const=10,
decay=0)
params = {'y': random_targets(y_test, ptc.nb_classes)}
x_test_adv = cl2m.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1)
self.assertTrue((target == y_pred_adv).any())
# Second attack
cl2m = CarliniL2Method(classifier=ptc,
targeted=False,
max_iter=100,
binary_search_steps=1,
learning_rate=1,
initial_const=10,
decay=0)
params = {'y': random_targets(y_test, ptc.nb_classes)}
x_test_adv = cl2m.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1)
self.assertTrue((target != y_pred_adv).any())
# Third attack
cl2m = CarliniL2Method(classifier=ptc,
targeted=False,
max_iter=100,
binary_search_steps=1,
learning_rate=1,
initial_const=10,
decay=0)
params = {}
x_test_adv = cl2m.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
y_pred = np.argmax(ptc.predict(x_test), axis=1)
y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1)
self.assertTrue((y_pred != y_pred_adv).any())
def robustness_check(object_storage_url,
object_storage_username,
object_storage_password,
data_bucket_name,
result_bucket_name,
model_id,
feature_testset_path='processed_data/X_test.npy',
label_testset_path='processed_data/y_test.npy',
clip_values=(0, 1),
nb_classes=2,
input_shape=(1, 3, 64, 64),
model_class_file='model.py',
model_class_name='model',
LossFn='',
Optimizer='',
epsilon=0.2):
url = re.compile(r"https?://")
cos = Minio(url.sub('', object_storage_url),
access_key=object_storage_username,
secret_key=object_storage_password)
dataset_filenamex = "X_test.npy"
dataset_filenamey = "y_test.npy"
weights_filename = "model.pt"
model_files = model_id + '/_submitted_code/model.zip'
cos.fget_object(data_bucket_name, feature_testset_path, dataset_filenamex)
cos.fget_object(data_bucket_name, label_testset_path, dataset_filenamey)
cos.fget_object(result_bucket_name, model_id + '/' + weights_filename,
weights_filename)
cos.fget_object(result_bucket_name, model_files, 'model.zip')
# Load PyTorch model definition from the source code.
zip_ref = zipfile.ZipFile('model.zip', 'r')
zip_ref.extractall('model_files')
zip_ref.close()
modulename = 'model_files.' + model_class_file.split('.')[0].replace(
'-', '_')
'''
We required users to define where the model class is located or follow
some naming convention we have provided.
'''
model_class = getattr(importlib.import_module(modulename),
model_class_name)
# load & compile model
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
model = model_class().to(device)
model.load_state_dict(torch.load(weights_filename, map_location=device))
# Define Loss and optimizer function for the PyTorch model
if LossFn:
loss_fn = eval(LossFn)
else:
loss_fn = torch.nn.CrossEntropyLoss()
if Optimizer:
optimizer = eval(Optimizer)
else:
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
# create pytorch classifier
classifier = PyTorchClassifier(clip_values, model, loss_fn, optimizer,
input_shape, nb_classes)
# load test dataset
x = np.load(dataset_filenamex)
y = np.load(dataset_filenamey)
# craft adversarial samples using FGSM
crafter = FastGradientMethod(classifier, eps=epsilon)
x_samples = crafter.generate(x)
# obtain all metrics (robustness score, perturbation metric, reduction in confidence)
metrics, y_pred_orig, y_pred_adv = get_metrics(model, x, x_samples, y)
print("metrics:", metrics)
return metrics
def test_input_shape(self):
# Start to test
ptc = PyTorchClassifier(None, self._model, self._loss_fn,
self._optimizer, (1, 28, 28), (10, ))
self.assertTrue(np.array(ptc.input_shape == (1, 28, 28)).all())
def test_nb_classes(self):
# Start to test
ptc = PyTorchClassifier(None, self._model, self._loss_fn,
self._optimizer, (1, 28, 28), (10, ))
self.assertTrue(ptc.nb_classes == 10)