batch_size = 512 epsilon = 0.3 if __name__ == '__main__': # print(use_cuda) ckpt_path = args.ckpt_path # Obtain the model object model = SmallCNN().to(device) # Define the loss function and the optimizer criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.5) # Initialize the classifier mnist_classifier = PyTorchClassifier(clip_values=(0, 1), model=model, loss=criterion, optimizer=optimizer, input_shape=(1, 28, 28), nb_classes=10) # Load the classifier model.load_state_dict(torch.load(ckpt_path)) # Test the classifier predictions = mnist_classifier.predict(test_dataset_array) accuracy = np.sum(np.argmax(predictions, axis=1) == test_label_dataset_array) / len(test_label_dataset_array) print('Accuracy before attack: {}%'.format(accuracy * 100)) # Craft the adversarial examples # PGD-20 # adv_crafter_pgd_40 = ProjectedGradientDescent(mnist_classifier, eps=epsilon, max_iter=40, batch_size=batch_size) #
fp = gzip.open('../../data/cifar/testing_labels.npy', 'rb') y_test = pickle.load(fp) # Obtain the model object model = ResNet18() criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=0.9, weight_decay=5e-4) # Initialize the classifier cifar_classifier = PyTorchClassifier(clip_values=(0, 1), model=model, loss=criterion, optimizer=optimizer, input_shape=(3, 32, 32), nb_classes=10) # Train the classifier #cifar_classifier.fit(x_train, y_train, batch_size=128, nb_epochs=10) state = load_data('../../data/cifar/cifar_target_classifier.npy') cifar_classifier.__setstate__(state) # Test the classifier predictions = cifar_classifier.predict(x_test) accuracy = np.sum( np.argmax(predictions, axis=1) == np.argmax(y_test, axis=1)) / len(y_test) print('Accuracy before attack: {}%'.format(accuracy * 100)) #x_test_adv = load_data('../../data/cifar/fgsm_adversarial.npy')
def test_ptclassifier(self): """ Third test with the PyTorchClassifier. :return: """ # Get MNIST (x_train, y_train), (x_test, y_test) = self.mnist x_train = np.swapaxes(x_train, 1, 3) x_test = np.swapaxes(x_test, 1, 3) # Define the network model = Model() # Define a loss function and optimizer loss_fn = nn.CrossEntropyLoss() optimizer = optim.Adam(model.parameters(), lr=0.01) # Get classifier ptc = PyTorchClassifier((0, 1), model, loss_fn, optimizer, (1, 28, 28), 10) ptc.fit(x_train, y_train, batch_size=BATCH_SIZE, nb_epochs=10) # First attack cl2m = CarliniL2Method(classifier=ptc, targeted=True, max_iter=100, binary_search_steps=1, learning_rate=1, initial_const=10, decay=0) params = {'y': random_targets(y_test, ptc.nb_classes)} x_test_adv = cl2m.generate(x_test, **params) self.assertFalse((x_test == x_test_adv).all()) self.assertTrue((x_test_adv <= 1.0001).all()) self.assertTrue((x_test_adv >= -0.0001).all()) target = np.argmax(params['y'], axis=1) y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1) self.assertTrue((target == y_pred_adv).any()) # Second attack cl2m = CarliniL2Method(classifier=ptc, targeted=False, max_iter=100, binary_search_steps=1, learning_rate=1, initial_const=10, decay=0) params = {'y': random_targets(y_test, ptc.nb_classes)} x_test_adv = cl2m.generate(x_test, **params) self.assertFalse((x_test == x_test_adv).all()) self.assertTrue((x_test_adv <= 1.0001).all()) self.assertTrue((x_test_adv >= -0.0001).all()) target = np.argmax(params['y'], axis=1) y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1) self.assertTrue((target != y_pred_adv).any()) # Third attack cl2m = CarliniL2Method(classifier=ptc, targeted=False, max_iter=100, binary_search_steps=1, learning_rate=1, initial_const=10, decay=0) params = {} x_test_adv = cl2m.generate(x_test, **params) self.assertFalse((x_test == x_test_adv).all()) self.assertTrue((x_test_adv <= 1.0001).all()) self.assertTrue((x_test_adv >= -0.0001).all()) y_pred = np.argmax(ptc.predict(x_test), axis=1) y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1) self.assertTrue((y_pred != y_pred_adv).any())
def robustness_check(object_storage_url, object_storage_username, object_storage_password, data_bucket_name, result_bucket_name, model_id, feature_testset_path='processed_data/X_test.npy', label_testset_path='processed_data/y_test.npy', clip_values=(0, 1), nb_classes=2, input_shape=(1, 3, 64, 64), model_class_file='model.py', model_class_name='model', LossFn='', Optimizer='', epsilon=0.2): url = re.compile(r"https?://") cos = Minio(url.sub('', object_storage_url), access_key=object_storage_username, secret_key=object_storage_password) dataset_filenamex = "X_test.npy" dataset_filenamey = "y_test.npy" weights_filename = "model.pt" model_files = model_id + '/_submitted_code/model.zip' cos.fget_object(data_bucket_name, feature_testset_path, dataset_filenamex) cos.fget_object(data_bucket_name, label_testset_path, dataset_filenamey) cos.fget_object(result_bucket_name, model_id + '/' + weights_filename, weights_filename) cos.fget_object(result_bucket_name, model_files, 'model.zip') # Load PyTorch model definition from the source code. zip_ref = zipfile.ZipFile('model.zip', 'r') zip_ref.extractall('model_files') zip_ref.close() modulename = 'model_files.' + model_class_file.split('.')[0].replace( '-', '_') ''' We required users to define where the model class is located or follow some naming convention we have provided. ''' model_class = getattr(importlib.import_module(modulename), model_class_name) # load & compile model device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') model = model_class().to(device) model.load_state_dict(torch.load(weights_filename, map_location=device)) # Define Loss and optimizer function for the PyTorch model if LossFn: loss_fn = eval(LossFn) else: loss_fn = torch.nn.CrossEntropyLoss() if Optimizer: optimizer = eval(Optimizer) else: optimizer = torch.optim.Adam(model.parameters(), lr=0.001) # create pytorch classifier classifier = PyTorchClassifier(clip_values, model, loss_fn, optimizer, input_shape, nb_classes) # load test dataset x = np.load(dataset_filenamex) y = np.load(dataset_filenamey) # craft adversarial samples using FGSM crafter = FastGradientMethod(classifier, eps=epsilon) x_samples = crafter.generate(x) # obtain all metrics (robustness score, perturbation metric, reduction in confidence) metrics, y_pred_orig, y_pred_adv = get_metrics(model, x, x_samples, y) print("metrics:", metrics) return metrics
def test_input_shape(self): # Start to test ptc = PyTorchClassifier(None, self._model, self._loss_fn, self._optimizer, (1, 28, 28), (10, )) self.assertTrue(np.array(ptc.input_shape == (1, 28, 28)).all())
def test_nb_classes(self): # Start to test ptc = PyTorchClassifier(None, self._model, self._loss_fn, self._optimizer, (1, 28, 28), (10, )) self.assertTrue(ptc.nb_classes == 10)