def test_pytorch_binary_pgd(art_warning, get_mnist_dataset):
"""
This test instantiates a binary classification PyTorch model, then attacks it using PGD
"""
class BasicModel(nn.Module):
def __init__(self):
super(BasicModel, self).__init__()
self.layer_1 = nn.Linear(20, 32)
self.layer_2 = nn.Linear(32, 1)
def forward(self, x):
x = F.relu(self.layer_1(x))
x = torch.sigmoid(self.layer_2(x))
return x
try:
device = "cpu"
x, y = sklearn.datasets.make_classification(n_samples=10000,
n_features=20,
n_informative=5,
n_redundant=2,
n_repeated=0,
n_classes=2)
train_x, test_x, train_y, test_y = sklearn.model_selection.train_test_split(
x, y, test_size=0.2)
train_x = test_x.astype(np.float32)
train_y = train_y.astype(np.float32)
test_x = test_x.astype(np.float32)
model = BasicModel()
loss_func = nn.BCELoss()
model.to(device)
opt = optim.Adam(model.parameters(), lr=0.001)
classifier = PyTorchClassifier(
model=model,
loss=loss_func,
optimizer=opt,
input_shape=(1, 28, 28),
nb_classes=2,
)
classifier.fit(train_x, train_y, batch_size=64, nb_epochs=3)
test_x_batch = test_x[0:16]
preds = classifier.predict(test_x_batch)
attacker = ProjectedGradientDescent(classifier, eps=0.5)
generated = attacker.generate(test_x_batch)
adv_predicted = classifier.predict(generated)
assert (adv_predicted != preds).all()
except ARTTestException as e:
art_warning(e)
def _test_preprocessing_defences_forward(
get_default_mnist_subset, image_dl_estimator, device_type, preprocessing_defences
):
(_, _), (x_test_mnist, y_test_mnist) = get_default_mnist_subset
classifier_, _ = image_dl_estimator()
clip_values = (0, 1)
criterion = nn.CrossEntropyLoss()
classifier = PyTorchClassifier(
clip_values=clip_values,
model=classifier_.model,
preprocessing_defences=preprocessing_defences,
loss=criterion,
input_shape=(1, 28, 28),
nb_classes=10,
device_type=device_type,
)
with torch.no_grad():
predictions_classifier = classifier.predict(x_test_mnist)
# Apply the same defences by hand
x_test_defense = x_test_mnist
for defence in preprocessing_defences:
x_test_defense, _ = defence(x_test_defense, y_test_mnist)
x_test_defense = torch.tensor(x_test_defense)
with torch.no_grad():
predictions_check = classifier_.model(x_test_defense)
predictions_check = predictions_check.cpu().numpy()
# Check that the prediction results match
np.testing.assert_array_almost_equal(predictions_classifier, predictions_check, decimal=4)
def _predict_classifier(self, x: np.ndarray, batch_size: int,
training_mode: bool, **kwargs) -> np.ndarray:
x = x.astype(ART_NUMPY_DTYPE)
return PyTorchClassifier.predict(self,
x=x,
batch_size=batch_size,
training_mode=training_mode,
**kwargs)
def test_pickle(art_warning, get_default_mnist_subset, image_dl_estimator):
try:
(x_train_mnist,
y_train_mnist), (x_test_mnist,
y_test_mnist) = get_default_mnist_subset
from art import config
full_path = os.path.join(config.ART_DATA_PATH, "my_classifier")
folder = os.path.split(full_path)[0]
if not os.path.exists(folder):
os.makedirs(folder)
# The model used within the common ART pytorch get_image_classifier_list does not support pickling
model = Model()
loss_fn = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.01)
myclassifier_2 = PyTorchClassifier(model=model,
clip_values=(0, 1),
loss=loss_fn,
optimizer=optimizer,
input_shape=(1, 28, 28),
nb_classes=10)
myclassifier_2.fit(x_train_mnist,
y_train_mnist,
batch_size=100,
nb_epochs=1)
pickle.dump(myclassifier_2, open(full_path, "wb"))
with open(full_path, "rb") as f:
loaded_model = pickle.load(f)
np.testing.assert_equal(myclassifier_2._clip_values,
loaded_model._clip_values)
assert myclassifier_2._channel_index == loaded_model._channel_index
assert set(myclassifier_2.__dict__.keys()) == set(
loaded_model.__dict__.keys())
# Test predict
predictions_1 = myclassifier_2.predict(x_test_mnist)
accuracy_1 = np.sum(
np.argmax(predictions_1, axis=1) == np.argmax(
y_test_mnist, axis=1)) / y_test_mnist.shape[0]
predictions_2 = loaded_model.predict(x_test_mnist)
accuracy_2 = np.sum(
np.argmax(predictions_2, axis=1) == np.argmax(
y_test_mnist, axis=1)) / y_test_mnist.shape[0]
assert accuracy_1 == accuracy_2
except ARTTestException as e:
art_warning(e)
def _predict_classifier(self, x: np.ndarray,
batch_size: int) -> np.ndarray:
x = x.astype(ART_NUMPY_DTYPE)
return PyTorchClassifier.predict(self, x=x, batch_size=batch_size)
def test_pytorch_iris(self):
"""
Third test for PyTorch.
:return:
"""
# Build PyTorchClassifier
victim_ptc = get_tabular_classifier_pt()
class Model(nn.Module):
"""
Create Iris model for PyTorch.
"""
def __init__(self):
super(Model, self).__init__()
self.fully_connected1 = nn.Linear(4, 10)
self.fully_connected2 = nn.Linear(10, 10)
self.fully_connected3 = nn.Linear(10, 3)
# pylint: disable=W0221
# disable pylint because of API requirements for function
def forward(self, x):
x = self.fully_connected1(x)
x = self.fully_connected2(x)
logit_output = self.fully_connected3(x)
return logit_output
# Define the network
model = Model()
# Define a loss function and optimizer
loss_fn = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.001)
# Get classifier
thieved_ptc = PyTorchClassifier(
model=model,
loss=loss_fn,
optimizer=optimizer,
input_shape=(4, ),
nb_classes=3,
clip_values=(0, 1),
channels_first=True,
)
# Create attack
copycat_cnn = CopycatCNN(
classifier=victim_ptc,
batch_size_fit=self.batch_size,
batch_size_query=self.batch_size,
nb_epochs=NB_EPOCHS,
nb_stolen=NB_STOLEN,
)
thieved_ptc = copycat_cnn.extract(x=self.x_train_iris,
thieved_classifier=thieved_ptc)
victim_preds = np.argmax(victim_ptc.predict(x=self.x_train_iris[:100]),
axis=1)
thieved_preds = np.argmax(
thieved_ptc.predict(x=self.x_train_iris[:100]), axis=1)
acc = np.sum(victim_preds == thieved_preds) / len(victim_preds)
self.assertGreater(acc, 0.3)
def test_pytorch_classifier(self):
"""
Third test with the PyTorchClassifier.
:return:
"""
x_train = np.reshape(self.x_train_mnist,
(self.x_train_mnist.shape[0], 1, 28, 28)).astype(
np.float32)
# Build PyTorchClassifier
victim_ptc = get_image_classifier_pt()
class Model(nn.Module):
"""
Create model for pytorch.
"""
def __init__(self):
super(Model, self).__init__()
self.conv = nn.Conv2d(in_channels=1,
out_channels=1,
kernel_size=7)
self.pool = nn.MaxPool2d(4, 4)
self.fullyconnected = nn.Linear(25, 10)
# pylint: disable=W0221
# disable pylint because of API requirements for function
def forward(self, x):
"""
Forward function to evaluate the model
:param x: Input to the model
:return: Prediction of the model
"""
x = self.conv(x)
x = torch.nn.functional.relu(x)
x = self.pool(x)
x = x.reshape(-1, 25)
x = self.fullyconnected(x)
x = torch.nn.functional.softmax(x, dim=1)
return x
# Define the network
model = Model()
# Define a loss function and optimizer
loss_fn = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.01)
# Get classifier
thieved_ptc = PyTorchClassifier(model=model,
loss=loss_fn,
optimizer=optimizer,
input_shape=(1, 28, 28),
nb_classes=10,
clip_values=(0, 1))
# Create attack
copycat_cnn = CopycatCNN(
classifier=victim_ptc,
batch_size_fit=self.batch_size,
batch_size_query=self.batch_size,
nb_epochs=NB_EPOCHS,
nb_stolen=NB_STOLEN,
)
thieved_ptc = copycat_cnn.extract(x=x_train,
thieved_classifier=thieved_ptc)
victim_preds = np.argmax(victim_ptc.predict(x=x_train[:100]), axis=1)
thieved_preds = np.argmax(thieved_ptc.predict(x=x_train[:100]), axis=1)
acc = np.sum(victim_preds == thieved_preds) / len(victim_preds)
self.assertGreater(acc, 0.3)
