def test_keras_imagenet_transfer():
classifier_module = import_module(
"armory.baseline_models.keras.inception_resnet_v2")
classifier_fn = getattr(classifier_module, "get_art_model")
classifier = classifier_fn(
model_kwargs={},
wrapper_kwargs={},
weights_file="inceptionresnetv2_imagenet_v1.h5",
)
preprocessing_fn = getattr(classifier_module, "preprocessing_fn")
dataset = adversarial_datasets.imagenet_adversarial(
split_type="adversarial",
epochs=1,
batch_size=100,
dataset_dir=DATASET_DIR,
preprocessing_fn=preprocessing_fn,
)
accuracy_clean = 0
accuracy_adv = 0
for _ in range(dataset.batches_per_epoch):
(x_clean, x_adv), y = dataset.get_batch()
predictions_clean = classifier.predict(x_clean)
accuracy_clean += np.sum(
np.argmax(predictions_clean, axis=1) == y) / len(y)
predictions_adv = classifier.predict(x_adv)
accuracy_adv += np.sum(
np.argmax(predictions_adv, axis=1) == y) / len(y)
assert (accuracy_clean / dataset.batches_per_epoch) > 0.75
assert (accuracy_adv / dataset.batches_per_epoch) < 0.73
def test_keras_imagenet():
classifier_module = import_module("armory.baseline_models.keras.resnet50")
classifier_fn = getattr(classifier_module, "get_art_model")
weights_path = maybe_download_weights_from_s3("resnet50_imagenet_v1.h5")
classifier = classifier_fn(model_kwargs={},
wrapper_kwargs={},
weights_path=weights_path)
dataset = adversarial_datasets.imagenet_adversarial(
split="adversarial",
epochs=1,
batch_size=100,
dataset_dir=DATASET_DIR,
)
accuracy_clean = 0
accuracy_adv = 0
for _ in range(dataset.batches_per_epoch):
(x_clean, x_adv), y = dataset.get_batch()
predictions_clean = classifier.predict(x_clean)
accuracy_clean += np.sum(
np.argmax(predictions_clean, axis=1) == y) / len(y)
predictions_adv = classifier.predict(x_adv)
accuracy_adv += np.sum(
np.argmax(predictions_adv, axis=1) == y) / len(y)
assert (accuracy_clean / dataset.batches_per_epoch) > 0.65
assert (accuracy_adv / dataset.batches_per_epoch) < 0.02
def test_imagenet_adv():
batch_size = 100
total_size = 1000
test_dataset = adversarial_datasets.imagenet_adversarial(
dataset_dir=DATASET_DIR,
split="adversarial",
batch_size=batch_size,
epochs=1,
adversarial_key="adversarial",
)
assert test_dataset.size == total_size
assert test_dataset.batch_size == batch_size
assert test_dataset.batches_per_epoch == (total_size // batch_size +
bool(total_size % batch_size))
x, y = test_dataset.get_batch()
for i in range(2):
assert x[i].shape == (batch_size, 224, 224, 3)
assert y.shape == (batch_size, )