def test_imagedatagen_interface(self):
train_size, batch_size = 20, 5
x_train, y_train = np.random.rand(
train_size, 28, 28, 1), np.random.randint(0,
2,
size=(train_size, 10))
datagen = ImageDataGenerator(
width_shift_range=0.075,
height_shift_range=0.075,
rotation_range=12,
shear_range=0.075,
zoom_range=0.05,
fill_mode="constant",
cval=0,
)
datagen.fit(x_train)
# Create wrapper and get batch
data_gen = KerasDataGenerator(datagen.flow(x_train,
y_train,
batch_size=batch_size),
size=None,
batch_size=batch_size)
x, y = data_gen.get_batch()
# Check return types
self.assertTrue(isinstance(x, np.ndarray))
self.assertTrue(isinstance(y, np.ndarray))
# Check shapes
self.assertEqual(x.shape, (batch_size, 28, 28, 1))
self.assertEqual(y.shape, (batch_size, 10))
def test_gen_interface(self):
gen = self._dummy_gen()
data_gen = KerasDataGenerator(gen, size=None, batch_size=5)
x, y = data_gen.get_batch()
# Check return types
self.assertTrue(isinstance(x, np.ndarray))
self.assertTrue(isinstance(y, np.ndarray))
# Check shapes
self.assertEqual(x.shape, (5, 28, 28, 1))
self.assertEqual(y.shape, (5, 10))
def _test_fit_image_generator(self, custom_activation=False):
from keras.preprocessing.image import ImageDataGenerator
from art.data_generators import KerasDataGenerator
x_train, y_train = self.mnist[0]
labels_test = np.argmax(self.mnist[1][1], axis=1)
classifier = KerasClassifier((0, 1),
self.model_mnist,
use_logits=False,
custom_activation=custom_activation)
acc = np.sum(
np.argmax(classifier.predict(self.mnist[1][0]), axis=1) ==
labels_test) / NB_TEST
logger.info('Accuracy: %.2f%%', (acc * 100))
keras_gen = ImageDataGenerator(width_shift_range=0.075,
height_shift_range=0.075,
rotation_range=12,
shear_range=0.075,
zoom_range=0.05,
fill_mode='constant',
cval=0)
keras_gen.fit(x_train)
data_gen = KerasDataGenerator(generator=keras_gen.flow(
x_train, y_train, batch_size=BATCH_SIZE),
size=NB_TRAIN,
batch_size=BATCH_SIZE)
classifier.fit_generator(generator=data_gen, nb_epochs=2)
acc2 = np.sum(
np.argmax(classifier.predict(self.mnist[1][0]), axis=1) ==
labels_test) / NB_TEST
logger.info('Accuracy: %.2f%%', (acc2 * 100))
self.assertTrue(acc2 >= .8 * acc)
def _image_data_generator(**kwargs):
(x_train_mnist, y_train_mnist), (_, _) = get_default_mnist_subset
if framework == "keras" or framework == "kerastf":
return KerasDataGenerator(
iterator=image_iterator,
size=x_train_mnist.shape[0],
batch_size=default_batch_size,
)
if framework == "tensorflow":
if not is_tf_version_2:
return TensorFlowDataGenerator(
sess=kwargs["sess"],
iterator=image_iterator,
iterator_type="initializable",
iterator_arg={},
size=x_train_mnist.shape[0],
batch_size=default_batch_size,
)
if framework == "pytorch":
return PyTorchDataGenerator(iterator=image_iterator,
size=x_train_mnist.shape[0],
batch_size=default_batch_size)
if framework == "mxnet":
return MXDataGenerator(iterator=image_iterator,
size=x_train_mnist.shape[0],
batch_size=default_batch_size)
def test_fit_image_generator(get_default_mnist_subset, default_batch_size, get_image_classifier_list):
(x_train_mnist, y_train_mnist), (x_test_mnist, y_test_mnist) = get_default_mnist_subset
classifier, _ = get_image_classifier_list(one_classifier=True)
labels_test = np.argmax(y_test_mnist, axis=1)
accuracy = np.sum(np.argmax(classifier.predict(x_test_mnist), axis=1) == labels_test) / x_test_mnist.shape[0]
logger.info("Accuracy: %.2f%%", (accuracy * 100))
keras_gen = ImageDataGenerator(
width_shift_range=0.075,
height_shift_range=0.075,
rotation_range=12,
shear_range=0.075,
zoom_range=0.05,
fill_mode="constant",
cval=0,
)
keras_gen.fit(x_train_mnist)
data_gen = KerasDataGenerator(iterator=keras_gen.flow(x_train_mnist, y_train_mnist,
batch_size=default_batch_size),
size=x_train_mnist.shape[0], batch_size=default_batch_size)
classifier.fit_generator(generator=data_gen, nb_epochs=5)
accuracy_2 = np.sum(np.argmax(classifier.predict(x_test_mnist), axis=1) == labels_test) / x_test_mnist.shape[0]
logger.info("Accuracy: %.2f%%", (accuracy_2 * 100))
assert accuracy == 0.32
np.testing.assert_array_almost_equal(accuracy_2, 0.35, decimal=0.06)
def _test_fit_generator(self, custom_activation=False):
from art.classifiers.keras import generator_fit
from art.data_generators import KerasDataGenerator
labels = np.argmax(self.mnist[1][1], axis=1)
classifier = KerasClassifier((0, 1),
self.model_mnist,
use_logits=False,
custom_activation=custom_activation)
acc = np.sum(
np.argmax(classifier.predict(self.mnist[1][0]), axis=1) ==
labels) / NB_TEST
logger.info('Accuracy: %.2f%%', (acc * 100))
gen = generator_fit(self.mnist[0][0],
self.mnist[0][1],
batch_size=BATCH_SIZE)
data_gen = KerasDataGenerator(generator=gen,
size=NB_TRAIN,
batch_size=BATCH_SIZE)
classifier.fit_generator(generator=data_gen, nb_epochs=2)
acc2 = np.sum(
np.argmax(classifier.predict(self.mnist[1][0]), axis=1) ==
labels) / NB_TEST
logger.info('Accuracy: %.2f%%', (acc2 * 100))
self.assertTrue(acc2 >= .8 * acc)
def test_fit_image_generator(self):
classifier = get_classifier_kr()
labels_test = np.argmax(self.y_test, axis=1)
accuracy = np.sum(
np.argmax(classifier.predict(self.x_test), axis=1) ==
labels_test) / NB_TEST
logger.info('Accuracy: %.2f%%', (accuracy * 100))
keras_gen = ImageDataGenerator(width_shift_range=0.075,
height_shift_range=0.075,
rotation_range=12,
shear_range=0.075,
zoom_range=0.05,
fill_mode='constant',
cval=0)
keras_gen.fit(self.x_train)
data_gen = KerasDataGenerator(generator=keras_gen.flow(
self.x_train, self.y_train, batch_size=BATCH_SIZE),
size=NB_TRAIN,
batch_size=BATCH_SIZE)
classifier.fit_generator(generator=data_gen, nb_epochs=2)
accuracy_2 = np.sum(
np.argmax(classifier.predict(self.x_test), axis=1) ==
labels_test) / NB_TEST
logger.info('Accuracy: %.2f%%', (accuracy_2 * 100))
self.assertEqual(accuracy, 0.32)
self.assertAlmostEqual(accuracy_2, 0.35, delta=0.02)
def test_fit_image_generator(self):
master_seed(seed=1234)
labels_test = np.argmax(self.y_test_mnist, axis=1)
classifier = get_image_classifier_kr_tf()
acc = np.sum(np.argmax(classifier.predict(self.x_test_mnist), axis=1) == labels_test) / self.n_test
logger.info("Accuracy: %.2f%%", (acc * 100))
keras_gen = ImageDataGenerator(
width_shift_range=0.075,
height_shift_range=0.075,
rotation_range=12,
shear_range=0.075,
zoom_range=0.05,
fill_mode="constant",
cval=0,
)
keras_gen.fit(self.x_train_mnist)
data_gen = KerasDataGenerator(
iterator=keras_gen.flow(self.x_train_mnist, self.y_train_mnist, batch_size=self.batch_size),
size=self.n_train, batch_size=self.batch_size)
classifier.fit_generator(generator=data_gen, nb_epochs=2)
acc2 = np.sum(np.argmax(classifier.predict(self.x_test_mnist), axis=1) == labels_test) / self.n_test
logger.info("Accuracy: %.2f%%", (acc2 * 100))
self.assertEqual(acc, 0.32)
self.assertAlmostEqual(acc2, 0.69, delta=0.02)
def setUp(self):
import keras
master_seed(42)
class DummySequence(keras.utils.Sequence):
def __init__(self):
self._size = 5
self._x = np.random.rand(self._size, 28, 28, 1)
self._y = np.random.randint(0, high=10, size=(self._size, 10))
def __len__(self):
return self._size
def __getitem__(self, idx):
return self._x[idx], self._y[idx]
sequence = DummySequence()
self.data_gen = KerasDataGenerator(sequence, size=5, batch_size=1)
def setUpClass(cls):
(x_train, y_train), (x_test, y_test), min_, max_ = load_mnist()
x_train, y_train = x_train[:NB_TRAIN], y_train[:NB_TRAIN]
cls.mnist = (x_train, y_train), (x_test, y_test), (min_, max_)
# Create simple keras model
import tensorflow as tf
tf_version = [int(v) for v in tf.__version__.split(".")]
if tf_version[0] == 2 and tf_version[1] >= 3:
tf.compat.v1.disable_eager_execution()
from tensorflow.keras import backend as k
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Flatten, Conv2D, MaxPooling2D
else:
import keras.backend as k
from keras.models import Sequential
from keras.layers import Dense, Flatten, Conv2D, MaxPooling2D
k.set_learning_phase(1)
model = Sequential()
model.add(
Conv2D(32,
kernel_size=(3, 3),
activation="relu",
input_shape=x_train.shape[1:]))
model.add(MaxPooling2D(pool_size=(3, 3)))
model.add(Flatten())
model.add(Dense(10, activation="softmax"))
model.compile(loss="categorical_crossentropy",
optimizer="adam",
metrics=["accuracy"])
from art.estimators.classification.keras import KerasClassifier
cls.classifier = KerasClassifier(model=model, clip_values=(min_, max_))
cls.classifier.fit(x_train, y_train, nb_epochs=1, batch_size=128)
cls.defence = ActivationDefence(cls.classifier, x_train, y_train)
datagen = ImageDataGenerator()
datagen.fit(x_train)
data_gen = KerasDataGenerator(datagen.flow(x_train,
y_train,
batch_size=NB_TRAIN),
size=NB_TRAIN,
batch_size=NB_TRAIN)
cls.defence_gen = ActivationDefence(cls.classifier,
None,
None,
generator=data_gen)
def test_fit_generator(get_default_mnist_subset, default_batch_size, get_image_classifier_list):
(x_train_mnist, y_train_mnist), (x_test_mnist, y_test_mnist) = get_default_mnist_subset
gen = generator_fit(x_train_mnist, y_train_mnist, batch_size=default_batch_size)
data_gen = KerasDataGenerator(iterator=gen, size=x_train_mnist.shape[0], batch_size=default_batch_size)
classifier, _ = get_image_classifier_list(one_classifier=True)
expected_values = {"pre_fit_accuracy": ExpectedValue(0.32, 0.06), "post_fit_accuracy": ExpectedValue(0.36, 0.06)}
backend_test_fit_generator(expected_values, classifier, data_gen, get_default_mnist_subset, nb_epochs=3)
def test_gen_keras_specific(self):
gen = self._dummy_gen()
data_gen = KerasDataGenerator(gen, size=None, batch_size=5)
iter_ = iter(data_gen.generator)
x, y = next(iter_)
# Check return types
self.assertTrue(isinstance(x, np.ndarray))
self.assertTrue(isinstance(y, np.ndarray))
# Check shapes
self.assertEqual(x.shape, (5, 28, 28, 1))
self.assertEqual(y.shape, (5, 10))
def test_fit_generator(self):
classifier = get_classifier_kr()
labels = np.argmax(self.y_test, axis=1)
accuracy = np.sum(np.argmax(classifier.predict(self.x_test), axis=1) == labels) / NB_TEST
logger.info('Accuracy: %.2f%%', (accuracy * 100))
gen = generator_fit(self.x_train, self.y_train, batch_size=BATCH_SIZE)
data_gen = KerasDataGenerator(generator=gen, size=NB_TRAIN, batch_size=BATCH_SIZE)
classifier.fit_generator(generator=data_gen, nb_epochs=3)
accuracy_2 = np.sum(np.argmax(classifier.predict(self.x_test), axis=1) == labels) / NB_TEST
logger.info('Accuracy: %.2f%%', (accuracy_2 * 100))
self.assertEqual(accuracy, 0.32)
self.assertAlmostEqual(accuracy_2, 0.36, delta=0.02)
def test_fit_generator(self):
labels = np.argmax(self.y_test_mnist, axis=1)
classifier = get_image_classifier_kr_tf()
acc = np.sum(np.argmax(classifier.predict(self.x_test_mnist), axis=1) == labels) / self.n_test
logger.info("Accuracy: %.2f%%", (acc * 100))
gen = generator_fit(self.x_train_mnist, self.y_train_mnist, batch_size=self.batch_size)
data_gen = KerasDataGenerator(iterator=gen, size=self.n_train, batch_size=self.batch_size)
classifier.fit_generator(generator=data_gen, nb_epochs=2)
acc2 = np.sum(np.argmax(classifier.predict(self.x_test_mnist), axis=1) == labels) / self.n_test
logger.info("Accuracy: %.2f%%", (acc2 * 100))
self.assertEqual(acc, 0.32)
self.assertAlmostEqual(acc2, 0.70, delta=0.15)
def _image_data_generator(**kwargs):
(x_train_mnist, y_train_mnist), (_, _) = get_default_mnist_subset
image_it = image_iterator()
data_generator = None
if framework == "keras" or framework == "kerastf":
data_generator = KerasDataGenerator(
iterator=image_it,
size=x_train_mnist.shape[0],
batch_size=default_batch_size,
)
if framework == "tensorflow1":
data_generator = TensorFlowDataGenerator(
sess=kwargs["sess"],
iterator=image_it,
iterator_type="initializable",
iterator_arg={},
size=x_train_mnist.shape[0],
batch_size=default_batch_size,
)
if framework == "pytorch":
data_generator = PyTorchDataGenerator(
iterator=image_it,
size=x_train_mnist.shape[0],
batch_size=default_batch_size)
if framework == "mxnet":
data_generator = MXDataGenerator(iterator=image_it,
size=x_train_mnist.shape[0],
batch_size=default_batch_size)
if data_generator is None:
raise ARTTestFixtureNotImplemented(
"framework {0} does not current have any data generator implemented",
image_data_generator.__name__,
framework,
)
return data_generator
def _image_data_generator(**kwargs):
(x_train_mnist, y_train_mnist), (_, _) = get_default_mnist_subset
image_it = image_iterator()
data_generator = None
if framework == "keras" or framework == "kerastf":
data_generator = KerasDataGenerator(
iterator=image_it,
size=x_train_mnist.shape[0],
batch_size=default_batch_size,
)
if framework == "tensorflow1":
data_generator = TensorFlowDataGenerator(
sess=kwargs["sess"],
iterator=image_it,
iterator_type="initializable",
iterator_arg={},
size=x_train_mnist.shape[0],
batch_size=default_batch_size,
)
if framework == "tensorflow2":
data_generator = TensorFlowV2DataGenerator(
iterator=image_it,
size=x_train_mnist.shape[0],
batch_size=default_batch_size,
)
if framework == "pytorch":
data_generator = PyTorchDataGenerator(
iterator=image_it,
size=x_train_mnist.shape[0],
batch_size=default_batch_size)
if framework == "mxnet":
data_generator = MXDataGenerator(iterator=image_it,
size=x_train_mnist.shape[0],
batch_size=default_batch_size)
return data_generator
def test_fit_generator(self):
from art.classifiers.keras import generator_fit
from art.data_generators import KerasDataGenerator
labels = np.argmax(self.mnist[1][1], axis=1)
acc = np.sum(
np.argmax(self.classifier.predict(self.mnist[1][0]), axis=1) ==
labels) / NB_TEST
logger.info('Accuracy: %.2f%%', (acc * 100))
gen = generator_fit(self.mnist[0][0], self.mnist[0][1], batch_size=100)
data_gen = KerasDataGenerator(generator=gen,
size=NB_TRAIN,
batch_size=100)
self.classifier.fit_generator(generator=data_gen, nb_epochs=2)
acc2 = np.sum(
np.argmax(self.classifier.predict(self.mnist[1][0]), axis=1) ==
labels) / NB_TEST
logger.info('Accuracy: %.2f%%', (acc2 * 100))
self.assertTrue(acc2 >= .8 * acc)
class TestKerasDataGenerator(unittest.TestCase):
def setUp(self):
import keras
master_seed(42)
class DummySequence(keras.utils.Sequence):
def __init__(self):
self._size = 5
self._x = np.random.rand(self._size, 28, 28, 1)
self._y = np.random.randint(0, high=10, size=(self._size, 10))
def __len__(self):
return self._size
def __getitem__(self, idx):
return self._x[idx], self._y[idx]
sequence = DummySequence()
self.data_gen = KerasDataGenerator(sequence, size=5, batch_size=1)
def tearDown(self):
import keras.backend as k
k.clear_session()
def test_gen_interface(self):
gen = self._dummy_gen()
data_gen = KerasDataGenerator(gen, size=None, batch_size=5)
x, y = data_gen.get_batch()
# Check return types
self.assertTrue(isinstance(x, np.ndarray))
self.assertTrue(isinstance(y, np.ndarray))
# Check shapes
self.assertEqual(x.shape, (5, 28, 28, 1))
self.assertEqual(y.shape, (5, 10))
def test_gen_keras_specific(self):
gen = self._dummy_gen()
data_gen = KerasDataGenerator(gen, size=None, batch_size=5)
iter_ = iter(data_gen.generator)
x, y = next(iter_)
# Check return types
self.assertTrue(isinstance(x, np.ndarray))
self.assertTrue(isinstance(y, np.ndarray))
# Check shapes
self.assertEqual(x.shape, (5, 28, 28, 1))
self.assertEqual(y.shape, (5, 10))
def test_sequence_keras_specific(self):
iter_ = iter(self.data_gen.generator)
x, y = next(iter_)
# Check return types
self.assertTrue(isinstance(x, np.ndarray))
self.assertTrue(isinstance(y, np.ndarray))
# Check shapes
self.assertEqual(x.shape, (28, 28, 1))
self.assertEqual(y.shape, (10,))
def test_sequence_interface(self):
x, y = self.data_gen.get_batch()
# Check return types
self.assertTrue(isinstance(x, np.ndarray))
self.assertTrue(isinstance(y, np.ndarray))
# Check shapes
self.assertEqual(x.shape, (28, 28, 1))
self.assertEqual(y.shape, (10,))
def test_imagedatagen_interface(self):
train_size, batch_size = 20, 5
x_train, y_train = np.random.rand(train_size, 28, 28, 1), np.random.randint(0, 2, size=(train_size, 10))
datagen = ImageDataGenerator(width_shift_range=0.075, height_shift_range=0.075, rotation_range=12,
shear_range=0.075, zoom_range=0.05, fill_mode='constant', cval=0)
datagen.fit(x_train)
# Create wrapper and get batch
data_gen = KerasDataGenerator(datagen.flow(x_train, y_train, batch_size=batch_size), size=None,
batch_size=batch_size)
x, y = data_gen.get_batch()
# Check return types
self.assertTrue(isinstance(x, np.ndarray))
self.assertTrue(isinstance(y, np.ndarray))
# Check shapes
self.assertEqual(x.shape, (batch_size, 28, 28, 1))
self.assertEqual(y.shape, (batch_size, 10))
def test_imagedatagen_keras_specific(self):
train_size, batch_size = 20, 5
x_train, y_train = np.random.rand(train_size, 28, 28, 1), np.random.randint(0, 2, size=(train_size, 10))
datagen = ImageDataGenerator(width_shift_range=0.075, height_shift_range=0.075, rotation_range=12,
shear_range=0.075, zoom_range=0.05, fill_mode='constant', cval=0)
datagen.fit(x_train)
# Create wrapper and get batch
data_gen = KerasDataGenerator(datagen.flow(x_train, y_train, batch_size=batch_size), size=None,
batch_size=batch_size)
x, y = next(data_gen.generator)
# Check return types
self.assertTrue(isinstance(x, np.ndarray))
self.assertTrue(isinstance(y, np.ndarray))
# Check shapes
self.assertEqual(x.shape, (batch_size, 28, 28, 1))
self.assertEqual(y.shape, (batch_size, 10))
@staticmethod
def _dummy_gen(size=5):
yield np.random.rand(size, 28, 28, 1), np.random.randint(low=0, high=10, size=(size, 10))
# Load data and normalize
(x_train, y_train), (x_test, y_test), min_, max_ = load_cifar10()
# Build a Keras image augmentation object and wrap it in ART
batch_size = 50
datagen = ImageDataGenerator(horizontal_flip=True,
width_shift_range=0.125,
height_shift_range=0.125,
fill_mode="constant",
cval=0.0)
datagen.fit(x_train)
art_datagen = KerasDataGenerator(
datagen.flow(x=x_train, y=y_train, batch_size=batch_size, shuffle=True),
size=x_train.shape[0],
batch_size=batch_size,
)
# Create a toy Keras CNN architecture & wrap it under ART interface
classifier = KerasClassifier(build_model(),
clip_values=(0, 1),
use_logits=False)
# Create attack for adversarial trainer; here, we use 2 attacks, both crafting adv examples on the target model
pgd = ProjectedGradientDescent(classifier,
eps=8,
eps_step=2,
max_iter=10,
num_random_init=20)
