def setUp(self):
import mxnet as mx
master_seed(42)
x = mx.random.uniform(shape=(10, 1, 5, 5))
y = mx.random.uniform(shape=10)
dataset = mx.gluon.data.dataset.ArrayDataset(x, y)
data_loader = mx.gluon.data.DataLoader(dataset, batch_size=5, shuffle=True)
self.data_gen = MXDataGenerator(data_loader, size=10, batch_size=5)
def test_fit_generator(self):
import mxnet as mx
from art.data_generators import MXDataGenerator
(x_train, y_train), (x_test, y_test) = self.mnist
acc = np.sum(
np.argmax(self.classifier.predict(x_test), axis=1) == np.argmax(
y_test, axis=1)) / NB_TEST
logger.info('Accuracy: %.2f%%', (acc * 100))
# Create MXNet dataset and loader
dataset = mx.gluon.data.dataset.ArrayDataset(x_train, y_train)
data_loader = mx.gluon.data.DataLoader(dataset,
batch_size=5,
shuffle=True)
data_gen = MXDataGenerator(data_loader, size=NB_TRAIN, batch_size=5)
# Fit model with generator
self.classifier.fit_generator(data_gen, nb_epochs=2)
acc2 = np.sum(
np.argmax(self.classifier.predict(x_test), axis=1) == np.argmax(
y_test, axis=1)) / NB_TEST
logger.info('Accuracy: %.2f%%', (acc * 100))
self.assertTrue(acc2 >= .8 * acc)
def _image_data_generator(**kwargs):
(x_train_mnist, y_train_mnist), (_, _) = get_default_mnist_subset
if framework == "keras" or framework == "kerastf":
return KerasDataGenerator(
iterator=image_iterator,
size=x_train_mnist.shape[0],
batch_size=default_batch_size,
)
if framework == "tensorflow":
if not is_tf_version_2:
return TensorFlowDataGenerator(
sess=kwargs["sess"],
iterator=image_iterator,
iterator_type="initializable",
iterator_arg={},
size=x_train_mnist.shape[0],
batch_size=default_batch_size,
)
if framework == "pytorch":
return PyTorchDataGenerator(iterator=image_iterator,
size=x_train_mnist.shape[0],
batch_size=default_batch_size)
if framework == "mxnet":
return MXDataGenerator(iterator=image_iterator,
size=x_train_mnist.shape[0],
batch_size=default_batch_size)
def test_fit_generator(self):
from art.data_generators import MXDataGenerator
acc = (np.sum(
np.argmax(self.classifier.predict(self.x_test_mnist), axis=1) ==
np.argmax(self.y_test_mnist, axis=1)) / self.n_test)
logger.info("Accuracy: %.2f%%", (acc * 100))
# Create MXNet dataset and loader
dataset = gluon.data.dataset.ArrayDataset(self.x_train_mnist,
self.y_train_mnist)
data_loader = gluon.data.DataLoader(dataset,
batch_size=5,
shuffle=True)
data_gen = MXDataGenerator(data_loader,
size=self.n_train,
batch_size=5)
# Fit model with generator
self.classifier.fit_generator(data_gen, nb_epochs=2)
acc2 = (np.sum(
np.argmax(self.classifier.predict(self.x_test_mnist), axis=1) ==
np.argmax(self.y_test_mnist, axis=1)) / self.n_test)
logger.info("Accuracy: %.2f%%", (acc * 100))
self.assertGreaterEqual(acc2, 0.8 * acc)
def setUpClass(cls):
import mxnet as mx
x = mx.random.uniform(shape=(10, 1, 5, 5))
y = mx.random.uniform(shape=10)
dataset = mx.gluon.data.dataset.ArrayDataset(x, y)
data_loader = mx.gluon.data.DataLoader(dataset,
batch_size=5,
shuffle=True)
cls.data_gen = MXDataGenerator(data_loader, size=10, batch_size=5)
class TestMXGenerator(unittest.TestCase):
def setUp(self):
import mxnet as mx
master_seed(seed=42, set_mxnet=True)
x = mx.random.uniform(shape=(10, 1, 5, 5))
y = mx.random.uniform(shape=10)
dataset = mx.gluon.data.dataset.ArrayDataset(x, y)
data_loader = mx.gluon.data.DataLoader(dataset,
batch_size=5,
shuffle=True)
self.data_gen = MXDataGenerator(data_loader, size=10, batch_size=5)
def test_gen_interface(self):
x, y = self.data_gen.get_batch()
# Check return types
self.assertTrue(isinstance(x, np.ndarray))
self.assertTrue(isinstance(y, np.ndarray))
# Check shapes
self.assertEqual(x.shape, (5, 1, 5, 5))
self.assertEqual(y.shape, (5, ))
def test_mxnet_specific(self):
import mxnet as mx
iter_ = iter(self.data_gen.iterator)
x, y = next(iter_)
# Check return types
self.assertTrue(isinstance(x, mx.ndarray.NDArray))
self.assertTrue(isinstance(y, mx.ndarray.NDArray))
# Check shapes
self.assertEqual(x.shape, (5, 1, 5, 5))
self.assertEqual(y.shape, (5, ))
def test_error(self):
with self.assertRaises(TypeError):
_ = MXDataGenerator("data_loader", size=10, batch_size=5)
def _image_data_generator(**kwargs):
(x_train_mnist, y_train_mnist), (_, _) = get_default_mnist_subset
image_it = image_iterator()
data_generator = None
if framework == "keras" or framework == "kerastf":
data_generator = KerasDataGenerator(
iterator=image_it,
size=x_train_mnist.shape[0],
batch_size=default_batch_size,
)
if framework == "tensorflow1":
data_generator = TensorFlowDataGenerator(
sess=kwargs["sess"],
iterator=image_it,
iterator_type="initializable",
iterator_arg={},
size=x_train_mnist.shape[0],
batch_size=default_batch_size,
)
if framework == "pytorch":
data_generator = PyTorchDataGenerator(
iterator=image_it,
size=x_train_mnist.shape[0],
batch_size=default_batch_size)
if framework == "mxnet":
data_generator = MXDataGenerator(iterator=image_it,
size=x_train_mnist.shape[0],
batch_size=default_batch_size)
if data_generator is None:
raise ARTTestFixtureNotImplemented(
"framework {0} does not current have any data generator implemented",
image_data_generator.__name__,
framework,
)
return data_generator
def _image_data_generator(**kwargs):
(x_train_mnist, y_train_mnist), (_, _) = get_default_mnist_subset
image_it = image_iterator()
data_generator = None
if framework == "keras" or framework == "kerastf":
data_generator = KerasDataGenerator(
iterator=image_it,
size=x_train_mnist.shape[0],
batch_size=default_batch_size,
)
if framework == "tensorflow1":
data_generator = TensorFlowDataGenerator(
sess=kwargs["sess"],
iterator=image_it,
iterator_type="initializable",
iterator_arg={},
size=x_train_mnist.shape[0],
batch_size=default_batch_size,
)
if framework == "tensorflow2":
data_generator = TensorFlowV2DataGenerator(
iterator=image_it,
size=x_train_mnist.shape[0],
batch_size=default_batch_size,
)
if framework == "pytorch":
data_generator = PyTorchDataGenerator(
iterator=image_it,
size=x_train_mnist.shape[0],
batch_size=default_batch_size)
if framework == "mxnet":
data_generator = MXDataGenerator(iterator=image_it,
size=x_train_mnist.shape[0],
batch_size=default_batch_size)
return data_generator
def test_error(self):
with self.assertRaises(TypeError):
_ = MXDataGenerator("data_loader", size=10, batch_size=5)