def setUp(self): import mxnet as mx master_seed(42) x = mx.random.uniform(shape=(10, 1, 5, 5)) y = mx.random.uniform(shape=10) dataset = mx.gluon.data.dataset.ArrayDataset(x, y) data_loader = mx.gluon.data.DataLoader(dataset, batch_size=5, shuffle=True) self.data_gen = MXDataGenerator(data_loader, size=10, batch_size=5)
def test_fit_generator(self): import mxnet as mx from art.data_generators import MXDataGenerator (x_train, y_train), (x_test, y_test) = self.mnist acc = np.sum( np.argmax(self.classifier.predict(x_test), axis=1) == np.argmax( y_test, axis=1)) / NB_TEST logger.info('Accuracy: %.2f%%', (acc * 100)) # Create MXNet dataset and loader dataset = mx.gluon.data.dataset.ArrayDataset(x_train, y_train) data_loader = mx.gluon.data.DataLoader(dataset, batch_size=5, shuffle=True) data_gen = MXDataGenerator(data_loader, size=NB_TRAIN, batch_size=5) # Fit model with generator self.classifier.fit_generator(data_gen, nb_epochs=2) acc2 = np.sum( np.argmax(self.classifier.predict(x_test), axis=1) == np.argmax( y_test, axis=1)) / NB_TEST logger.info('Accuracy: %.2f%%', (acc * 100)) self.assertTrue(acc2 >= .8 * acc)
def _image_data_generator(**kwargs): (x_train_mnist, y_train_mnist), (_, _) = get_default_mnist_subset if framework == "keras" or framework == "kerastf": return KerasDataGenerator( iterator=image_iterator, size=x_train_mnist.shape[0], batch_size=default_batch_size, ) if framework == "tensorflow": if not is_tf_version_2: return TensorFlowDataGenerator( sess=kwargs["sess"], iterator=image_iterator, iterator_type="initializable", iterator_arg={}, size=x_train_mnist.shape[0], batch_size=default_batch_size, ) if framework == "pytorch": return PyTorchDataGenerator(iterator=image_iterator, size=x_train_mnist.shape[0], batch_size=default_batch_size) if framework == "mxnet": return MXDataGenerator(iterator=image_iterator, size=x_train_mnist.shape[0], batch_size=default_batch_size)
def test_fit_generator(self): from art.data_generators import MXDataGenerator acc = (np.sum( np.argmax(self.classifier.predict(self.x_test_mnist), axis=1) == np.argmax(self.y_test_mnist, axis=1)) / self.n_test) logger.info("Accuracy: %.2f%%", (acc * 100)) # Create MXNet dataset and loader dataset = gluon.data.dataset.ArrayDataset(self.x_train_mnist, self.y_train_mnist) data_loader = gluon.data.DataLoader(dataset, batch_size=5, shuffle=True) data_gen = MXDataGenerator(data_loader, size=self.n_train, batch_size=5) # Fit model with generator self.classifier.fit_generator(data_gen, nb_epochs=2) acc2 = (np.sum( np.argmax(self.classifier.predict(self.x_test_mnist), axis=1) == np.argmax(self.y_test_mnist, axis=1)) / self.n_test) logger.info("Accuracy: %.2f%%", (acc * 100)) self.assertGreaterEqual(acc2, 0.8 * acc)
def setUpClass(cls): import mxnet as mx x = mx.random.uniform(shape=(10, 1, 5, 5)) y = mx.random.uniform(shape=10) dataset = mx.gluon.data.dataset.ArrayDataset(x, y) data_loader = mx.gluon.data.DataLoader(dataset, batch_size=5, shuffle=True) cls.data_gen = MXDataGenerator(data_loader, size=10, batch_size=5)
class TestMXGenerator(unittest.TestCase): def setUp(self): import mxnet as mx master_seed(seed=42, set_mxnet=True) x = mx.random.uniform(shape=(10, 1, 5, 5)) y = mx.random.uniform(shape=10) dataset = mx.gluon.data.dataset.ArrayDataset(x, y) data_loader = mx.gluon.data.DataLoader(dataset, batch_size=5, shuffle=True) self.data_gen = MXDataGenerator(data_loader, size=10, batch_size=5) def test_gen_interface(self): x, y = self.data_gen.get_batch() # Check return types self.assertTrue(isinstance(x, np.ndarray)) self.assertTrue(isinstance(y, np.ndarray)) # Check shapes self.assertEqual(x.shape, (5, 1, 5, 5)) self.assertEqual(y.shape, (5, )) def test_mxnet_specific(self): import mxnet as mx iter_ = iter(self.data_gen.iterator) x, y = next(iter_) # Check return types self.assertTrue(isinstance(x, mx.ndarray.NDArray)) self.assertTrue(isinstance(y, mx.ndarray.NDArray)) # Check shapes self.assertEqual(x.shape, (5, 1, 5, 5)) self.assertEqual(y.shape, (5, )) def test_error(self): with self.assertRaises(TypeError): _ = MXDataGenerator("data_loader", size=10, batch_size=5)
def _image_data_generator(**kwargs): (x_train_mnist, y_train_mnist), (_, _) = get_default_mnist_subset image_it = image_iterator() data_generator = None if framework == "keras" or framework == "kerastf": data_generator = KerasDataGenerator( iterator=image_it, size=x_train_mnist.shape[0], batch_size=default_batch_size, ) if framework == "tensorflow1": data_generator = TensorFlowDataGenerator( sess=kwargs["sess"], iterator=image_it, iterator_type="initializable", iterator_arg={}, size=x_train_mnist.shape[0], batch_size=default_batch_size, ) if framework == "pytorch": data_generator = PyTorchDataGenerator( iterator=image_it, size=x_train_mnist.shape[0], batch_size=default_batch_size) if framework == "mxnet": data_generator = MXDataGenerator(iterator=image_it, size=x_train_mnist.shape[0], batch_size=default_batch_size) if data_generator is None: raise ARTTestFixtureNotImplemented( "framework {0} does not current have any data generator implemented", image_data_generator.__name__, framework, ) return data_generator
def _image_data_generator(**kwargs): (x_train_mnist, y_train_mnist), (_, _) = get_default_mnist_subset image_it = image_iterator() data_generator = None if framework == "keras" or framework == "kerastf": data_generator = KerasDataGenerator( iterator=image_it, size=x_train_mnist.shape[0], batch_size=default_batch_size, ) if framework == "tensorflow1": data_generator = TensorFlowDataGenerator( sess=kwargs["sess"], iterator=image_it, iterator_type="initializable", iterator_arg={}, size=x_train_mnist.shape[0], batch_size=default_batch_size, ) if framework == "tensorflow2": data_generator = TensorFlowV2DataGenerator( iterator=image_it, size=x_train_mnist.shape[0], batch_size=default_batch_size, ) if framework == "pytorch": data_generator = PyTorchDataGenerator( iterator=image_it, size=x_train_mnist.shape[0], batch_size=default_batch_size) if framework == "mxnet": data_generator = MXDataGenerator(iterator=image_it, size=x_train_mnist.shape[0], batch_size=default_batch_size) return data_generator
def test_error(self): with self.assertRaises(TypeError): _ = MXDataGenerator("data_loader", size=10, batch_size=5)