def test_mnist_load(self):
"""Loading MNIST dataset"""
dataset_train = spn.MNISTDataset(subset=spn.MNISTDataset.Subset.TRAIN,
format=spn.ImageFormat.INT,
num_epochs=1,
batch_size=100,
shuffle=False,
ratio=1,
crop=0,
num_threads=1,
allow_smaller_final_batch=True,
classes=None)
dataset_test = spn.MNISTDataset(subset=spn.MNISTDataset.Subset.TEST,
format=spn.ImageFormat.INT,
num_epochs=1,
batch_size=100,
shuffle=False,
ratio=1,
crop=0,
num_threads=1,
allow_smaller_final_batch=True,
classes=None)
dataset_all = spn.MNISTDataset(subset=spn.MNISTDataset.Subset.ALL,
format=spn.ImageFormat.INT,
num_epochs=1,
batch_size=100,
shuffle=False,
ratio=1,
crop=0,
num_threads=1,
allow_smaller_final_batch=True,
classes=None)
dataset_train.load_data()
self.assertEqual(dataset_train.samples.shape, (60000, 784))
self.assertIs(dataset_train.samples.dtype.type, np.uint8)
self.assertEqual(dataset_train.labels.shape, (60000, 1))
self.assertIs(dataset_train.labels.dtype.type, np.dtype(np.int).type)
dataset_test.load_data()
self.assertEqual(dataset_test.samples.shape, (10000, 784))
self.assertIs(dataset_test.samples.dtype.type, np.uint8)
self.assertEqual(dataset_test.labels.shape, (10000, 1))
self.assertIs(dataset_test.labels.dtype.type, np.dtype(np.int).type)
dataset_all.load_data()
self.assertEqual(dataset_all.samples.shape, (70000, 784))
self.assertIs(dataset_all.samples.dtype.type, np.uint8)
self.assertEqual(dataset_all.labels.shape, (70000, 1))
self.assertIs(dataset_all.labels.dtype.type, np.dtype(np.int).type)
def test_int_ratiocrop(self):
dataset = spn.MNISTDataset(subset=spn.MNISTDataset.Subset.TEST,
format=spn.ImageFormat.INT,
num_epochs=1,
batch_size=1,
shuffle=False,
ratio=2,
crop=2,
num_threads=1,
allow_smaller_final_batch=True,
classes=None)
img, label = self.generic_dataset_test(dataset)
true_img = np.array([[
11, 4, 2, 7, 9, 9, 9, 9, 10, 11, 13, 93, 110, 56, 26, 25, 25, 24,
15, 11, 19, 164, 192, 218, 217, 222, 220, 223, 162, 13, 11, 11, 11,
31, 51, 62, 54, 189, 173, 8, 11, 10, 10, 9, 7, 0, 62, 244, 59, 6,
11, 11, 11, 11, 10, 7, 192, 156, 5, 11, 11, 11, 11, 12, 4, 64, 226,
40, 7, 11, 11, 11, 11, 8, 24, 218, 141, 3, 11, 11, 11, 11, 11, 2,
148, 220, 25, 8, 11, 11, 11, 11, 6, 67, 255, 74, 3, 12, 11, 11
]],
dtype=np.uint8)
true_label = np.array([[7]], dtype=np.int)
np.testing.assert_array_equal(img, true_img)
np.testing.assert_array_equal(label, true_label)
def test_int_noproc(self):
dataset = spn.MNISTDataset(subset=spn.MNISTDataset.Subset.TEST,
format=spn.ImageFormat.INT,
num_epochs=1,
batch_size=1,
shuffle=False,
ratio=1,
crop=0,
num_threads=1,
allow_smaller_final_batch=True,
classes=None)
img, label = self.generic_dataset_test(dataset)
true_img = np.array([[
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 84, 185, 159, 151, 60, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222, 254, 254, 254, 254,
241, 198, 198, 198, 198, 198, 198, 198, 198, 170, 52, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 67, 114, 72, 114, 163, 227, 254, 225, 254,
254, 254, 250, 229, 254, 254, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 17, 66, 14, 67, 67, 67, 59, 21, 236, 254, 106, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 83, 253, 209, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 22, 233, 255, 83, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 254, 238, 44,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 59, 249, 254, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 254, 187, 5, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 205, 248, 58, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 126, 254, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 75, 251, 240, 57, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 221, 254, 166, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
3, 203, 254, 219, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 38, 254, 254, 77, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 224, 254, 115, 1,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 133, 254, 254, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 61, 242, 254, 254, 52, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 254, 254, 219,
40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 121, 254, 207, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0
]],
dtype=np.uint8)
true_label = np.array([[7]], dtype=np.int)
np.testing.assert_array_equal(img, true_img)
np.testing.assert_array_equal(label, true_label)
def test_binary_noproc(self):
dataset = spn.MNISTDataset(subset=spn.MNISTDataset.Subset.TEST,
format=spn.ImageFormat.BINARY,
num_epochs=1,
batch_size=1,
shuffle=False,
ratio=1,
crop=0,
num_threads=1,
allow_smaller_final_batch=True,
classes=None)
img, label = self.generic_dataset_test(dataset)
true_img = np.array([[
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
]],
dtype=np.uint8)
true_label = np.array([[7]], dtype=np.int)
np.testing.assert_array_equal(img, true_img)
np.testing.assert_array_equal(label, true_label)
def test_mnist_classes(self):
dataset = spn.MNISTDataset(subset=spn.MNISTDataset.Subset.TEST,
format=spn.ImageFormat.INT,
num_epochs=1, batch_size=100, shuffle=False,
ratio=1, crop=0, num_threads=1,
allow_smaller_final_batch=True,
classes=[1, 3, 5])
dataset.load_data()
self.assertEqual(dataset.samples.shape, (3037, 784))
self.assertIs(dataset.samples.dtype.type, np.uint8)
self.assertEqual(dataset.labels.shape, (3037, 1))
self.assertIs(dataset.labels.dtype.type, np.dtype(np.int).type)
self.assertEqual(set(dataset.labels.flatten()), {1, 3, 5})
def test_float_noproc(self):
dataset = spn.MNISTDataset(subset=spn.MNISTDataset.Subset.TEST,
format=spn.ImageFormat.FLOAT,
num_epochs=1,
batch_size=1,
shuffle=False,
ratio=1,
crop=0,
num_threads=1,
allow_smaller_final_batch=True,
classes=None)
img, label = self.generic_dataset_test(dataset)
true_img = np.array([[
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0.32941177, 0.72549021, 0.62352943, 0.59215689, 0.23529412,
0.14117648, 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0.87058824, 0.99607843, 0.99607843,
0.99607843, 0.99607843, 0.94509804, 0.7764706, 0.7764706,
0.7764706, 0.7764706, 0.7764706, 0.7764706, 0.7764706, 0.7764706,
0.66666669, 0.20392157, 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0.26274511, 0.44705883, 0.28235295, 0.44705883, 0.63921571,
0.89019608, 0.99607843, 0.88235295, 0.99607843, 0.99607843,
0.99607843, 0.98039216, 0.89803922, 0.99607843, 0.99607843,
0.54901963, 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0.06666667, 0.25882354, 0.05490196, 0.26274511,
0.26274511, 0.26274511, 0.23137255, 0.08235294, 0.9254902,
0.99607843, 0.41568628, 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.32549021,
0.99215686, 0.81960785, 0.07058824, 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0.08627451, 0.9137255, 1., 0.32549021, 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0.50588238, 0.99607843, 0.93333334, 0.17254902, 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0.23137255, 0.97647059, 0.99607843, 0.24313726, 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0.52156866, 0.99607843, 0.73333335, 0.01960784, 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0.03529412, 0.80392158, 0.97254902, 0.22745098, 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0.49411765, 0.99607843, 0.71372551, 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0.29411766, 0.98431373, 0.94117647, 0.22352941,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0.07450981, 0.86666667, 0.99607843,
0.65098041, 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0.01176471, 0.79607844,
0.99607843, 0.85882354, 0.13725491, 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0.14901961, 0.99607843, 0.99607843, 0.3019608, 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0.12156863, 0.87843138, 0.99607843, 0.4509804, 0.00392157, 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0.52156866, 0.99607843, 0.99607843, 0.20392157,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0.23921569, 0.94901961, 0.99607843,
0.99607843, 0.20392157, 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.47450981,
0.99607843, 0.99607843, 0.85882354, 0.15686275, 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0.47450981, 0.99607843, 0.81176472, 0.07058824, 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0.
]],
dtype=np.float32)
true_label = np.array([[7]], dtype=np.int)
np.testing.assert_allclose(img, true_img, atol=0.00001)
np.testing.assert_array_equal(label, true_label)