def __init__(self, n_val=10000, val_lower=0.0, val_upper=1.0):
h5_path = fetch_syn_digits()
if h5_path is not None:
f = tables.open_file(h5_path, mode='r')
train_X_u8 = f.root.syn_digits.train_X_u8
train_y = f.root.syn_digits.train_y
self.test_X_u8 = f.root.syn_digits.test_X_u8
self.test_y = f.root.syn_digits.test_y
if n_val == 0 or n_val is None:
self.train_X_u8 = train_X_u8
self.train_y = train_y
self.val_X_u8 = np.zeros((0, 3, 32, 32), dtype=np.uint8)
self.val_y = np.zeros((0, ), dtype=np.int32)
else:
self.train_X_u8 = train_X_u8[:-n_val]
self.train_y = train_y[:-n_val]
self.val_X_u8 = train_X_u8[-n_val:]
self.val_y = train_y[-n_val:]
else:
raise RuntimeError('Could not load Syn-Digits dataset')
self.train_X = ImageArrayUInt8ToFloat32(self.train_X_u8, val_lower,
val_upper)
self.val_X = ImageArrayUInt8ToFloat32(self.val_X_u8, val_lower,
val_upper)
self.test_X = ImageArrayUInt8ToFloat32(self.test_X_u8, val_lower,
val_upper)
def __init__(self,
n_val=2935,
shuffle_seed=12345,
val_lower=0.0,
val_upper=1.0):
h5_path = os.path.join(get_data_dir('gtsrb'), 'gtsrb.h5')
if not os.path.exists(h5_path):
raise RuntimeError(
'Could not load GTSRB from {}; please run '
'\'prepare_gtsrb.py\' to create it'.format(h5_path))
f = tables.open_file(h5_path, mode='r')
train_X_u8 = f.root.gtsrb.train_X_u8
train_y = f.root.gtsrb.train_y
test_X_u8 = f.root.gtsrb.test_X_u8
test_y = f.root.gtsrb.test_y
shuffle_rng = np.random.RandomState(shuffle_seed)
train_ndx = shuffle_rng.permutation(len(train_X_u8))
test_ndx = shuffle_rng.permutation(len(test_X_u8))
train_X_u8 = train_X_u8[:][train_ndx]
train_y = train_y[:][train_ndx]
test_X_u8 = test_X_u8[:][test_ndx]
test_y = test_y[:][test_ndx]
if n_val == 0 or n_val is None:
self.train_X_u8, self.train_y = train_X_u8, train_y
self.val_X_u8 = np.zeros((0, 3, 40, 40), dtype=np.uint8)
self.val_y = np.zeros((0, ), dtype=np.int32)
else:
self.train_X_u8, self.val_X_u8 = train_X_u8[:-n_val], train_X_u8[
-n_val:]
self.train_y, self.val_y = train_y[:-n_val], train_y[-n_val:]
self.test_X_u8 = test_X_u8
self.test_y = test_y
self.n_classes = 43
self.train_X = ImageArrayUInt8ToFloat32(self.train_X_u8, val_lower,
val_upper)
self.val_X = ImageArrayUInt8ToFloat32(self.val_X_u8, val_lower,
val_upper)
self.test_X = ImageArrayUInt8ToFloat32(self.test_X_u8, val_lower,
val_upper)
def test_ImageArrayUInt8ToFloat32():
from batchup.image.utils import ImageArrayUInt8ToFloat32
x = np.array([[0, 127, 255], [30, 60, 90]], dtype=np.uint8)
xf32 = x.astype(np.float32) * np.float32(1.0 / 255.0)
assert x.shape == (2, 3)
x_wrapped = ImageArrayUInt8ToFloat32(x)
x_wrapped_cen = ImageArrayUInt8ToFloat32(x, val_lower=-1.0,
val_upper=1.0)
assert x_wrapped.shape == (2, 3)
assert len(x_wrapped) == 2
assert np.allclose(x_wrapped[...], xf32)
# Check that scaling the range works
xf32_cen = x.astype(np.float32) * np.float32(2.0 / 255.0) - 1.0
assert np.allclose(x_wrapped_cen[...], xf32_cen)
# An array-like object without a shape parameter
class ArrayLike (object):
def __len__(self):
return 10
def __getitem__(self, index):
return np.arange(10).astype(np.uint8)[index]
# Check the basics
a = ArrayLike()
a_wrapped = ImageArrayUInt8ToFloat32(a)
assert len(a_wrapped) == 10
assert a_wrapped[0] == 0.0
# Now ensure that accessing the `shape` attribute raises `AttributeError`
with pytest.raises(AttributeError):
shape = a_wrapped.shape
del shape
def __init__(self,
n_val=10000,
n_test=10000,
shuffle_seed=12345,
val_lower=0.0,
val_upper=1.0):
h5_path = os.path.join(get_data_dir('syn_signs'), 'syn_signs.h5')
if not os.path.exists(h5_path):
raise RuntimeError(
'Could not load Syn-Signs from {}; please run '
'\'prepare_synsigns.py\' to create it'.format(h5_path))
f = tables.open_file(h5_path, mode='r')
X_u8 = f.root.syn_signs.X_u8
y = f.root.syn_signs.y
shuffle_rng = np.random.RandomState(shuffle_seed)
ndx = shuffle_rng.permutation(len(X_u8))
X_u8 = X_u8[:][ndx]
y = y[:][ndx]
n_vt = n_val + n_test
self.train_X_u8 = X_u8[:-n_vt]
self.train_y = y[:-n_vt]
valtest_X_u8 = X_u8[-n_vt:]
valtest_y = y[-n_vt:]
self.val_X_u8 = valtest_X_u8[:n_val]
self.val_y = valtest_y[:n_val]
self.test_X_u8 = valtest_X_u8[n_val:]
self.test_y = valtest_y[n_val:]
self.n_classes = 43
self.train_X = ImageArrayUInt8ToFloat32(self.train_X_u8, val_lower,
val_upper)
self.val_X = ImageArrayUInt8ToFloat32(self.val_X_u8, val_lower,
val_upper)
self.test_X = ImageArrayUInt8ToFloat32(self.test_X_u8, val_lower,
val_upper)
def load_mnistm(zero_centre=False, val=False):
#
# Load mnistm
#
setname = 'mnist_m'
# pdb.set_trace()
d_set = mnist_m(setname=setname)
print('Loading ' + setname + '...')
print('train...')
imglist = os.listdir(d_set.trainpath)
for i, dirstr in enumerate(imglist):
imglist2 = os.listdir(os.path.join(d_set.trainpath, dirstr))
for k, imgstr in enumerate(imglist2):
img = cv2.imread(os.path.join(d_set.trainpath, dirstr, imgstr))
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img = cv2.resize(img, (d_set.height, d_set.width),
interpolation=cv2.INTER_AREA)
d_set.train_X.append(img)
d_set.train_y.append(int(dirstr))
if (k % 50) == 0:
print("{0:02.02f}%".format(
(i + (k / len(imglist2))) / len(imglist) * 100),
end='\r')
print('test...')
imglist = os.listdir(d_set.testpath)
for i, dirstr in enumerate(imglist):
imglist2 = os.listdir(os.path.join(d_set.testpath, dirstr))
for k, imgstr in enumerate(imglist2):
img = cv2.imread(os.path.join(d_set.testpath, dirstr, imgstr))
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img = cv2.resize(img, (d_set.height, d_set.width),
interpolation=cv2.INTER_AREA)
d_set.test_X.append(img)
d_set.test_y.append(int(dirstr))
if (k % 50) == 0:
print("{0:02.02f}%".format(
(i + (k / len(imglist2))) / len(imglist) * 100),
end='\r')
d_set.train_X = np.array(d_set.train_X)
d_set.train_X = ImageArrayUInt8ToFloat32(d_set.train_X, d_set.val_lower,
d_set.val_upper)
d_set.train_y = np.asarray(d_set.train_y)
d_set.train_y = ImageArrayUInt8ToFloat32(d_set.train_y, d_set.val_lower,
d_set.val_upper)
d_set.test_X = np.array(d_set.test_X)
d_set.test_X = ImageArrayUInt8ToFloat32(d_set.test_X, d_set.val_lower,
d_set.val_upper)
d_set.test_y = np.asarray(d_set.test_y)
d_set.test_y = ImageArrayUInt8ToFloat32(d_set.test_y, d_set.val_lower,
d_set.val_upper)
d_set.train_X = np.swapaxes(d_set.train_X, 1, 3)
d_set.train_X = np.swapaxes(d_set.train_X, 2, 3)
d_set.test_X = np.swapaxes(d_set.test_X, 1, 3)
d_set.test_X = np.swapaxes(d_set.test_X, 2, 3)
d_set.train_X = d_set.train_X[:]
d_set.train_y = d_set.train_y[:]
d_set.test_X = d_set.test_X[:]
d_set.test_y = d_set.test_y[:]
print('{}: train: X.shape={}, y.shape={}, '
'test: X.shape={}, y.shape={}'.format(setname, d_set.train_X.shape,
d_set.train_y.shape,
d_set.test_X.shape,
d_set.test_y.shape))
print('{}: train: X.min={}, X.max={}'.format(setname, d_set.train_X.min(),
d_set.train_X.max()))
return d_set
def load_Officedata(setname, name, zero_centre=False, val=False):
#
# Load Office data
#
d_set = Office(setname=setname, name=name)
print('Loading ' + setname + name + '...')
imglist = os.listdir(d_set.path)
for i, dirstr in enumerate(imglist):
imglist2 = os.listdir(os.path.join(d_set.path, dirstr))
for k, imgstr in enumerate(imglist2):
img = cv2.imread(os.path.join(d_set.path, dirstr, imgstr))
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img = cv2.resize(img, (d_set.height, d_set.width),
interpolation=cv2.INTER_AREA)
# split train and test
if randrange(10) > 1:
d_set.train_X.append(img)
d_set.train_y.append(i)
else:
d_set.test_X.append(img)
d_set.test_y.append(i)
if (k % 50) == 0:
print("{0:02.02f}%".format(
(i + (k / len(imglist2))) / len(imglist) * 100),
end='\r')
d_set.train_X = np.array(d_set.train_X)
d_set.train_X = ImageArrayUInt8ToFloat32(d_set.train_X, d_set.val_lower,
d_set.val_upper)
d_set.train_y = np.asarray(d_set.train_y)
d_set.test_X = np.array(d_set.test_X)
d_set.test_X = ImageArrayUInt8ToFloat32(d_set.test_X, d_set.val_lower,
d_set.val_upper)
d_set.test_y = np.asarray(d_set.test_y)
d_set.train_X = d_set.train_X[:]
d_set.train_y = d_set.train_y[:]
d_set.test_X = d_set.test_X[:]
d_set.test_y = d_set.test_y[:]
d_set.train_X = np.swapaxes(d_set.train_X, 1, 3)
d_set.train_X = np.swapaxes(d_set.train_X, 2, 3)
d_set.test_X = np.swapaxes(d_set.test_X, 1, 3)
d_set.test_X = np.swapaxes(d_set.test_X, 2, 3)
print('{}{}: train: X.shape={}, y.shape={}, '
'test: X.shape={}, y.shape={}'.format(setname, name,
d_set.train_X.shape,
d_set.train_y.shape,
d_set.test_X.shape,
d_set.test_y.shape))
print('{}{}: train: X.min={}, X.max={}'.format(setname, name,
d_set.train_X.min(),
d_set.train_X.max()))
return d_set
def load_trafficdata(setname, zero_centre=False, val=False):
#
# Load Traffic Data
#
d_set = trafficdata(setname=setname)
print('Loading ' + setname + '...')
print('train...')
imglist = os.listdir(d_set.trainpath)
if setname == "cityscapes":
imglist = [file for file in imglist if file.endswith(".jpg")]
if setname == "GTA":
imglist = [file for file in imglist if file.endswith(".png")]
for i, dirstr in enumerate(imglist):
img = cv2.imread(os.path.join(d_set.trainpath, dirstr))
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img1 = img[:, 0:int(img.shape[0] / 2)]
img2 = img[:, int(img.shape[0] / 2):img.shape[0]]
img1 = cv2.resize(img1, (d_set.height, d_set.width),
interpolation=cv2.INTER_AREA)
img2 = cv2.resize(img2, (d_set.height, d_set.width),
interpolation=cv2.INTER_AREA)
d_set.train_X.append(img1)
d_set.train_y.append(img2)
if (i % 50) == 0:
print("{0:02.02f}%".format(i / len(imglist) * 100), end='\r')
print('test...')
imglist = os.listdir(d_set.testpath)
if setname == "cityscapes":
imglist = [file for file in imglist if file.endswith(".jpg")]
if setname == "GTA":
imglist = [file for file in imglist if file.endswith(".png")]
for i, dirstr in enumerate(imglist):
img = cv2.imread(os.path.join(d_set.testpath, dirstr))
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img1 = img[:, 0:int(img.shape[0] / 2)]
img2 = img[:, int(img.shape[0] / 2):img.shape[0]]
img1 = cv2.resize(img1, (d_set.height, d_set.width),
interpolation=cv2.INTER_AREA)
img2 = cv2.resize(img2, (d_set.height, d_set.width),
interpolation=cv2.INTER_AREA)
d_set.test_X.append(img1)
d_set.test_y.append(img2)
if (i % 50) == 0:
print("{0:02.02f}%".format(i / len(imglist) * 100), end='\r')
d_set.train_X = np.array(d_set.train_X)
d_set.train_X = ImageArrayUInt8ToFloat32(d_set.train_X, d_set.val_lower,
d_set.val_upper)
d_set.train_y = np.asarray(d_set.train_y)
d_set.train_y = ImageArrayUInt8ToFloat32(d_set.train_y, d_set.val_lower,
d_set.val_upper)
d_set.test_X = np.array(d_set.test_X)
d_set.test_X = ImageArrayUInt8ToFloat32(d_set.test_X, d_set.val_lower,
d_set.val_upper)
d_set.test_y = np.asarray(d_set.test_y)
d_set.test_y = ImageArrayUInt8ToFloat32(d_set.test_y, d_set.val_lower,
d_set.val_upper)
d_set.train_X = np.swapaxes(d_set.train_X, 1, 3)
d_set.train_X = np.swapaxes(d_set.train_X, 2, 3)
d_set.test_X = np.swapaxes(d_set.test_X, 1, 3)
d_set.test_X = np.swapaxes(d_set.test_X, 2, 3)
d_set.train_X = d_set.train_X[:]
d_set.train_y = d_set.train_y[:]
d_set.test_X = d_set.test_X[:]
d_set.test_y = d_set.test_y[:]
print('{}: train: X.shape={}, y.shape={}, '
'test: X.shape={}, y.shape={}'.format(setname, d_set.train_X.shape,
d_set.train_y.shape,
d_set.test_X.shape,
d_set.test_y.shape))
print('{}: train: X.min={}, X.max={}'.format(setname, d_set.train_X.min(),
d_set.train_X.max()))
return d_set