def get_data():
train = BatchData(dataset.Mnist('train'), 128)
test = BatchData(dataset.Mnist('test'), 256, remainder=True)
train = PrintData(train)
return train, test
def get_data():
def f(dp):
im = dp[0][:, :, None]
onehot = np.eye(10)[dp[1]]
return [im, onehot]
train = BatchData(MapData(dataset.Mnist('train'), f), 128)
test = BatchData(MapData(dataset.Mnist('test'), f), 256)
return train, test
def get_data():
train = BatchData(dataset.Mnist('train'), 128)
# 若 remainder 为 True, 剩余不足256大小的样本也会成为一个小 batch
test = BatchData(dataset.Mnist('test'), 256, remainder=True)
# Behave like an identity mapping, but print shape and
# range of the first few datapoints.
train = PrintData(train)
return train, test
def get_data():
# We don't need any fancy data loading for this simple example.
# See dataflow tutorial at https://tensorpack.readthedocs.io/tutorial/dataflow.html
train = BatchData(dataset.Mnist('train'), 128)
test = BatchData(dataset.Mnist('test'), 256, remainder=True)
train = PrintData(train)
return train, test
def get_data():
def f(dp):
im = dp[0][:, :, None]
onehot = np.zeros(10, dtype='int32')
onehot[dp[1]] = 1
return [im, onehot]
train = BatchData(MapData(dataset.Mnist('train'), f), 128)
test = BatchData(MapData(dataset.Mnist('test'), f), 256)
return train, test
def get_input_mnist():
train, test = dataset.Mnist('train'), dataset.Mnist('test', shuffle=False)
def preprocess(x):
image, label = x
image = np.expand_dims(image, axis=-1) # Add a channels dimension
onehot = np.zeros(10)
onehot[label] = 1.0
return image, onehot
return MapData(train,
preprocess), MapData(test,
preprocess), ((28, 28, 1), (10, ))
def get_data():
# Dataflow / Input src
# Batch size = hyperparam. others found (cross validation)
train = BatchData(dataset.Mnist('train'), batch_size / num_gpus)
test = BatchData(dataset.Mnist('test'),
2 * batch_size / num_gpus,
remainder=True)
# train = PrintData(train)
print("Testing Dataflow Speed ...")
print(TestDataSpeed(dataset.Mnist('train')).start())
print("Ended Dataflow test")
return train, test
def get_data():
# ds = ConcatData([dataset.Mnist('train'), dataset.Mnist('test')])
ds = ConcatData([dataset.Mnist('test')])
ds = BatchData(ds, BATCH)
# ds = MapData(ds, lambda dp: [dp[0]]) # only use the image
ds = MapData(ds, lambda dp: [dp[0], dp[1]]) # only use the image
return ds
def get_data(train_or_test):
isTrain = train_or_test == 'train'
ds = dataset.Mnist(train_or_test)
# ds = dataset.Mnist(train_or_test, dir='data')
ds = BatchData(ds, BATCH_SIZE, remainder=not isTrain)
if isTrain:
ds = PrefetchData(ds, 3, 2)
return ds
def get_data(train_or_test, batch_size):
isTrain = train_or_test == 'train'
ds = dataset.Mnist(train_or_test, isTrain)
datasize = len(ds)
if isTrain:
augmentors = [imgaug.Rotation(45.0)]
else:
augmentors = [imgaug.Rotation(45.0)]
ds = AugmentImageComponent(ds, augmentors)
ds = BatchData(ds, batch_size, remainder=not isTrain)
if isTrain:
ds = PrefetchData(ds, 3, 2)
return ds, datasize
def get_data(isTrain):
ds = dataset.Mnist('train' if isTrain else 'test')
# create augmentation for both training and testing
augs = [
imgaug.MapImage(lambda x: x * 255.0),
imgaug.RandomResize((0.7, 1.2), (0.7, 1.2)),
imgaug.RotationAndCropValid(45),
imgaug.RandomPaste((IMAGE_SIZE, IMAGE_SIZE)),
imgaug.SaltPepperNoise(white_prob=0.01, black_prob=0.01)
]
ds = AugmentImageComponent(ds, augs)
ds = JoinData([ds, ds])
# stack the two digits into two channels, and label it with the sum
ds = MapData(ds, lambda dp: [np.stack([dp[0], dp[2]], axis=2), dp[1] + dp[3]])
ds = BatchData(ds, 128)
return ds
def get_data():
dataset_train = BatchData(DisturbLabel(dataset.Mnist('train'), args.prob),
128)
dataset_test = BatchData(dataset.Mnist('test'), 256, remainder=True)
return dataset_train, dataset_test
# img = next(self.dataset)[0]
# fig.add_subplot(2, 1, 1)
# plt.imshow(img)
# rec = self.decoder(self.encoder(img[None, ...])[0])[0][0].reshape((IMAGE_SIZE, IMAGE_SIZE))
# fig.add_subplot(2, 1, 2)
# plt.imshow(rec)
# plt.show()
if __name__ == '__main__':
# automatically setup the directory train_log/mnist-convnet for logging
logger.auto_set_dir()
dataset_train, dataset_test = get_data()
evaluator = Evaluator(dataset.Mnist('test'))
# How many iterations you want in each epoch.
# This len(data) is the default value.
steps_per_epoch = len(dataset_train)
# get the config which contains everything necessary in a training
config = TrainConfig(
model=Model(),
# The input source for training. FeedInput is slow, this is just for demo purpose.
# In practice it's best to use QueueInput or others. See tutorials for details.
data=FeedInput(dataset_train),
callbacks=[
ScheduledHyperParamSetter('learning_rate', [(50, 1e-3),
(500, 1e-4)]),
ModelSaver(), # save the model after every epoch
def get_data(train_or_test='train'):
isTrain = train_or_test == 'train'
# ds = dataset.Mnist(train_or_test)
ds = dataset.Mnist(train_or_test, shuffle=False)
ds = BatchData(ds, BATCH_SIZE, remainder=not isTrain)
return ds
def get_test_data(batch=128):
ds = dataset.Mnist('test')
ds = BatchData(ds, batch)
return ds
def get_data():
ds = ConcatData([dataset.Mnist('train'), dataset.Mnist('test')])
ds = BatchData(ds, BATCH)
return ds
def get_data():
train = BatchData(dataset.Mnist('train'), 10000)
test = BatchData(dataset.Mnist('test'), 256, remainder=True)
return train, test