def __init__(self, mode, batch_size=256, shuffle=False, num_workers=25, cache=50000, collate_fn=default_collate, drop_last=False, cuda=False): # enumerate standard imagenet augmentors imagenet_augmentors = fbresnet_augmentor(mode == 'train') # load the lmdb if we can find it lmdb_loc = os.path.join(os.environ['IMAGENET'], 'ILSVRC-%s.lmdb' % mode) ds = td.LMDBData(lmdb_loc, shuffle=False) ds = td.LocallyShuffleData(ds, cache) ds = td.PrefetchData(ds, 5000, 1) ds = td.LMDBDataPoint(ds) ds = td.MapDataComponent(ds, lambda x: cv2.imdecode(x, cv2.IMREAD_COLOR), 0) ds = td.AugmentImageComponent(ds, imagenet_augmentors) ds = td.PrefetchDataZMQ(ds, num_workers) self.ds = td.BatchData(ds, batch_size) self.ds.reset_state() self.batch_size = batch_size self.num_workers = num_workers self.cuda = cuda
def __init__(self, mode, batch_size=256, shuffle=False, num_workers=25, cache=50000, collate_fn=default_collate, remainder=False, cuda=False, transform=None): # enumerate standard imagenet augmentors #imagenet_augmentors = fbresnet_augmentor(mode == 'train') imagenet_augmentors = [ImgAugTVCompose(transform)] # load the lmdb if we can find it lmdb_loc = os.path.join(os.environ['IMAGENET'], 'ILSVRC-%s.lmdb' % mode) ds = td.LMDBData(lmdb_loc, shuffle=False) if mode == 'train': ds = td.LocallyShuffleData(ds, cache) ds = td.PrefetchData(ds, 5000, 1) ds = td.LMDBDataPoint(ds) #ds = td.MapDataComponent(ds, lambda x: cv2.imdecode(x, cv2.IMREAD_COLOR), 0) ds = td.MapDataComponent( ds, lambda x: np.asarray(Image.open(io.BytesIO(x)).convert('RGB')), 0) ds = td.AugmentImageComponent(ds, imagenet_augmentors) ds = td.PrefetchDataZMQ(ds, num_workers) self.ds = td.BatchData(ds, batch_size, remainder=remainder) self.ds.reset_state() self.batch_size = batch_size self.num_workers = num_workers self.cuda = cuda
def __init__(self, mode, do_aug, batch_size=256, shuffle=False, num_workers=25, cache=50000, cuda=False, out_tensor=True, data_transforms=None): # enumerate standard imagenet augmentors imagenet_augmentors = fbresnet_augmentor(do_aug) # load the lmdb if we can find it lmdb_loc = os.path.join(os.environ['IMAGENET'], 'ILSVRC-%s.lmdb' % mode) ds = td.LMDBSerializer.load(lmdb_loc, shuffle=shuffle) #ds = td.LMDBData(lmdb_loc, shuffle=False) #ds = td.LocallyShuffleData(ds, cache) #ds = td.PrefetchData(ds, 5000, 1) #ds = td.LMDBDataPoint(ds) ds = td.MapDataComponent( ds, lambda x: cv2.imdecode(x, cv2.IMREAD_COLOR)[:, :, ::-1], 0) ds = td.AugmentImageComponent(ds, imagenet_augmentors) ds = td.MultiProcessRunnerZMQ(ds, num_workers) self.ds = td.BatchData(ds, batch_size) self.ds.reset_state() self.batch_size = batch_size self.num_workers = num_workers self.cuda = cuda self.out_tensor = out_tensor # data_transforms should be present only when out_tensor=True # data_transforms typically consists of # PIL Image transforms, ToTensor(), Normalize(): # normalize = transforms.Compose( [ # transforms.ToTensor(), # transforms.Normalize(mean=[0.485, 0.456, 0.406], # std=[0.229, 0.224, 0.225]) ] ) self.data_transforms = data_transforms print("Loaded '%s'." % lmdb_loc)
def plate_generator_pipline(pg, num_proc=10): augmentors = df.imgaug.AugmentorList([ df.imgaug.RandomOrderAug([ df.imgaug.BrightnessScale((0.6, 1.4), clip=False), df.imgaug.Contrast((0.6, 1.4), clip=False), df.imgaug.Saturation(0.4, rgb=False), # rgb-bgr conversion for the constants copied from fb.resnet.torch df.imgaug.Lighting( std=0.1, eigval=np.asarray([0.2175, 0.0188, 0.0045][::-1]) * 255.0, eigvec=np.array( [[-0.5675, 0.7192, 0.4009], [-0.5808, -0.0045, -0.8140], [-0.5836, -0.6948, 0.4203]], dtype='float32')[::-1, ::-1]) ]), df.imgaug.RandomOrderAug([ df.imgaug.RandomApplyAug(df.imgaug.GaussianNoise(45), 0.5), df.imgaug.RandomApplyAug(df.imgaug.GaussianBlur(), 0.5), df.imgaug.RandomApplyAug(df.imgaug.JpegNoise(), 0.5), # # df.imgaug.RandomApplyAug(df.imgaug.Grayscale(keepshape=True), 0.2), df.imgaug.RandomApplyAug(df.imgaug.SaltPepperNoise(), 0.5), df.imgaug.RandomApplyAug(MotionBlur(15), 1.0), ]), df.imgaug.RandomOrderAug([ df.imgaug.RandomApplyAug( df.imgaug.Affine(scale=(0.8, 1.0), translate_frac=(0.05, 0.05), rotate_max_deg=10, shear=35), 0.5), ]), ]) ds = df.DataFromGenerator(pg) ds = df.AugmentImageComponent(ds, augmentors) # ds = df.MultiProcessRunnerZMQ(ds, num_proc=num_proc) return ds
import tensorpack.dataflow as df if __name__ == '__main__': ds = df.dataset.Mnist('train') augmentors = [ df.imgaug.RandomApplyAug( df.imgaug.RandomResize((0.8, 1.2), (0.8, 1.2)), 0.3), df.imgaug.RandomApplyAug(df.imgaug.RotationAndCropValid(15), 0.5), df.imgaug.RandomApplyAug( df.imgaug.SaltPepperNoise(white_prob=0.01, black_prob=0.01), 0.25), df.imgaug.Resize((28, 28)), df.imgaug.CenterPaste((32, 32)), df.imgaug.RandomCrop((28, 28)), df.imgaug.MapImage(lambda x: x.reshape(28, 28, 1)) ] ds = df.AugmentImageComponent(ds, augmentors) ds = df.BatchData(ds, batch_size=32, remainder=False) ds = df.PrefetchData(ds, nr_prefetch=12, nr_proc=2) ds = df.PrintData(ds) df.send_dataflow_zmq(ds, 'tcp://localhost:2222')
import tensorflow as tf import tensorpack.dataflow as df if __name__ == '__main__': # prepare dataset ds = df.dataset.Mnist('train') augmentors_variation = [ df.imgaug.Resize((28, 28)), df.imgaug.CenterPaste((32, 32)), df.imgaug.RandomCrop((28, 28)), df.imgaug.MapImage(lambda v: v.reshape(784)) ] ds = df.AugmentImageComponent(ds, augmentors_variation) ds = df.PrefetchData(ds, nr_prefetch=12, nr_proc=4) ds = df.BatchData(ds, batch_size=128, remainder=False, use_list=False) # create the model x = tf.placeholder(tf.float32, [None, 784]) W = tf.Variable(tf.ones([784, 10])) b = tf.Variable(tf.zeros([10])) y = tf.matmul(x, W) + b y_ = tf.placeholder(tf.int64, [None]) cross_entropy = tf.reduce_mean( tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y_, logits=y)) global_step = tf.train.get_or_create_global_step() train_op = tf.train.GradientDescentOptimizer(learning_rate=0.01).minimize( cross_entropy, global_step=global_step) correct_prediction = tf.equal(tf.argmax(y, 1), y_) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
else: logging.basicConfig(level=logging.INFO, format='[%(asctime)s %(levelname)s] %(message)s', filename=args.log_filename) if args.augment: augmentors = fbresnet_augmentor(isTrain=True) else: augmentors = [ df.imgaug.Resize((128, 128)), ] ds = dataflow.dataset.ILSVRC12( args.service_code, 'train', shuffle=True).parallel(num_threads=args.threads) ds = df.AugmentImageComponent(ds, augmentors, copy=False) ds = df.PrefetchDataZMQ(ds, nr_proc=args.process) if args.view: ds = dataflow.utils.image.Viewer(ds, lambda x: x[1] == 4, 'label-4', prob=1.0, pos=(0, (128 + 64) * 0)) ds = dataflow.utils.image.Viewer(ds, lambda x: x[1] == 16, 'label-16', prob=1.0, pos=(0, (128 + 64) * 1)) ds = dataflow.utils.image.Viewer(ds, lambda x: x[1] == 32, 'label-32',
df.imgaug.RandomOrderAug([ df.imgaug.RandomApplyAug( df.imgaug.BrightnessScale((0.8, 1.2), clip=False), 0.5), df.imgaug.RandomApplyAug( df.imgaug.Contrast((0.8, 1.2), clip=False), 0.5), # df.imgaug.RandomApplyAug(df.imgaug.Saturation(0.4, rgb=False), 0.5), ]), ] augmentors_default = [ df.imgaug.Resize((32, 32)), df.imgaug.MapImage(lambda x: x.reshape(32, 32, 1)) ] # keep original image at index 1 ds = df.MapData( ds, lambda datapoint: [datapoint[0], datapoint[0]] + datapoint[1:]) ds = df.AugmentImageComponent(ds, augmentors_variation + augmentors_default) ds = df.AugmentImageComponent(ds, augmentors_default, index=1) ds = df.PrefetchData(ds, nr_prefetch=12, nr_proc=4) ds = df.PrintData(ds) ds = df.BatchData(ds, batch_size=32, remainder=False, use_list=True) ds = df.PrintData(ds) for minibatch in ds.get_data(): images, originals, labels = minibatch image, original, label = images[0], originals[0], labels[0] name = '{:02d}'.format(label) cv2.namedWindow(name) cv2.moveWindow(name, 0, 25 + 128 * label) display_image = np.concatenate((image, original), axis=1)