def get_dataflow(path, is_train, img_path=None): ds = CocoPose(path, img_path, is_train) # read data from lmdb if is_train: ds = MapData(ds, read_image_url) ds = MapDataComponent(ds, pose_random_scale) ds = MapDataComponent(ds, pose_rotation) ds = MapDataComponent(ds, pose_flip) ds = MapDataComponent(ds, pose_resize_shortestedge_random) ds = MapDataComponent(ds, pose_crop_random) ds = MapData(ds, pose_to_img) # augs = [ # imgaug.RandomApplyAug(imgaug.RandomChooseAug([ # imgaug.GaussianBlur(max_size=3) # ]), 0.7) # ] # ds = AugmentImageComponent(ds, augs) ds = PrefetchData(ds, 1000, multiprocessing.cpu_count() * 4) else: ds = MultiThreadMapData(ds, nr_thread=16, map_func=read_image_url, buffer_size=1000) ds = MapDataComponent(ds, pose_resize_shortestedge_fixed) ds = MapDataComponent(ds, pose_crop_center) ds = MapData(ds, pose_to_img) ds = PrefetchData(ds, 100, multiprocessing.cpu_count() // 4) return ds
def get_dataflow_batch(is_train, batchsize): ds = get_dataflow(is_train) ds = PrefetchData(ds, 1000, multiprocessing.cpu_count()) ds = BatchData(ds, batchsize) ds = PrefetchData(ds, 10, 4) return ds
def get_mnist_data(is_train, image_size, batchsize): ds = MNISTCh('train' if is_train else 'test', shuffle=True) if is_train: augs = [ imgaug.RandomApplyAug(imgaug.RandomResize((0.8, 1.2), (0.8, 1.2)), 0.3), imgaug.RandomApplyAug(imgaug.RotationAndCropValid(15), 0.5), imgaug.RandomApplyAug( imgaug.SaltPepperNoise(white_prob=0.01, black_prob=0.01), 0.25), imgaug.Resize((224, 224), cv2.INTER_AREA) ] ds = AugmentImageComponent(ds, augs) ds = PrefetchData(ds, 128 * 10, multiprocessing.cpu_count()) ds = BatchData(ds, batchsize) ds = PrefetchData(ds, 256, 4) else: # no augmentation, only resizing augs = [ imgaug.Resize((image_size, image_size), cv2.INTER_CUBIC), ] ds = AugmentImageComponent(ds, augs) ds = BatchData(ds, batchsize) ds = PrefetchData(ds, 20, 2) return ds
def get_dataflow_batch(path, is_train, batchsize, img_path=None): logger.info('dataflow img_path=%s' % img_path) ds = get_dataflow(path, is_train, img_path=img_path) ds = BatchData(ds, batchsize) if is_train: ds = PrefetchData(ds, 10, 2) else: ds = PrefetchData(ds, 50, 2) return ds
def get_data(name, data_dir, meta_dir, gpu_nums): isTrain = True if 'train' in name else False ds = PascalVOC12(data_dir, meta_dir, name, shuffle=True) if isTrain: ds = MapData(ds, RandomResize) if isTrain: shape_aug = [ RandomCropWithPadding(args.crop_size, IGNORE_LABEL), Flip(horiz=True), ] else: shape_aug = [] ds = AugmentImageComponents(ds, shape_aug, (0, 1), copy=False) def f(ds): image, label = ds m = np.array([104, 116, 122]) const_arr = np.resize(m, (1, 1, 3)) # NCHW image = image - const_arr return image, label ds = MapData(ds, f) if isTrain: ds = BatchData(ds, args.batch_size * gpu_nums) ds = PrefetchData(ds, nr_prefetch=batch_size * 3, nr_proc=2) else: ds = BatchData(ds, 1) return ds
def get_dataflow(path, is_train): ds = CocoPoseLMDB(path, is_train) # read data from lmdb if is_train: ds = MapDataComponent(ds, pose_random_scale) ds = MapDataComponent(ds, pose_rotation) ds = MapDataComponent(ds, pose_flip) ds = MapDataComponent(ds, pose_resize_shortestedge_random) ds = MapDataComponent(ds, pose_crop_random) ds = MapData(ds, pose_to_img) augs = [ imgaug.RandomApplyAug( imgaug.RandomChooseAug([ imgaug.SaltPepperNoise(white_prob=0.01, black_prob=0.01), imgaug.RandomOrderAug([ imgaug.BrightnessScale((0.8, 1.2), clip=False), imgaug.Contrast((0.8, 1.2), clip=False), # imgaug.Saturation(0.4, rgb=True), ]), ]), 0.7), ] ds = AugmentImageComponent(ds, augs) else: ds = MapDataComponent(ds, pose_resize_shortestedge_fixed) ds = MapDataComponent(ds, pose_crop_center) ds = MapData(ds, pose_to_img) ds = PrefetchData(ds, 1000, multiprocessing.cpu_count()) return ds
def get_remote_dataflow(port, nr_prefetch=1000, nr_thread=1): ipc = 'ipc:///tmp/ipc-socket' tcp = 'tcp://0.0.0.0:%d' % port data_loader = RemoteDataZMQ(ipc, tcp, hwm=10000) data_loader = BatchData(data_loader, batch_size=hp.train.batch_size) data_loader = PrefetchData(data_loader, nr_prefetch, nr_thread) return data_loader
def get_dataflow(path, is_train): ds = CocoPoseLMDB(path, is_train) # read data from lmdb if is_train: ds = MapDataComponent(ds, pose_random_scale) ds = MapDataComponent(ds, pose_rotation) ds = MapDataComponent(ds, pose_flip) ds = MapDataComponent(ds, pose_resize_shortestedge_random) ds = MapDataComponent(ds, pose_crop_random) ds = MapData(ds, pose_to_img) augs = [ imgaug.RandomApplyAug( imgaug.RandomChooseAug([ imgaug.BrightnessScale((0.6, 1.4), clip=False), imgaug.Contrast((0.7, 1.4), clip=False), imgaug.GaussianBlur(max_size=3) ]), 0.7), ] ds = AugmentImageComponent(ds, augs) else: ds = MapDataComponent(ds, pose_resize_shortestedge_fixed) ds = MapDataComponent(ds, pose_crop_center) ds = MapData(ds, pose_to_img) ds = PrefetchData(ds, 1000, multiprocessing.cpu_count()) return ds
def get_ilsvrc_data_alexnet(is_train, image_size, batchsize, directory): if is_train: if not directory.startswith('/'): ds = ILSVRCTTenthTrain(directory) else: ds = ILSVRC12(directory, 'train') augs = [ imgaug.RandomApplyAug(imgaug.RandomResize((0.9, 1.2), (0.9, 1.2)), 0.7), imgaug.RandomApplyAug(imgaug.RotationAndCropValid(15), 0.7), imgaug.RandomApplyAug( imgaug.RandomChooseAug([ imgaug.SaltPepperNoise(white_prob=0.01, black_prob=0.01), imgaug.RandomOrderAug([ imgaug.BrightnessScale((0.8, 1.2), clip=False), imgaug.Contrast((0.8, 1.2), clip=False), # imgaug.Saturation(0.4, rgb=True), ]), ]), 0.7), imgaug.Flip(horiz=True), imgaug.ResizeShortestEdge(256, cv2.INTER_CUBIC), imgaug.RandomCrop((224, 224)), ] ds = AugmentImageComponent(ds, augs) ds = PrefetchData(ds, 1000, multiprocessing.cpu_count()) ds = BatchData(ds, batchsize) ds = PrefetchData(ds, 10, 4) else: if not directory.startswith('/'): ds = ILSVRCTenthValid(directory) else: ds = ILSVRC12(directory, 'val') ds = AugmentImageComponent(ds, [ imgaug.ResizeShortestEdge(224, cv2.INTER_CUBIC), imgaug.CenterCrop((224, 224)), ]) ds = PrefetchData(ds, 100, multiprocessing.cpu_count()) ds = BatchData(ds, batchsize) return ds
def __call__(self, n_prefetch=1000, n_thread=1): df = self df = BatchData(df, self.batch_size) df = PrefetchData(df, n_prefetch, n_thread) return df
def dataflow(self, nr_prefetch=1000, nr_thread=1): ds = self ds = BatchData(ds, self.batch_size) ds = PrefetchData(ds, nr_prefetch, nr_thread) return ds
def get_dataflow_batch(path, is_train, batchsize): ds = get_dataflow(path, is_train) ds = BatchData(ds, batchsize) ds = PrefetchData(ds, 10, 2) return ds