def get_train_dataflow(datadir, args): parallel = min(args.process_nums, multiprocessing.cpu_count() // 2) ds = ILSVRC12Files(datadir, shuffle=False) def mapf(dp): fname, cls = dp im_0 = ReadImageToCVMat(fname[0], args) im_1 = ReadImageToCVMat(fname[1], args) im_0 = occ_augImage(im_0) im_1 = occ_augImage(im_1) im_0 = transform(im_0, args) im_1 = transform(im_1, args) return np.array([im_0, im_1]), np.array([cls, cls]) ds = MultiThreadMapData(ds, parallel, mapf, buffer_size=min(2000, ds.size()), strict=True) # ds = MultiProcessRunnerZMQ(ds, parallel) # ds = BatchData(ds, batch_size, remainder=True) # do not fork() under MPI return ds
def make_data(self, ds,is_training=True): if is_training: ds = MultiThreadMapData(ds, 10, self.train_map_func, buffer_size=200, strict=True) else: ds = MultiThreadMapData(ds, 5, self.val_map_func, buffer_size=200, strict=True) ds = BatchData(ds, cfg.TRAIN.num_gpu * cfg.TRAIN.batch_size, remainder=True,use_list=False) ds = MultiProcessPrefetchData(ds, 100,2) ds.reset_state() ds=ds.get_data() ########### # ds = data_set.shuffle(buffer_size=512) # shuffle before loading images # ds = ds.repeat(cfg.TRAIN.epoch) # if is_training: # ds = ds.map(self.train_map_func, num_parallel_calls=multiprocessing.cpu_count()) # decouple the heavy map_fn # else: # ds = ds.map(self.val_map_func, num_parallel_calls=multiprocessing.cpu_count()) # decouple the heavy map_fn # ds = ds.batch( # cfg.TRAIN.num_gpu * cfg.TRAIN.batch_size) # TODO: consider using tf.contrib.map_and_batch # # ds = ds.prefetch(5 * cfg.TRAIN.num_gpu) # iterator = ds.make_one_shot_iterator() # one_element = iterator.get_next() # images, labels = one_element return ds
def get_simple_val_dataflow(src, batch_size=32, augmentors=None, parallel=None): if augmentors is None: augmentors = fbresnet_augmentor(False) assert isinstance(augmentors, list) if parallel is None: parallel = min(40, multiprocessing.cpu_count()) aug = imgaug.AugmentorList(augmentors) def mapf(dp): im, cls = dp im_new = im * 255 im_new = im_new[:, :, ::-1] im_new = im_new.astype('uint8') return im_new, cls im_new = aug.augment(im_new) return im_new, cls ds = MultiThreadMapData(src, parallel, mapf, buffer_size=min(2000, src.size()), strict=True) ds = BatchData(ds, batch_size, remainder=True) # do not fork() under MPI return ds
def get_imagenet_dataflow( datadir, name, batch_size, augmentors=None, parallel=None): """ Args: augmentors (list[imgaug.Augmentor]): Defaults to `fbresnet_augmentor(isTrain)` Returns: A DataFlow which produces BGR images and labels. """ assert name in ['train', 'val', 'test'] isTrain = name == 'train' assert datadir is not None if augmentors is None: augmentors = fbresnet_augmentor(isTrain) assert isinstance(augmentors, list) augmentors = AugmentorList(augmentors) if parallel is None: parallel = min(40, multiprocessing.cpu_count() // 2) # assuming hyperthreading def mapf(dp): fname, label = dp img = cv2.imread(fname) img = augmentors.augment(img) return img, label if isTrain: ds = dataset.ILSVRC12Files(datadir, name, shuffle=True) ds = MultiProcessMapDataZMQ(ds, parallel, mapf, buffer_size=2000) ds = BatchData(ds, batch_size, remainder=False) else: ds = dataset.ILSVRC12Files(datadir, name, shuffle=False) ds = MultiThreadMapData(ds, parallel, mapf, buffer_size=2000, strict=True) ds = BatchData(ds, batch_size, remainder=True) ds = PrefetchDataZMQ(ds, 1) return ds
def get_dataflow(path, is_train, img_path=None): ds = CocoPose(path, img_path, is_train) # read data from lmdb if is_train: ds = MapData(ds, read_image_url) ds = MapDataComponent(ds, pose_random_scale) ds = MapDataComponent(ds, pose_rotation) ds = MapDataComponent(ds, pose_flip) ds = MapDataComponent(ds, pose_resize_shortestedge_random) ds = MapDataComponent(ds, pose_crop_random) ds = MapData(ds, pose_to_img) # augs = [ # imgaug.RandomApplyAug(imgaug.RandomChooseAug([ # imgaug.GaussianBlur(max_size=3) # ]), 0.7) # ] # ds = AugmentImageComponent(ds, augs) ds = PrefetchData(ds, 1000, multiprocessing.cpu_count() * 1) else: ds = MultiThreadMapData(ds, nr_thread=16, map_func=read_image_url, buffer_size=1000) ds = MapDataComponent(ds, pose_resize_shortestedge_fixed) ds = MapDataComponent(ds, pose_crop_center) ds = MapData(ds, pose_to_img) ds = PrefetchData(ds, 100, multiprocessing.cpu_count() // 4) return ds
def get_imagenet_dataflow(datadir, name, batch_size, augmentors): """ See explanations in the tutorial: http://tensorpack.readthedocs.io/en/latest/tutorial/efficient-dataflow.html """ assert name in ['train', 'val', 'test'] assert datadir is not None assert isinstance(augmentors, list) isTrain = name == 'train' cpu = min(30, multiprocessing.cpu_count()) meta_dir = './ilsvrc_metadata' if isTrain: ds = dataset.ILSVRC12(datadir, name, meta_dir=meta_dir, shuffle=True) ds = AugmentImageComponent(ds, augmentors, copy=False) ds = PrefetchDataZMQ(ds, cpu) ds = BatchData(ds, batch_size, remainder=False) else: ds = dataset.ILSVRC12Files(datadir, name, meta_dir=meta_dir, shuffle=False) aug = imgaug.AugmentorList(augmentors) def mapf(dp): fname, cls = dp im = cv2.imread(fname, cv2.IMREAD_COLOR) im = aug.augment(im) return im, cls ds = MultiThreadMapData(ds, cpu, mapf, buffer_size=2000, strict=True) ds = BatchData(ds, batch_size, remainder=True) ds = PrefetchDataZMQ(ds, 1) return ds
def get_val_dataflow( datadir, batch_size, augmentors, parallel=None, num_splits=None, split_index=None): assert datadir is not None assert isinstance(augmentors, list) if parallel is None: parallel = min(40, multiprocessing.cpu_count()) if num_splits is None: ds = dataset.ILSVRC12Files(datadir, 'val', shuffle=False) else: assert split_index < num_splits files = dataset.ILSVRC12Files(datadir, 'val', shuffle=False) files.reset_state() files = list(files.get_data()) logger.info("#ValidationData = {}".format(len(files))) split_size = len(files) // num_splits start, end = split_size * split_index, split_size * (split_index + 1) end = min(end, len(files)) logger.info("#ValidationSplit = {} - {}".format(start, end)) files = files[start: end] ds = DataFromList(files, shuffle=False) aug = imgaug.AugmentorList(augmentors) def mapf(dp): fname, cls = dp im = cv2.imread(fname, cv2.IMREAD_COLOR) im = aug.augment(im) return im, cls ds = MultiThreadMapData(ds, parallel, mapf, buffer_size=2000, strict=True) ds = BatchData(ds, batch_size, remainder=True) # ds = PrefetchDataZMQ(ds, 1) # do not fork() under MPI return ds
def get_iNaturalist_dataflow( datadir, name, batch_size, augmentors, parallel=None): """ See explanations in the tutorial: http://tensorpack.readthedocs.io/en/latest/tutorial/efficient-dataflow.html """ assert name in ['train', 'val', 'test'] assert datadir is not None assert isinstance(augmentors, list) isTrain = name == 'train' if parallel is None: parallel = min(40, multiprocessing.cpu_count() // 2) # assuming hyperthreading if isTrain: ds = dataset.iNaturalist(datadir, name, shuffle=True) ds = AugmentImageComponent(ds, augmentors, copy=False) if parallel < 16: logger.warn("DataFlow may become the bottleneck when too few processes are used.") ds = PrefetchDataZMQ(ds, parallel) ds = BatchData(ds, batch_size, remainder=False) else: ds = dataset.iNaturalistFiles(datadir, name, shuffle=False) aug = imgaug.AugmentorList(augmentors) def mapf(dp): fname, cls = dp im = cv2.imread(fname, cv2.IMREAD_COLOR) im = aug.augment(im) return im, cls ds = MultiThreadMapData(ds, parallel, mapf, buffer_size=2000, strict=True) ds = BatchData(ds, batch_size, remainder=True) ds = PrefetchDataZMQ(ds, 1) return ds
def get_train_dataflow(batch_size=2): print("In train dataflow") roidbs = list(itertools.chain.from_iterable(DatasetRegistry.get(x).training_roidbs() for x in cfg.DATA.TRAIN)) print_class_histogram(roidbs) print("Done loading roidbs") # Filter out images that have no gt boxes, but this filter shall not be applied for testing. # The model does support training with empty images, but it is not useful for COCO. num = len(roidbs) roidbs = list(filter(lambda img: len(img["boxes"][img["is_crowd"] == 0]) > 0, roidbs)) logger.info( "Filtered {} images which contain no non-crowd groudtruth boxes. Total #images for training: {}".format( num - len(roidbs), len(roidbs) ) ) aspect_grouping = [1] aspect_ratios = [float(x["height"]) / float(x["width"]) for x in roidbs] group_ids = _quantize(aspect_ratios, aspect_grouping) ds = AspectGroupingDataFlow(roidbs, group_ids, batch_size=batch_size, drop_uneven=True) preprocess = TrainingDataPreprocessor() buffer_size = cfg.DATA.NUM_WORKERS * 10 # ds = MultiProcessMapData(ds, cfg.DATA.NUM_WORKERS, preprocess, buffer_size=buffer_size) ds = MultiThreadMapData(ds, cfg.DATA.NUM_WORKERS, preprocess, buffer_size=buffer_size) ds.reset_state() # to get an infinite data flow ds = RepeatedData(ds, num=-1) dataiter = ds.__iter__() return dataiter
def get_imagenet_dataflow(datadir, name, batch_size, parallel=None): """ Get a standard imagenet training/evaluation dataflow, for linear classifier tuning. """ assert name in ['train', 'val'] isTrain = name == 'train' assert datadir is not None augmentors = get_basic_augmentor(isTrain) augmentors = imgaug.AugmentorList(augmentors) if parallel is None: parallel = min(50, mp.cpu_count()) def mapper(dp): fname, label = dp img = cv2.imread(fname) img = augmentors.augment(img) return img, label if isTrain: ds = dataset.ILSVRC12Files(datadir, name, shuffle=True) ds = MultiProcessMapAndBatchDataZMQ(ds, parallel, mapper, batch_size, buffer_size=7000) else: ds = dataset.ILSVRC12Files(datadir, name, shuffle=False) ds = MultiThreadMapData(ds, parallel, mapper, buffer_size=2000, strict=True) ds = BatchData(ds, batch_size, remainder=True) ds = MultiProcessRunnerZMQ(ds, 1) return ds
def get_batch_train_dataflow(roidbs, batch_size): """ Tensorpack batch text dataflow. """ batched_roidbs = [] batch = [] for i, d in enumerate(roidbs): if i % batch_size == 0: if len(batch) == batch_size: batched_roidbs.append(batch) batch = [] batch.append(d) def preprocess(roidb_batch): """ Tensorpack batch text data preprocess function. """ datapoint_list = [] for roidb in roidb_batch: filename, label, mask, bbox, polygon = roidb['filename'], roidb[ 'label'], roidb['mask'], roidb['bbox'], roidb['polygon'] img = cv2.imread(filename) img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) image = affine_transform(img, polygon) # img = img[bbox[0]:bbox[2], bbox[1]:bbox[3], :] if image.shape[0]<cfg.stride/2 or image.shape[1]<cfg.stride/2 else image img = img if image.shape[0] < cfg.stride / 2 or image.shape[ 1] < cfg.stride / 2 else image largest_side = np.random.randint(cfg.crop_min_size, cfg.image_size) img = aspect_preserving_resize(img, largest_side) img, crop_bbox = padding_image(img, cfg.image_size) normalized_bbox = [coord / cfg.image_size for coord in crop_bbox] img = img.astype("float32") / 255. ret = { "image": img, "label": label, "mask": mask, "normalized_bbox": normalized_bbox } datapoint_list.append(ret) batched_datapoint = {"is_training": True, "dropout_keep_prob": 0.5} for stackable_field in ["image", "label", "mask", "normalized_bbox"]: batched_datapoint[stackable_field] = np.stack( [d[stackable_field] for d in datapoint_list]) return batched_datapoint ds = DataFromList(batched_roidbs, shuffle=True) ds = MultiThreadMapData(ds, cfg.num_threads, preprocess) # ds = PrefetchData(ds, 100, multiprocessing.cpu_count() // 4) return ds
def get_test_dataflow(datadir, args): parallel = min(args.process_nums, multiprocessing.cpu_count()) ds = ILSVRC12Files(datadir, shuffle=False) def mapf(dp): fname, cls = dp im_0 = ReadImageToCVMat(fname, args) im_0 = transform(im_0, args) return np.array(im_0), np.array(cls) ds = MultiThreadMapData(ds, parallel, mapf, buffer_size=min(2000, ds.size()), strict=True) # ds = BatchData(ds, batch_size, remainder=True) # do not fork() under MPI return ds
def get_train_dataflow(): """ Return a training dataflow. Each datapoint consists of the following: An image: (h, w, 3), 1 or more pairs of (anchor_labels, anchor_boxes): anchor_labels: (h', w', NA) anchor_boxes: (h', w', NA, 4) gt_boxes: (N, 4) gt_labels: (N,) If MODE_MASK, gt_masks: (N, h, w) """ roidbs = list( itertools.chain.from_iterable( DatasetRegistry.get(x).training_roidbs() for x in cfg.DATA.TRAIN)) print( "---------------------------------------------------------------- data.py:343" ) print_class_histogram(roidbs) # Filter out images that have no gt boxes, but this filter shall not be applied for testing. # The model does support training with empty images, but it is not useful for COCO. num = len(roidbs) roidbs = list( filter(lambda img: len(img["boxes"][img["is_crowd"] == 0]) > 0, roidbs)) logger.info( "Filtered {} images which contain no non-crowd groudtruth boxes. Total #images for training: {}" .format(num - len(roidbs), len(roidbs))) ds = DataFromList(roidbs, shuffle=True) preprocess = TrainingDataPreprocessor(cfg) if cfg.DATA.NUM_WORKERS > 0: if cfg.TRAINER == "horovod": # one dataflow for each process, therefore don't need large buffer buffer_size = cfg.DATA.NUM_WORKERS * 10 ds = MultiThreadMapData(ds, cfg.DATA.NUM_WORKERS, preprocess, buffer_size=buffer_size) # MPI does not like fork() else: buffer_size = cfg.DATA.NUM_WORKERS * 20 ds = MultiProcessMapData(ds, cfg.DATA.NUM_WORKERS, preprocess, buffer_size=buffer_size) else: ds = MapData(ds, preprocess) return ds
def get_imagenet_dataflow(datadir, name, batch_size, augmentors, parallel=None): """ See explanations in the tutorial: http://tensorpack.readthedocs.io/en/latest/tutorial/efficient-dataflow.html """ assert name in ['train', 'val', 'test'] assert datadir is not None assert isinstance(augmentors, list) isTrain = name == 'train' if parallel is None: parallel = min(40, multiprocessing.cpu_count()) if isTrain: ds = dataset.ILSVRC12(datadir, name, shuffle=True) ds = AugmentImageComponent(ds, augmentors, copy=False) if parallel < 16: logger.warn( "DataFlow may become the bottleneck when too few processes are used." ) ds = PrefetchDataZMQ(ds, parallel) ds = BatchData(ds, batch_size, remainder=False) else: ds = dataset.ILSVRC12Files(datadir, name, shuffle=False) aug = imgaug.AugmentorList(augmentors) def mapf(dp): fname, cls = dp jpeg_filename = os.path.basename(fname) jpeg_dirname = os.path.basename(os.path.dirname(fname)) zip_filepath = os.path.dirname(fname) + '.zip' f = zipfile.ZipFile(zip_filepath, 'r') compress_jpeg = np.fromstring(f.read( os.path.join(jpeg_dirname, jpeg_filename)), dtype=np.uint8) im = cv2.imdecode(compress_jpeg, cv2.IMREAD_COLOR) #im = cv2.imread(fname, cv2.IMREAD_COLOR) im = aug.augment(im) return im, cls ds = MultiThreadMapData(ds, parallel, mapf, buffer_size=2000, strict=True) ds = BatchData(ds, batch_size, remainder=True) ds = PrefetchDataZMQ(ds, 1) return ds
def get_dataflow(path, is_train): ds = SynthHands(path, is_train) # read data from lmdb if is_train: ds = MapData(ds, read_image_url) ds = MapData(ds, pose_to_img) ds = PrefetchData(ds, 1000, multiprocessing.cpu_count() * 1) else: ds = MultiThreadMapData(ds, num_thread=16, map_func=read_image_url, buffer_size=1000) ds = MapData(ds, pose_to_img) ds = PrefetchData(ds, 100, multiprocessing.cpu_count() // 4) return ds
def build_iter(self,samples): map_func=partial(self._map_func,is_training=self.training_flag) ds = DataFromList(samples, shuffle=True) ds = MultiThreadMapData(ds, self.thread_num, map_func, buffer_size=self.buffer_size) ds = BatchData(ds, self.num_gpu * self.batch_size) ds = MultiProcessPrefetchData(ds, self.prefetch_size, self.process_num) ds.reset_state() ds = ds.get_data() return ds
def get_imagenet_dataflow(datadir, name, batch_size, augmentors=None, parallel=None): """ Args: augmentors (list[imgaug.Augmentor]): Defaults to `fbresnet_augmentor(isTrain)` Returns: A DataFlow which produces BGR images and labels. See explanations in the tutorial: http://tensorpack.readthedocs.io/tutorial/efficient-dataflow.html """ assert name in ['train', 'val', 'test'] isTrain = name == 'train' assert datadir is not None if augmentors is None: augmentors = fbresnet_augmentor(isTrain) assert isinstance(augmentors, list) if parallel is None: parallel = min(40, multiprocessing.cpu_count() // 2) # assuming hyperthreading if isTrain: ds = dataset.ILSVRC12(datadir, name, shuffle=True) ds = AugmentImageComponent(ds, augmentors, copy=False) if parallel < 16: logger.warn( "DataFlow may become the bottleneck when too few processes are used." ) ds = MultiProcessRunnerZMQ(ds, parallel) ds = BatchData(ds, batch_size, remainder=False) else: ds = dataset.ILSVRC12Files(datadir, name, shuffle=False) aug = imgaug.AugmentorList(augmentors) def mapf(dp): fname, cls = dp im = cv2.imread(fname, cv2.IMREAD_COLOR) im = aug.augment(im) return im, cls ds = MultiThreadMapData(ds, parallel, mapf, buffer_size=2000, strict=True) ds = BatchData(ds, batch_size, remainder=True) ds = MultiProcessRunnerZMQ(ds, 1) return ds
def get_val_dataflow(datadir, batch_size, augmentors=None, parallel=None, num_splits=None, split_index=None, dataname="val"): if augmentors is None: augmentors = fbresnet_augmentor(False) assert datadir is not None assert isinstance(augmentors, list) if parallel is None: parallel = min(40, multiprocessing.cpu_count()) if num_splits is None: ds = dataset.ILSVRC12Files(datadir, dataname, shuffle=True) else: # shard validation data assert False assert split_index < num_splits files = dataset.ILSVRC12Files(datadir, dataname, shuffle=True) files.reset_state() files = list(files.get_data()) logger.info("Number of validation data = {}".format(len(files))) split_size = len(files) // num_splits start, end = split_size * split_index, split_size * (split_index + 1) end = min(end, len(files)) logger.info("Local validation split = {} - {}".format(start, end)) files = files[start:end] ds = DataFromList(files, shuffle=True) aug = imgaug.AugmentorList(augmentors) def mapf(dp): fname, cls = dp im = cv2.imread(fname, cv2.IMREAD_COLOR) #from BGR to RGB im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) im = aug.augment(im) return im, cls ds = MultiThreadMapData(ds, parallel, mapf, buffer_size=min(2000, ds.size()), strict=True) ds = BatchData(ds, batch_size, remainder=False) ds = RepeatedData(ds, num=-1) # do not fork() under MPI return ds
def get_imagenet_dataflow(datadir, name, batch_size, augmentors, parallel=None): """ See explanations in the tutorial: http://tensorpack.readthedocs.io/en/latest/tutorial/efficient-dataflow.html """ assert name in ['train', 'val', 'test'] assert datadir is not None assert isinstance(augmentors, list) isTrain = name == 'train' if parallel is None: parallel = min(40, multiprocessing.cpu_count() // 6) if isTrain: ds = dataset.ILSVRC12(datadir, name, shuffle=True) ds = AugmentImageComponent(ds, augmentors, copy=False) if parallel < 16: logger.warning( "DataFlow may become the bottleneck when too few processes are used." ) ds = PrefetchData(ds, 1000, parallel) ds = BatchData(ds, batch_size, remainder=False) else: ds = dataset.ILSVRC12Files(datadir, name, shuffle=False) aug = imgaug.AugmentorList(augmentors) def mapf(dp): fname, cls = dp im = np.zeros((256, 256, 3), dtype=np.uint8) for _ in range(30): try: im = cv2.imread(fname, cv2.IMREAD_COLOR) im = aug.augment(im) break except Exception as e: logger.warning(str(e), 'file=', fname) time.sleep(1) return im, cls ds = MultiThreadMapData(ds, parallel, mapf, buffer_size=2000, strict=True) ds = BatchData(ds, batch_size, remainder=True) ds = PrefetchData(ds, 100, 1) return ds
def get_imagenet_dataflow(datadir, name, batch_size, augmentors, parallel=None): #获取图像网络数据流 """ See explanations in the tutorial: http://tensorpack.readthedocs.io/en/latest/tutorial/efficient-dataflow.html """ assert name in ['train', 'val', 'test'] assert datadir is not None assert isinstance(augmentors, list) isTrain = name == 'train' if parallel is None: # 如果不是并行的话 parallel = min(40, multiprocessing.cpu_count() // 2) # assuming hyperthreading 超线程? 获取当前计算机cpu数量 if isTrain: # dataset:创建一个在数据流上运行的预测器,并且拿出一个batch? ds = dataset.ILSVRC12(datadir, name, shuffle=True) ds = AugmentImageComponent(ds, augmentors, copy=False) # 使用共享的增强参数在多个组件上应用图像增强器 if parallel < 16: # 如果少于16个的话 logger.warn( "DataFlow may become the bottleneck when too few processes are used." ) ds = PrefetchDataZMQ(ds, parallel) # 实现高效的数据流水线 ds = BatchData(ds, batch_size, remainder=False) # 取一个batch? else: # 如果是测试时,增强图像,加速对数据流的读取操作等 # 与ILSVRC12相同,但生成图像的文件名而不是np array。 ds = dataset.ILSVRC12Files(datadir, name, shuffle=False) aug = imgaug.AugmentorList(augmentors) def mapf(dp): fname, cls = dp im = cv2.imread(fname, cv2.IMREAD_COLOR ) # cv2.IMREAD_COLOR : 默认使用该种标识。加载一张彩色图片,忽视它的透明度 im = aug.augment(im) # 增强图像 return im, cls ds = MultiThreadMapData(ds, parallel, mapf, buffer_size=2000, strict=True) # 并行加速? ds = BatchData(ds, batch_size, remainder=True) # 取一个batch? ds = PrefetchDataZMQ(ds, 1) return ds
def process_s2b_data(df, batch_size, num_threads): """ Perform preprocessing for the avatar synth data. :param df: An AvatarSynthDataFlow. :param batch_size: The minibatch size. :param num_threads: The number of threads to read and process data. :return: A dataflow with extra processing steps applied. """ augmentor = imgaug.AugmentorList([imgaug.MinMaxNormalize(min=-1, max=1)]) def get_imgs(dp): """ :param dp: A datapoint tuple, (path_to_face.jpg, path_to_bitmoji.jpg) """ face_img = augmentor.augment(imread(dp[0])) bitmoji_img = augmentor.augment(imread(dp[1])) if len(face_img.shape) == 2: face_img = np.stack([face_img] * 3, axis=-1) if len(bitmoji_img.shape) == 2: bitmoji_img = np.stack([bitmoji_img] * 3, axis=-1) return [face_img, bitmoji_img] df = MultiThreadMapData(df, nr_thread=num_threads, map_func=get_imgs, buffer_size=min(df.size(), 200)) df = PrefetchDataZMQ(df, nr_proc=num_threads) # TODO: switch back to remainder=True when s2b input batch size switched back to None df = BatchData(df, batch_size, remainder=False) # df = BatchData(df, batch_size, remainder=True) return df
def get_train_dataflow(roidb): """ Tensorpack text dataflow. """ ds = DataFromList(roidb, shuffle=True) preprocess = TextDataPreprocessor(cfg) buffer_size = cfg.num_threads * 10 ds = MultiThreadMapData(ds, cfg.num_threads, preprocess, buffer_size=buffer_size) # ds = MultiProcessMapData(ds, cfg.num_workers, preprocess, buffer_size=buffer_size) ds = PrefetchData(ds, 100, multiprocessing.cpu_count() // 4) #ds = BatchData(ds, cfg.batch_size, remainder=True) return ds
def get_imagenet_dataflow(datadir, is_train, batch_size, augmentors, parallel=None): """ See explanations in the tutorial: http://tensorpack.readthedocs.io/en/latest/tutorial/efficient-dataflow.html """ assert datadir is not None assert isinstance(augmentors, list) if parallel is None: parallel = min(40, multiprocessing.cpu_count() // 2) # assuming hyperthreading if is_train: ds = dataset.ILSVRC12(datadir, "train", shuffle=True) ds = AugmentImageComponent(ds, augmentors, copy=False) if parallel < 16: logging.warning( "DataFlow may become the bottleneck when too few processes are used." ) ds = PrefetchDataZMQ(ds, parallel) ds = BatchData(ds, batch_size, remainder=False) else: ds = dataset.ILSVRC12Files(datadir, "val", shuffle=False) aug = imgaug.AugmentorList(augmentors) def mapf(dp): fname, cls = dp im = cv2.imread(fname, cv2.IMREAD_COLOR) im = np.flip(im, axis=2) # print("fname={}".format(fname)) im = aug.augment(im) return im, cls ds = MultiThreadMapData(ds, parallel, mapf, buffer_size=2000, strict=True) # ds = MapData(ds, mapf) ds = BatchData(ds, batch_size, remainder=True) ds = PrefetchDataZMQ(ds, 1) # ds = PrefetchData(ds, 1) return ds
def get_data(self, name, num_gpu): gpu_batch = self.batch_size // num_gpu assert name in ['train', 'val', 'test'] isTrain = name == 'train' augmentors = fbresnet_augmentor(isTrain) assert isinstance(augmentors, list) parallel = min(40, multiprocessing.cpu_count() // 2) # assuming hyperthreading if isTrain: ds = dataset.ILSVRC12(self.datadir, name, shuffle=True, dir_structure='train') ds = AugmentImageComponent(ds, augmentors, copy=False) ds = MultiProcessRunnerZMQ(ds, parallel) ds = BatchData(ds, gpu_batch, remainder=False) #ds = QueueInput(ds) else: ds = dataset.ILSVRC12Files(self.datadir, name, shuffle=False, dir_structure='train') aug = imgaug.AugmentorList(augmentors) def mapf(dp): fname, cls = dp im = cv2.imread(fname, cv2.IMREAD_COLOR) im = aug.augment(im) return im, cls ds = MultiThreadMapData(ds, parallel, mapf, buffer_size=2000, strict=True) ds = BatchData(ds, gpu_batch, remainder=True) ds = MultiProcessRunnerZMQ(ds, 1) if num_gpu == 1: ds = QueueInput(ds) return ds
def get_random_loader(ds, isTrain, batch_size, augmentors, parallel=None): """ DataFlow data (Random Read) Args: augmentors (list[imgaug.Augmentor]): Defaults to `fbresnet_augmentor(isTrain)` Returns: A DataFlow which produces BGR images and labels. See explanations in the tutorial: http://tensorpack.readthedocs.io/tutorial/efficient-dataflow.html """ assert isinstance(augmentors, list) aug = imgaug.AugmentorList(augmentors) if parallel is None: parallel = min(40, multiprocessing.cpu_count() // 2) # assuming hyperthreading if isTrain: ds = AugmentImageComponent(ds, aug, copy=False) if parallel < 16: logger.warn( "DataFlow may become the bottleneck when too few processes are used." ) ds = MultiProcessRunnerZMQ(ds, parallel) ds = BatchData(ds, batch_size, remainder=False) else: def mapf(dp): fname, cls = dp im = cv2.imread(fname, cv2.IMREAD_COLOR) im = aug.augment(im) return im, cls ds = MultiThreadMapData(ds, parallel, mapf, buffer_size=2000, strict=True) ds = BatchData(ds, batch_size, remainder=True) ds = MultiProcessRunnerZMQ(ds, 1) return ds
def dump_imdb(ds, output_path, parallel=None): """ Create a Single-File LMDB from raw images. """ if parallel is None: parallel = min(40, multiprocessing.cpu_count() // 2) # assuming hyperthreading def mapf(dp): fname, label = dp with open(fname, 'rb') as f: bytes = f.read() bytes = np.asarray(bytearray(bytes), dtype='uint8') return bytes, label ds = MultiThreadMapData(ds, 1, mapf, buffer_size=2000, strict=True) ds = MultiProcessRunnerZMQ(ds, num_proc=parallel) LMDBSerializer.save(ds, output_path)
def get_dataflow(path, is_train=True, img_path=None,sigma=8.0,output_shape=(1440,2560), numparts=5,translation=False,scale=False,rotation=True, mins=0.25,maxs=1.2,mina=-np.pi,maxa=np.pi ,ilumination=0.0,image_type='RGB'): print('Creating images from',path) numparts, skeleton = get_skeleton_from_json(path) #numparts + 1 because need to the background ds = CocoPose(path, img_path, is_train, numparts=numparts + 1, sigma=sigma,skeleton=skeleton, output_shape=output_shape, translation=translation,scale=scale,rotation=rotation, mins=mins,maxs=maxs,mina=mina,maxa=maxa, ilumination=ilumination,image_type=image_type ) # read data from lmdb if is_train: #ds = MapData(ds, read_image_url) ds = MultiThreadMapData(ds, nr_thread=8, map_func=read_image_url, buffer_size=10) ds = MapDataComponent(ds, get_augmented_image) #ds = MapDataComponent(ds, pose_rotation) #ds = MapDataComponent(ds, pose_flip) #ds = MapDataComponent(ds, pose_resize_shortestedge_random) #ds = MapDataComponent(ds, pose_crop_random) #logger.info('Out of new augmenter') ds = MapData(ds, pose_to_img) #logger.info('Out pose to img') # augs = [ # imgaug.RandomApplyAug(imgaug.RandomChooseAug([ # imgaug.GaussianBlur(max_size=3) # ]), 0.7) # ] # ds = AugmentImageComponent(ds, augs) ds = PrefetchData(ds, 10, multiprocessing.cpu_count() * 1) else: #ds = MultiThreadMapData(ds, nr_thread=4, map_func=read_image_url, buffer_size=10) ds = MapData(ds, read_image_url) #ds = MapDataComponent(ds, pose_resize_shortestedge_fixed) #ds = MapDataComponent(ds, pose_crop_center) ds = MapData(ds, pose_to_img) ds = PrefetchData(ds, 10, multiprocessing.cpu_count() // 4) return ds
def process_avatar_synth_data(df, batch_size, num_threads): """ Perform preprocessing for the avatar synth data. :param df: An AvatarSynthDataFlow. :param batch_size: The minibatch size. :param num_threads: The number of threads to read and process data. :return: A dataflow with extra processing steps applied. """ augmentor = imgaug.AugmentorList([imgaug.MinMaxNormalize(min=-1, max=1)]) df = MultiThreadMapData( df, nr_thread=num_threads, map_func=lambda dp: [np.load(dp[0]), augmentor.augment(imread(dp[1]))]) # df = MapData(df, lambda dp: [np.load(dp[0]), augmentor.augment(imread(dp[1]))]) df = PrefetchDataZMQ(df, nr_proc=num_threads) df = BatchData(df, batch_size, remainder=True) return df
def get_train_dataflow(): """ Return a training dataflow. Each datapoint consists of the following: An image: (h, w, 3), 1 or more pairs of (anchor_labels, anchor_boxes): anchor_labels: (h', w', NA) anchor_boxes: (h', w', NA, 4) gt_boxes: (N, 4) gt_labels: (N,) If MODE_MASK, gt_masks: (N, h, w) """ imgs = COCODetection.load_many(cfg.DATA.BASEDIR, cfg.DATA.TRAIN, add_gt=True, add_mask=cfg.MODE_MASK) """ To train on your own data, change this to your loader. Produce "imgs" as a list of dict, in the dict the following keys are needed for training: height, width: integer file_name: str, full path to the image boxes: numpy array of kx4 floats class: numpy array of k integers is_crowd: k booleans. Use k False if you don't know what it means. segmentation: k lists of numpy arrays (one for each box). Each list of numpy array corresponds to the mask for one instance. Each numpy array in the list is a polygon of shape Nx2, because one mask can be represented by N polygons. If your segmentation annotations are originally masks rather than polygons, either convert it, or the augmentation code below will need to be changed or skipped accordingly. """ # Valid training images should have at least one fg box. # But this filter shall not be applied for testing. num = len(imgs) imgs = list( filter(lambda img: len(img['boxes'][img['is_crowd'] == 0]) > 0, imgs)) logger.info( "Filtered {} images which contain no non-crowd groudtruth boxes. Total #images for training: {}" .format(num - len(imgs), len(imgs))) ds = DataFromList(imgs, shuffle=True) aug = imgaug.AugmentorList([ CustomResize(cfg.PREPROC.SHORT_EDGE_SIZE, cfg.PREPROC.MAX_SIZE), imgaug.Flip(horiz=True) ]) def preprocess(img): fname, boxes, klass, is_crowd = img['file_name'], img['boxes'], img[ 'class'], img['is_crowd'] boxes = np.copy(boxes) im = cv2.imread(fname, cv2.IMREAD_COLOR) assert im is not None, fname im = im.astype('float32') # assume floatbox as input assert boxes.dtype == np.float32, "Loader has to return floating point boxes!" # augmentation: im, params = aug.augment_return_params(im) points = box_to_point8(boxes) points = aug.augment_coords(points, params) boxes = point8_to_box(points) assert np.min(np_area(boxes)) > 0, "Some boxes have zero area!" # rpn anchor: try: if cfg.MODE_FPN: multilevel_anchor_inputs = get_multilevel_rpn_anchor_input( im, boxes, is_crowd) anchor_inputs = itertools.chain.from_iterable( multilevel_anchor_inputs) else: # anchor_labels, anchor_boxes anchor_inputs = get_rpn_anchor_input(im, boxes, is_crowd) assert len(anchor_inputs) == 2 boxes = boxes[is_crowd == 0] # skip crowd boxes in training target klass = klass[is_crowd == 0] if not len(boxes): raise MalformedData("No valid gt_boxes!") except MalformedData as e: log_once( "Input {} is filtered for training: {}".format(fname, str(e)), 'warn') return None ret = [im] + list(anchor_inputs) + [boxes, klass] if cfg.MODE_MASK: # augmentation will modify the polys in-place segmentation = copy.deepcopy(img['segmentation']) segmentation = [ segmentation[k] for k in range(len(segmentation)) if not is_crowd[k] ] assert len(segmentation) == len(boxes) # Apply augmentation on polygon coordinates. # And produce one image-sized binary mask per box. masks = [] for polys in segmentation: polys = [aug.augment_coords(p, params) for p in polys] masks.append( segmentation_to_mask(polys, im.shape[0], im.shape[1])) masks = np.asarray(masks, dtype='uint8') # values in {0, 1} ret.append(masks) # from viz import draw_annotation, draw_mask # viz = draw_annotation(im, boxes, klass) # for mask in masks: # viz = draw_mask(viz, mask) # tpviz.interactive_imshow(viz) return ret if cfg.TRAINER == 'horovod': ds = MultiThreadMapData(ds, 5, preprocess) # MPI does not like fork() else: ds = MultiProcessMapDataZMQ(ds, 10, preprocess) return ds
def get_train_dataflow_YCBV(): """ Return a training dataflow. Each datapoint consists of the following: An image: (h, w, 3), 1 or more pairs of (anchor_labels, anchor_boxes): anchor_labels: (h', w', NA) anchor_boxes: (h', w', NA, 4) gt_boxes: (N, 4) gt_labels: (N,) If MODE_MASK, gt_masks: (N, h, w) """ img_ids = YCBVDetectionDataset().load_training_image_ids(cfg.DATA.TRAIN) # print_class_histogram(roidbs) # Valid training images should have at least one fg box. # But this filter shall not be applied for testing. # num = len(img_ids) # roidbs = list(filter(lambda img: len(img['boxes'][img['is_crowd'] == 0]) > 0, roidbs)) # logger.info("Filtered {} images which contain no non-crowd groudtruth boxes. Total #images for training: {}".format( # num - len(roidbs), len(roidbs))) ds = DataFromList(img_ids, shuffle=True) # aug = imgaug.AugmentorList( # [CustomResize(cfg.PREPROC.TRAIN_SHORT_EDGE_SIZE, cfg.PREPROC.MAX_SIZE)]) def preprocess(image_id): roidb = YCBVDetectionDataset().load_single_roidb(image_id) fname, boxes, klass, is_crowd = roidb['file_name'], roidb[ 'boxes'], roidb['class'], roidb['is_crowd'] boxes = np.copy(boxes) im = cv2.imread(fname, cv2.IMREAD_COLOR) assert im is not None, fname im = im.astype('float32') height, width = im.shape[:2] # assume floatbox as input assert boxes.dtype == np.float32, "Loader has to return floating point boxes!" if not cfg.DATA.ABSOLUTE_COORD: boxes[:, 0::2] *= width boxes[:, 1::2] *= height # augmentation: # im, params = aug.augment_return_params(im) points = box_to_point8(boxes) # points = aug.augment_coords(points, params) boxes = point8_to_box(points) assert np.min(np_area(boxes)) > 0, "Some boxes have zero area!" ret = {'image': im} # rpn anchor: try: if cfg.MODE_FPN: multilevel_anchor_inputs = get_multilevel_rpn_anchor_input( im, boxes, is_crowd) for i, (anchor_labels, anchor_boxes) in enumerate(multilevel_anchor_inputs): ret['anchor_labels_lvl{}'.format(i + 2)] = anchor_labels ret['anchor_boxes_lvl{}'.format(i + 2)] = anchor_boxes else: # anchor_labels, anchor_boxes ret['anchor_labels'], ret[ 'anchor_boxes'] = get_rpn_anchor_input( im, boxes, is_crowd) boxes = boxes[is_crowd == 0] # skip crowd boxes in training target klass = klass[is_crowd == 0] ret['gt_boxes'] = boxes ret['gt_labels'] = klass if not len(boxes): raise MalformedData("No valid gt_boxes!") except MalformedData as e: log_once( "Input {} is filtered for training: {}".format(fname, str(e)), 'warn') return None if cfg.MODE_MASK: # augmentation will modify the polys in-place segmentation = copy.deepcopy(roidb['segmentation']) segmentation = [ segmentation[k] for k in range(len(segmentation)) if not is_crowd[k] ] assert len(segmentation) == len(boxes) # Apply augmentation on polygon coordinates. # And produce one image-sized binary mask per box. ret['gt_masks'] = segmentation # from viz import draw_annotation, draw_mask # viz = draw_annotation(im, boxes, klass) # for mask in masks: # viz = draw_mask(viz, mask) # tpviz.interactive_imshow(viz) return ret if cfg.TRAINER == 'horovod': ds = MultiThreadMapData(ds, 5, preprocess) # MPI does not like fork() else: ds = MultiProcessMapDataZMQ(ds, 10, preprocess) return ds