def __init__(self, ds: DataFlow, values: Union[np.ndarray, Sequence[Any]], index: int = 0) -> None: MapDataComponent.__init__(self, ds, self._map_component, index) self._values = np.asanyarray(values)
def get_eval_dataflow(name, shard=0, num_shards=1): """ Args: name (str): name of the dataset to evaluate shard, num_shards: to get subset of evaluation data """ roidbs = COCODetection.load_many(cfg.DATA.BASEDIR, name, add_gt=False) """ To inference on your own data, change this to your loader. Produce "roidbs" as a list of dict, in the dict the following keys are needed for training: file_name: str, full path to the image id: an id of this image """ num_imgs = len(roidbs) img_per_shard = num_imgs // num_shards img_range = (shard * img_per_shard, (shard + 1) * img_per_shard if shard + 1 < num_shards else num_imgs) # no filter for training ds = DataFromListOfDict(roidbs[img_range[0]: img_range[1]], ['file_name', 'id']) def f(fname): im = cv2.imread(fname, cv2.IMREAD_COLOR) assert im is not None, fname return im ds = MapDataComponent(ds, f, 0) # Evaluation itself may be multi-threaded, therefore don't add prefetch here. return ds
def get_wider_eval_dataflow(shard=0, num_shards=1, augment=False): """ Args: shard, num_shards: to get subset of evaluation data """ roidbs_test = load_many(cfg.WIDER.BASEDIR, 'test', augment) roidbs = roidbs_test # for key in roidbs[4].keys(): # print(key) num_imgs = len(roidbs) img_per_shard = num_imgs // num_shards img_range = (shard * img_per_shard, (shard + 1) * img_per_shard if shard + 1 < num_shards else num_imgs) # no filter for training ds = DataFromListOfDict(roidbs[img_range[0]:img_range[1]], [ 'img', 'bbox', 'id', 'male', 'longhair', 'sunglass', 'hat', 'tshirt', 'longsleeve', 'formal', 'shorts', 'jeans', 'skirt', 'facemask', 'logo', 'stripe', 'longpants' ]) def f(fname): im = cv2.imread(fname, cv2.IMREAD_COLOR) assert im is not None, fname return im ds = MapDataComponent(ds, f, 0) # Evaluation itself may be multi-threaded, therefore don't add prefetch here. return roidbs, ds
def get_eval_dataflow(shard=0, num_shards=1): """ Args: shard, num_shards: to get subset of evaluation data """ prw = PRWDataset(cfg.DATA.BASEDIR) imgs = prw.load('test') num_imgs = len(imgs) img_per_shard = num_imgs // num_shards img_range = (shard * img_per_shard, (shard + 1) * img_per_shard if shard + 1 < num_shards else num_imgs) # no filter for training # test if it can repeat keys ds = DataFromListOfDict(imgs[img_range[0]:img_range[1]], ['file_name', 'file_name']) def f(fname): im = cv2.imread(fname, cv2.IMREAD_COLOR) assert im is not None, fname return im ds = MapDataComponent(ds, f, 0) # Evaluation itself may be multi-threaded, therefore don't add prefetch here. return ds
def read_and_augment_images(ds): def mapf(dp): fname = dp[0] im = cv2.imread(fname, cv2.IMREAD_COLOR).astype('float32') assert im is not None, dp[0] dp[0] = im # assume floatbox as input assert dp[1].dtype == np.float32 dp[1] = box_to_point8(dp[1]) dp.append(fname) return dp ds = MapData(ds, mapf) augs = [ CustomResize(config.SHORT_EDGE_SIZE, config.MAX_SIZE), imgaug.Flip(horiz=True) ] ds = AugmentImageComponents(ds, augs, index=(0, ), coords_index=(1, )) def unmapf(points): boxes = point8_to_box(points) return boxes ds = MapDataComponent(ds, unmapf, 1) return ds
def get_eval_dataflow(name, shard=0, num_shards=1): """ Args: name (str): name of the dataset to evaluate shard, num_shards: to get subset of evaluation data """ roidbs = DatasetRegistry.get(name).inference_roidbs() logger.info("Found {} images for inference.".format(len(roidbs))) num_imgs = len(roidbs) img_per_shard = num_imgs // num_shards img_range = (shard * img_per_shard, (shard + 1) * img_per_shard if shard + 1 < num_shards else num_imgs) # no filter for training ds = DataFromListOfDict(roidbs[img_range[0]:img_range[1]], ["file_name", "image_id"]) def f(fname): im = cv2.imread(fname, cv2.IMREAD_COLOR) assert im is not None, fname return im ds = MapDataComponent(ds, f, 0) # Evaluation itself may be multi-threaded, therefore don't add prefetch here. return ds
def get_eval_dataflow_YCBV(name, shard=0, num_shards=1): """ Args: name (str): name of the dataset to evaluate shard, num_shards: to get subset of evaluation data """ roidbs = YCBVDetectionDataset().load_inference_image_ids(name) num_imgs = len(roidbs) img_per_shard = num_imgs // num_shards img_range = (shard * img_per_shard, (shard + 1) * img_per_shard if shard + 1 < num_shards else num_imgs) # no filter for training ds = DataFromListOfDict(roidbs[img_range[0]:img_range[1]], ['file_name', 'image_id']) def f(fname): im = cv2.imread(fname, cv2.IMREAD_COLOR) assert im is not None, fname return im ds = MapDataComponent(ds, f, 0) # Evaluation itself may be multi-threaded, therefore don't add prefetch here. return ds
def get_eval_dataflow(shard=0, num_shards=1): """ Args: shard, num_shards: to get subset of evaluation data """ imgs = COCODetection.load_many(cfg.DATA.BASEDIR, cfg.DATA.VAL, add_gt=False) num_imgs = len(imgs) img_per_shard = num_imgs // num_shards img_range = (shard * img_per_shard, (shard + 1) * img_per_shard if shard + 1 < num_shards else num_imgs) # no filter for training ds = DataFromListOfDict(imgs[img_range[0]:img_range[1]], ['file_name', 'id']) def f(fname): im = cv2.imread(fname, cv2.IMREAD_COLOR) assert im is not None, fname return im ds = MapDataComponent(ds, f, 0) # Evaluation itself may be multi-threaded, therefore don't add prefetch here. return ds
def get_eval_unlabeled_dataflow(name, shard=0, num_shards=1, return_size=False): """ Return a training dataflow. Each datapoint consists of the following: An image: (h, w, 3), 1 or more pairs of (anchor_labels, anchor_boxes): anchor_labels: (h', w', NA) anchor_boxes: (h', w', NA, 4) gt_boxes: (N, 4) gt_labels: (N,) If MODE_MASK, gt_masks: (N, h, w) """ if isinstance(name, (list, tuple)) and len(name) > 1: if "VOC" not in name[0]: assert "VOC" not in name[ 1], "VOC has to be put before coco in cfg.DATA.TRAIN" roidbs = [] for x in name: _roidbs = DatasetRegistry.get(x).training_roidbs() print_class_histogram(_roidbs) roidbs.extend(_roidbs) # roidbs = list(itertools.chain.from_iterable(DatasetRegistry.get(x).training_roidbs() for x in name)) logger.info("Merged roidbs from {}".format(name)) print_class_histogram(roidbs) else: if isinstance(name, (list, tuple)): name = name[0] roidbs = DatasetRegistry.get(name).training_roidbs() print_class_histogram(roidbs) num_imgs = len(roidbs) img_per_shard = num_imgs // num_shards img_range = (shard * img_per_shard, (shard + 1) * img_per_shard if shard + 1 < num_shards else num_imgs) logger.info("Found {} images for inference.".format(img_range[1] - img_range[0] + 1)) # no filter for training ds = DataFromListOfDict(roidbs[img_range[0]:img_range[1]], ["file_name", "image_id"]) def f(fname): im = cv2.imread(fname, cv2.IMREAD_COLOR) assert im is not None, fname return im ds = MapDataComponent(ds, f, 0) # Evaluation itself may be multi-threaded, therefore don't add prefetch # here. if return_size: return ds, num_imgs return ds
def get_eval_dataflow(): imgs = COCODetection.load_many(config.BASEDIR, config.VAL_DATASET, add_gt=False) # no filter for training ds = DataFromListOfDict(imgs, ['file_name', 'id']) def f(fname): im = cv2.imread(fname, cv2.IMREAD_COLOR) assert im is not None, fname return im ds = MapDataComponent(ds, f, 0) return ds
def get_test_dataflow(): imgs = BRATS_SEG.load_many(config.BASEDIR, config.TEST_DATASET, add_gt=False) # no filter for training ds = DataFromListOfDict(imgs, ['file_name', 'id', 'preprocessed']) def f(data): volume_list, label, weight, original_shape, bbox = data batch = sampler3d_whole(volume_list, label, weight, original_shape, bbox) return batch ds = MapDataComponent(ds, f, 2) ds = PrefetchDataZMQ(ds, 1) return ds
def get_eval_dataflow(): imgs = COCODetection.load_many(cfg.DATA.BASEDIR, cfg.DATA.VAL, add_gt=False) # no filter for training ds = DataFromListOfDict(imgs, ['file_name', 'id']) def f(fname): im = cv2.imread(fname, cv2.IMREAD_COLOR) assert im is not None, fname return im ds = MapDataComponent(ds, f, 0) if cfg.TRAINER != 'horovod': ds = PrefetchDataZMQ(ds, 1) return ds
def get_eval_dataflow(): #if config.CROSS_VALIDATION: imgs = BRATS_SEG.load_from_file(config.BASEDIR, config.VAL_DATASET) # no filter for training ds = DataFromListOfDict(imgs, ['file_name', 'id', 'preprocessed']) def f(data): volume_list, label, weight, original_shape, bbox = data batch = sampler3d_whole(volume_list, label, weight, original_shape, bbox) return batch ds = MapDataComponent(ds, f, 2) ds = PrefetchDataZMQ(ds, 1) return ds
def get_augmented_speech_commands_data(subset, options, do_multiprocess=True, shuffle=True): isTrain = subset == 'train' and do_multiprocess shuffle = shuffle if shuffle is not None else isTrain ds = SpeechCommandsDataFlow( os.path.join(options.data_dir, 'speech_commands_v0.02'), subset, shuffle, None) if isTrain: add_noise_func = functools.partial(_add_noise, noises=ds.noises) ds = MapDataComponent(ds, _pad_or_clip_to_desired_sample, index=0) ds = MapDataComponent(ds, _to_float, index=0) if isTrain: ds = MapDataComponent(ds, _time_shift, index=0) ds = MapData(ds, add_noise_func) ds = BatchData(ds, options.batch_size // options.nr_gpu, remainder=not isTrain) if do_multiprocess: ds = PrefetchData(ds, 4, 4) return ds
def get_test_dataflow(add_mask=True): """ Return a training dataflow. Each datapoint is: image, fm_labels, fm_boxes, gt_boxes, gt_class [, masks] """ imgs = Detection.load_many( config.BASEDIR, config.VAL_DATASET, add_gt=False, add_mask=add_mask) # no filter for training ds = DataFromListOfDict(imgs, ['image_data', 'id']) def f(image): im = cv2.imread(image) return im ds = MapDataComponent(ds, f, 0) ds = PrefetchDataZMQ(ds, 1) return ds
def get_sequential_loader(ds, isTrain, batch_size, augmentors, parallel=None): """ Load a Single-File LMDB (Sequential Read) Args: augmentors (list[imgaug.Augmentor]): Defaults to `fbresnet_augmentor(isTrain)` Returns: A LMDBData which produces BGR images and labels. See explanations in the tutorial: http://tensorpack.readthedocs.io/tutorial/efficient-dataflow.html """ assert isinstance(augmentors, list) aug = imgaug.AugmentorList(augmentors) if parallel is None: parallel = min(40, multiprocessing.cpu_count() // 2) # assuming hyperthreading if isTrain: ds = LocallyShuffleData(ds, 50000) ds = MapDataComponent(ds, lambda x: cv2.imdecode(x, cv2.IMREAD_COLOR), 0) ds = AugmentImageComponent(ds, aug, copy=False) if parallel < 16: logger.warn( "DataFlow may become the bottleneck when too few processes are used." ) ds = BatchData(ds, batch_size, remainder=False, use_list=True) ds = MultiProcessRunnerZMQ(ds, parallel) else: def mapper(data): im, label = data im = cv2.imdecode(im, cv2.IMREAD_COLOR) im = aug.augment(im) return im, label ds = MultiProcessMapDataZMQ(ds, parallel, mapper, buffer_size=2000, strict=True) ds = BatchData(ds, batch_size, remainder=True, use_list=True) return ds
def get_eval_dataflow(name, is_aws, is_gcs, shard=0, num_shards=1): """ Args: name (str): name of the dataset to evaluate shard, num_shards: to get subset of evaluation data """ roidbs = DatasetRegistry.get(name).inference_roidbs() logger.info("Found {} images for inference.".format(len(roidbs))) num_imgs = len(roidbs) img_per_shard = num_imgs // num_shards img_range = ( shard * img_per_shard, (shard + 1) * img_per_shard if shard + 1 < num_shards else num_imgs, ) # no filter for training ds = DataFromListOfDict(roidbs[img_range[0] : img_range[1]], ["file_name", "image_id"]) if is_aws: s3 = boto3.resource("s3") elif is_gcs: c = storage.Client.create_anonymous_client() bucket = c.get_bucket("determined-ai-coco-dataset") def f(fname): if is_aws: s3_object = s3.meta.client.get_object(Bucket="determined-ai-coco-dataset", Key=fname) im = cv2.imdecode( np.asarray(bytearray(s3_object["Body"].read()), dtype=np.uint8), cv2.IMREAD_COLOR, ) elif is_gcs: blob = bucket.blob(fname) s = download_gcs_blob_with_backoff(blob) im = cv2.imdecode(np.asarray(bytearray(s), dtype=np.uint8), cv2.IMREAD_COLOR) else: im = cv2.imread(fname, cv2.IMREAD_COLOR) assert im is not None, fname return im ds = MapDataComponent(ds, f, 0) # Evaluation itself may be multi-threaded, therefore don't add prefetch here. return ds
def get_eval_dataflow(batch_size=0, shard=0, num_shards=1): ''' ''' imgs = load_many_from_db(cfg.DATA.NAME, add_gt=True, is_train=False) if num_shards > 1: num_imgs = len(imgs) img_per_shard = num_imgs // num_shards s, e = shard * img_per_shard, min(num_imgs, (shard + 1) * img_per_shard) imgs = imgs[s:e] assert len(imgs) % batch_size == 0, \ 'len(img) must be multiples of batch_size, {}, {}'.format(len(imgs), batch_size) # imgs = COCODetection.load_many(cfg.DATA.BASEDIR, cfg.DATA.VAL, add_gt=False) # no filter for training # ds = DataFromList(imgs, shuffle=False) ds = DataFromListOfDict(imgs, ['fn_img', 'id']) if batch_size <= 0: batch_size = cfg.PREPROC.EVAL_BATCH_SIZE assert batch_size > 0, 'Batch size should be greater than 0' hh, ww = cfg.PREPROC.INPUT_SHAPE_EVAL mean_bgr = np.array(cfg.PREPROC.PIXEL_MEAN[::-1]) aug = CropPadTransform(0, 0, ww, hh, mean_bgr) def f(fname): im = cv2.imread(fname, cv2.IMREAD_COLOR) assert im is not None, fname scale = min(ww / float(im.shape[1]), hh / float(im.shape[0])) im = cv2.resize(im, (0, 0), fx=scale, fy=scale) im = aug.apply_image(im) im = cv2.resize(im, (ww, hh)) return im ds = MapDataComponent(ds, f, 0) ds = BatchData(ds, batch_size, use_list=False) return ds
def __init__(self, ds, values, index=0): MapDataComponent.__init__(self, ds, self._map_component, index) self._values = values
def get_inat_augmented_data(subset, options, lmdb_dir=None, year='2018', do_multiprocess=True, do_validation=False, is_train=None, shuffle=None, n_allow=None): input_size = options.input_size if options.input_size else 224 isTrain = is_train if is_train is not None else (subset == 'train' and do_multiprocess) shuffle = shuffle if shuffle is not None else isTrain postfix = "" if n_allow is None else "_allow_{}".format(n_allow) #TODO: Parameterize the cv split to be consider #Currently hardcoding to 1 cv = 1 # When do_validation is True it will expect *cv_train and *cv_val lmdbs # Currently the cv_train split is always used if isTrain: postfix += '_cv_train_{}'.format(cv) elif do_validation: subset = 'train' postfix += '_cv_val_{}'.format(cv) if lmdb_dir == None: lmdb_path = os.path.join(options.data_dir, 'inat_lmdb', 'inat2018_{}{}.lmdb'.format(subset, postfix)) else: lmdb_path = os.path.join( options.data_dir, lmdb_dir, 'inat{}_{}{}.lmdb'.format(year, subset, postfix)) ds = LMDBData(lmdb_path, shuffle=False) if shuffle: ds = LocallyShuffleData(ds, 1024 * 80) # This is 64G~80G in memory images ds = PrefetchData(ds, 1024 * 8, 1) # prefetch around 8 G ds = LMDBDataPoint(ds) ds = MapDataComponent(ds, lambda x: cv2.imdecode(x, cv2.IMREAD_COLOR), 0) # BGR uint8 data if isTrain: class Resize(imgaug.ImageAugmentor): """ crop 8%~100% of the original image See `Going Deeper with Convolutions` by Google. """ def _augment(self, img, _): h, w = img.shape[:2] area = h * w for _ in range(10): targetArea = self.rng.uniform(0.08, 1.0) * area aspectR = self.rng.uniform(0.75, 1.333) ww = int(np.sqrt(targetArea * aspectR)) hh = int(np.sqrt(targetArea / aspectR)) if self.rng.uniform() < 0.5: ww, hh = hh, ww if hh <= h and ww <= w: x1 = 0 if w == ww else self.rng.randint(0, w - ww) y1 = 0 if h == hh else self.rng.randint(0, h - hh) out = img[y1:y1 + hh, x1:x1 + ww] out = cv2.resize(out, (input_size, input_size), interpolation=cv2.INTER_CUBIC) return out out = cv2.resize(img, (input_size, input_size), interpolation=cv2.INTER_CUBIC) return out augmentors = [ Resize(), imgaug.RandomOrderAug([ imgaug.Brightness(30, clip=False), imgaug.Contrast((0.8, 1.2), clip=False), imgaug.Saturation(0.4), # rgb-bgr conversion imgaug.Lighting(0.1, eigval=[0.2175, 0.0188, 0.0045][::-1], eigvec=np.array([[-0.5675, 0.7192, 0.4009], [-0.5808, -0.0045, -0.8140], [-0.5836, -0.6948, 0.4203]], dtype='float32')[::-1, ::-1]) ]), imgaug.Clip(), imgaug.Flip(horiz=True), imgaug.ToUint8() ] else: augmentors = [ imgaug.ResizeShortestEdge(256), imgaug.CenterCrop((input_size, input_size)), imgaug.ToUint8() ] ds = AugmentImageComponent(ds, augmentors, copy=False) if do_multiprocess: ds = PrefetchDataZMQ(ds, min(24, multiprocessing.cpu_count())) ds = BatchData(ds, options.batch_size // options.nr_gpu, remainder=not isTrain) return ds
5000, random=False, dtype='uint8') df_val = FakeData([[64, 224, 224, 3], [64, 1000]], 5000, random=False) else: batch_size = TOTAL_BATCH_SIZE // num_gpu assert args.data is not None df_train = get_imagenet_dataflow(args.data, 'train', batch_size, fbresnet_augmentor(True)) df_val = get_imagenet_dataflow(args.data, 'val', batch_size, fbresnet_augmentor(False)) def one_hot(label): return np.eye(1000)[label] df_train = MapDataComponent(df_train, one_hot, 1) df_val = MapDataComponent(df_val, one_hot, 1) M = KerasModel( resnet50, inputs_desc=[InputDesc(tf.uint8, [None, 224, 224, 3], 'images')], targets_desc=[InputDesc(tf.float32, [None, 1000], 'labels')], input=df_train, trainer=SyncMultiGPUTrainerReplicated(num_gpu)) lr = tf.get_variable('learning_rate', initializer=0.1, trainable=False) tf.summary.scalar('lr', lr) M.compile(optimizer=tf.train.MomentumOptimizer(lr, 0.9, use_nesterov=True), loss='categorical_crossentropy', metrics='categorical_accuracy')
def get_tiny_imagenet_augmented_data(subset, options, do_multiprocess=True, is_train=None, shuffle=None): isTrain = is_train if is_train is not None else (subset == 'train' and do_multiprocess) shuffle = shuffle if shuffle is not None else isTrain lmdb_path = os.path.join(options.data_dir, 'tiny_imagenet_lmdb', 'tiny_imagenet_{}.lmdb'.format(subset)) # since tiny imagenet is small (200MB zipped) we can shuffle all directly. # we skipped the LocallyShuffleData and PrefetchData routine. ds = LMDBData(lmdb_path, shuffle=shuffle) ds = LMDBDataPoint(ds) ds = MapDataComponent(ds, lambda x: cv2.imdecode(x, cv2.IMREAD_COLOR), 0) img_size = 64 if isTrain: class Resize(imgaug.ImageAugmentor): """ crop 8%~100% of the original image See `Going Deeper with Convolutions` by Google. """ def _augment(self, img, _): h, w = img.shape[:2] area = h * w for _ in range(10): targetArea = self.rng.uniform(0.3, 1.0) * area aspectR = self.rng.uniform(0.75, 1.333) ww = int(np.sqrt(targetArea * aspectR)) hh = int(np.sqrt(targetArea / aspectR)) if self.rng.uniform() < 0.5: ww, hh = hh, ww if hh <= h and ww <= w: x1 = 0 if w == ww else self.rng.randint(0, w - ww) y1 = 0 if h == hh else self.rng.randint(0, h - hh) out = img[y1:y1 + hh, x1:x1 + ww] out = cv2.resize(out, (img_size, img_size), interpolation=cv2.INTER_CUBIC) return out out = cv2.resize(img, (img_size, img_size), interpolation=cv2.INTER_CUBIC) return out augmentors = [ Resize(), imgaug.RandomOrderAug( [imgaug.Brightness(30, clip=False), imgaug.Contrast((0.8, 1.2), clip=False), imgaug.Saturation(0.4), # rgb-bgr conversion imgaug.Lighting(0.1, eigval=[0.2175, 0.0188, 0.0045][::-1], eigvec=np.array( [[-0.5675, 0.7192, 0.4009], [-0.5808, -0.0045, -0.8140], [-0.5836, -0.6948, 0.4203]], dtype='float32')[::-1, ::-1] )]), imgaug.Clip(), imgaug.Flip(horiz=True), imgaug.ToUint8() ] else: augmentors = [ imgaug.ResizeShortestEdge(72), imgaug.CenterCrop((img_size, img_size)), imgaug.ToUint8() ] ds = AugmentImageComponent(ds, augmentors, copy=False) ds = BatchData(ds, options.batch_size // options.nr_gpu, remainder=not isTrain) if do_multiprocess: ds = PrefetchData(ds, nr_prefetch=4, nr_proc=4) return ds