def __init__(self,
ds: DataFlow,
values: Union[np.ndarray, Sequence[Any]],
index: int = 0) -> None:
MapDataComponent.__init__(self, ds, self._map_component, index)
self._values = np.asanyarray(values)
def get_eval_dataflow(name, shard=0, num_shards=1):
"""
Args:
name (str): name of the dataset to evaluate
shard, num_shards: to get subset of evaluation data
"""
roidbs = COCODetection.load_many(cfg.DATA.BASEDIR, name, add_gt=False)
"""
To inference on your own data, change this to your loader.
Produce "roidbs" as a list of dict, in the dict the following keys are needed for training:
file_name: str, full path to the image
id: an id of this image
"""
num_imgs = len(roidbs)
img_per_shard = num_imgs // num_shards
img_range = (shard * img_per_shard, (shard + 1) * img_per_shard if shard + 1 < num_shards else num_imgs)
# no filter for training
ds = DataFromListOfDict(roidbs[img_range[0]: img_range[1]], ['file_name', 'id'])
def f(fname):
im = cv2.imread(fname, cv2.IMREAD_COLOR)
assert im is not None, fname
return im
ds = MapDataComponent(ds, f, 0)
# Evaluation itself may be multi-threaded, therefore don't add prefetch here.
return ds
def get_wider_eval_dataflow(shard=0, num_shards=1, augment=False):
"""
Args:
shard, num_shards: to get subset of evaluation data
"""
roidbs_test = load_many(cfg.WIDER.BASEDIR, 'test', augment)
roidbs = roidbs_test
# for key in roidbs[4].keys():
# print(key)
num_imgs = len(roidbs)
img_per_shard = num_imgs // num_shards
img_range = (shard * img_per_shard, (shard + 1) *
img_per_shard if shard + 1 < num_shards else num_imgs)
# no filter for training
ds = DataFromListOfDict(roidbs[img_range[0]:img_range[1]], [
'img', 'bbox', 'id', 'male', 'longhair', 'sunglass', 'hat', 'tshirt',
'longsleeve', 'formal', 'shorts', 'jeans', 'skirt', 'facemask', 'logo',
'stripe', 'longpants'
])
def f(fname):
im = cv2.imread(fname, cv2.IMREAD_COLOR)
assert im is not None, fname
return im
ds = MapDataComponent(ds, f, 0)
# Evaluation itself may be multi-threaded, therefore don't add prefetch here.
return roidbs, ds
def get_eval_dataflow(shard=0, num_shards=1):
"""
Args:
shard, num_shards: to get subset of evaluation data
"""
prw = PRWDataset(cfg.DATA.BASEDIR)
imgs = prw.load('test')
num_imgs = len(imgs)
img_per_shard = num_imgs // num_shards
img_range = (shard * img_per_shard, (shard + 1) *
img_per_shard if shard + 1 < num_shards else num_imgs)
# no filter for training
# test if it can repeat keys
ds = DataFromListOfDict(imgs[img_range[0]:img_range[1]],
['file_name', 'file_name'])
def f(fname):
im = cv2.imread(fname, cv2.IMREAD_COLOR)
assert im is not None, fname
return im
ds = MapDataComponent(ds, f, 0)
# Evaluation itself may be multi-threaded, therefore don't add prefetch here.
return ds
def read_and_augment_images(ds):
def mapf(dp):
fname = dp[0]
im = cv2.imread(fname, cv2.IMREAD_COLOR).astype('float32')
assert im is not None, dp[0]
dp[0] = im
# assume floatbox as input
assert dp[1].dtype == np.float32
dp[1] = box_to_point8(dp[1])
dp.append(fname)
return dp
ds = MapData(ds, mapf)
augs = [
CustomResize(config.SHORT_EDGE_SIZE, config.MAX_SIZE),
imgaug.Flip(horiz=True)
]
ds = AugmentImageComponents(ds, augs, index=(0, ), coords_index=(1, ))
def unmapf(points):
boxes = point8_to_box(points)
return boxes
ds = MapDataComponent(ds, unmapf, 1)
return ds
def get_eval_dataflow(name, shard=0, num_shards=1):
"""
Args:
name (str): name of the dataset to evaluate
shard, num_shards: to get subset of evaluation data
"""
roidbs = DatasetRegistry.get(name).inference_roidbs()
logger.info("Found {} images for inference.".format(len(roidbs)))
num_imgs = len(roidbs)
img_per_shard = num_imgs // num_shards
img_range = (shard * img_per_shard, (shard + 1) *
img_per_shard if shard + 1 < num_shards else num_imgs)
# no filter for training
ds = DataFromListOfDict(roidbs[img_range[0]:img_range[1]],
["file_name", "image_id"])
def f(fname):
im = cv2.imread(fname, cv2.IMREAD_COLOR)
assert im is not None, fname
return im
ds = MapDataComponent(ds, f, 0)
# Evaluation itself may be multi-threaded, therefore don't add prefetch here.
return ds
def get_eval_dataflow_YCBV(name, shard=0, num_shards=1):
"""
Args:
name (str): name of the dataset to evaluate
shard, num_shards: to get subset of evaluation data
"""
roidbs = YCBVDetectionDataset().load_inference_image_ids(name)
num_imgs = len(roidbs)
img_per_shard = num_imgs // num_shards
img_range = (shard * img_per_shard, (shard + 1) *
img_per_shard if shard + 1 < num_shards else num_imgs)
# no filter for training
ds = DataFromListOfDict(roidbs[img_range[0]:img_range[1]],
['file_name', 'image_id'])
def f(fname):
im = cv2.imread(fname, cv2.IMREAD_COLOR)
assert im is not None, fname
return im
ds = MapDataComponent(ds, f, 0)
# Evaluation itself may be multi-threaded, therefore don't add prefetch here.
return ds
def get_eval_dataflow(shard=0, num_shards=1):
"""
Args:
shard, num_shards: to get subset of evaluation data
"""
imgs = COCODetection.load_many(cfg.DATA.BASEDIR,
cfg.DATA.VAL,
add_gt=False)
num_imgs = len(imgs)
img_per_shard = num_imgs // num_shards
img_range = (shard * img_per_shard, (shard + 1) *
img_per_shard if shard + 1 < num_shards else num_imgs)
# no filter for training
ds = DataFromListOfDict(imgs[img_range[0]:img_range[1]],
['file_name', 'id'])
def f(fname):
im = cv2.imread(fname, cv2.IMREAD_COLOR)
assert im is not None, fname
return im
ds = MapDataComponent(ds, f, 0)
# Evaluation itself may be multi-threaded, therefore don't add prefetch here.
return ds
def get_eval_unlabeled_dataflow(name,
shard=0,
num_shards=1,
return_size=False):
"""
Return a training dataflow. Each datapoint consists of the following:
An image: (h, w, 3),
1 or more pairs of (anchor_labels, anchor_boxes):
anchor_labels: (h', w', NA)
anchor_boxes: (h', w', NA, 4)
gt_boxes: (N, 4)
gt_labels: (N,)
If MODE_MASK, gt_masks: (N, h, w)
"""
if isinstance(name, (list, tuple)) and len(name) > 1:
if "VOC" not in name[0]:
assert "VOC" not in name[
1], "VOC has to be put before coco in cfg.DATA.TRAIN"
roidbs = []
for x in name:
_roidbs = DatasetRegistry.get(x).training_roidbs()
print_class_histogram(_roidbs)
roidbs.extend(_roidbs)
# roidbs = list(itertools.chain.from_iterable(DatasetRegistry.get(x).training_roidbs() for x in name))
logger.info("Merged roidbs from {}".format(name))
print_class_histogram(roidbs)
else:
if isinstance(name, (list, tuple)):
name = name[0]
roidbs = DatasetRegistry.get(name).training_roidbs()
print_class_histogram(roidbs)
num_imgs = len(roidbs)
img_per_shard = num_imgs // num_shards
img_range = (shard * img_per_shard, (shard + 1) *
img_per_shard if shard + 1 < num_shards else num_imgs)
logger.info("Found {} images for inference.".format(img_range[1] -
img_range[0] + 1))
# no filter for training
ds = DataFromListOfDict(roidbs[img_range[0]:img_range[1]],
["file_name", "image_id"])
def f(fname):
im = cv2.imread(fname, cv2.IMREAD_COLOR)
assert im is not None, fname
return im
ds = MapDataComponent(ds, f, 0)
# Evaluation itself may be multi-threaded, therefore don't add prefetch
# here.
if return_size:
return ds, num_imgs
return ds
def get_eval_dataflow():
imgs = COCODetection.load_many(config.BASEDIR, config.VAL_DATASET, add_gt=False)
# no filter for training
ds = DataFromListOfDict(imgs, ['file_name', 'id'])
def f(fname):
im = cv2.imread(fname, cv2.IMREAD_COLOR)
assert im is not None, fname
return im
ds = MapDataComponent(ds, f, 0)
return ds
def get_test_dataflow():
imgs = BRATS_SEG.load_many(config.BASEDIR, config.TEST_DATASET, add_gt=False)
# no filter for training
ds = DataFromListOfDict(imgs, ['file_name', 'id', 'preprocessed'])
def f(data):
volume_list, label, weight, original_shape, bbox = data
batch = sampler3d_whole(volume_list, label, weight, original_shape, bbox)
return batch
ds = MapDataComponent(ds, f, 2)
ds = PrefetchDataZMQ(ds, 1)
return ds
def get_eval_dataflow():
imgs = COCODetection.load_many(cfg.DATA.BASEDIR, cfg.DATA.VAL, add_gt=False)
# no filter for training
ds = DataFromListOfDict(imgs, ['file_name', 'id'])
def f(fname):
im = cv2.imread(fname, cv2.IMREAD_COLOR)
assert im is not None, fname
return im
ds = MapDataComponent(ds, f, 0)
if cfg.TRAINER != 'horovod':
ds = PrefetchDataZMQ(ds, 1)
return ds
def get_eval_dataflow():
#if config.CROSS_VALIDATION:
imgs = BRATS_SEG.load_from_file(config.BASEDIR, config.VAL_DATASET)
# no filter for training
ds = DataFromListOfDict(imgs, ['file_name', 'id', 'preprocessed'])
def f(data):
volume_list, label, weight, original_shape, bbox = data
batch = sampler3d_whole(volume_list, label, weight, original_shape, bbox)
return batch
ds = MapDataComponent(ds, f, 2)
ds = PrefetchDataZMQ(ds, 1)
return ds
def get_augmented_speech_commands_data(subset,
options,
do_multiprocess=True,
shuffle=True):
isTrain = subset == 'train' and do_multiprocess
shuffle = shuffle if shuffle is not None else isTrain
ds = SpeechCommandsDataFlow(
os.path.join(options.data_dir, 'speech_commands_v0.02'), subset,
shuffle, None)
if isTrain:
add_noise_func = functools.partial(_add_noise, noises=ds.noises)
ds = MapDataComponent(ds, _pad_or_clip_to_desired_sample, index=0)
ds = MapDataComponent(ds, _to_float, index=0)
if isTrain:
ds = MapDataComponent(ds, _time_shift, index=0)
ds = MapData(ds, add_noise_func)
ds = BatchData(ds,
options.batch_size // options.nr_gpu,
remainder=not isTrain)
if do_multiprocess:
ds = PrefetchData(ds, 4, 4)
return ds
def get_test_dataflow(add_mask=True):
"""
Return a training dataflow. Each datapoint is:
image, fm_labels, fm_boxes, gt_boxes, gt_class [, masks]
"""
imgs = Detection.load_many(
config.BASEDIR, config.VAL_DATASET, add_gt=False, add_mask=add_mask)
# no filter for training
ds = DataFromListOfDict(imgs, ['image_data', 'id'])
def f(image):
im = cv2.imread(image)
return im
ds = MapDataComponent(ds, f, 0)
ds = PrefetchDataZMQ(ds, 1)
return ds
def get_sequential_loader(ds, isTrain, batch_size, augmentors, parallel=None):
""" Load a Single-File LMDB (Sequential Read)
Args:
augmentors (list[imgaug.Augmentor]): Defaults to `fbresnet_augmentor(isTrain)`
Returns: A LMDBData which produces BGR images and labels.
See explanations in the tutorial:
http://tensorpack.readthedocs.io/tutorial/efficient-dataflow.html
"""
assert isinstance(augmentors, list)
aug = imgaug.AugmentorList(augmentors)
if parallel is None:
parallel = min(40,
multiprocessing.cpu_count() //
2) # assuming hyperthreading
if isTrain:
ds = LocallyShuffleData(ds, 50000)
ds = MapDataComponent(ds, lambda x: cv2.imdecode(x, cv2.IMREAD_COLOR),
0)
ds = AugmentImageComponent(ds, aug, copy=False)
if parallel < 16:
logger.warn(
"DataFlow may become the bottleneck when too few processes are used."
)
ds = BatchData(ds, batch_size, remainder=False, use_list=True)
ds = MultiProcessRunnerZMQ(ds, parallel)
else:
def mapper(data):
im, label = data
im = cv2.imdecode(im, cv2.IMREAD_COLOR)
im = aug.augment(im)
return im, label
ds = MultiProcessMapDataZMQ(ds,
parallel,
mapper,
buffer_size=2000,
strict=True)
ds = BatchData(ds, batch_size, remainder=True, use_list=True)
return ds
def get_eval_dataflow(name, is_aws, is_gcs, shard=0, num_shards=1):
"""
Args:
name (str): name of the dataset to evaluate
shard, num_shards: to get subset of evaluation data
"""
roidbs = DatasetRegistry.get(name).inference_roidbs()
logger.info("Found {} images for inference.".format(len(roidbs)))
num_imgs = len(roidbs)
img_per_shard = num_imgs // num_shards
img_range = (
shard * img_per_shard,
(shard + 1) * img_per_shard if shard + 1 < num_shards else num_imgs,
)
# no filter for training
ds = DataFromListOfDict(roidbs[img_range[0] : img_range[1]], ["file_name", "image_id"])
if is_aws:
s3 = boto3.resource("s3")
elif is_gcs:
c = storage.Client.create_anonymous_client()
bucket = c.get_bucket("determined-ai-coco-dataset")
def f(fname):
if is_aws:
s3_object = s3.meta.client.get_object(Bucket="determined-ai-coco-dataset", Key=fname)
im = cv2.imdecode(
np.asarray(bytearray(s3_object["Body"].read()), dtype=np.uint8), cv2.IMREAD_COLOR,
)
elif is_gcs:
blob = bucket.blob(fname)
s = download_gcs_blob_with_backoff(blob)
im = cv2.imdecode(np.asarray(bytearray(s), dtype=np.uint8), cv2.IMREAD_COLOR)
else:
im = cv2.imread(fname, cv2.IMREAD_COLOR)
assert im is not None, fname
return im
ds = MapDataComponent(ds, f, 0)
# Evaluation itself may be multi-threaded, therefore don't add prefetch here.
return ds
def get_eval_dataflow(batch_size=0, shard=0, num_shards=1):
'''
'''
imgs = load_many_from_db(cfg.DATA.NAME, add_gt=True, is_train=False)
if num_shards > 1:
num_imgs = len(imgs)
img_per_shard = num_imgs // num_shards
s, e = shard * img_per_shard, min(num_imgs, (shard + 1) * img_per_shard)
imgs = imgs[s:e]
assert len(imgs) % batch_size == 0, \
'len(img) must be multiples of batch_size, {}, {}'.format(len(imgs), batch_size)
# imgs = COCODetection.load_many(cfg.DATA.BASEDIR, cfg.DATA.VAL, add_gt=False)
# no filter for training
# ds = DataFromList(imgs, shuffle=False)
ds = DataFromListOfDict(imgs, ['fn_img', 'id'])
if batch_size <= 0:
batch_size = cfg.PREPROC.EVAL_BATCH_SIZE
assert batch_size > 0, 'Batch size should be greater than 0'
hh, ww = cfg.PREPROC.INPUT_SHAPE_EVAL
mean_bgr = np.array(cfg.PREPROC.PIXEL_MEAN[::-1])
aug = CropPadTransform(0, 0, ww, hh, mean_bgr)
def f(fname):
im = cv2.imread(fname, cv2.IMREAD_COLOR)
assert im is not None, fname
scale = min(ww / float(im.shape[1]), hh / float(im.shape[0]))
im = cv2.resize(im, (0, 0), fx=scale, fy=scale)
im = aug.apply_image(im)
im = cv2.resize(im, (ww, hh))
return im
ds = MapDataComponent(ds, f, 0)
ds = BatchData(ds, batch_size, use_list=False)
return ds
def __init__(self, ds, values, index=0):
MapDataComponent.__init__(self, ds, self._map_component, index)
self._values = values
def get_inat_augmented_data(subset,
options,
lmdb_dir=None,
year='2018',
do_multiprocess=True,
do_validation=False,
is_train=None,
shuffle=None,
n_allow=None):
input_size = options.input_size if options.input_size else 224
isTrain = is_train if is_train is not None else (subset == 'train'
and do_multiprocess)
shuffle = shuffle if shuffle is not None else isTrain
postfix = "" if n_allow is None else "_allow_{}".format(n_allow)
#TODO: Parameterize the cv split to be consider
#Currently hardcoding to 1
cv = 1
# When do_validation is True it will expect *cv_train and *cv_val lmdbs
# Currently the cv_train split is always used
if isTrain:
postfix += '_cv_train_{}'.format(cv)
elif do_validation:
subset = 'train'
postfix += '_cv_val_{}'.format(cv)
if lmdb_dir == None:
lmdb_path = os.path.join(options.data_dir, 'inat_lmdb',
'inat2018_{}{}.lmdb'.format(subset, postfix))
else:
lmdb_path = os.path.join(
options.data_dir, lmdb_dir,
'inat{}_{}{}.lmdb'.format(year, subset, postfix))
ds = LMDBData(lmdb_path, shuffle=False)
if shuffle:
ds = LocallyShuffleData(ds,
1024 * 80) # This is 64G~80G in memory images
ds = PrefetchData(ds, 1024 * 8, 1) # prefetch around 8 G
ds = LMDBDataPoint(ds)
ds = MapDataComponent(ds, lambda x: cv2.imdecode(x, cv2.IMREAD_COLOR),
0) # BGR uint8 data
if isTrain:
class Resize(imgaug.ImageAugmentor):
"""
crop 8%~100% of the original image
See `Going Deeper with Convolutions` by Google.
"""
def _augment(self, img, _):
h, w = img.shape[:2]
area = h * w
for _ in range(10):
targetArea = self.rng.uniform(0.08, 1.0) * area
aspectR = self.rng.uniform(0.75, 1.333)
ww = int(np.sqrt(targetArea * aspectR))
hh = int(np.sqrt(targetArea / aspectR))
if self.rng.uniform() < 0.5:
ww, hh = hh, ww
if hh <= h and ww <= w:
x1 = 0 if w == ww else self.rng.randint(0, w - ww)
y1 = 0 if h == hh else self.rng.randint(0, h - hh)
out = img[y1:y1 + hh, x1:x1 + ww]
out = cv2.resize(out, (input_size, input_size),
interpolation=cv2.INTER_CUBIC)
return out
out = cv2.resize(img, (input_size, input_size),
interpolation=cv2.INTER_CUBIC)
return out
augmentors = [
Resize(),
imgaug.RandomOrderAug([
imgaug.Brightness(30, clip=False),
imgaug.Contrast((0.8, 1.2), clip=False),
imgaug.Saturation(0.4),
# rgb-bgr conversion
imgaug.Lighting(0.1,
eigval=[0.2175, 0.0188, 0.0045][::-1],
eigvec=np.array([[-0.5675, 0.7192, 0.4009],
[-0.5808, -0.0045, -0.8140],
[-0.5836, -0.6948, 0.4203]],
dtype='float32')[::-1, ::-1])
]),
imgaug.Clip(),
imgaug.Flip(horiz=True),
imgaug.ToUint8()
]
else:
augmentors = [
imgaug.ResizeShortestEdge(256),
imgaug.CenterCrop((input_size, input_size)),
imgaug.ToUint8()
]
ds = AugmentImageComponent(ds, augmentors, copy=False)
if do_multiprocess:
ds = PrefetchDataZMQ(ds, min(24, multiprocessing.cpu_count()))
ds = BatchData(ds,
options.batch_size // options.nr_gpu,
remainder=not isTrain)
return ds
5000,
random=False,
dtype='uint8')
df_val = FakeData([[64, 224, 224, 3], [64, 1000]], 5000, random=False)
else:
batch_size = TOTAL_BATCH_SIZE // num_gpu
assert args.data is not None
df_train = get_imagenet_dataflow(args.data, 'train', batch_size,
fbresnet_augmentor(True))
df_val = get_imagenet_dataflow(args.data, 'val', batch_size,
fbresnet_augmentor(False))
def one_hot(label):
return np.eye(1000)[label]
df_train = MapDataComponent(df_train, one_hot, 1)
df_val = MapDataComponent(df_val, one_hot, 1)
M = KerasModel(
resnet50,
inputs_desc=[InputDesc(tf.uint8, [None, 224, 224, 3], 'images')],
targets_desc=[InputDesc(tf.float32, [None, 1000], 'labels')],
input=df_train,
trainer=SyncMultiGPUTrainerReplicated(num_gpu))
lr = tf.get_variable('learning_rate', initializer=0.1, trainable=False)
tf.summary.scalar('lr', lr)
M.compile(optimizer=tf.train.MomentumOptimizer(lr, 0.9, use_nesterov=True),
loss='categorical_crossentropy',
metrics='categorical_accuracy')
def get_tiny_imagenet_augmented_data(subset, options,
do_multiprocess=True, is_train=None, shuffle=None):
isTrain = is_train if is_train is not None else (subset == 'train' and do_multiprocess)
shuffle = shuffle if shuffle is not None else isTrain
lmdb_path = os.path.join(options.data_dir,
'tiny_imagenet_lmdb', 'tiny_imagenet_{}.lmdb'.format(subset))
# since tiny imagenet is small (200MB zipped) we can shuffle all directly.
# we skipped the LocallyShuffleData and PrefetchData routine.
ds = LMDBData(lmdb_path, shuffle=shuffle)
ds = LMDBDataPoint(ds)
ds = MapDataComponent(ds, lambda x: cv2.imdecode(x, cv2.IMREAD_COLOR), 0)
img_size = 64
if isTrain:
class Resize(imgaug.ImageAugmentor):
"""
crop 8%~100% of the original image
See `Going Deeper with Convolutions` by Google.
"""
def _augment(self, img, _):
h, w = img.shape[:2]
area = h * w
for _ in range(10):
targetArea = self.rng.uniform(0.3, 1.0) * area
aspectR = self.rng.uniform(0.75, 1.333)
ww = int(np.sqrt(targetArea * aspectR))
hh = int(np.sqrt(targetArea / aspectR))
if self.rng.uniform() < 0.5:
ww, hh = hh, ww
if hh <= h and ww <= w:
x1 = 0 if w == ww else self.rng.randint(0, w - ww)
y1 = 0 if h == hh else self.rng.randint(0, h - hh)
out = img[y1:y1 + hh, x1:x1 + ww]
out = cv2.resize(out, (img_size, img_size), interpolation=cv2.INTER_CUBIC)
return out
out = cv2.resize(img, (img_size, img_size), interpolation=cv2.INTER_CUBIC)
return out
augmentors = [
Resize(),
imgaug.RandomOrderAug(
[imgaug.Brightness(30, clip=False),
imgaug.Contrast((0.8, 1.2), clip=False),
imgaug.Saturation(0.4),
# rgb-bgr conversion
imgaug.Lighting(0.1,
eigval=[0.2175, 0.0188, 0.0045][::-1],
eigvec=np.array(
[[-0.5675, 0.7192, 0.4009],
[-0.5808, -0.0045, -0.8140],
[-0.5836, -0.6948, 0.4203]],
dtype='float32')[::-1, ::-1]
)]),
imgaug.Clip(),
imgaug.Flip(horiz=True),
imgaug.ToUint8()
]
else:
augmentors = [
imgaug.ResizeShortestEdge(72),
imgaug.CenterCrop((img_size, img_size)),
imgaug.ToUint8()
]
ds = AugmentImageComponent(ds, augmentors, copy=False)
ds = BatchData(ds, options.batch_size // options.nr_gpu, remainder=not isTrain)
if do_multiprocess:
ds = PrefetchData(ds, nr_prefetch=4, nr_proc=4)
return ds
