def get_train_dataflow(datadir, args):
parallel = min(args.process_nums, multiprocessing.cpu_count() // 2)
ds = ILSVRC12Files(datadir, shuffle=False)
def mapf(dp):
fname, cls = dp
im_0 = ReadImageToCVMat(fname[0], args)
im_1 = ReadImageToCVMat(fname[1], args)
im_0 = occ_augImage(im_0)
im_1 = occ_augImage(im_1)
im_0 = transform(im_0, args)
im_1 = transform(im_1, args)
return np.array([im_0, im_1]), np.array([cls, cls])
ds = MultiThreadMapData(ds,
parallel,
mapf,
buffer_size=min(2000, ds.size()),
strict=True)
# ds = MultiProcessRunnerZMQ(ds, parallel)
# ds = BatchData(ds, batch_size, remainder=True)
# do not fork() under MPI
return ds
def make_data(self, ds,is_training=True):
if is_training:
ds = MultiThreadMapData(ds, 10, self.train_map_func, buffer_size=200, strict=True)
else:
ds = MultiThreadMapData(ds, 5, self.val_map_func, buffer_size=200, strict=True)
ds = BatchData(ds, cfg.TRAIN.num_gpu * cfg.TRAIN.batch_size, remainder=True,use_list=False)
ds = MultiProcessPrefetchData(ds, 100,2)
ds.reset_state()
ds=ds.get_data()
###########
# ds = data_set.shuffle(buffer_size=512) # shuffle before loading images
# ds = ds.repeat(cfg.TRAIN.epoch)
# if is_training:
# ds = ds.map(self.train_map_func, num_parallel_calls=multiprocessing.cpu_count()) # decouple the heavy map_fn
# else:
# ds = ds.map(self.val_map_func, num_parallel_calls=multiprocessing.cpu_count()) # decouple the heavy map_fn
# ds = ds.batch(
# cfg.TRAIN.num_gpu * cfg.TRAIN.batch_size) # TODO: consider using tf.contrib.map_and_batch
#
# ds = ds.prefetch(5 * cfg.TRAIN.num_gpu)
# iterator = ds.make_one_shot_iterator()
# one_element = iterator.get_next()
# images, labels = one_element
return ds
def get_simple_val_dataflow(src,
batch_size=32,
augmentors=None,
parallel=None):
if augmentors is None:
augmentors = fbresnet_augmentor(False)
assert isinstance(augmentors, list)
if parallel is None:
parallel = min(40, multiprocessing.cpu_count())
aug = imgaug.AugmentorList(augmentors)
def mapf(dp):
im, cls = dp
im_new = im * 255
im_new = im_new[:, :, ::-1]
im_new = im_new.astype('uint8')
return im_new, cls
im_new = aug.augment(im_new)
return im_new, cls
ds = MultiThreadMapData(src,
parallel,
mapf,
buffer_size=min(2000, src.size()),
strict=True)
ds = BatchData(ds, batch_size, remainder=True)
# do not fork() under MPI
return ds
def get_imagenet_dataflow(
datadir, name, batch_size,
augmentors=None, parallel=None):
"""
Args:
augmentors (list[imgaug.Augmentor]): Defaults to `fbresnet_augmentor(isTrain)`
Returns: A DataFlow which produces BGR images and labels.
"""
assert name in ['train', 'val', 'test']
isTrain = name == 'train'
assert datadir is not None
if augmentors is None:
augmentors = fbresnet_augmentor(isTrain)
assert isinstance(augmentors, list)
augmentors = AugmentorList(augmentors)
if parallel is None:
parallel = min(40, multiprocessing.cpu_count() // 2) # assuming hyperthreading
def mapf(dp):
fname, label = dp
img = cv2.imread(fname)
img = augmentors.augment(img)
return img, label
if isTrain:
ds = dataset.ILSVRC12Files(datadir, name, shuffle=True)
ds = MultiProcessMapDataZMQ(ds, parallel, mapf, buffer_size=2000)
ds = BatchData(ds, batch_size, remainder=False)
else:
ds = dataset.ILSVRC12Files(datadir, name, shuffle=False)
ds = MultiThreadMapData(ds, parallel, mapf, buffer_size=2000, strict=True)
ds = BatchData(ds, batch_size, remainder=True)
ds = PrefetchDataZMQ(ds, 1)
return ds
def get_dataflow(path, is_train, img_path=None):
ds = CocoPose(path, img_path, is_train) # read data from lmdb
if is_train:
ds = MapData(ds, read_image_url)
ds = MapDataComponent(ds, pose_random_scale)
ds = MapDataComponent(ds, pose_rotation)
ds = MapDataComponent(ds, pose_flip)
ds = MapDataComponent(ds, pose_resize_shortestedge_random)
ds = MapDataComponent(ds, pose_crop_random)
ds = MapData(ds, pose_to_img)
# augs = [
# imgaug.RandomApplyAug(imgaug.RandomChooseAug([
# imgaug.GaussianBlur(max_size=3)
# ]), 0.7)
# ]
# ds = AugmentImageComponent(ds, augs)
ds = PrefetchData(ds, 1000, multiprocessing.cpu_count() * 1)
else:
ds = MultiThreadMapData(ds,
nr_thread=16,
map_func=read_image_url,
buffer_size=1000)
ds = MapDataComponent(ds, pose_resize_shortestedge_fixed)
ds = MapDataComponent(ds, pose_crop_center)
ds = MapData(ds, pose_to_img)
ds = PrefetchData(ds, 100, multiprocessing.cpu_count() // 4)
return ds
def get_imagenet_dataflow(datadir, name, batch_size, augmentors):
"""
See explanations in the tutorial:
http://tensorpack.readthedocs.io/en/latest/tutorial/efficient-dataflow.html
"""
assert name in ['train', 'val', 'test']
assert datadir is not None
assert isinstance(augmentors, list)
isTrain = name == 'train'
cpu = min(30, multiprocessing.cpu_count())
meta_dir = './ilsvrc_metadata'
if isTrain:
ds = dataset.ILSVRC12(datadir, name, meta_dir=meta_dir, shuffle=True)
ds = AugmentImageComponent(ds, augmentors, copy=False)
ds = PrefetchDataZMQ(ds, cpu)
ds = BatchData(ds, batch_size, remainder=False)
else:
ds = dataset.ILSVRC12Files(datadir,
name,
meta_dir=meta_dir,
shuffle=False)
aug = imgaug.AugmentorList(augmentors)
def mapf(dp):
fname, cls = dp
im = cv2.imread(fname, cv2.IMREAD_COLOR)
im = aug.augment(im)
return im, cls
ds = MultiThreadMapData(ds, cpu, mapf, buffer_size=2000, strict=True)
ds = BatchData(ds, batch_size, remainder=True)
ds = PrefetchDataZMQ(ds, 1)
return ds
def get_val_dataflow(
datadir, batch_size,
augmentors, parallel=None,
num_splits=None, split_index=None):
assert datadir is not None
assert isinstance(augmentors, list)
if parallel is None:
parallel = min(40, multiprocessing.cpu_count())
if num_splits is None:
ds = dataset.ILSVRC12Files(datadir, 'val', shuffle=False)
else:
assert split_index < num_splits
files = dataset.ILSVRC12Files(datadir, 'val', shuffle=False)
files.reset_state()
files = list(files.get_data())
logger.info("#ValidationData = {}".format(len(files)))
split_size = len(files) // num_splits
start, end = split_size * split_index, split_size * (split_index + 1)
end = min(end, len(files))
logger.info("#ValidationSplit = {} - {}".format(start, end))
files = files[start: end]
ds = DataFromList(files, shuffle=False)
aug = imgaug.AugmentorList(augmentors)
def mapf(dp):
fname, cls = dp
im = cv2.imread(fname, cv2.IMREAD_COLOR)
im = aug.augment(im)
return im, cls
ds = MultiThreadMapData(ds, parallel, mapf, buffer_size=2000, strict=True)
ds = BatchData(ds, batch_size, remainder=True)
# ds = PrefetchDataZMQ(ds, 1)
# do not fork() under MPI
return ds
def get_iNaturalist_dataflow(
datadir, name, batch_size,
augmentors, parallel=None):
"""
See explanations in the tutorial:
http://tensorpack.readthedocs.io/en/latest/tutorial/efficient-dataflow.html
"""
assert name in ['train', 'val', 'test']
assert datadir is not None
assert isinstance(augmentors, list)
isTrain = name == 'train'
if parallel is None:
parallel = min(40, multiprocessing.cpu_count() // 2) # assuming hyperthreading
if isTrain:
ds = dataset.iNaturalist(datadir, name, shuffle=True)
ds = AugmentImageComponent(ds, augmentors, copy=False)
if parallel < 16:
logger.warn("DataFlow may become the bottleneck when too few processes are used.")
ds = PrefetchDataZMQ(ds, parallel)
ds = BatchData(ds, batch_size, remainder=False)
else:
ds = dataset.iNaturalistFiles(datadir, name, shuffle=False)
aug = imgaug.AugmentorList(augmentors)
def mapf(dp):
fname, cls = dp
im = cv2.imread(fname, cv2.IMREAD_COLOR)
im = aug.augment(im)
return im, cls
ds = MultiThreadMapData(ds, parallel, mapf, buffer_size=2000, strict=True)
ds = BatchData(ds, batch_size, remainder=True)
ds = PrefetchDataZMQ(ds, 1)
return ds
def get_train_dataflow(batch_size=2):
print("In train dataflow")
roidbs = list(itertools.chain.from_iterable(DatasetRegistry.get(x).training_roidbs() for x in cfg.DATA.TRAIN))
print_class_histogram(roidbs)
print("Done loading roidbs")
# Filter out images that have no gt boxes, but this filter shall not be applied for testing.
# The model does support training with empty images, but it is not useful for COCO.
num = len(roidbs)
roidbs = list(filter(lambda img: len(img["boxes"][img["is_crowd"] == 0]) > 0, roidbs))
logger.info(
"Filtered {} images which contain no non-crowd groudtruth boxes. Total #images for training: {}".format(
num - len(roidbs), len(roidbs)
)
)
aspect_grouping = [1]
aspect_ratios = [float(x["height"]) / float(x["width"]) for x in roidbs]
group_ids = _quantize(aspect_ratios, aspect_grouping)
ds = AspectGroupingDataFlow(roidbs, group_ids, batch_size=batch_size, drop_uneven=True)
preprocess = TrainingDataPreprocessor()
buffer_size = cfg.DATA.NUM_WORKERS * 10
# ds = MultiProcessMapData(ds, cfg.DATA.NUM_WORKERS, preprocess, buffer_size=buffer_size)
ds = MultiThreadMapData(ds, cfg.DATA.NUM_WORKERS, preprocess, buffer_size=buffer_size)
ds.reset_state()
# to get an infinite data flow
ds = RepeatedData(ds, num=-1)
dataiter = ds.__iter__()
return dataiter
def get_imagenet_dataflow(datadir, name, batch_size, parallel=None):
"""
Get a standard imagenet training/evaluation dataflow, for linear classifier tuning.
"""
assert name in ['train', 'val']
isTrain = name == 'train'
assert datadir is not None
augmentors = get_basic_augmentor(isTrain)
augmentors = imgaug.AugmentorList(augmentors)
if parallel is None:
parallel = min(50, mp.cpu_count())
def mapper(dp):
fname, label = dp
img = cv2.imread(fname)
img = augmentors.augment(img)
return img, label
if isTrain:
ds = dataset.ILSVRC12Files(datadir, name, shuffle=True)
ds = MultiProcessMapAndBatchDataZMQ(ds,
parallel,
mapper,
batch_size,
buffer_size=7000)
else:
ds = dataset.ILSVRC12Files(datadir, name, shuffle=False)
ds = MultiThreadMapData(ds,
parallel,
mapper,
buffer_size=2000,
strict=True)
ds = BatchData(ds, batch_size, remainder=True)
ds = MultiProcessRunnerZMQ(ds, 1)
return ds
def get_batch_train_dataflow(roidbs, batch_size):
"""
Tensorpack batch text dataflow.
"""
batched_roidbs = []
batch = []
for i, d in enumerate(roidbs):
if i % batch_size == 0:
if len(batch) == batch_size:
batched_roidbs.append(batch)
batch = []
batch.append(d)
def preprocess(roidb_batch):
"""
Tensorpack batch text data preprocess function.
"""
datapoint_list = []
for roidb in roidb_batch:
filename, label, mask, bbox, polygon = roidb['filename'], roidb[
'label'], roidb['mask'], roidb['bbox'], roidb['polygon']
img = cv2.imread(filename)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
image = affine_transform(img, polygon)
# img = img[bbox[0]:bbox[2], bbox[1]:bbox[3], :] if image.shape[0]<cfg.stride/2 or image.shape[1]<cfg.stride/2 else image
img = img if image.shape[0] < cfg.stride / 2 or image.shape[
1] < cfg.stride / 2 else image
largest_side = np.random.randint(cfg.crop_min_size, cfg.image_size)
img = aspect_preserving_resize(img, largest_side)
img, crop_bbox = padding_image(img, cfg.image_size)
normalized_bbox = [coord / cfg.image_size for coord in crop_bbox]
img = img.astype("float32") / 255.
ret = {
"image": img,
"label": label,
"mask": mask,
"normalized_bbox": normalized_bbox
}
datapoint_list.append(ret)
batched_datapoint = {"is_training": True, "dropout_keep_prob": 0.5}
for stackable_field in ["image", "label", "mask", "normalized_bbox"]:
batched_datapoint[stackable_field] = np.stack(
[d[stackable_field] for d in datapoint_list])
return batched_datapoint
ds = DataFromList(batched_roidbs, shuffle=True)
ds = MultiThreadMapData(ds, cfg.num_threads, preprocess)
# ds = PrefetchData(ds, 100, multiprocessing.cpu_count() // 4)
return ds
def get_test_dataflow(datadir, args):
parallel = min(args.process_nums, multiprocessing.cpu_count())
ds = ILSVRC12Files(datadir, shuffle=False)
def mapf(dp):
fname, cls = dp
im_0 = ReadImageToCVMat(fname, args)
im_0 = transform(im_0, args)
return np.array(im_0), np.array(cls)
ds = MultiThreadMapData(ds,
parallel,
mapf,
buffer_size=min(2000, ds.size()),
strict=True)
# ds = BatchData(ds, batch_size, remainder=True)
# do not fork() under MPI
return ds
def get_train_dataflow():
"""
Return a training dataflow. Each datapoint consists of the following:
An image: (h, w, 3),
1 or more pairs of (anchor_labels, anchor_boxes):
anchor_labels: (h', w', NA)
anchor_boxes: (h', w', NA, 4)
gt_boxes: (N, 4)
gt_labels: (N,)
If MODE_MASK, gt_masks: (N, h, w)
"""
roidbs = list(
itertools.chain.from_iterable(
DatasetRegistry.get(x).training_roidbs() for x in cfg.DATA.TRAIN))
print(
"---------------------------------------------------------------- data.py:343"
)
print_class_histogram(roidbs)
# Filter out images that have no gt boxes, but this filter shall not be applied for testing.
# The model does support training with empty images, but it is not useful for COCO.
num = len(roidbs)
roidbs = list(
filter(lambda img: len(img["boxes"][img["is_crowd"] == 0]) > 0,
roidbs))
logger.info(
"Filtered {} images which contain no non-crowd groudtruth boxes. Total #images for training: {}"
.format(num - len(roidbs), len(roidbs)))
ds = DataFromList(roidbs, shuffle=True)
preprocess = TrainingDataPreprocessor(cfg)
if cfg.DATA.NUM_WORKERS > 0:
if cfg.TRAINER == "horovod":
# one dataflow for each process, therefore don't need large buffer
buffer_size = cfg.DATA.NUM_WORKERS * 10
ds = MultiThreadMapData(ds,
cfg.DATA.NUM_WORKERS,
preprocess,
buffer_size=buffer_size)
# MPI does not like fork()
else:
buffer_size = cfg.DATA.NUM_WORKERS * 20
ds = MultiProcessMapData(ds,
cfg.DATA.NUM_WORKERS,
preprocess,
buffer_size=buffer_size)
else:
ds = MapData(ds, preprocess)
return ds
def get_imagenet_dataflow(datadir,
name,
batch_size,
augmentors,
parallel=None):
"""
See explanations in the tutorial:
http://tensorpack.readthedocs.io/en/latest/tutorial/efficient-dataflow.html
"""
assert name in ['train', 'val', 'test']
assert datadir is not None
assert isinstance(augmentors, list)
isTrain = name == 'train'
if parallel is None:
parallel = min(40, multiprocessing.cpu_count())
if isTrain:
ds = dataset.ILSVRC12(datadir, name, shuffle=True)
ds = AugmentImageComponent(ds, augmentors, copy=False)
if parallel < 16:
logger.warn(
"DataFlow may become the bottleneck when too few processes are used."
)
ds = PrefetchDataZMQ(ds, parallel)
ds = BatchData(ds, batch_size, remainder=False)
else:
ds = dataset.ILSVRC12Files(datadir, name, shuffle=False)
aug = imgaug.AugmentorList(augmentors)
def mapf(dp):
fname, cls = dp
jpeg_filename = os.path.basename(fname)
jpeg_dirname = os.path.basename(os.path.dirname(fname))
zip_filepath = os.path.dirname(fname) + '.zip'
f = zipfile.ZipFile(zip_filepath, 'r')
compress_jpeg = np.fromstring(f.read(
os.path.join(jpeg_dirname, jpeg_filename)),
dtype=np.uint8)
im = cv2.imdecode(compress_jpeg, cv2.IMREAD_COLOR)
#im = cv2.imread(fname, cv2.IMREAD_COLOR)
im = aug.augment(im)
return im, cls
ds = MultiThreadMapData(ds,
parallel,
mapf,
buffer_size=2000,
strict=True)
ds = BatchData(ds, batch_size, remainder=True)
ds = PrefetchDataZMQ(ds, 1)
return ds
def get_dataflow(path, is_train):
ds = SynthHands(path, is_train) # read data from lmdb
if is_train:
ds = MapData(ds, read_image_url)
ds = MapData(ds, pose_to_img)
ds = PrefetchData(ds, 1000, multiprocessing.cpu_count() * 1)
else:
ds = MultiThreadMapData(ds, num_thread=16, map_func=read_image_url, buffer_size=1000)
ds = MapData(ds, pose_to_img)
ds = PrefetchData(ds, 100, multiprocessing.cpu_count() // 4)
return ds
def build_iter(self,samples):
map_func=partial(self._map_func,is_training=self.training_flag)
ds = DataFromList(samples, shuffle=True)
ds = MultiThreadMapData(ds, self.thread_num, map_func, buffer_size=self.buffer_size)
ds = BatchData(ds, self.num_gpu * self.batch_size)
ds = MultiProcessPrefetchData(ds, self.prefetch_size, self.process_num)
ds.reset_state()
ds = ds.get_data()
return ds
def get_imagenet_dataflow(datadir,
name,
batch_size,
augmentors=None,
parallel=None):
"""
Args:
augmentors (list[imgaug.Augmentor]): Defaults to `fbresnet_augmentor(isTrain)`
Returns: A DataFlow which produces BGR images and labels.
See explanations in the tutorial:
http://tensorpack.readthedocs.io/tutorial/efficient-dataflow.html
"""
assert name in ['train', 'val', 'test']
isTrain = name == 'train'
assert datadir is not None
if augmentors is None:
augmentors = fbresnet_augmentor(isTrain)
assert isinstance(augmentors, list)
if parallel is None:
parallel = min(40,
multiprocessing.cpu_count() //
2) # assuming hyperthreading
if isTrain:
ds = dataset.ILSVRC12(datadir, name, shuffle=True)
ds = AugmentImageComponent(ds, augmentors, copy=False)
if parallel < 16:
logger.warn(
"DataFlow may become the bottleneck when too few processes are used."
)
ds = MultiProcessRunnerZMQ(ds, parallel)
ds = BatchData(ds, batch_size, remainder=False)
else:
ds = dataset.ILSVRC12Files(datadir, name, shuffle=False)
aug = imgaug.AugmentorList(augmentors)
def mapf(dp):
fname, cls = dp
im = cv2.imread(fname, cv2.IMREAD_COLOR)
im = aug.augment(im)
return im, cls
ds = MultiThreadMapData(ds,
parallel,
mapf,
buffer_size=2000,
strict=True)
ds = BatchData(ds, batch_size, remainder=True)
ds = MultiProcessRunnerZMQ(ds, 1)
return ds
def get_val_dataflow(datadir,
batch_size,
augmentors=None,
parallel=None,
num_splits=None,
split_index=None,
dataname="val"):
if augmentors is None:
augmentors = fbresnet_augmentor(False)
assert datadir is not None
assert isinstance(augmentors, list)
if parallel is None:
parallel = min(40, multiprocessing.cpu_count())
if num_splits is None:
ds = dataset.ILSVRC12Files(datadir, dataname, shuffle=True)
else:
# shard validation data
assert False
assert split_index < num_splits
files = dataset.ILSVRC12Files(datadir, dataname, shuffle=True)
files.reset_state()
files = list(files.get_data())
logger.info("Number of validation data = {}".format(len(files)))
split_size = len(files) // num_splits
start, end = split_size * split_index, split_size * (split_index + 1)
end = min(end, len(files))
logger.info("Local validation split = {} - {}".format(start, end))
files = files[start:end]
ds = DataFromList(files, shuffle=True)
aug = imgaug.AugmentorList(augmentors)
def mapf(dp):
fname, cls = dp
im = cv2.imread(fname, cv2.IMREAD_COLOR)
#from BGR to RGB
im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
im = aug.augment(im)
return im, cls
ds = MultiThreadMapData(ds,
parallel,
mapf,
buffer_size=min(2000, ds.size()),
strict=True)
ds = BatchData(ds, batch_size, remainder=False)
ds = RepeatedData(ds, num=-1)
# do not fork() under MPI
return ds
def get_imagenet_dataflow(datadir,
name,
batch_size,
augmentors,
parallel=None):
"""
See explanations in the tutorial:
http://tensorpack.readthedocs.io/en/latest/tutorial/efficient-dataflow.html
"""
assert name in ['train', 'val', 'test']
assert datadir is not None
assert isinstance(augmentors, list)
isTrain = name == 'train'
if parallel is None:
parallel = min(40, multiprocessing.cpu_count() // 6)
if isTrain:
ds = dataset.ILSVRC12(datadir, name, shuffle=True)
ds = AugmentImageComponent(ds, augmentors, copy=False)
if parallel < 16:
logger.warning(
"DataFlow may become the bottleneck when too few processes are used."
)
ds = PrefetchData(ds, 1000, parallel)
ds = BatchData(ds, batch_size, remainder=False)
else:
ds = dataset.ILSVRC12Files(datadir, name, shuffle=False)
aug = imgaug.AugmentorList(augmentors)
def mapf(dp):
fname, cls = dp
im = np.zeros((256, 256, 3), dtype=np.uint8)
for _ in range(30):
try:
im = cv2.imread(fname, cv2.IMREAD_COLOR)
im = aug.augment(im)
break
except Exception as e:
logger.warning(str(e), 'file=', fname)
time.sleep(1)
return im, cls
ds = MultiThreadMapData(ds,
parallel,
mapf,
buffer_size=2000,
strict=True)
ds = BatchData(ds, batch_size, remainder=True)
ds = PrefetchData(ds, 100, 1)
return ds
def get_imagenet_dataflow(datadir,
name,
batch_size,
augmentors,
parallel=None): #获取图像网络数据流
"""
See explanations in the tutorial:
http://tensorpack.readthedocs.io/en/latest/tutorial/efficient-dataflow.html
"""
assert name in ['train', 'val', 'test']
assert datadir is not None
assert isinstance(augmentors, list)
isTrain = name == 'train'
if parallel is None: # 如果不是并行的话
parallel = min(40,
multiprocessing.cpu_count() //
2) # assuming hyperthreading 超线程？ 获取当前计算机cpu数量
if isTrain:
# dataset:创建一个在数据流上运行的预测器，并且拿出一个batch？
ds = dataset.ILSVRC12(datadir, name, shuffle=True)
ds = AugmentImageComponent(ds, augmentors,
copy=False) # 使用共享的增强参数在多个组件上应用图像增强器
if parallel < 16: # 如果少于16个的话
logger.warn(
"DataFlow may become the bottleneck when too few processes are used."
)
ds = PrefetchDataZMQ(ds, parallel) # 实现高效的数据流水线
ds = BatchData(ds, batch_size, remainder=False) # 取一个batch？
else:
# 如果是测试时,增强图像，加速对数据流的读取操作等
# 与ILSVRC12相同，但生成图像的文件名而不是np array。
ds = dataset.ILSVRC12Files(datadir, name, shuffle=False)
aug = imgaug.AugmentorList(augmentors)
def mapf(dp):
fname, cls = dp
im = cv2.imread(fname, cv2.IMREAD_COLOR
) # cv2.IMREAD_COLOR : 默认使用该种标识。加载一张彩色图片,忽视它的透明度
im = aug.augment(im) # 增强图像
return im, cls
ds = MultiThreadMapData(ds,
parallel,
mapf,
buffer_size=2000,
strict=True) # 并行加速？
ds = BatchData(ds, batch_size, remainder=True) # 取一个batch?
ds = PrefetchDataZMQ(ds, 1)
return ds
def process_s2b_data(df, batch_size, num_threads):
"""
Perform preprocessing for the avatar synth data.
:param df: An AvatarSynthDataFlow.
:param batch_size: The minibatch size.
:param num_threads: The number of threads to read and process data.
:return: A dataflow with extra processing steps applied.
"""
augmentor = imgaug.AugmentorList([imgaug.MinMaxNormalize(min=-1, max=1)])
def get_imgs(dp):
"""
:param dp: A datapoint tuple, (path_to_face.jpg, path_to_bitmoji.jpg)
"""
face_img = augmentor.augment(imread(dp[0]))
bitmoji_img = augmentor.augment(imread(dp[1]))
if len(face_img.shape) == 2:
face_img = np.stack([face_img] * 3, axis=-1)
if len(bitmoji_img.shape) == 2:
bitmoji_img = np.stack([bitmoji_img] * 3, axis=-1)
return [face_img, bitmoji_img]
df = MultiThreadMapData(df,
nr_thread=num_threads,
map_func=get_imgs,
buffer_size=min(df.size(), 200))
df = PrefetchDataZMQ(df, nr_proc=num_threads)
# TODO: switch back to remainder=True when s2b input batch size switched back to None
df = BatchData(df, batch_size, remainder=False)
# df = BatchData(df, batch_size, remainder=True)
return df
def get_train_dataflow(roidb):
"""
Tensorpack text dataflow.
"""
ds = DataFromList(roidb, shuffle=True)
preprocess = TextDataPreprocessor(cfg)
buffer_size = cfg.num_threads * 10
ds = MultiThreadMapData(ds, cfg.num_threads, preprocess, buffer_size=buffer_size)
# ds = MultiProcessMapData(ds, cfg.num_workers, preprocess, buffer_size=buffer_size)
ds = PrefetchData(ds, 100, multiprocessing.cpu_count() // 4)
#ds = BatchData(ds, cfg.batch_size, remainder=True)
return ds
def get_imagenet_dataflow(datadir,
is_train,
batch_size,
augmentors,
parallel=None):
"""
See explanations in the tutorial:
http://tensorpack.readthedocs.io/en/latest/tutorial/efficient-dataflow.html
"""
assert datadir is not None
assert isinstance(augmentors, list)
if parallel is None:
parallel = min(40,
multiprocessing.cpu_count() //
2) # assuming hyperthreading
if is_train:
ds = dataset.ILSVRC12(datadir, "train", shuffle=True)
ds = AugmentImageComponent(ds, augmentors, copy=False)
if parallel < 16:
logging.warning(
"DataFlow may become the bottleneck when too few processes are used."
)
ds = PrefetchDataZMQ(ds, parallel)
ds = BatchData(ds, batch_size, remainder=False)
else:
ds = dataset.ILSVRC12Files(datadir, "val", shuffle=False)
aug = imgaug.AugmentorList(augmentors)
def mapf(dp):
fname, cls = dp
im = cv2.imread(fname, cv2.IMREAD_COLOR)
im = np.flip(im, axis=2)
# print("fname={}".format(fname))
im = aug.augment(im)
return im, cls
ds = MultiThreadMapData(ds,
parallel,
mapf,
buffer_size=2000,
strict=True)
# ds = MapData(ds, mapf)
ds = BatchData(ds, batch_size, remainder=True)
ds = PrefetchDataZMQ(ds, 1)
# ds = PrefetchData(ds, 1)
return ds
def get_data(self, name, num_gpu):
gpu_batch = self.batch_size // num_gpu
assert name in ['train', 'val', 'test']
isTrain = name == 'train'
augmentors = fbresnet_augmentor(isTrain)
assert isinstance(augmentors, list)
parallel = min(40,
multiprocessing.cpu_count() //
2) # assuming hyperthreading
if isTrain:
ds = dataset.ILSVRC12(self.datadir,
name,
shuffle=True,
dir_structure='train')
ds = AugmentImageComponent(ds, augmentors, copy=False)
ds = MultiProcessRunnerZMQ(ds, parallel)
ds = BatchData(ds, gpu_batch, remainder=False)
#ds = QueueInput(ds)
else:
ds = dataset.ILSVRC12Files(self.datadir,
name,
shuffle=False,
dir_structure='train')
aug = imgaug.AugmentorList(augmentors)
def mapf(dp):
fname, cls = dp
im = cv2.imread(fname, cv2.IMREAD_COLOR)
im = aug.augment(im)
return im, cls
ds = MultiThreadMapData(ds,
parallel,
mapf,
buffer_size=2000,
strict=True)
ds = BatchData(ds, gpu_batch, remainder=True)
ds = MultiProcessRunnerZMQ(ds, 1)
if num_gpu == 1:
ds = QueueInput(ds)
return ds
def get_random_loader(ds, isTrain, batch_size, augmentors, parallel=None):
""" DataFlow data (Random Read)
Args:
augmentors (list[imgaug.Augmentor]): Defaults to `fbresnet_augmentor(isTrain)`
Returns: A DataFlow which produces BGR images and labels.
See explanations in the tutorial:
http://tensorpack.readthedocs.io/tutorial/efficient-dataflow.html
"""
assert isinstance(augmentors, list)
aug = imgaug.AugmentorList(augmentors)
if parallel is None:
parallel = min(40,
multiprocessing.cpu_count() //
2) # assuming hyperthreading
if isTrain:
ds = AugmentImageComponent(ds, aug, copy=False)
if parallel < 16:
logger.warn(
"DataFlow may become the bottleneck when too few processes are used."
)
ds = MultiProcessRunnerZMQ(ds, parallel)
ds = BatchData(ds, batch_size, remainder=False)
else:
def mapf(dp):
fname, cls = dp
im = cv2.imread(fname, cv2.IMREAD_COLOR)
im = aug.augment(im)
return im, cls
ds = MultiThreadMapData(ds,
parallel,
mapf,
buffer_size=2000,
strict=True)
ds = BatchData(ds, batch_size, remainder=True)
ds = MultiProcessRunnerZMQ(ds, 1)
return ds
def dump_imdb(ds, output_path, parallel=None):
"""
Create a Single-File LMDB from raw images.
"""
if parallel is None:
parallel = min(40,
multiprocessing.cpu_count() //
2) # assuming hyperthreading
def mapf(dp):
fname, label = dp
with open(fname, 'rb') as f:
bytes = f.read()
bytes = np.asarray(bytearray(bytes), dtype='uint8')
return bytes, label
ds = MultiThreadMapData(ds, 1, mapf, buffer_size=2000, strict=True)
ds = MultiProcessRunnerZMQ(ds, num_proc=parallel)
LMDBSerializer.save(ds, output_path)
def get_dataflow(path, is_train=True, img_path=None,sigma=8.0,output_shape=(1440,2560),
numparts=5,translation=False,scale=False,rotation=True,
mins=0.25,maxs=1.2,mina=-np.pi,maxa=np.pi ,ilumination=0.0,image_type='RGB'):
print('Creating images from',path)
numparts, skeleton = get_skeleton_from_json(path)
#numparts + 1 because need to the background
ds = CocoPose(path, img_path, is_train, numparts=numparts + 1, sigma=sigma,skeleton=skeleton,
output_shape=output_shape, translation=translation,scale=scale,rotation=rotation,
mins=mins,maxs=maxs,mina=mina,maxa=maxa, ilumination=ilumination,image_type=image_type
) # read data from lmdb
if is_train:
#ds = MapData(ds, read_image_url)
ds = MultiThreadMapData(ds, nr_thread=8, map_func=read_image_url, buffer_size=10)
ds = MapDataComponent(ds, get_augmented_image)
#ds = MapDataComponent(ds, pose_rotation)
#ds = MapDataComponent(ds, pose_flip)
#ds = MapDataComponent(ds, pose_resize_shortestedge_random)
#ds = MapDataComponent(ds, pose_crop_random)
#logger.info('Out of new augmenter')
ds = MapData(ds, pose_to_img)
#logger.info('Out pose to img')
# augs = [
# imgaug.RandomApplyAug(imgaug.RandomChooseAug([
# imgaug.GaussianBlur(max_size=3)
# ]), 0.7)
# ]
# ds = AugmentImageComponent(ds, augs)
ds = PrefetchData(ds, 10, multiprocessing.cpu_count() * 1)
else:
#ds = MultiThreadMapData(ds, nr_thread=4, map_func=read_image_url, buffer_size=10)
ds = MapData(ds, read_image_url)
#ds = MapDataComponent(ds, pose_resize_shortestedge_fixed)
#ds = MapDataComponent(ds, pose_crop_center)
ds = MapData(ds, pose_to_img)
ds = PrefetchData(ds, 10, multiprocessing.cpu_count() // 4)
return ds
def process_avatar_synth_data(df, batch_size, num_threads):
"""
Perform preprocessing for the avatar synth data.
:param df: An AvatarSynthDataFlow.
:param batch_size: The minibatch size.
:param num_threads: The number of threads to read and process data.
:return: A dataflow with extra processing steps applied.
"""
augmentor = imgaug.AugmentorList([imgaug.MinMaxNormalize(min=-1, max=1)])
df = MultiThreadMapData(
df,
nr_thread=num_threads,
map_func=lambda dp: [np.load(dp[0]),
augmentor.augment(imread(dp[1]))])
# df = MapData(df, lambda dp: [np.load(dp[0]), augmentor.augment(imread(dp[1]))])
df = PrefetchDataZMQ(df, nr_proc=num_threads)
df = BatchData(df, batch_size, remainder=True)
return df
def get_train_dataflow():
"""
Return a training dataflow. Each datapoint consists of the following:
An image: (h, w, 3),
1 or more pairs of (anchor_labels, anchor_boxes):
anchor_labels: (h', w', NA)
anchor_boxes: (h', w', NA, 4)
gt_boxes: (N, 4)
gt_labels: (N,)
If MODE_MASK, gt_masks: (N, h, w)
"""
imgs = COCODetection.load_many(cfg.DATA.BASEDIR,
cfg.DATA.TRAIN,
add_gt=True,
add_mask=cfg.MODE_MASK)
"""
To train on your own data, change this to your loader.
Produce "imgs" as a list of dict, in the dict the following keys are needed for training:
height, width: integer
file_name: str, full path to the image
boxes: numpy array of kx4 floats
class: numpy array of k integers
is_crowd: k booleans. Use k False if you don't know what it means.
segmentation: k lists of numpy arrays (one for each box).
Each list of numpy array corresponds to the mask for one instance.
Each numpy array in the list is a polygon of shape Nx2,
because one mask can be represented by N polygons.
If your segmentation annotations are originally masks rather than polygons,
either convert it, or the augmentation code below will need to be
changed or skipped accordingly.
"""
# Valid training images should have at least one fg box.
# But this filter shall not be applied for testing.
num = len(imgs)
imgs = list(
filter(lambda img: len(img['boxes'][img['is_crowd'] == 0]) > 0, imgs))
logger.info(
"Filtered {} images which contain no non-crowd groudtruth boxes. Total #images for training: {}"
.format(num - len(imgs), len(imgs)))
ds = DataFromList(imgs, shuffle=True)
aug = imgaug.AugmentorList([
CustomResize(cfg.PREPROC.SHORT_EDGE_SIZE, cfg.PREPROC.MAX_SIZE),
imgaug.Flip(horiz=True)
])
def preprocess(img):
fname, boxes, klass, is_crowd = img['file_name'], img['boxes'], img[
'class'], img['is_crowd']
boxes = np.copy(boxes)
im = cv2.imread(fname, cv2.IMREAD_COLOR)
assert im is not None, fname
im = im.astype('float32')
# assume floatbox as input
assert boxes.dtype == np.float32, "Loader has to return floating point boxes!"
# augmentation:
im, params = aug.augment_return_params(im)
points = box_to_point8(boxes)
points = aug.augment_coords(points, params)
boxes = point8_to_box(points)
assert np.min(np_area(boxes)) > 0, "Some boxes have zero area!"
# rpn anchor:
try:
if cfg.MODE_FPN:
multilevel_anchor_inputs = get_multilevel_rpn_anchor_input(
im, boxes, is_crowd)
anchor_inputs = itertools.chain.from_iterable(
multilevel_anchor_inputs)
else:
# anchor_labels, anchor_boxes
anchor_inputs = get_rpn_anchor_input(im, boxes, is_crowd)
assert len(anchor_inputs) == 2
boxes = boxes[is_crowd == 0] # skip crowd boxes in training target
klass = klass[is_crowd == 0]
if not len(boxes):
raise MalformedData("No valid gt_boxes!")
except MalformedData as e:
log_once(
"Input {} is filtered for training: {}".format(fname, str(e)),
'warn')
return None
ret = [im] + list(anchor_inputs) + [boxes, klass]
if cfg.MODE_MASK:
# augmentation will modify the polys in-place
segmentation = copy.deepcopy(img['segmentation'])
segmentation = [
segmentation[k] for k in range(len(segmentation))
if not is_crowd[k]
]
assert len(segmentation) == len(boxes)
# Apply augmentation on polygon coordinates.
# And produce one image-sized binary mask per box.
masks = []
for polys in segmentation:
polys = [aug.augment_coords(p, params) for p in polys]
masks.append(
segmentation_to_mask(polys, im.shape[0], im.shape[1]))
masks = np.asarray(masks, dtype='uint8') # values in {0, 1}
ret.append(masks)
# from viz import draw_annotation, draw_mask
# viz = draw_annotation(im, boxes, klass)
# for mask in masks:
# viz = draw_mask(viz, mask)
# tpviz.interactive_imshow(viz)
return ret
if cfg.TRAINER == 'horovod':
ds = MultiThreadMapData(ds, 5, preprocess)
# MPI does not like fork()
else:
ds = MultiProcessMapDataZMQ(ds, 10, preprocess)
return ds
def get_train_dataflow_YCBV():
"""
Return a training dataflow. Each datapoint consists of the following:
An image: (h, w, 3),
1 or more pairs of (anchor_labels, anchor_boxes):
anchor_labels: (h', w', NA)
anchor_boxes: (h', w', NA, 4)
gt_boxes: (N, 4)
gt_labels: (N,)
If MODE_MASK, gt_masks: (N, h, w)
"""
img_ids = YCBVDetectionDataset().load_training_image_ids(cfg.DATA.TRAIN)
# print_class_histogram(roidbs)
# Valid training images should have at least one fg box.
# But this filter shall not be applied for testing.
# num = len(img_ids)
# roidbs = list(filter(lambda img: len(img['boxes'][img['is_crowd'] == 0]) > 0, roidbs))
# logger.info("Filtered {} images which contain no non-crowd groudtruth boxes. Total #images for training: {}".format(
# num - len(roidbs), len(roidbs)))
ds = DataFromList(img_ids, shuffle=True)
# aug = imgaug.AugmentorList(
# [CustomResize(cfg.PREPROC.TRAIN_SHORT_EDGE_SIZE, cfg.PREPROC.MAX_SIZE)])
def preprocess(image_id):
roidb = YCBVDetectionDataset().load_single_roidb(image_id)
fname, boxes, klass, is_crowd = roidb['file_name'], roidb[
'boxes'], roidb['class'], roidb['is_crowd']
boxes = np.copy(boxes)
im = cv2.imread(fname, cv2.IMREAD_COLOR)
assert im is not None, fname
im = im.astype('float32')
height, width = im.shape[:2]
# assume floatbox as input
assert boxes.dtype == np.float32, "Loader has to return floating point boxes!"
if not cfg.DATA.ABSOLUTE_COORD:
boxes[:, 0::2] *= width
boxes[:, 1::2] *= height
# augmentation:
# im, params = aug.augment_return_params(im)
points = box_to_point8(boxes)
# points = aug.augment_coords(points, params)
boxes = point8_to_box(points)
assert np.min(np_area(boxes)) > 0, "Some boxes have zero area!"
ret = {'image': im}
# rpn anchor:
try:
if cfg.MODE_FPN:
multilevel_anchor_inputs = get_multilevel_rpn_anchor_input(
im, boxes, is_crowd)
for i, (anchor_labels,
anchor_boxes) in enumerate(multilevel_anchor_inputs):
ret['anchor_labels_lvl{}'.format(i + 2)] = anchor_labels
ret['anchor_boxes_lvl{}'.format(i + 2)] = anchor_boxes
else:
# anchor_labels, anchor_boxes
ret['anchor_labels'], ret[
'anchor_boxes'] = get_rpn_anchor_input(
im, boxes, is_crowd)
boxes = boxes[is_crowd == 0] # skip crowd boxes in training target
klass = klass[is_crowd == 0]
ret['gt_boxes'] = boxes
ret['gt_labels'] = klass
if not len(boxes):
raise MalformedData("No valid gt_boxes!")
except MalformedData as e:
log_once(
"Input {} is filtered for training: {}".format(fname, str(e)),
'warn')
return None
if cfg.MODE_MASK:
# augmentation will modify the polys in-place
segmentation = copy.deepcopy(roidb['segmentation'])
segmentation = [
segmentation[k] for k in range(len(segmentation))
if not is_crowd[k]
]
assert len(segmentation) == len(boxes)
# Apply augmentation on polygon coordinates.
# And produce one image-sized binary mask per box.
ret['gt_masks'] = segmentation
# from viz import draw_annotation, draw_mask
# viz = draw_annotation(im, boxes, klass)
# for mask in masks:
# viz = draw_mask(viz, mask)
# tpviz.interactive_imshow(viz)
return ret
if cfg.TRAINER == 'horovod':
ds = MultiThreadMapData(ds, 5, preprocess)
# MPI does not like fork()
else:
ds = MultiProcessMapDataZMQ(ds, 10, preprocess)
return ds
