def get_data(name, data_dir, meta_dir, gpu_nums):
isTrain = name == 'train'
ds = PascalVOC12(data_dir, meta_dir, name, shuffle=True)
if isTrain:#special augmentation
shape_aug = [RandomResize(xrange=(0.7, 1.5), yrange=(0.7, 1.5),
aspect_ratio_thres=0.15),
RandomCropWithPadding(args.crop_size,IGNORE_LABEL),
Flip(horiz=True),
]
else:
shape_aug = []
ds = AugmentImageComponents(ds, shape_aug, (0, 1), copy=False)
def f(ds):
image, label = ds
m = np.array([104, 116, 122])
const_arr = np.resize(m, (1,1,3)) # NCHW
image = image - const_arr
return image, label
ds = MapData(ds, f)
if isTrain:
ds = BatchData(ds, args.batch_size*gpu_nums)
ds = PrefetchDataZMQ(ds, 1)
else:
ds = BatchData(ds, 1)
return ds
def get_data(name, meta_dir, gpu_nums):
isTrain = name == 'train'
ds = Cityscapes(meta_dir, name, shuffle=True)
if isTrain:
ds = MapData(ds, RandomResize)
if isTrain:
shape_aug = [
RandomCropWithPadding(args.crop_size, IGNORE_LABEL),
Flip(horiz=True),
]
else:
shape_aug = []
ds = AugmentImageComponents(ds, shape_aug, (0, 1), copy=False)
def f(ds):
image, label = ds
m = np.array([104, 116, 122])
const_arr = np.resize(m, (1, 1, 3)) # NCHW
image = image - const_arr
return image, label
ds = MapData(ds, f)
if isTrain:
ds = BatchData(ds, args.batch_size * gpu_nums)
#ds = PrefetchDataZMQ(ds, 1)
else:
ds = BatchData(ds, 1)
return ds
def get_data(name, data_dir, meta_dir, gpu_nums):
isTrain = True if 'train' in name else False
def imgread(ds):
img, label = ds
img = cv2.imread(img, cv2.IMREAD_COLOR)
label = cv2.imread(label, cv2.IMREAD_GRAYSCALE)
return img, label
if isTrain:
#ds = LMDBData('/data2/dataset/cityscapes/cityscapes_train.lmdb', shuffle=True)
#ds = FakeData([[batch_size, CROP_HEIGHT, CROP_HEIGHT, 3], [batch_size, CROP_HEIGHT, CROP_HEIGHT, 1]], 5000, random=False, dtype='uint8')
ds = PascalVOC12Files(data_dir, meta_dir, name, shuffle=True)
ds = MultiThreadMapData(ds,4,imgread, buffer_size= 2)
#ds = PrefetchDataZMQ(MapData(ds, ImageDecode), 1) #imagedecode is heavy
ds = MapData(ds, RandomResize)
else:
ds = PascalVOC12Files(data_dir, meta_dir, name, shuffle=False)
ds = MultiThreadMapData(ds, 4, imgread, buffer_size= 2)
if isTrain:
shape_aug = [
RandomCropWithPadding(args.crop_size,IGNORE_LABEL),
Flip(horiz=True),
]
ds = AugmentImageComponents(ds, shape_aug, (0, 1), copy=False)
def reduce_mean_rgb(ds):
image, label = ds
m = np.array([104, 116, 122])
const_arr = np.resize(m, (1,1,3)) # NCHW
image = image - const_arr
return image, label
def MxnetPrepare(ds):
data, label = ds
data = np.transpose(data, (0, 3, 1, 2)) # NCHW
label = label[:, :, :, None]
label = np.transpose(label, (0, 3, 1, 2)) # NCHW
dl = [[mx.nd.array(data[args.batch_size * i:args.batch_size * (i + 1)])] for i in
range(gpu_nums)] # multi-gpu distribute data, time-consuming!!!
ll = [[mx.nd.array(label[args.batch_size * i:args.batch_size * (i + 1)])] for i in
range(gpu_nums)]
return dl, ll
#ds = MapData(ds, reduce_mean_rgb)
ds = MultiThreadMapData(ds, 4, reduce_mean_rgb, buffer_size=2)
if isTrain:
ds = FastBatchData(ds, args.batch_size*gpu_nums)
ds = MapData(ds, MxnetPrepare)
#ds = PrefetchDataZMQ(ds, 1)
else:
ds = BatchData(ds, 1)
return ds
def get_data(name, meta_dir, gpu_nums):
isTrain = True if 'train' in name else False
m = np.array([104, 116, 122])
const_arr = np.resize(m, (1, 1, 3)) # NCHW
const_arr = np.zeros(
(args.crop_size[0], args.crop_size[1], 3)) + const_arr #broadcast
if isTrain:
#ds = FakeData([[1024, 2048, 3], [ 1024, 2048]], 5000, random=False, dtype='uint8')
#ds = FakeData([[CROP_HEIGHT, CROP_HEIGHT, 3], [CROP_HEIGHT, CROP_HEIGHT]], 5000,random=False, dtype='uint8')
ds = CityscapesFiles(base_dir, meta_dir, name, shuffle=True)
parallel = min(3, multiprocessing.cpu_count())
augmentors = [
RandomCropWithPadding(args.crop_size),
Flip(horiz=True),
]
aug = imgaug.AugmentorList(augmentors)
def mapf(ds):
img, label = ds
img = cv2.imread(img, cv2.IMREAD_COLOR)
label = cv2.imread(label, cv2.IMREAD_GRAYSCALE)
img, params = aug.augment_return_params(img)
label = aug._augment(label, params)
img = img - const_arr # very time-consuming
return img, label
#ds = MapData(ds, mapf)
ds = MultiThreadMapData(ds,
parallel,
mapf,
buffer_size=500,
strict=True)
#ds = MapData(ds, reduce_mean_rgb)
ds = BatchData(ds, args.batch_size * gpu_nums)
#ds = PrefetchDataZMQ(ds, 1)
else:
def imgread(ds):
img, label = ds
img = cv2.imread(img, cv2.IMREAD_COLOR)
label = cv2.imread(label, cv2.IMREAD_GRAYSCALE)
return [img, label]
ds = CityscapesFiles(base_dir, meta_dir, name, shuffle=False)
ds = MapData(ds, imgread)
ds = BatchData(ds, 1)
return ds
def get_data(name, base_dir, meta_dir, batch_size):
isTrain = True if 'train' in name else False
ds = dataset.PSSD(base_dir, meta_dir, name, shuffle=True)
if isTrain: #special augmentation
shape_aug = [
imgaug.RandomResize(xrange=(0.7, 1.5),
yrange=(0.7, 1.5),
aspect_ratio_thres=0.15),
RandomCropWithPadding(CROP_SIZE, IGNORE_LABEL),
imgaug.Flip(horiz=True),
]
else:
shape_aug = []
ds = AugmentImageComponents(ds, shape_aug, (0, 1), copy=False)
if isTrain:
ds = BatchData(ds, batch_size)
ds = PrefetchDataZMQ(ds, 1)
else:
ds = BatchData(ds, 1)
return ds
