def __init__(self, params, num_threads, device_id):
super(DaliPipeline, self).__init__(params.batch_size,
num_threads,
device_id,
seed=12)
dii = DaliInputIterator(params, device_id)
self.no_copy = params.no_copy
if self.no_copy:
print("Use Zero Copy ES")
self.source = ops.ExternalSource(device="gpu",
source=dii,
num_outputs=2,
layout=["DHWC", "DHWC"],
no_copy=self.no_copy)
self.do_rotate = True if params.rotate_input == 1 else False
print("Enable Rotation" if self.do_rotate else "Disable Rotation")
self.rng_angle = ops.Uniform(device="cpu", range=[-1.5, 2.5])
self.icast = ops.Cast(device="cpu", dtype=types.INT32)
self.fcast = ops.Cast(device="cpu", dtype=types.FLOAT)
self.rotate1 = ops.Rotate(device="gpu",
axis=(1, 0, 0),
interp_type=types.INTERP_LINEAR)
self.rotate2 = ops.Rotate(device="gpu",
axis=(0, 1, 0),
interp_type=types.INTERP_LINEAR)
self.rotate3 = ops.Rotate(device="gpu",
axis=(0, 0, 1),
interp_type=types.INTERP_LINEAR)
self.transpose = ops.Transpose(device="gpu", perm=[3, 0, 1, 2])
def __init__(self,
imageset_dir,
image_size,
random_shuffle,
batch_size=4,
num_threads=2,
device_id=0):
super(ImagePipeline, self).__init__(batch_size,
num_threads,
device_id,
seed=12)
self.imageset_dir = imageset_dir
self.random_shuffle = random_shuffle
eii = ExternalInputIterator(root=self.imageset_dir,
batch_size=self.batch_size,
random_shuffle=self.random_shuffle)
self.iterator = iter(eii)
self.num_inputs = len(self.iterator.files)
self.input_image = ops.ExternalSource()
self.input_mask = ops.ExternalSource()
self.decode_image = ops.ImageDecoder(device="mixed",
output_type=types.RGB)
self.decode_mask = ops.ImageDecoder(device="mixed",
output_type=types.GRAY)
# The rest of pre-processing is done on the GPU
self.res = ops.Resize(device="gpu",
resize_x=image_size,
resize_y=image_size)
self.flip = ops.Flip(device="gpu", horizontal=1, vertical=0)
rotate_degree = random.random() * 2 * 10 - 10
self.rotate_image = ops.Rotate(
device="gpu",
angle=rotate_degree,
interp_type=types.DALIInterpType.INTERP_LINEAR)
self.rotate_mask = ops.Rotate(
device="gpu",
angle=rotate_degree,
interp_type=types.DALIInterpType.INTERP_NN)
self.cmnp_image = ops.CropMirrorNormalize(
device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NCHW,
image_type=types.RGB,
mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
self.cmnp_mask = ops.CropMirrorNormalize(device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NCHW,
image_type=types.GRAY,
mean=[0],
std=[255])
def __init__(self, params, num_threads, device_id):
super(DaliPipeline, self).__init__(params.batch_size,
num_threads,
device_id,
seed=12)
with h5py.File(params.data_path, 'r') as f:
# load hydro and clean up
Hydro = f['Hydro'][...]
self.Hydro = types.Constant(Hydro,
shape=Hydro.shape,
layout="DHWC",
device="cpu")
del Hydro
# load nbody and clean up
Nbody = f['Nbody'][...]
self.Nbody = types.Constant(Nbody,
shape=Nbody.shape,
layout="DHWC",
device="cpu")
del Nbody
#self.ndummy = np.zeros((20, 20, 20, 4), dtype=np.float32)
#self.hdummy = np.zeros((20, 20, 20, 5), dtype=np.float32)
#self.Nbody = types.Constant(self.ndummy, shape = self.ndummy.shape, layout = "DHWC", device="cpu")
#self.Hydro = types.Constant(self.hdummy, shape = self.hdummy.shape, layout = "DHWC", device="cpu")
#self.Nbody = ops.Constant(fdata = self.ndummy.flatten().tolist(), shape = self.ndummy.shape, layout = "DHWC", device = "cpu")
#self.Hydro = ops.Constant(fdata = self.hdummy.flatten().tolist(), shape = self.hdummy.shape, layout = "DHWC", device = "cpu")
self.do_rotate = True if params.rotate_input == 1 else False
print("Enable Rotation" if self.do_rotate else "Disable Rotation")
self.rng_angle = ops.Uniform(device="cpu", range=[-1.5, 2.5])
self.rng_pos = ops.Uniform(device="cpu", range=[0., 1.])
self.icast = ops.Cast(device="cpu", dtype=types.INT32)
self.fcast = ops.Cast(device="cpu", dtype=types.FLOAT)
self.crop = ops.Crop(device="cpu",
crop_d=params.data_size,
crop_h=params.data_size,
crop_w=params.data_size)
self.rotate1 = ops.Rotate(device="gpu",
axis=(1, 0, 0),
interp_type=types.INTERP_LINEAR)
self.rotate2 = ops.Rotate(device="gpu",
axis=(0, 1, 0),
interp_type=types.INTERP_LINEAR)
self.rotate3 = ops.Rotate(device="gpu",
axis=(0, 0, 1),
interp_type=types.INTERP_LINEAR)
self.transpose = ops.Transpose(device="gpu", perm=[3, 0, 1, 2])
def __init__(self,
root,
split,
batch_size,
device_id,
dali_cpu=False,
local_rank=0,
world_size=1,
num_workers=2,
augment=False,
resize=False,
resize_size=314,
crop=True,
crop_size=314,
shuffle=True,
flip=True,
rotate=False,
rotate_angle=10.0):
super(VOCDali, self).__init__(batch_size,
num_threads=num_workers,
device_id=device_id,
seed=12 + device_id)
self.iterator = iter(VOCIter(batch_size, split, root, shuffle))
self.split = split
assert self.split in ['train', 'val']
dali_device = "gpu" if not dali_cpu else "cpu"
self.input = ops.ExternalSource()
self.input_label = ops.ExternalSource()
self.is_flip = flip
self.is_rotate = rotate
self.is_augment = augment
self.is_crop = crop
self.is_resize = resize
self.decode = ops.ImageDecoder(device='mixed', output_type=types.RGB)
# self.cast = ops.Cast(device='gpu', dtype=types.INT32)
self.rng = ops.Uniform(range=(0., 1.))
self.coin = ops.CoinFlip(probability=0.5)
# 定义大小
self.resize = ops.Resize(device="gpu",
resize_x=resize_size,
resize_y=resize_size,
interp_type=types.INTERP_TRIANGULAR)
# 定义旋转
self.rotate = ops.Rotate(device="gpu", angle=rotate_angle)
# 定义翻转
self.flip = ops.Flip(device="gpu", vertical=1, horizontal=0)
# 定义剪裁
self.crop = ops.Crop(device=dali_device,
crop_h=crop_size,
crop_w=crop_size)
# 定义正则化
self.cmnp = ops.CropMirrorNormalize(
device=dali_device,
output_dtype=types.FLOAT,
output_layout=types.NCHW,
image_type=types.RGB,
mean=[0.45734706 * 255, 0.43338275 * 255, 0.40058118 * 255],
std=[0.23965294 * 255, 0.23532275 * 255, 0.2398498 * 255])
def __new__(cls,
fill_value=0,
interp_type='linear',
keep_size=True,
**kwargs):
"""Create a ``Rotate`` operator.
Parameters
----------
fill_value : number, optional
The value to fill the empty regions.
interp_type : str, optional, default='linear'
The interpolation method.
keep_size : bool, optional, default=True
Whether to keep the original image size.
Returns
------
nvidia.dali.ops.Rotate
The operator.
"""
if isinstance(interp_type, six.string_types):
interp_type = getattr(types, 'INTERP_' + interp_type.upper())
return ops.Rotate(fill_value=fill_value,
interp_type=interp_type,
keep_size=keep_size,
device=context.get_device_type(),
**kwargs)
def __init__(self,
device,
batch_size,
output_type,
input_type,
fixed_size=None,
num_threads=3,
device_id=0,
num_gpus=1):
super(RotatePipeline, self).__init__(batch_size,
num_threads,
device_id,
seed=7865,
exec_async=False,
exec_pipelined=False)
self.name = device
self.input = ops.CaffeReader(path=caffe_db_folder,
shard_id=device_id,
num_shards=num_gpus)
self.decode = ops.ImageDecoder(device="cpu", output_type=types.RGB)
if input_type != dali.types.UINT8:
self.cast = ops.Cast(device=device, dtype=input_type)
else:
self.cast = None
# TODO(michalz): When we move from Support to CPU operators, replace hardcoded angle
# with one taken from the distribution below
# self.uniform = ops.Uniform(range = (-180.0, 180.0), seed = 42);
self.rotate = ops.Rotate(device=device,
size=fixed_size,
angle=30,
fill_value=42,
output_dtype=output_type)
def __init__(self,
device,
batch_size,
output_type,
input_type,
fixed_size=None,
num_threads=3,
device_id=0,
num_gpus=1):
super(RotatePipeline, self).__init__(batch_size,
num_threads,
device_id,
seed=7865,
exec_async=False,
exec_pipelined=False)
self.name = device
self.input = ops.readers.Caffe(path=caffe_db_folder,
shard_id=device_id,
num_shards=num_gpus)
self.decode = ops.ImageDecoder(device="cpu", output_type=types.RGB)
if input_type != dali.types.UINT8:
self.cast = ops.Cast(device=device, dtype=input_type)
else:
self.cast = None
self.uniform = ops.random.Uniform(range=(-180.0, 180.0), seed=42)
self.rotate = ops.Rotate(device=device,
size=fixed_size,
fill_value=42,
dtype=output_type)
def __init__(self, batch_size, num_threads, device_id, data_dir, crop):
super(HybridTrainPipe, self).__init__(batch_size, num_threads,
device_id, data_dir, crop)
self.pad = ops.Paste(device="gpu",
fill_value=0,
ratio=1.1,
min_canvas_size=crop)
self.res = ops.RandomResizedCrop(device="gpu",
size=crop,
random_area=[0.9, 1.1],
random_aspect_ratio=1.33333)
self.cutmix = ops.PythonFunction(function=cut_mixe_image,
num_outputs=2,
device='gpu')
self.cmnp = ops.CropMirrorNormalize(
device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NCHW,
image_type=types.RGB,
mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
self.coin = ops.CoinFlip(probability=0.5)
self.rotated = ops.Rotate(device="gpu", keep_size=True)
self.rotated_rng = ops.Uniform(range=(-5.0, 5.0))
self.brightness = ops.Brightness(device="gpu")
self.brightness_rng = ops.Uniform(range=(0.8, 1.2))
self.reshape = ops.Reshape(device="gpu", layout="HWC")
self.one_hot = ops.OneHot(num_classes=3,
dtype=types.INT32,
device="cpu")
self.jitter_rng = ops.CoinFlip(probability=0.3)
self.jittered = ops.Jitter(device="gpu")
def __init__(self, root_dir, batch_size, num_threads, device_id,
use_shift_scale=False,
num_shards=None, shard_id=None):
super().__init__(batch_size, num_threads, device_id, seed=12)
self.random_angle = ops.Uniform(range=(0, 360.0))
self.random = ops.Uniform(range=(0.5, 1.5))
self.random_coin = ops.CoinFlip()
self.input = ops.FileReader(
file_root=root_dir, random_shuffle=True,
num_shards=num_shards, shard_id=shard_id,
)
self.decode = ops.ImageDecoder(device='mixed')
self.rotate = ops.Rotate(device='gpu', interp_type=types.INTERP_LINEAR)
self.crop = ops.Crop(device='gpu', crop=(224, 224))
self.use_shift_scale = use_shift_scale
if self.use_shift_scale:
self.shift_scale = ops.RandomResizedCrop(
device='gpu',
size=(224, 224),
interp_type=types.INTERP_LINEAR,
random_area=(0.3, 1.0),
)
self.flip = ops.Flip(device='gpu')
self.color_twist = ops.ColorTwist(device='gpu')
def __init__(self,
DATA_PATH,
input_height,
batch_size,
copies,
stage,
num_threads,
device_id,
seed=1729):
super(SimCLRTransform, self).__init__(batch_size,
num_threads,
device_id,
seed=seed)
#this lets our pytorch compat function find the length of our dataset
self.num_samples = len(ImageFolder(DATA_PATH))
self.copies = copies
self.input_height = input_height
self.stage = stage
self.input = ops.FileReader(file_root=DATA_PATH,
random_shuffle=True,
seed=seed)
self.to_int64 = ops.Cast(dtype=types.INT64, device="gpu")
self.to_int32_cpu = ops.Cast(dtype=types.INT32, device="cpu")
self.coin = ops.random.CoinFlip(probability=0.5)
self.uniform = ops.random.Uniform(range=[0.6, 0.9])
self.blur_amt = ops.random.Uniform(values=[
float(i) for i in range(1, int(0.1 * self.input_height), 2)
])
self.angles = ops.random.Uniform(range=[0, 360])
self.cast = ops.Cast(dtype=types.FLOAT, device='gpu')
self.decode = ops.ImageDecoder(device='mixed', output_type=types.RGB)
self.crop = ops.RandomResizedCrop(size=self.input_height,
minibatch_size=batch_size,
random_area=[0.75, 1.0],
device="gpu")
self.resize = ops.Resize(resize_x=self.input_height,
resize_y=self.input_height,
device="gpu")
self.flip = ops.Flip(vertical=self.coin(),
horizontal=self.coin(),
device="gpu")
self.colorjit_gray = ops.ColorTwist(brightness=self.uniform(),
contrast=self.uniform(),
hue=self.uniform(),
saturation=self.uniform(),
device="gpu")
self.blur = ops.GaussianBlur(window_size=self.to_int32_cpu(
self.blur_amt()),
device="gpu")
self.rotate = ops.Rotate(
angle=self.angles(),
keep_size=True,
interp_type=types.DALIInterpType.INTERP_LINEAR,
device="gpu")
self.swapaxes = ops.Transpose(perm=[2, 0, 1], device="gpu")
def aug(self, input_img):
rot_angle = uniform(self.rot_rand[0], self.rot_rand[1])
cont_un = uniform(self.cont_rand[0], self.cont_rand[1])
bri_un = uniform(self.bri_rand[0], self.bri_rand[1])
sat_un = uniform(self.sat_rand[0], self.sat_rand[1])
hue_un = uniform(self.hue_rand[0], self.hue_rand[1])
print(rot_angle)
self.augmentations["rotate"] = ops.Rotate(
device="gpu",
angle=rot_angle,
fill_value=255.,
interp_type=types.INTERP_LINEAR)
self.augmentations["contrast"] = ops.Contrast(device="gpu",
contrast=cont_un)
self.augmentations["brightness"] = ops.Brightness(device="gpu",
brightness=bri_un)
self.augmentations["saturation"] = ops.Saturation(device="gpu",
saturation=sat_un)
self.augmentations["hue"] = ops.Hue(device="gpu", hue=hue_un)
now_n = randint(self.aug_num_rad[0], self.aug_num_rad[1])
aug_list = list(self.augmentations.values())
shuffle(aug_list)
aug_list = aug_list[:now_n]
for now_aug in aug_list:
input_img = now_aug(input_img)
return input_img
def __init__(self,
batch_size,
num_threads,
device_id,
data_dir,
crop,
size=256,
dali_cpu=False,
local_rank=0,
world_size=1):
super(HybridTrainPipe, self).__init__(batch_size,
num_threads,
device_id,
seed=12 + device_id)
dali_device = "gpu"
self.input = ops.FileReader(file_root=data_dir,
shard_id=local_rank,
num_shards=world_size,
random_shuffle=True)
self.decode = ops.ImageDecoder(device="mixed", output_type=types.RGB)
# self.res = ops.Resize(device="gpu", resize_x=size, resize_y=size, interp_type=types.INTERP_LINEAR)
self.res = ops.Resize(device="gpu",
resize_shorter=size,
interp_type=types.INTERP_LINEAR)
self.rescrop = ops.RandomResizedCrop(device="gpu",
size=crop,
random_area=[0.08, 1.25])
self.bc = ops.BrightnessContrast(device="gpu",
brightness=0.5,
contrast=0.6)
# Will flip vertically with prob of 0.1
self.vert_flip = ops.Flip(device='gpu', horizontal=0)
self.vert_coin = ops.CoinFlip(probability=0.4)
self.transform_source = ops.ExternalSource()
self.warp_keep_size = ops.WarpAffine(
device="gpu",
# size # keep original canvas size
interp_type=types.INTERP_LINEAR # use linear interpolation
)
# My workaround for Dali not supporting random affine transforms:
# a "synthetic random" warp affine transform.
# Rotate within a narrow range with probability of 0.075
self.rotate = ops.Rotate(device='gpu')
self.rotate_range = ops.Uniform(range=(-20.0, 20.0))
self.rotate_coin = ops.CoinFlip(probability=0.075)
self.cmnp = ops.CropMirrorNormalize(
device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NCHW,
image_type=types.RGB,
mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
self.coin = ops.CoinFlip(probability=0.5)
print('DALI "{0}" variant'.format(dali_device))
def __init__(self, batch_size, num_threads, device_id):
super(TFRecordPipeline, self).__init__(batch_size, num_threads,
device_id)
self.input = ops.TFRecordReader(
path=tfrecord,
index_path=tfrecord_idx,
features={
"image/encoded":
tfrec.FixedLenFeature((), tfrec.string, ""),
'image/filename':
tfrec.FixedLenFeature([], tfrec.string, ''),
'image/height':
tfrec.FixedLenFeature([1], tfrec.int64, -1),
'image/width':
tfrec.FixedLenFeature([1], tfrec.int64, -1),
'image/colorspace':
tfrec.FixedLenFeature([], tfrec.string, ''),
'image/channels':
tfrec.FixedLenFeature([1], tfrec.int64, -1),
'image/format':
tfrec.FixedLenFeature([], tfrec.string, ''),
'image/class/label':
tfrec.FixedLenFeature([1], tfrec.int64, -1),
'image/class/synset':
tfrec.FixedLenFeature([], tfrec.string, ''),
'image/class/text':
tfrec.FixedLenFeature([], tfrec.string, ''),
'image/object/bbox/xmin':
tfrec.VarLenFeature(tfrec.float32, 0.0),
'image/object/bbox/ymin':
tfrec.VarLenFeature(tfrec.float32, 0.0),
'image/object/bbox/xmax':
tfrec.VarLenFeature(tfrec.float32, 0.0),
'image/object/bbox/ymax':
tfrec.VarLenFeature(tfrec.float32, 0.0),
'image/object/bbox/label':
tfrec.FixedLenFeature([1], tfrec.int64, -1)
})
self.decode = ops.ImageDecoder(device="cpu", output_type=types.RGB)
self.resize = ops.Resize(device="cpu", resize_x=512., resize_y=512.)
self.vert_flip = ops.Flip(device="cpu", horizontal=0)
self.vert_coin = ops.CoinFlip(probability=0.5)
self.rotate = ops.Rotate(device='cpu', interp_type=types.INTERP_NN)
self.rotate_range = ops.Uniform(range=(-7, 7))
self.rotate_coin = ops.CoinFlip(probability=0.2)
self.cmnp = ops.CropMirrorNormalize(device="cpu",
output_dtype=types.FLOAT,
crop=(512, 512),
image_type=types.RGB,
mean=[0., 0., 0.],
std=[1., 1., 1.])
self.mirror_coin = ops.CoinFlip(probability=0.5)
self.uniform = ops.Uniform(range=(0.0, 1.0))
self.iter = 0
def __init__(self, batch_size, img_dir, json_dir, num_threads=2, device_id=0, num_gpus=1, resize=None,
augment=False, shuffle=True):
"""
Args:
batch_size: batch size for output at the first dim.
num_threads: int, number of cpu working threads.
device_id: int, the slice number of gpu.
num_gpus: int, number of multiple gpu.
img_dir: str, dir path where the images are stored.
json_dir: str, json path for coco dataset.
resize(optional): default int, if other format please modify function params in ops.Resize.
Output:
(images, captions) pair stacked by batch_size. The output shape of images will be NCHW with type of float.
Note that the output type of captions will be a list of numpy which is encoded from the original string
caption. To use it in the custom model, one needs to decode the numpy into string by .tostring() function
or .tobytes().decode() function. .tostring will get a bytes type result while .tobytes.decode will directly
get the string.
Notes:
param 'device' in ops functions instruct which device will process the data. optional in 'mixed'/'cpu'/'gpu',
for detail please see DALI documentation online.
"""
super(COCOCaptionPipeline, self).__init__(batch_size, num_threads, device_id, seed=15)
self.coco_itr = COCOCaptionInputIterator(batch_size, device_id, num_gpus, img_dir, json_dir, shuffle=shuffle)
self.iterator = iter(self.coco_itr)
self.input = ops.ExternalSource()
self.caption = ops.ExternalSource()
self.img_id = ops.ExternalSource()
self.decode = ops.ImageDecoder(device="mixed", output_type=types.RGB)
self.augment = augment
if resize is not None:
if isinstance(resize, tuple):
resx, resy = resize
elif isinstance(resize, int):
resx = resy = resize
else:
resx = resy = 0.
self.res = ops.Resize(device="gpu", resize_x=resx, resize_y=resy)
else:
self.res = None
if augment:
self.cmnp = ops.CropMirrorNormalize(device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NHWC,
image_type=types.RGB,
mean=MEAN,
std=STD)
self.cf = ops.CoinFlip()
self.rotate = ops.Rotate(device="gpu")
self.rng = ops.Uniform(range=(-10.0, 10.0))
def __init__(self, batch_size, num_threads, device_id):
super(TestPipeline, self).__init__(batch_size, num_threads, device_id)
self.input = ops.SegFileReader(file_root='data',
file_list='data/file_list.txt',
random_shuffle=True)
self.decodeMulti = ops.TiffDecoder()
self.rotate = ops.Rotate(device='gpu',
interp_type=types.INTERP_NN,
keep_size=True)
self.rotate_range = ops.Uniform(range=(-27, 27))
self.transpose = ops.Transpose(perm=[1, 2, 0])
self.transposeBack = ops.Transpose(device='gpu', perm=[2, 0, 1])
def __init__(self,
batch_size,
device,
data_dir,
mean,
std,
device_id=0,
shard_id=0,
num_shards=1,
num_threads=4,
seed=0):
super(DaliTransformsTrainPipeline,
self).__init__(batch_size, num_threads, device_id, seed)
# should we make this drive the device flags?
self.reader = ops.FileReader(file_root=data_dir,
shard_id=shard_id,
num_shards=num_shards,
random_shuffle=True)
self.decode = ops.ImageDecoder(device='mixed',
output_type=types.RGB,
memory_stats=True)
self.resize = ops.Resize(device=device,
size=[200, 300],
interp_type=types.INTERP_TRIANGULAR)
self.centrecrop = ops.Crop(device=device, crop=[100, 100])
self.randomcrop = ops.RandomResizedCrop(device=device, size=[80, 80])
self.hz_coin = ops.CoinFlip(probability=0.5)
self.horizontalflip = ops.Flip(device=device)
self.rotate_angle = ops.Uniform(range=[-90, 90])
self.rotate_coin = ops.CoinFlip(probability=0.5)
self.rotate = ops.Rotate(device=device, keep_size=True)
self.vt_coin = ops.CoinFlip(probability=0.5)
self.verticalflip = ops.Flip(device=device, horizontal=0)
self.normalize = ops.CropMirrorNormalize(device=device,
dtype=types.FLOAT,
output_layout=types.NCHW) #,
#mean=mean*255, std=std*255)
#self.normalize = ops.Normalize(device=device, dtype=types.FLOAT)#, mean=mean, stddev=std)
self.to_int64 = ops.Cast(dtype=types.INT64, device=device)
def __init__(self, db_prefix, input_shape, batch_size, data_params,
for_train, num_threads, device_id, num_shards):
super(HybridRecPipe, self).__init__(batch_size,
num_threads,
device_id,
seed=12 + device_id,
prefetch_queue_depth=2)
self.for_train = for_train
self.input = ops.MXNetReader(
path=[db_prefix + '.rec'],
index_path=[db_prefix + '.idx'],
random_shuffle=data_params['shuffle'] if for_train else False,
shard_id=device_id,
num_shards=num_shards)
self.decode = ops.ImageDecoder(device="mixed", output_type=types.RGB)
self.cmnp = ops.CropMirrorNormalize(
device="gpu",
dtype=types.FLOAT,
output_layout=types.NCHW,
crop=(input_shape[1], input_shape[2]),
mean=data_params['mean'] if isinstance(data_params['mean'], list)
else [data_params['mean'] for i in range(input_shape[0])],
std=data_params['std'] if isinstance(data_params['std'], list) else
[data_params['std'] for i in range(input_shape[0])])
if self.for_train:
self.rotate = ops.Rotate(device="gpu",
interp_type=types.INTERP_LINEAR)
self.color = ops.ColorTwist(device='gpu')
self.rng_angle = ops.Uniform(
range=(-float(data_params['max_rotate_angle']),
+float(data_params['max_rotate_angle'])))
self.rng_contrast = ops.Uniform(
range=(1.0 - data_params['contrast'],
1.0 + data_params['contrast']))
self.rng_brightness = ops.Uniform(
range=(1.0 - data_params['brightness'],
1.0 + data_params['brightness']))
self.rng_saturation = ops.Uniform(
range=(1.0 - data_params['saturation'],
1.0 + data_params['saturation']))
self.rng_hue = ops.Uniform(range=(1.0 - data_params['hue'],
1.0 + data_params['hue']))
self.coin = ops.CoinFlip(
probability=0.5) if data_params['rand_mirror'] else 0
def __init__(self, batch_size, output_size, num_threads, device_id, images_directory):
super(HybridPipelineTrain, self).__init__(batch_size, num_threads, device_id, seed = 12)
self.input = ops.FileReader(file_root = images_directory, random_shuffle = True, initial_fill = 21)
self.decode = ops.ImageDecoder(device = "mixed", output_type = types.RGB)
self.rotate = ops.Rotate(device = "gpu")
self.RRC = ops.RandomResizedCrop(size = output_size,
random_area = [0.4, 1.0],
random_aspect_ratio = [0.5, 1.5],
device="gpu"
)
self.cmn = ops.CropMirrorNormalize(
device="gpu",
dtype=types.FLOAT,
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
output_layout="HWC"
)
self.rng = ops.random.Uniform(range = (-25.0, 25.0))
self.coin = ops.random.CoinFlip(probability = 0.5)
self.flip = ops.Flip(device = "gpu")
def __init__(self,
p: float = .5,
angle_limit: Union[List, float] = 45.,
fill_value: float = 0.):
"""Initialization
Args:
p (float, optional): Probability to apply this transformation.. Defaults to .5.
angle_limit (Union[List,float], optional): Range for changing angle in [min,max] value format. If provided as a single float, the range will be (-limit, limit). Defaults to 45..
fill_value (float, optional): [description]. Defaults to 0..
"""
angle_limit = _check_and_convert_limit_value(angle_limit, None, 0)
self.angle_uniform = ops.Uniform(range=angle_limit)
self.rotate = ops.Rotate(device='gpu', fill_value=0., keep_size=True)
self.rng = ops.CoinFlip(probability=p)
self.bool = ops.Cast(dtype=types.DALIDataType.BOOL)
def __init__(self, params, num_threads, device_id):
super(DaliPipeline, self).__init__(params.batch_size,
num_threads,
device_id,
seed=12)
dii = DaliInputIterator(params, device_id)
dpi = DaliParameterIterator(params)
self.no_copy = params.no_copy
if self.no_copy:
print("Use Zero Copy ES")
self.source = ops.ExternalSource(device="gpu",
source=dii,
num_outputs=2,
layout=["DHWC", "DHWC"],
no_copy=self.no_copy)
self.params = ops.ExternalSource(device="cpu",
source=dpi,
num_outputs=2)
self.do_rotate = True if params.rotate_input == 1 else False
print("Enable Rotation" if self.do_rotate else "Disable Rotation")
self.rotate = ops.Rotate(device="gpu", interp_type=types.INTERP_LINEAR)
self.transpose = ops.Transpose(device="gpu", perm=[3, 0, 1, 2])
def __init__(self,
batch_size,
num_threads,
device_id,
external_data,
is_train=True):
super(ExternalSourcePipeline, self).__init__(batch_size,
num_threads,
device_id,
seed=12)
self.is_train = is_train
self.input = ops.ExternalSource()
self.angle_rng = ops.Uniform(range=(-10.0, 10.0))
self.rotate = ops.Rotate(device="gpu")
self.flip_rng = ops.CoinFlip(probability=0.5)
self.flip = ops.Flip(device='gpu')
self.slice = ops.Slice()
self.external_data = external_data
self.iterator = iter(self.external_data)
def __init__(self, data_iterator, batch_size, num_threads, device_id, size, eval_enabled=False):
super(ExternalSourcePipeline, self).__init__(batch_size, num_threads, device_id)
self.data_iterator = data_iterator
self.eval_enabled = eval_enabled
self.ops.SequenceReader(device="cpu", file_root)
self.iim1 = ops.ExternalSource()
self.iim2 = ops.ExternalSource()
self.iim3 = ops.ExternalSource()
self.decode = ops.ImageDecoder(device="mixed", output_type=types.RGB)
self.int = ops.Resize(device="gpu", resize_x=size[0]*2, resize_y=size[1]*2, image_type=types.RGB, interp_type=types.INTERP_CUBIC)
self.res = ops.Resize(device="gpu", resize_x=size[0], resize_y=size[1], image_type=types.RGB, interp_type=types.INTERP_LINEAR)
self.down = ops.Resize(device="gpu", resize_x=size[0]//2, resize_y=size[1]//2, image_type=types.RGB, interp_type=types.INTERP_LANCZOS3)
self.up = ops.Resize(device="gpu", resize_x=size[0], resize_y=size[1], image_type=types.RGB, interp_type=types.INTERP_CUBIC)
self.rotate = ops.Rotate(device='gpu', interp_type=types.INTERP_LINEAR, keep_size=True)
self.uniform = ops.Uniform(range = (-50., 50.))
self.finish = ops.CropMirrorNormalize(device="gpu", std=255., mean=0., output_dtype=types.FLOAT, image_type=types.RGB)
def __init__(self, params, num_threads, device_id):
super(DaliPipeline, self).__init__(params.batch_size,
num_threads,
device_id,
seed=12)
dii = DaliInputIterator(params)
self.no_copy = params.no_copy
if self.no_copy:
print("Use Zero Copy ES")
self.source = ops.ExternalSource(source = dii, num_outputs = 2, layout = ["DHWC", "DHWC"], no_copy = self.no_copy)
self.do_rotate = True if params.rotate_input==1 else False
print("Enable Rotation" if self.do_rotate else "Disable Rotation")
self.rng_angle = ops.Uniform(device = "cpu",
range = [0., 180.])
self.rng_axis = ops.Uniform(device = "cpu",
range = [-1., 1.],
shape=(3))
self.rotate = ops.Rotate(device = "gpu",
interp_type = types.INTERP_LINEAR,
keep_size=True)
self.transpose = ops.Transpose(device = "gpu",
perm=[3,0,1,2])
self.crop = ops.Crop(device = "gpu",
crop = (dii.size, dii.size, dii.size))
def __init__(self,
batch_size,
num_threads,
device_id,
image_size,
tfrecord_path,
index_path,
config,
shard_id=0):
super(CommonPipeline, self).__init__(batch_size, num_threads,
device_id)
self.image_size = image_size
self.input = self._input(tfrecord_path, index_path, shard_id=shard_id)
# The nvjpeg decoder throws an error for some unsupported jpegs.
# until this is fixed, we'll use the host decoder, which runs on the
# CPU.
# self.decode = ops.nvJPEGDecoder(device="mixed",
# output_type=types.RGB)
self.decode = ops.HostDecoder(device="cpu", output_type=types.RGB)
self.resize = ops.Resize(device="gpu",
image_type=types.RGB,
interp_type=types.INTERP_LINEAR,
resize_x=image_size,
resize_y=image_size)
self.resize_large = ops.Resize(device="gpu",
image_type=types.RGB,
interp_type=types.INTERP_LINEAR,
resize_x=image_size * config.zoom_scale,
resize_y=image_size * config.zoom_scale)
self.color_twist = ops.ColorTwist(device="gpu", )
self.crop_mirror_normalize = ops.CropMirrorNormalize(
device="gpu",
crop=image_size,
output_dtype=types.FLOAT,
image_type=types.RGB,
output_layout=types.DALITensorLayout.NHWC,
mean=122.5,
std=255.0)
self.crop = ops.Crop(
device="gpu",
crop=image_size,
)
self.cast = ops.Cast(device="gpu", dtype=types.DALIDataType.INT64)
self.rotate = ops.Rotate(device="gpu", fill_value=0)
self.flip = ops.Flip(device="gpu")
self.coin = ops.CoinFlip(probability=0.5)
self.rotate_rng = ops.Uniform(range=(config.rotate_angle_min,
config.rotate_angle_max))
self.crop_x_rng = ops.Uniform(range=(0.0, config.crop_x_max))
self.crop_y_rng = ops.Uniform(range=(0.0, config.crop_y_max))
self.hue_rng = ops.Uniform(range=(config.hue_min, config.hue_max))
self.contrast_rng = ops.Uniform(range=(config.contrast_min,
config.contrast_max))
self.saturation_rng = ops.Uniform(range=(config.saturation_min,
config.saturation_max))
self.brightness_rng = ops.Uniform(range=(config.brightness_min,
config.brightness_max))
self.iter = 0
def __init__(self, batch_size, num_threads, device_id):
super(RRGPUPipeline, self).__init__(batch_size, num_threads, device_id, seed = 12)
self.input = ops.FileReader(file_root = image_dir, random_shuffle = True, initial_fill = 21)
self.decode = ops.nvJPEGDecoder(device = "mixed", output_type = types.RGB)
self.rotate = ops.Rotate(device = "gpu")
self.rng = ops.Uniform(range = (-10.0, 10.0))
def __init__(self, batch_size, num_threads, device_id):
super(RandomRotatedSimplePipeline, self).__init__(batch_size, num_threads, device_id, seed = 12)
self.input = ops.FileReader(file_root = image_dir, random_shuffle = True, initial_fill = 21)
self.decode = ops.HostDecoder(output_type = types.RGB)
self.rotate = ops.Rotate()
self.rng = ops.Uniform(range = (-10.0, 10.0))
