def __init__(self, batch_size, num_threads, device_id):
super(CommonPipeline, self).__init__(batch_size, num_threads,
device_id)
self.decode = ops.nvJPEGDecoder(device="mixed", output_type=types.RGB)
self.resize = ops.Resize(device="gpu",
image_type=types.RGB,
interp_type=types.INTERP_LINEAR)
self.cmn = ops.CropMirrorNormalize(device="gpu",
output_dtype=types.FLOAT,
crop=(227, 227),
image_type=types.RGB,
mean=[128., 128., 128.],
std=[1., 1., 1.])
self.uniform = ops.Uniform(range=(0.0, 1.0))
self.resize_rng = ops.Uniform(range=(256, 480))
def __init__(self, batch_size, num_threads, device_id):
super(HybridPipe, self).__init__(batch_size, num_threads, device_id, seed = 12)
self.input = ops.CaffeReader(path = caffe_db_folder, random_shuffle = True)
self.decode = ops.nvJPEGDecoder(device = "mixed", output_type = types.RGB)
self.cmnp = ops.CropMirrorNormalize(device = "gpu",
output_dtype = types.FLOAT,
output_layout = types.NHWC,
crop = (224, 224),
image_type = types.RGB,
mean = [128., 128., 128.],
std = [1., 1., 1.])
self.rotate = ops.Rotate(device = "gpu", angle = 45.0,
fill_value = 128,
interp_type=types.INTERP_LINEAR)
self.uniform = ops.Uniform(range = (0.0,1.0))
self.iter = 0
def __init__(self, batch_size, num_threads, device_id, num_gpus, data_paths, prefetch, fp16, nhwc):
super(RN50Pipeline, self).__init__(batch_size, num_threads, device_id, prefetch_queue_depth=prefetch)
self.input = ops.FileReader(file_root = data_paths[0], shard_id = device_id, num_shards = num_gpus)
self.decode_gpu = ops.ImageDecoder(device = "mixed", output_type = types.RGB)
self.res = ops.RandomResizedCrop(device="gpu", size =(224,224))
layout = types.args.nhwc if nhwc else types.NCHW
out_type = types.FLOAT16 if fp16 else types.FLOAT
self.cmnp = ops.CropMirrorNormalize(device="gpu",
dtype=out_type,
output_layout=layout,
crop=(224, 224),
mean=[0.485 * 255,0.456 * 255,0.406 * 255],
std=[0.229 * 255,0.224 * 255,0.225 * 255])
self.coin = ops.random.CoinFlip(probability=0.5)
def __init__(self, image_dir, batch_size, num_threads, device_id, num_gpus=1, seed=1, train=True):
super(DataPipeline, self).__init__(
batch_size, num_threads, device_id, seed=seed)
self.input = ops.FileReader(file_root=image_dir,
random_shuffle=True, num_shards=num_gpus, shard_id=device_id)
self.decode = ops.ImageDecoder(device='mixed', output_type=types.RGB)
self.rrc = ops.RandomResizedCrop(device='gpu', size=(128, 128))
self.cmn = ops.CropMirrorNormalize(device='gpu',
crop=(128, 128),
image_type=types.RGB,
mean=[0.5*256, 0.5*256, 0.5*256],
std=[0.5*256, 0.5*256, 0.5*256],
)
self.coin = ops.CoinFlip(probability=0.5)
self.res = ops.Resize(device="gpu", resize_shorter=256)
self.train = train
def __init__(self, batch_size, num_threads, device_id, pipelined = True, exec_async = True):
super(C2Pipe, self).__init__(batch_size,
num_threads,
device_id,
exec_pipelined=pipelined,
exec_async=exec_async)
self.input = ops.ExternalSource()
self.decode = ops.ImageDecoder(device = 'cpu', output_type = types.RGB)
self.rcm = ops.FastResizeCropMirror(crop = (224, 224))
self.np = ops.CropMirrorNormalize(device = "gpu",
dtype = types.FLOAT16,
mean = [128., 128., 128.],
std = [1., 1., 1.])
self.uniform = ops.Uniform(range = (0., 1.))
self.resize_uniform = ops.Uniform(range = (256., 480.))
self.mirror = ops.CoinFlip(probability = 0.5)
def __init__(self, file_tfrecord, batch_size, num_workers, device_id=0):
super().__init__(batch_size, num_workers, device_id)
self.input = ops.TFRecordReader(
path=file_tfrecord.as_posix(),
index_path=(file_tfrecord.parent /
(file_tfrecord.stem + '.idx')).as_posix(),
features={'encoded': tfrec.FixedLenFeature((), tfrec.string, "")})
self.decode = ops.ImageDecoder(device='mixed', output_type=types.RGB)
self.cmnp = ops.CropMirrorNormalize(device='gpu',
output_dtype=types.FLOAT,
output_layout=types.NCHW,
image_type=types.RGB,
mean=[124, 116, 104],
std=[58, 57, 57])
def __init__(self, data_root, data_list, sampler, crop, size):
super(ImageNetValPipeV2, self).__init__()
self.mc_input = ops.McReader(file_root=data_root,
file_list=data_list,
sampler_index=list(sampler))
self.decode = ops.ImageDecoder(device="mixed", output_type=types.RGB)
self.res = ops.Resize(device="gpu", resize_shorter=size)
self.cmnp = ops.CropMirrorNormalize(device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NCHW,
crop=(crop, crop),
image_type=types.RGB,
mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
def __init__(self, batch_size, num_threads, device_id):
super(HybridPipe, self).__init__(batch_size,
num_threads,
device_id,
seed = 12)
self.input = ops.CaffeReader(path = caffe_db_folder, random_shuffle = True)
self.decode = ops.nvJPEGDecoder(device = "mixed", output_type = types.RGB)
self.cmnp = ops.CropMirrorNormalize(device = "gpu",
output_dtype = types.FLOAT,
crop = (224, 224),
image_type = types.RGB,
mean = [128., 128., 128.],
std = [1., 1., 1.])
self.coin = ops.CoinFlip()
self.uniform = ops.Uniform(range = (0.0,1.0))
self.iter = 0
def __init__(
self,
name,
batch_size,
num_threads,
device_id,
num_shards,
shard_id,
root=os.path.expanduser('~/.mxnet/datasets/face'),
):
super().__init__(batch_size, num_threads, device_id, seed=12)
idx_files = [os.path.join(root, name, "train.idx")]
rec_files = [os.path.join(root, name, "train.rec")]
prop = open(os.path.join(root, name, "property"),
"r").read().strip().split(',')
assert len(prop) == 3
self.num_classes = int(prop[0])
self.image_size = [int(prop[1]), int(prop[2])]
self.size = 0
for idx_file in idx_files:
with open(idx_file, "r") as f:
self.size += len(list(f.readlines()))
self._input = ops.MXNetReader(path=rec_files,
index_path=idx_files,
random_shuffle=True,
num_shards=num_shards,
shard_id=shard_id,
seed=12,
tensor_init_bytes=self.image_size[0] *
self.image_size[1] * 8)
self._decode = ops.ImageDecoder(device="mixed", output_type=types.RGB)
self._cmnp = ops.CropMirrorNormalize(device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NCHW,
crop=self.image_size,
image_type=types.RGB,
mean=[0., 0., 0.],
std=[255., 255., 255.])
self._contrast = ops.Contrast(device="gpu")
self._saturation = ops.Saturation(device="gpu")
self._brightness = ops.Brightness(device="gpu")
self._uniform = ops.Uniform(range=(0.7, 1.3))
self._coin = ops.CoinFlip(probability=0.5)
def __init__(self, data_paths, num_shards, batch_size, num_threads,
device_id, prefetch, fp16, random_shuffle, nhwc,
dont_use_mmap, decoder_type, decoder_cache_params,
reader_queue_depth, shard_id):
super(CommonPipeline, self).__init__(batch_size,
num_threads,
device_id,
random_shuffle,
prefetch_queue_depth=prefetch)
if decoder_type == 'roi':
print('Using nvJPEG with ROI decoding')
self.decode_gpu = ops.decoders.ImageRandomCrop(
device="mixed", output_type=types.RGB)
self.res = ops.Resize(device="gpu", resize_x=224, resize_y=224)
elif decoder_type == 'cached':
assert decoder_cache_params['cache_enabled'] == True
cache_size = decoder_cache_params['cache_size']
cache_threshold = decoder_cache_params['cache_threshold']
cache_type = decoder_cache_params['cache_type']
print(
'Using nvJPEG with cache (size : {} threshold: {}, type: {})'.
format(cache_size, cache_threshold, cache_type))
self.decode_gpu = ops.decoders.Image(
device="mixed",
output_type=types.RGB,
cache_size=cache_size,
cache_threshold=cache_threshold,
cache_type=cache_type,
cache_debug=False)
self.res = ops.RandomResizedCrop(device="gpu", size=(224, 224))
else:
print('Using nvJPEG')
self.decode_gpu = ops.decoders.Image(device="mixed",
output_type=types.RGB)
self.res = ops.RandomResizedCrop(device="gpu", size=(224, 224))
layout = types.NHWC if nhwc else types.NCHW
out_type = types.FLOAT16 if fp16 else types.FLOAT
self.cmnp = ops.CropMirrorNormalize(
device="gpu",
dtype=out_type,
output_layout=layout,
crop=(224, 224),
mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
self.coin = ops.random.CoinFlip(probability=0.5)
def __new__(
cls,
crop=None,
mirror=None,
mean=0.,
std=1.,
dtype='float32',
output_layout='NCHW',
**kwargs
):
"""Create a ``CropMirrorNormalize`` operator.
Parameters
----------
crop : Sequence[int], optional
The cropped spatial dimensions for output.
mirror : {0, 1}, optional
Whether to apply the horizontal flip.
mean : Union[float, Sequence[float]], optional
The values to subtract.
std : Union[float, Sequence[float]], optional
The values to divide after subtraction.
dtype : {'float16', 'float32'}, optional
The data type of output.
output_layout : {'NCHW', 'NHWC'}, optional
The data format of output.
Returns
-------
nvidia.dali.ops.CropMirrorNormalize
The operator.
"""
if isinstance(dtype, six.string_types):
dtype = getattr(types, dtype.upper())
if isinstance(output_layout, six.string_types):
output_layout = getattr(types, output_layout.upper())
return ops.CropMirrorNormalize(
crop=crop,
mirror=mirror,
mean=mean,
std=std,
dtype=dtype,
output_layout=output_layout,
device=context.get_device_type(),
**kwargs
)
def __init__(self,
batch_size,
num_threads,
device_id,
data_dir,
crop,
dali_cpu=False):
super(HybridTrainPipe, self).__init__(batch_size,
num_threads,
device_id,
seed=12 + device_id)
# MXnet rec reader
self.input = ops.MXNetReader(path=join(data_dir, "train_label.rec"),
index_path=join(data_dir,
"train_label.idx"),
random_shuffle=True,
shard_id=args.local_rank,
num_shards=args.world_size)
#let user decide which pipeline works him bets for RN version he runs
dali_device = 'cpu' if dali_cpu else 'gpu'
decoder_device = 'cpu' if dali_cpu else 'mixed'
# This padding sets the size of the internal nvJPEG buffers to be able to handle all images from full-sized ImageNet
# without additional reallocations
device_memory_padding = 211025920 if decoder_device == 'mixed' else 0
host_memory_padding = 140544512 if decoder_device == 'mixed' else 0
self.decode = ops.ImageDecoderRandomCrop(
device=decoder_device,
output_type=types.RGB,
device_memory_padding=device_memory_padding,
host_memory_padding=host_memory_padding,
random_aspect_ratio=[0.8, 1.25],
random_area=[0.1, 1.0],
num_attempts=100)
self.res = ops.Resize(device=dali_device,
resize_x=crop,
resize_y=crop,
interp_type=types.INTERP_TRIANGULAR)
self.cmnp = ops.CropMirrorNormalize(
device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NCHW,
crop=(crop, crop),
image_type=types.RGB,
mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
self.coin = ops.CoinFlip(probability=0.5)
print('DALI "{0}" variant'.format(dali_device))
def __init__(self, batch_size, num_threads, device_id, num_gpus, cfg):
super(CommonPipeline, self).__init__(batch_size, num_threads,
device_id)
self.decode = ops.ImageDecoder(device="mixed", output_type=types.RGB)
self.resize = ops.Resize(device="gpu",
image_type=types.RGB,
interp_type=types.INTERP_LINEAR)
self.cmn = ops.CropMirrorNormalize(
device="gpu",
output_dtype=types.FLOAT,
# crop=(227, 227),
image_type=types.RGB,
mean=cfg.mean,
std=cfg.std)
self.uniform = ops.Uniform(range=(0.0, 1.0))
self.resize_rng = ops.Uniform(range=(512, 512))
self.cast = ops.Cast(device="gpu", dtype=types.UINT8)
def __init__(self,
batch_size,
num_threads,
device_id,
data_dir,
crop,
dali_cpu=False,
args_new=None):
super(HybridUnifiedTrainPipe, self).__init__(batch_size,
num_threads,
device_id,
seed=12 + device_id)
self.input = ops.FileReader(file_root=data_dir,
shard_id=args_new.local_rank,
num_shards=args_new.num_procs,
shuffle_after_epoch=True,
cache_size=args.cache_size)
dali_device = 'cpu' if dali_cpu else 'gpu'
decoder_device = 'cpu' if dali_cpu else 'mixed'
# This padding sets the size of the internal nvJPEG buffers to be able to handle all images from full-sized ImageNet
# without additional reallocations
device_memory_padding = 211025920 if decoder_device == 'mixed' else 0
host_memory_padding = 140544512 if decoder_device == 'mixed' else 0
self.decode = ops.ImageDecoderRandomCrop(
device=decoder_device,
output_type=types.RGB,
device_memory_padding=device_memory_padding,
host_memory_padding=host_memory_padding,
random_aspect_ratio=[0.8, 1.25],
random_area=[0.1, 1.0],
num_attempts=100)
self.res = ops.Resize(device=dali_device,
resize_x=crop,
resize_y=crop,
interp_type=types.INTERP_TRIANGULAR)
self.cmnp = ops.CropMirrorNormalize(
device=dali_device,
output_dtype=types.FLOAT,
output_layout=types.NCHW,
crop=(crop, crop),
image_type=types.RGB,
mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
self.coin = ops.CoinFlip(probability=0.5)
print('DALI "{0}" variant, shards={1}'.format(dali_device,
args_new.num_procs))
def __init__(self, function, device, num_outputs=1):
super(PythonFunctionPipeline, self).__init__(BATCH_SIZE,
NUM_WORKERS,
DEVICE_ID,
seed=SEED,
exec_async=False,
exec_pipelined=False)
self.device = device
self.reader = ops.readers.File(file_root=images_dir)
self.decode = ops.decoders.Image(device='cpu', output_type=types.RGB)
self.norm = ops.CropMirrorNormalize(std=255.,
mean=0.,
device=device,
output_layout="HWC")
self.func = ops.PythonFunction(device=device,
function=function,
num_outputs=num_outputs)
def __init__(self, batch_size, num_threads, device_id, crop, size):
super(ImageNetValPipe, self).__init__(batch_size,
num_threads,
device_id,
seed=12 + device_id)
self.data_input = ops.ExternalSource()
self.label_input = ops.ExternalSource()
self.decode = ops.nvJPEGDecoder(device="mixed", output_type=types.RGB)
self.res = ops.Resize(device="gpu", resize_shorter=size)
self.cmnp = ops.CropMirrorNormalize(
device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NCHW,
crop=(crop, crop),
image_type=types.RGB,
mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
def __init__(self, device, batch_size, num_threads=1, device_id=0, num_gpus=1,
dtype = types.FLOAT, output_layout = "HWC",
mirror_probability = 0.0, mean=[0., 0., 0.], std=[1., 1., 1.], pad_output=False):
super(CropMirrorNormalizePipeline, self).__init__(batch_size, num_threads, device_id, seed=7865)
self.device = device
self.input = ops.CaffeReader(path = caffe_db_folder, shard_id = device_id, num_shards = num_gpus)
self.decode = ops.ImageDecoder(device = "cpu", output_type = types.RGB)
self.cmn = ops.CropMirrorNormalize(device = self.device,
dtype = dtype,
output_layout = output_layout,
crop = (224, 224),
crop_pos_x = 0.3,
crop_pos_y = 0.2,
mean = mean,
std = std,
pad_output = pad_output)
self.coin = ops.CoinFlip(probability=mirror_probability, seed=7865)
def __init__(self,
batch_size,
num_threads,
device_id,
data_dir,
crop,
dali_cpu=False,
fp16=False):
super(HybridTrainPipe, self).__init__(batch_size,
num_threads,
device_id,
seed=12 + device_id)
self.input = ops.FileReader(file_root=data_dir,
shard_id=args.local_rank,
num_shards=args.world_size,
random_shuffle=True)
#let user decide which pipeline works him bets for RN version he runs
if dali_cpu:
dali_device = "cpu"
self.decode = ops.HostDecoder(device=dali_device,
output_type=types.RGB)
else:
dali_device = "gpu"
# This padding sets the size of the internal nvJPEG buffers to be able to handle all images from full-sized ImageNet
# without additional reallocations
self.decode = ops.nvJPEGDecoder(device="mixed",
output_type=types.RGB,
device_memory_padding=211025920,
host_memory_padding=140544512)
self.rrc = ops.RandomResizedCrop(device=dali_device, size=(crop, crop))
output_dtype = types.FLOAT
if fp16:
output_dtype = types.FLOAT16
self.cmnp = ops.CropMirrorNormalize(
device=dali_device, # 1: changed from "cpu" to dali_device
output_dtype=output_dtype, # 2: output to fp16 when fp16 training
output_layout=types.NCHW,
crop=(crop, crop),
image_type=types.RGB,
mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
self.coin = ops.CoinFlip(probability=0.5)
print('DALI "{0}" variant'.format(dali_device))
def __init__(self, batch_size, num_threads, device_id, num_gpus):
super(CaffeReadPipeline, self).__init__(batch_size, num_threads, device_id)
self.input = ops.CaffeReader(path = lmdb_folder,
random_shuffle = True, shard_id = device_id, num_shards = num_gpus)
self.decode = ops.ImageDecoder(device = "mixed", output_type = types.RGB)
self.resize = ops.Resize(device = "gpu",
image_type = types.RGB,
interp_type = types.INTERP_LINEAR)
self.cmn = ops.CropMirrorNormalize(device = "gpu",
output_dtype = types.FLOAT,
crop = (227, 227),
image_type = types.RGB,
mean = [128., 128., 128.],
std = [1., 1., 1.])
self.uniform = ops.Uniform(range = (0.0, 1.0))
self.resize_rng = ops.Uniform(range = (256, 480))
def __init__(self, batch_size, num_threads, device_id):
super(CMNvsCropPipe, self).__init__(batch_size, num_threads, device_id, seed = 12)
self.input = ops.CaffeReader(path = caffe_db_folder, shard_id = device_id, num_shards = 1)
self.decode = ops.nvJPEGDecoder(device = "mixed", output_type = types.RGB)
self.cmn = ops.CropMirrorNormalize(device = "gpu",
output_layout = types.NHWC,
output_dtype = types.FLOAT,
crop = (224, 224),
image_type = types.RGB,
mean = [0., 0., 0.],
std = [1., 1., 1.])
self.crop = ops.Crop(device = "gpu",
crop = (224, 224),
image_type = types.RGB)
self.uniform = ops.Uniform(range = (0.0, 1.0))
self.cast = ops.Cast(device = "gpu",
dtype = types.INT32)
def __init__(self,
batch_size,
device,
data_dir,
mean,
std,
device_id=0,
shard_id=0,
num_shards=1,
num_threads=4,
seed=0):
super(DaliTransformsTrainPipeline,
self).__init__(batch_size, num_threads, device_id, seed)
# should we make this drive the device flags?
self.reader = ops.FileReader(file_root=data_dir,
shard_id=shard_id,
num_shards=num_shards,
random_shuffle=True)
self.decode = ops.ImageDecoder(device='mixed',
output_type=types.RGB,
memory_stats=True)
self.resize = ops.Resize(device=device,
size=[200, 300],
interp_type=types.INTERP_TRIANGULAR)
self.centrecrop = ops.Crop(device=device, crop=[100, 100])
self.randomcrop = ops.RandomResizedCrop(device=device, size=[80, 80])
self.hz_coin = ops.CoinFlip(probability=0.5)
self.horizontalflip = ops.Flip(device=device)
self.rotate_angle = ops.Uniform(range=[-90, 90])
self.rotate_coin = ops.CoinFlip(probability=0.5)
self.rotate = ops.Rotate(device=device, keep_size=True)
self.vt_coin = ops.CoinFlip(probability=0.5)
self.verticalflip = ops.Flip(device=device, horizontal=0)
self.normalize = ops.CropMirrorNormalize(device=device,
dtype=types.FLOAT,
output_layout=types.NCHW) #,
#mean=mean*255, std=std*255)
#self.normalize = ops.Normalize(device=device, dtype=types.FLOAT)#, mean=mean, stddev=std)
self.to_int64 = ops.Cast(dtype=types.INT64, device=device)
def __init__(self, batch_size, num_threads, device_id, data_dir):
super(HybridPipe, self).__init__(batch_size, num_threads, device_id)
self.input = ops.Caffe2Reader(path=data_dir,
shard_id=args.rank,
num_shards=args.world_size)
self.decode = ops.nvJPEGDecoder(device="mixed", output_type=types.RGB)
self.rrc = ops.RandomResizedCrop(device="gpu", size=(224, 224))
self.cmnp = ops.CropMirrorNormalize(
device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NCHW,
image_type=types.RGB,
crop=(224, 224),
mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
self.coin = ops.CoinFlip(probability=0.5)
def __init__(self, device_id, crop, size, rank, seed_rank, options):
super(TrainPipeline, self).__init__(options.batchsize,
4,
device_id,
prefetch_queue_depth=3,
set_affinity=True,
seed=options.seed + seed_rank)
self.input = ops.FileReader(file_root=options.train_dir,
shard_id=rank,
num_shards=size,
random_shuffle=True)
# This padding sets the size of the internal nvJPEG buffers to be able to handle all images from full-sized ImageNet
# without additional reallocations
random_aspect_ratio = [0.75, 4. / 3.]
random_area = [0.08, 1.0]
self.decode = ops.ImageDecoderRandomCrop(
device="mixed",
output_type=types.RGB,
device_memory_padding=211025920,
host_memory_padding=140544512,
random_aspect_ratio=random_aspect_ratio,
random_area=random_area,
num_attempts=100,
seed=options.seed + seed_rank + 1641)
self.res = ops.Resize(device="gpu",
resize_x=crop,
resize_y=crop,
interp_type=types.INTERP_TRIANGULAR)
dtype = types.FLOAT16 if options.fp16 else types.FLOAT
layout = types.NCHW
padding = False
img_mean, img_std = get_mean_std(options)
self.cmnp = ops.CropMirrorNormalize(device="gpu",
output_dtype=dtype,
output_layout=layout,
crop=(crop, crop),
image_type=types.RGB,
mean=img_mean,
std=img_std,
pad_output=padding,
seed=options.seed + seed_rank +
1223)
self.coin = ops.CoinFlip(probability=0.5,
seed=options.seed + seed_rank + 412)
def __init__(self,
batch_size,
num_threads,
device_id,
data_dir,
crop,
seed,
dali_cpu=False):
super(TinyImageNetHybridTrainPipe,
self).__init__(batch_size, num_threads, device_id, seed)
if torch.distributed.is_initialized():
local_rank = torch.distributed.get_rank()
world_size = torch.distributed.get_world_size()
else:
local_rank = 0
world_size = 1
self.input = ops.FileReader(file_root=data_dir,
shard_id=local_rank,
num_shards=world_size,
pad_last_batch=True,
random_shuffle=False,
shuffle_after_epoch=True)
# decide to work on cpu or gpu
dali_device = 'cpu' if dali_cpu else 'gpu'
decoder_device = 'cpu' if dali_cpu else 'mixed'
self.decode = ops.ImageDecoder(device=decoder_device,
output_type=types.RGB)
self.res = ops.RandomResizedCrop(device=dali_device,
size=crop,
random_aspect_ratio=[0.75, 4. / 3],
random_area=[0.08, 1.0],
num_attempts=100,
interp_type=types.INTERP_TRIANGULAR)
self.cmnp = ops.CropMirrorNormalize(
device='gpu',
output_dtype=types.FLOAT,
output_layout=types.NCHW,
crop=(crop, crop),
image_type=types.RGB,
mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
self.coin = ops.CoinFlip(probability=0.5)
def __init__(self,
args,
batch_size,
num_threads,
device_id,
rec_path,
idx_path,
shard_id,
num_shards,
crop_shape,
nvjpeg_padding,
prefetch_queue=3,
resize_shp=None,
output_layout=types.NCHW,
pad_output=True,
dtype='float16',
dali_cpu=False):
super(HybridValPipe,
self).__init__(batch_size,
num_threads,
device_id,
seed=12 + device_id,
prefetch_queue_depth=prefetch_queue)
self.input = ops.MXNetReader(path=[rec_path],
index_path=[idx_path],
random_shuffle=False,
shard_id=shard_id,
num_shards=num_shards)
dali_device, decoder_device = get_device_names(dali_cpu)
self.decode = ops.ImageDecoder(device=decoder_device,
output_type=types.RGB,
device_memory_padding=nvjpeg_padding,
host_memory_padding=nvjpeg_padding)
self.resize = ops.Resize(
device=dali_device,
resize_shorter=resize_shp) if resize_shp else None
self.cmnp = ops.CropMirrorNormalize(
device="gpu",
dtype=types.FLOAT16 if dtype == 'float16' else types.FLOAT,
output_layout=output_layout,
crop=crop_shape,
pad_output=pad_output,
mean=args.rgb_mean,
std=args.rgb_std)
def __init__(self, db_folder, batch_size, num_threads, device_id,
num_gpus):
super(MXNetReaderPipeline, self).__init__(batch_size, num_threads,
device_id)
self.input = ops.MXNetReader(path=[db_folder + "train.rec"],
index_path=[db_folder + "train.idx"],
random_shuffle=True,
shard_id=device_id,
num_shards=num_gpus)
dali_device = "gpu"
self.decode = ops.ImageDecoder(device="mixed", output_type=types.RGB)
self.cmn = ops.CropMirrorNormalize(device="gpu",
output_dtype=types.FLOAT,
image_type=types.RGB,
mean=[128., 128., 128.],
std=[1., 1., 1.])
self.coin = ops.CoinFlip(probability=0.5)
def __init__(self, batch_size, num_threads, device_id, num_gpus, db_dir):
super(TrainPipe, self).__init__(batch_size, num_threads, device_id)
self.input = ops.MXNetReader(path=[db_dir + "train.rec"],
index_path=[db_dir + "train.idx"],
random_shuffle=False,
shard_id=device_id,
num_shards=num_gpus)
self.decode = ops.ImageDecoder(device="mixed", output_type=types.RGB)
# self.blur = ops.GaussianBlur(device="cpu", sigma=5.0, window_size=5)
# self.reshape = ops.Reshape(device="cpu", layout="HWC")
self.normalize = ops.CropMirrorNormalize(device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NCHW,
mean=[123.0, 116.0, 103.0],
std=[100.0, 100.0, 100.0])
self.transform_source = ops.ExternalSource()
self.warp_gpu = ops.WarpAffine(device="gpu",
interp_type=types.INTERP_LINEAR)
def __init__(self, file_root, annotations_file,
batch_size, num_threads, device_id=0, num_gpus=1,
mean=(123.675, 116.28, 103.53), stddev=(1., 1., 1.),
random_shuffle=True):
super(COCOPipeline, self).__init__(batch_size, num_threads, device_id, seed = 15)
self.input = ops.COCOReader(file_root=file_root, annotations_file=annotations_file,
shard_id=device_id, num_shards=num_gpus, ratio=True,
skip_empty=True, prefetch_queue_depth=32, random_shuffle=True)
self.decode = ops.ImageDecoder(device='mixed', output_type=types.BGR)
self.resize = ops.Resize(device='gpu', max_size=1216, resize_shorter=800)
self.flip = ops.CoinFlip(device='cpu')
self.bbox_flip = ops.BbFlip(device='gpu')
self.CMN = ops.CropMirrorNormalize(device='gpu', mean=mean, std=stddev, output_layout='HWC')
self.image_pad = ops.Pad(device='gpu', fill_value=0, axes=(0,1), shape=(1216, 1216))
self.bbox_pad = ops.Pad(device='gpu', fill_value=0, axes=(0,), shape=(100,))
self.label_pad = ops.Pad(device='gpu', fill_value=-1, axes=(0,), shape=(100,))
self.get_shape = ops.Shapes(device='gpu')
self.float_cast = ops.Cast(device='gpu', dtype=types.FLOAT)
def __init__(self, batch_size, num_threads, device_id, data_dir, crop, size, seed=12, local_rank=0, world_size=1,
spos_pre=False, shuffle=False):
super(HybridValPipe, self).__init__(batch_size,
num_threads, device_id, seed=seed + device_id)
color_space_type = types.BGR if spos_pre else types.RGB
self.input = ops.FileReader(file_root=data_dir, shard_id=local_rank, num_shards=world_size,
random_shuffle=shuffle)
self.decode = ops.ImageDecoder(
device="mixed", output_type=color_space_type)
self.res = ops.Resize(device="gpu", resize_shorter=size,
interp_type=types.INTERP_LINEAR if spos_pre else types.INTERP_TRIANGULAR)
self.cmnp = ops.CropMirrorNormalize(device="gpu",
dtype=types.FLOAT,
output_layout=types.NCHW,
crop=(crop, crop),
mean=0. if spos_pre else [
0.485 * 255, 0.456 * 255, 0.406 * 255],
std=1. if spos_pre else [0.229 * 255, 0.224 * 255, 0.225 * 255])
def __init__(self,
batch_size,
num_threads,
device_id,
data_dir,
crop,
num_shards,
dali_cpu=False):
super(HybridTrainPipe, self).__init__(batch_size,
num_threads,
device_id,
seed=12 + device_id)
self.input = ops.MXNetReader(
path=[os.path.join(data_dir, "_train.rec")],
index_path=[os.path.join(data_dir, "_train.idx")],
random_shuffle=True,
shard_id=device_id,
num_shards=num_shards)
# self.input = ops.FileReader(file_root=data_dir, shard_id=0, num_shards=4, random_shuffle=True)
# let user decide which pipeline works him bets for RN version he runs
if dali_cpu:
dali_device = "cpu"
self.decode = ops.HostDecoder(device=dali_device,
output_type=types.RGB)
else:
dali_device = "gpu"
# This padding sets the size of the internal nvJPEG buffers to be able to handle all images from full-sized ImageNet
# without additional reallocations
self.decode = ops.nvJPEGDecoder(device="mixed",
output_type=types.RGB)
self.rrc = ops.RandomResizedCrop(device=dali_device, size=(crop, crop))
self.cmnp = ops.CropMirrorNormalize(
device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NCHW,
crop=(crop, crop),
image_type=types.RGB,
mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
self.coin = ops.CoinFlip(probability=0.5)
print('DALI "{0}" variant'.format(dali_device))
