def __init__(self, batch_size, num_threads, device_id):
super(CommonPipeline, self).__init__(batch_size, num_threads,
device_id)
self.decode_gpu = ops.nvJPEGDecoder(device="mixed",
output_type=types.RGB)
self.decode_host = ops.HostDecoder(device="cpu", output_type=types.RGB)
def __init__(self, tfrec_filenames, tfrec_idx_filenames, batch_size,
num_threads, device_id, set_affinity, prefetch_queue_depth):
super(Dali_CPU_Pipe,
self).__init__(batch_size,
num_threads,
device_id,
set_affinity=set_affinity,
prefetch_queue_depth=prefetch_queue_depth)
self.input = ops.TFRecordReader(path=tfrec_filenames,
index_path=tfrec_idx_filenames,
initial_fill=10000,
features={
"image/encoded":
tfrec.FixedLenFeature(
(), tfrec.string, ""),
'image/class/label':
tfrec.FixedLenFeature([1],
tfrec.int64,
-1)
})
self.decode = ops.HostDecoder(output_type=types.RGB)
self.resize = ops.Resize(device="cpu", resize_shorter=_RESIZE_MIN)
self.cmnp = ops.CropMirrorNormalize(device="cpu",
output_dtype=types.FLOAT,
crop=(INPUT_SIZE, INPUT_SIZE),
image_type=types.RGB,
mean=_CHANNEL_MEANS,
std=[58.395, 57.120, 57.375],
output_layout=dali.types.NCHW)
self.iter = 0
def __init__(self,
file_list,
file_root,
crop_size,
batch_size,
num_threads,
device_id,
random_shuffle=False,
seed=-1,
mean=None,
std=None,
num_samples=None):
super(DaliPipelineVal, self).__init__(batch_size,
num_threads,
device_id,
seed=seed)
crop_size = _pair(crop_size)
if mean is None:
mean = [0.485 * 255, 0.456 * 255, 0.406 * 255]
if std is None:
std = [0.229 * 255, 0.224 * 255, 0.225 * 255]
if num_samples is None:
initial_fill = 512
else:
initial_fill = min(512, num_samples)
self.loader = ops.FileReader(file_root=file_root,
file_list=file_list,
random_shuffle=random_shuffle,
initial_fill=initial_fill)
self.decode = ops.HostDecoder()
self.resize = ops.Resize(device='gpu', resize_x=256, resize_y=256)
self.cmnorm = ops.CropMirrorNormalize(device='gpu',
crop=list(crop_size),
mean=mean,
std=std)
def __init__(self, batch_size, num_threads, device_id):
print(image_dir)
super(SimplePipeline, self).__init__(batch_size, num_threads, device_id, seed = 12)
self.input = ops.FileReader(file_root = image_dir)
# instead of path to file directory file with pairs image_name image_label_value can be provided
# self.input = ops.FileReader(file_root = image_dir, file_list = image_dir + '/file_list.txt')
self.decode = ops.HostDecoder(output_type = types.RGB)
def __init__(self, batch_size, num_threads, device_id, prefetch, seed):
super(CommonPipeline, self).__init__(batch_size,
num_threads,
device_id,
prefetch_queue_depth=prefetch)
self.decode_cpu = ops.HostDecoder(device="cpu", output_type=types.RGB)
self.decode_crop = ops.HostDecoderSlice(device="cpu",
output_type=types.RGB)
self.crop = ops.SSDRandomCrop(device="cpu", num_attempts=1, seed=seed)
self.crop2 = ops.RandomBBoxCrop(
device="cpu",
aspect_ratio=[0.5, 2.0],
thresholds=[0, 0.1, 0.3, 0.5, 0.7, 0.9],
scaling=[0.3, 1.0],
ltrb=True,
seed=seed)
self.slice_cpu = ops.Slice(device="cpu")
self.slice_gpu = ops.Slice(device="gpu")
self.flip_cpu = ops.Flip(device="cpu")
self.bb_flip_cpu = ops.BbFlip(device="cpu", ltrb=True)
self.flip_gpu = ops.Flip(device="gpu")
self.bb_flip_gpu = ops.BbFlip(device="gpu", ltrb=True)
def __init__(self,
device,
batch_size,
num_threads=1,
device_id=0,
num_gpus=1,
is_fused_decoder=False):
super(CropPipeline, self).__init__(batch_size, num_threads, device_id)
self.is_fused_decoder = is_fused_decoder
self.device = device
self.input = ops.CaffeReader(path=caffe_db_folder,
shard_id=device_id,
num_shards=num_gpus)
if self.is_fused_decoder:
self.decode = ops.HostDecoderCrop(device="cpu",
crop=(224, 224),
crop_pos_x=0.3,
crop_pos_y=0.2,
output_type=types.RGB)
else:
self.decode = ops.HostDecoder(device="cpu", output_type=types.RGB)
self.crop = ops.Crop(device=self.device,
crop=(224, 224),
crop_pos_x=0.3,
crop_pos_y=0.2,
image_type=types.RGB)
def __init__(self, device, batch_size, num_threads=1, device_id=0, num_gpus=1,
is_new_cmn = False, output_dtype = types.FLOAT, output_layout = types.NHWC,
mirror_probability = 0.0, mean=[0., 0., 0.], std=[1., 1., 1.], pad_output=False):
super(CropMirrorNormalizePipeline, self).__init__(batch_size, num_threads, device_id, seed=7865)
self.device = device
self.is_new_cmn = is_new_cmn
self.input = ops.CaffeReader(path = caffe_db_folder, shard_id = device_id, num_shards = num_gpus)
self.decode = ops.HostDecoder(device = "cpu", output_type = types.RGB)
if self.is_new_cmn:
self.cmn = ops.NewCropMirrorNormalize(device = self.device,
output_dtype = output_dtype,
output_layout = output_layout,
crop = (224, 224),
crop_pos_x = 0.3,
crop_pos_y = 0.2,
image_type = types.RGB,
mean = mean,
std = std,
pad_output = pad_output)
else:
self.cmn = ops.CropMirrorNormalize(device = self.device,
output_dtype = output_dtype,
output_layout = output_layout,
crop = (224, 224),
crop_pos_x = 0.3,
crop_pos_y = 0.2,
image_type = types.RGB,
mean = mean,
std = std,
pad_output = pad_output)
self.coin = ops.CoinFlip(probability=mirror_probability)
class C2Pipe(Pipeline):
def __init__(self,
batch_size,
num_threads,
device_id,
pipelined=True,
async=True):
super(C2Pipe, self).__init__(batch_size,
num_threads,
device_id,
exec_pipelined=pipelined,
exec_async=async)
self.input = ops.ExternalSource()
self.decode = ops.HostDecoder(output_type=types.RGB)
self.rcm = ops.FastResizeCropMirror(crop=[224, 224])
self.np = ops.NormalizePermute(device="gpu",
output_dtype=types.FLOAT16,
mean=[128., 128., 128.],
std=[1., 1., 1.],
height=224,
width=224,
image_type=types.RGB)
self.uniform = ops.Uniform(range=(0., 1.))
self.resize_uniform = ops.Uniform(range=(256., 480.))
self.mirror = ops.CoinFlip(probability=0.5)
self.iter = 0
def __init__(self, file_list, file_root, crop_size,
batch_size, num_threads, device_id,
random_shuffle=True, seed=-1, mean=None, std=None,
num_samples=None):
super(DaliPipelineTrain, self).__init__(batch_size, num_threads,
device_id, seed=seed)
crop_size = _pair(crop_size)
if mean is None:
mean = [0.485 * 255, 0.456 * 255, 0.406 * 255]
if std is None:
std = [0.229 * 255, 0.224 * 255, 0.225 * 255]
if num_samples is None:
initial_fill = 4096
else:
initial_fill = min(4096, num_samples)
self.loader = ops.FileReader(file_root=file_root, file_list=file_list,
random_shuffle=random_shuffle,
initial_fill=initial_fill)
self.decode = ops.HostDecoder()
self.resize = ops.Resize(device="gpu", resize_x=256, resize_y=256)
# self.hue = ops.Hue(device="gpu")
# self.bright = ops.Brightness(device="gpu")
# self.cntrst = ops.Contrast(device="gpu")
# self.rotate = ops.Rotate(device="gpu")
# self.jitter = ops.Jitter(device="gpu")
random_area = (crop_size[0] / 256) * (crop_size[1] / 256)
random_area = _pair(random_area)
random_aspect_ratio = _pair(1.0)
self.rrcrop = ops.RandomResizedCrop(
device="gpu", size=crop_size, random_area=random_area,
random_aspect_ratio=random_aspect_ratio)
self.cmnorm = ops.CropMirrorNormalize(
device="gpu", crop=list(crop_size), mean=mean, std=std)
self.coin = ops.CoinFlip(probability=0.5)
def __init__(self, batch_size, num_threads, device_id, data_dir,
crop, size, dali_cpu=False):
super(DALIValPipe, self).__init__(
batch_size, num_threads, device_id, seed=12 + device_id
)
self.input = ops.FileReader(
file_root=data_dir, shard_id=device_id,
num_shards=1, random_shuffle=True
)
#let user decide which pipeline works him bets for RN version he runs
if dali_cpu:
dali_device = "cpu"
self.decode = ops.HostDecoder(
device=dali_device, output_type=types.RGB)
else:
dali_device = "gpu"
# This padding sets the size of the internal nvJPEG buffers to be
# able to handle all images from full-sized ImageNet
# without additional reallocations
self.decode = ops.nvJPEGDecoder(
device="mixed", output_type=types.RGB,
device_memory_padding=211025920, host_memory_padding=140544512
)
self.res = ops.Resize(
device=dali_device, resize_shorter=size)
self.cmnp = ops.CropMirrorNormalize(
device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NCHW,
crop=(crop, crop),
image_type=types.RGB,
mean=[0.485 * 255,0.456 * 255,0.406 * 255],
std=[0.229 * 255,0.224 * 255,0.225 * 255])
print('DALI "{0}" variant'.format(dali_device))
def __init__(self, batch_size, num_threads, device_id):
super(SimplePipeline, self).__init__(batch_size,
num_threads,
device_id,
seed=12)
self.input = ops.FileReader(file_root=FLAGS.image_dir)
self.decode = ops.HostDecoder(output_type=types.RGB)
def __init__(self,
device,
batch_size,
num_threads=1,
device_id=0,
num_gpus=1,
should_perform_cast=False):
super(CropCastPipeline, self).__init__(batch_size, num_threads,
device_id)
self.should_perform_cast = should_perform_cast
self.device = device
self.input = ops.CaffeReader(path=caffe_db_folder,
shard_id=device_id,
num_shards=num_gpus)
self.decode = ops.HostDecoder(device="cpu", output_type=types.RGB)
if self.should_perform_cast:
self.crop = ops.Crop(device=self.device,
crop=(224, 224),
crop_pos_x=0.3,
crop_pos_y=0.2,
image_type=types.RGB,
output_dtype=types.FLOAT)
self.crop2 = ops.Crop(device=self.device,
crop=(224, 224),
crop_pos_x=0.0,
crop_pos_y=0.0,
image_type=types.RGB,
output_dtype=types.UINT8)
else:
self.crop = ops.Crop(device=self.device,
crop=(224, 224),
crop_pos_x=0.3,
crop_pos_y=0.2,
image_type=types.RGB)
def __init__(self, batch_size, num_threads, device_id, num_gpus):
super(HybridPipe, self).__init__(batch_size,
num_threads,
device_id)
self.input = ops.CaffeReader(path = caffe_db_folder, shard_id = device_id, num_shards = num_gpus)
self.decode = ops.HostDecoder(device = "cpu", output_type = types.RGB)
self.dump_cpu = ops.DumpImage(device = "cpu", suffix = "cpu")
self.dump_gpu = ops.DumpImage(device = "gpu", suffix = "gpu")
def __init__(self, function, batch_size, num_threads=1, device_id=0):
super(PythonOperatorPipeline, self).__init__(batch_size, num_threads, device_id,
exec_async=False,
exec_pipelined=False,
seed=1234)
self.input = ops.CaffeReader(path = caffe_db_folder, random_shuffle=False)
self.decode = ops.HostDecoder(device = 'cpu', output_type = types.RGB)
self.python_function = ops.PythonFunction(function=function)
def __init__(self, batch_size, num_threads, device_id):
super(ShuffledSimplePipeline, self).__init__(batch_size,
num_threads,
device_id,
seed=12)
self.input = ops.FileReader(file_root=FLAGS.image_dir,
random_shuffle=True,
initial_fill=21)
self.decode = ops.HostDecoder(output_type=types.RGB)
def __init__(self, batch_size,function, num_threads=1, device_id=0, num_gpus=1 ):
super(WaterPythonPipeline, self).__init__(batch_size,
num_threads,
device_id,
exec_async=False,
exec_pipelined=False)
self.input = ops.CaffeReader(path = caffe_db_folder, shard_id = device_id, num_shards = num_gpus)
self.decode = ops.HostDecoder(device = "cpu", output_type = types.RGB)
self.water = ops.PythonFunction(function=function)
def __init__(self, batch_size, num_threads=1, device_id=0):
super(CommonPipeline, self).__init__(batch_size,
num_threads,
device_id,
exec_async=False,
exec_pipelined=False,
seed=1234)
self.input = ops.CaffeReader(path=caffe_db_folder,
random_shuffle=False)
self.decode = ops.HostDecoder(device='cpu', output_type=types.RGB)
def __init__(self, batch_size, num_threads, device_id, _seed, image_dir):
super(CommonPipeline, self).__init__(batch_size,
num_threads,
device_id,
seed=_seed,
exec_async=False,
exec_pipelined=False)
self.input = ops.FileReader(file_root=image_dir)
self.decode = ops.HostDecoder(output_type=types.RGB)
self.resize = ops.PythonFunction(function=resize)
def __init__(self, device, batch_size, num_threads=1, device_id=0, num_gpus=1, decoder_only=False):
super(NoCropPipeline, self).__init__(batch_size, num_threads, device_id)
self.decoder_only = decoder_only
self.device = device
self.input = ops.CaffeReader(path = caffe_db_folder, shard_id = device_id, num_shards = num_gpus)
self.decode = ops.HostDecoder(device = "cpu", output_type = types.RGB)
if not self.decoder_only:
self.cmn = ops.NewCropMirrorNormalize(device = self.device,
image_type = types.RGB,
output_dtype = types.UINT8,
output_layout = types.NHWC)
def __init__(self,
batch_size=1,
file_root="/data_dir/540p/frames/train",
sequence_length=3):
super(SimpleSequencePipeline, self).__init__(batch_size,
num_threads=1,
device_id=0,
seed=12)
self.input = ops.SequenceReader(file_root=file_root,
sequence_length=sequence_length)
self.decode = ops.HostDecoder(output_type=types.RGB,
decode_sequences=True)
def __init__(self,
batch_size,
device_id,
file_root,
annotations_file,
num_gpus,
output_fp16=False,
output_nhwc=False,
pad_output=False,
num_threads=1,
seed=15):
super(COCOPipeline, self).__init__(batch_size=batch_size,
device_id=device_id,
num_threads=num_threads,
seed=seed)
if torch.distributed.is_initialized():
shard_id = torch.distributed.get_rank()
else:
shard_id = 0
self.input = ops.COCOReader(file_root=file_root,
annotations_file=annotations_file,
shard_id=shard_id,
num_shards=num_gpus,
ratio=True,
ltrb=True,
random_shuffle=True,
skip_empty=True)
self.decode = ops.HostDecoder(device="cpu", output_type=types.RGB)
# Augumentation techniques
self.crop = ops.SSDRandomCrop(device="cpu", num_attempts=1)
self.twist = ops.ColorTwist(device="gpu")
self.resize = ops.Resize(device="gpu", resize_x=300, resize_y=300)
output_dtype = types.FLOAT16 if output_fp16 else types.FLOAT
output_layout = types.NHWC if output_nhwc else types.NCHW
self.normalize = ops.CropMirrorNormalize(device="gpu",
crop=(300, 300),
mean=[0.0, 0.0, 0.0],
std=[255.0, 255.0, 255.0],
mirror=0,
output_dtype=output_dtype,
output_layout=output_layout,
pad_output=pad_output)
# Random variables
self.rng1 = ops.Uniform(range=[0.5, 1.5])
self.rng2 = ops.Uniform(range=[0.875, 1.125])
self.rng3 = ops.Uniform(range=[-0.5, 0.5])
def __init__(self,
args,
batch_size,
num_threads,
device_id,
rec_path,
idx_path,
shard_id,
num_shards,
crop_shape,
nvjpeg_padding,
prefetch_queue=3,
resize_shp=None,
output_layout=types.NCHW,
pad_output=True,
dtype='float16',
dali_cpu=False):
super(HybridValPipe,
self).__init__(batch_size,
num_threads,
device_id,
seed=12 + device_id,
prefetch_queue_depth=prefetch_queue)
self.input = ops.MXNetReader(path=[rec_path],
index_path=[idx_path],
random_shuffle=False,
shard_id=shard_id,
num_shards=num_shards)
if dali_cpu:
dali_device = "cpu"
self.decode = ops.HostDecoder(device=dali_device,
output_type=types.RGB)
else:
dali_device = "gpu"
self.decode = ops.nvJPEGDecoder(
device="mixed",
output_type=types.RGB,
device_memory_padding=nvjpeg_padding,
host_memory_padding=nvjpeg_padding)
self.resize = ops.Resize(
device=dali_device,
resize_shorter=resize_shp) if resize_shp else None
self.cmnp = ops.CropMirrorNormalize(
device="gpu",
output_dtype=types.FLOAT16 if dtype == 'float16' else types.FLOAT,
output_layout=output_layout,
crop=crop_shape,
pad_output=pad_output,
image_type=types.RGB,
mean=args.rgb_mean,
std=args.rgb_std)
def __init__(self, data_path, batch_size, num_threads, device_id, decoder):
super(DecoderPipeline, self).__init__(batch_size,
num_threads,
device_id,
prefetch_queue_depth=1)
self.input = ops.FileReader(file_root=data_path,
shard_id=0,
num_shards=1)
if decoder == 'nvJPEGDecoder':
self.decode = ops.nvJPEGDecoder(device="mixed",
output_type=types.RGB)
elif decoder == 'HostDecoder':
self.decode = ops.HostDecoder(device="cpu", output_type=types.RGB)
def __init__(self,
device,
batch_size,
num_threads=1,
device_id=0,
num_gpus=1,
is_vertical=0,
is_horizontal=1):
super(FlipPipeline, self).__init__(batch_size, num_threads, device_id)
self.device = device
self.input = ops.CaffeReader(path=caffe_db_folder,
shard_id=device_id,
num_shards=num_gpus)
self.decode = ops.HostDecoder(device="cpu", output_type=types.RGB)
self.flip = ops.Flip(device=self.device,
vertical=is_vertical,
horizontal=is_horizontal)
def __init__(self,
batch_size,
num_threads,
device_id,
data_dir,
crop,
dali_cpu=False,
fp16=False):
super(HybridTrainPipe, self).__init__(batch_size,
num_threads,
device_id,
seed=12 + device_id)
self.input = ops.FileReader(file_root=data_dir,
shard_id=args.local_rank,
num_shards=args.world_size,
random_shuffle=True)
#let user decide which pipeline works him bets for RN version he runs
if dali_cpu:
dali_device = "cpu"
self.decode = ops.HostDecoder(device=dali_device,
output_type=types.RGB)
else:
dali_device = "gpu"
# This padding sets the size of the internal nvJPEG buffers to be able to handle all images from full-sized ImageNet
# without additional reallocations
self.decode = ops.nvJPEGDecoder(device="mixed",
output_type=types.RGB,
device_memory_padding=211025920,
host_memory_padding=140544512)
self.rrc = ops.RandomResizedCrop(device=dali_device, size=(crop, crop))
output_dtype = types.FLOAT
if fp16:
output_dtype = types.FLOAT16
self.cmnp = ops.CropMirrorNormalize(
device=dali_device, # 1: changed from "cpu" to dali_device
output_dtype=output_dtype, # 2: output to fp16 when fp16 training
output_layout=types.NCHW,
crop=(crop, crop),
image_type=types.RGB,
mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
self.coin = ops.CoinFlip(probability=0.5)
print('DALI "{0}" variant'.format(dali_device))
def __init__(self,
batch_size,
num_threads,
device_id,
data_dir,
crop,
num_shards,
dali_cpu=False):
super(HybridTrainPipe, self).__init__(batch_size,
num_threads,
device_id,
seed=12 + device_id)
self.input = ops.MXNetReader(
path=[os.path.join(data_dir, "_train.rec")],
index_path=[os.path.join(data_dir, "_train.idx")],
random_shuffle=True,
shard_id=device_id,
num_shards=num_shards)
# self.input = ops.FileReader(file_root=data_dir, shard_id=0, num_shards=4, random_shuffle=True)
# let user decide which pipeline works him bets for RN version he runs
if dali_cpu:
dali_device = "cpu"
self.decode = ops.HostDecoder(device=dali_device,
output_type=types.RGB)
else:
dali_device = "gpu"
# This padding sets the size of the internal nvJPEG buffers to be able to handle all images from full-sized ImageNet
# without additional reallocations
self.decode = ops.nvJPEGDecoder(device="mixed",
output_type=types.RGB)
self.rrc = ops.RandomResizedCrop(device=dali_device, size=(crop, crop))
self.cmnp = ops.CropMirrorNormalize(
device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NCHW,
crop=(crop, crop),
image_type=types.RGB,
mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
self.coin = ops.CoinFlip(probability=0.5)
print('DALI "{0}" variant'.format(dali_device))
def __init__(self,
batch_size,
num_threads,
device_id,
data_dir,
crop,
size,
num_shards,
dali_cpu=False):
super(HybridValPipe, self).__init__(batch_size,
num_threads,
device_id,
seed=12 + device_id)
self.input = ops.MXNetReader(
path=[os.path.join(data_dir, "_val.rec")],
index_path=[os.path.join(data_dir, "_val.idx")],
random_shuffle=False,
shard_id=device_id,
num_shards=num_shards)
# self.input = ops.FileReader(file_root=data_dir, shard_id=0, num_shards=4, random_shuffle=False)
if dali_cpu:
dali_device = "cpu"
self.decode = ops.HostDecoder(device=dali_device,
output_type=types.RGB)
else:
dali_device = "gpu"
# This padding sets the size of the internal nvJPEG buffers to be able to handle all images from full-sized ImageNet
# without additional reallocations
self.decode = ops.nvJPEGDecoder(device="mixed",
output_type=types.RGB)
# self.decode = ops.nvJPEGDecoder(device="mixed", output_type=types.RGB)
self.res = ops.Resize(device=dali_device, resize_shorter=size)
self.cmnp = ops.CropMirrorNormalize(
device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NCHW,
crop=(crop, crop),
image_type=types.RGB,
mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
def __init__(self, batch_size, num_threads, device_id, num_gpus):
super(HybridPipe, self).__init__(batch_size,
num_threads,
device_id)
self.input = ops.CaffeReader(path = caffe_db_folder, shard_id = device_id, num_shards = num_gpus)
self.decode = ops.HostDecoder(device = "cpu", output_type = types.RGB)
self.cmnp_nhwc = ops.CropMirrorNormalize(device = "gpu",
output_dtype = types.FLOAT,
output_layout = types.NHWC,
crop = (224, 224),
image_type = types.RGB,
mean = [128., 128., 128.],
std = [1., 1., 1.])
self.cmnp_nchw = ops.CropMirrorNormalize(device = "gpu",
output_dtype = types.FLOAT,
output_layout = types.NCHW,
crop = (224, 224),
image_type = types.RGB,
mean = [128., 128., 128.],
std = [1., 1., 1.])
def __init__(self, device, batch_size, pos_size_iter, num_threads=1, device_id=0, is_fused_decoder=False):
super(SlicePipeline, self).__init__(batch_size,
num_threads,
device_id,
seed=1234)
self.is_fused_decoder = is_fused_decoder
self.pos_size_iter = pos_size_iter
self.device = device
self.input = ops.CaffeReader(path = caffe_db_folder, random_shuffle=False)
self.input_crop_pos = ops.ExternalSource()
self.input_crop_size = ops.ExternalSource()
if self.is_fused_decoder:
self.decode = ops.HostDecoderSlice(device = 'cpu',
output_type = types.RGB)
else:
self.decode = ops.HostDecoder(device = "cpu",
output_type = types.RGB)
self.slice = ops.Slice(device = device,
image_type = types.RGB)
def __init__(self, args, batch_size, num_threads, device_id, rec_path, idx_path,
shard_id, num_shards, crop_shape, nvjpeg_padding, prefetch_queue=3,
output_layout=types.NCHW, pad_output=True, dtype='float16', dali_cpu=False):
super(HybridTrainPipe, self).__init__(batch_size, num_threads, device_id, seed=12 + device_id, prefetch_queue_depth = prefetch_queue)
self.input = ops.MXNetReader(path=[rec_path], index_path=[idx_path],
random_shuffle=args.shuffle, shard_id=shard_id, num_shards=num_shards)
if dali_cpu:
dali_device = "cpu"
if args.dali_fuse_decoder:
self.decode = ops.HostDecoderRandomCrop(device=dali_device, output_type=types.RGB)
else:
self.decode = ops.HostDecoder(device=dali_device, output_type=types.RGB)
else:
dali_device = "gpu"
if args.dali_fuse_decoder:
# self.decode = ops.nvJPEGDecoderRandomCrop(device="mixed", output_type=types.RGB,
self.decode = ops.ImageDecoderRandomCrop(device="mixed", output_type=types.RGB,
random_area=args.random_area, random_aspect_ratio=args.random_aspect_ratio,
device_memory_padding=nvjpeg_padding, host_memory_padding=nvjpeg_padding)
else:
# self.decode = ops.nvJPEGDecoder(device="mixed", output_type=types.RGB,
self.decode = ops.ImageDecoder(device="mixed", output_type=types.RGB,
device_memory_padding=nvjpeg_padding, host_memory_padding=nvjpeg_padding)
if args.dali_fuse_decoder:
self.resize = ops.Resize(device=dali_device, resize_x=crop_shape[1], resize_y=crop_shape[0])
else:
self.resize = ops.RandomResizedCrop(device=dali_device, size=crop_shape, random_area=args.random_area,
random_aspect_ratio=args.random_aspect_ratio)
self.cmnp = ops.CropMirrorNormalize(device="gpu",
output_dtype=types.FLOAT16 if dtype == 'float16' else types.FLOAT,
output_layout=output_layout, crop=crop_shape, pad_output=pad_output,
image_type=types.RGB, mean=args.rgb_mean, std=args.rgb_std)
if args.random_mirror:
self.coin = ops.CoinFlip(probability=0.5)
