def __init__(self, params, num_threads, device_id):
super(DaliPipeline, self).__init__(params.batch_size,
num_threads,
device_id,
seed=12)
dii = DaliInputIterator(params, device_id)
self.no_copy = params.no_copy
if self.no_copy:
print("Use Zero Copy ES")
self.source = ops.ExternalSource(device="gpu",
source=dii,
num_outputs=2,
layout=["DHWC", "DHWC"],
no_copy=self.no_copy)
self.do_rotate = True if params.rotate_input == 1 else False
print("Enable Rotation" if self.do_rotate else "Disable Rotation")
self.rng_angle = ops.Uniform(device="cpu", range=[-1.5, 2.5])
self.icast = ops.Cast(device="cpu", dtype=types.INT32)
self.fcast = ops.Cast(device="cpu", dtype=types.FLOAT)
self.rotate1 = ops.Rotate(device="gpu",
axis=(1, 0, 0),
interp_type=types.INTERP_LINEAR)
self.rotate2 = ops.Rotate(device="gpu",
axis=(0, 1, 0),
interp_type=types.INTERP_LINEAR)
self.rotate3 = ops.Rotate(device="gpu",
axis=(0, 0, 1),
interp_type=types.INTERP_LINEAR)
self.transpose = ops.Transpose(device="gpu", perm=[3, 0, 1, 2])
def __init__(self,
batch_size,
layout,
iterator,
pos_size_iter,
num_threads=1,
device_id=0,
num_gpus=1,
axes=None,
axis_names=None,
normalized_anchor=True,
normalized_shape=True,
input_type=types.FLOAT,
output_type=None):
super().__init__(batch_size,
num_threads,
device_id,
seed=12345,
exec_async=False,
exec_pipelined=False)
self.device = "cpu"
self.layout = layout
self.iterator = iterator
self.pos_size_iter = pos_size_iter
self.inputs = ops.ExternalSource()
self.input_crop_pos = ops.ExternalSource()
self.input_crop_size = ops.ExternalSource()
self.cast_in = ops.Cast(dtype=input_type)
function = partial(slice_func_helper, axes, axis_names, self.layout,
normalized_anchor, normalized_shape)
self.slice = ops.PythonFunction(function=function,
output_layouts=layout)
self.output_type = output_type
if self.output_type is not None:
self.cast_out = ops.Cast(dtype=output_type)
def __init__(self,
DATA_PATH,
input_height,
batch_size,
copies,
stage,
num_threads,
device_id,
seed=1729):
super(SimCLRTransform, self).__init__(batch_size,
num_threads,
device_id,
seed=seed)
#this lets our pytorch compat function find the length of our dataset
self.num_samples = len(ImageFolder(DATA_PATH))
self.copies = copies
self.input_height = input_height
self.stage = stage
self.input = ops.FileReader(file_root=DATA_PATH,
random_shuffle=True,
seed=seed)
self.to_int64 = ops.Cast(dtype=types.INT64, device="gpu")
self.to_int32_cpu = ops.Cast(dtype=types.INT32, device="cpu")
self.coin = ops.random.CoinFlip(probability=0.5)
self.uniform = ops.random.Uniform(range=[0.6, 0.9])
self.blur_amt = ops.random.Uniform(values=[
float(i) for i in range(1, int(0.1 * self.input_height), 2)
])
self.angles = ops.random.Uniform(range=[0, 360])
self.cast = ops.Cast(dtype=types.FLOAT, device='gpu')
self.decode = ops.ImageDecoder(device='mixed', output_type=types.RGB)
self.crop = ops.RandomResizedCrop(size=self.input_height,
minibatch_size=batch_size,
random_area=[0.75, 1.0],
device="gpu")
self.resize = ops.Resize(resize_x=self.input_height,
resize_y=self.input_height,
device="gpu")
self.flip = ops.Flip(vertical=self.coin(),
horizontal=self.coin(),
device="gpu")
self.colorjit_gray = ops.ColorTwist(brightness=self.uniform(),
contrast=self.uniform(),
hue=self.uniform(),
saturation=self.uniform(),
device="gpu")
self.blur = ops.GaussianBlur(window_size=self.to_int32_cpu(
self.blur_amt()),
device="gpu")
self.rotate = ops.Rotate(
angle=self.angles(),
keep_size=True,
interp_type=types.DALIInterpType.INTERP_LINEAR,
device="gpu")
self.swapaxes = ops.Transpose(perm=[2, 0, 1], device="gpu")
def __init__(self,
batch_size,
num_threads,
device_id,
file_list,
sequence_length,
seg_num,
seg_length,
resize_shorter_scale,
crop_target_size,
is_training=False,
initial_prefetch_size=10,
num_shards=1,
shard_id=0,
dali_mean=0.,
dali_std=1.0):
super(VideoPipe, self).__init__(batch_size, num_threads, device_id)
self.input = ops.VideoReader(device="gpu",
file_list=file_list,
sequence_length=sequence_length,
seg_num=seg_num,
seg_length=seg_length,
is_training=is_training,
num_shards=num_shards,
shard_id=shard_id,
random_shuffle=is_training,
initial_fill=initial_prefetch_size)
# the sequece data read by ops.VideoReader is of shape [F, H, W, C]
# Because the ops.Resize does not support sequence data,
# it will be transposed into [H, W, F, C],
# then reshaped to [H, W, FC], and then resized like a 2-D image.
self.transpose = ops.Transpose(device="gpu", perm=[1, 2, 0, 3])
self.reshape = ops.Reshape(device="gpu",
rel_shape=[1.0, 1.0, -1],
layout='HWC')
self.resize = ops.Resize(device="gpu",
resize_shorter=resize_shorter_scale)
# crops and mirror are applied by ops.CropMirrorNormalize.
# Normalization will be implemented in paddle due to the difficulty of dimension broadcast,
# It is not sure whether dimension broadcast can be implemented correctly by dali, just take the Paddle Op instead.
self.pos_rng_x = ops.Uniform(range=(0.0, 1.0))
self.pos_rng_y = ops.Uniform(range=(0.0, 1.0))
self.mirror_generator = ops.Uniform(range=(0.0, 1.0))
self.cast_mirror = ops.Cast(dtype=types.DALIDataType.INT32)
self.crop_mirror_norm = ops.CropMirrorNormalize(
device="gpu",
crop=[crop_target_size, crop_target_size],
mean=dali_mean,
std=dali_std)
self.reshape_back = ops.Reshape(device="gpu",
shape=[
seg_num, seg_length * 3,
crop_target_size, crop_target_size
],
layout='FCHW')
self.cast_label = ops.Cast(device="gpu",
dtype=types.DALIDataType.INT64)
def __init__(self, params, num_threads, device_id):
super(DaliPipeline, self).__init__(params.batch_size,
num_threads,
device_id,
seed=12)
with h5py.File(params.data_path, 'r') as f:
# load hydro and clean up
Hydro = f['Hydro'][...]
self.Hydro = types.Constant(Hydro,
shape=Hydro.shape,
layout="DHWC",
device="cpu")
del Hydro
# load nbody and clean up
Nbody = f['Nbody'][...]
self.Nbody = types.Constant(Nbody,
shape=Nbody.shape,
layout="DHWC",
device="cpu")
del Nbody
#self.ndummy = np.zeros((20, 20, 20, 4), dtype=np.float32)
#self.hdummy = np.zeros((20, 20, 20, 5), dtype=np.float32)
#self.Nbody = types.Constant(self.ndummy, shape = self.ndummy.shape, layout = "DHWC", device="cpu")
#self.Hydro = types.Constant(self.hdummy, shape = self.hdummy.shape, layout = "DHWC", device="cpu")
#self.Nbody = ops.Constant(fdata = self.ndummy.flatten().tolist(), shape = self.ndummy.shape, layout = "DHWC", device = "cpu")
#self.Hydro = ops.Constant(fdata = self.hdummy.flatten().tolist(), shape = self.hdummy.shape, layout = "DHWC", device = "cpu")
self.do_rotate = True if params.rotate_input == 1 else False
print("Enable Rotation" if self.do_rotate else "Disable Rotation")
self.rng_angle = ops.Uniform(device="cpu", range=[-1.5, 2.5])
self.rng_pos = ops.Uniform(device="cpu", range=[0., 1.])
self.icast = ops.Cast(device="cpu", dtype=types.INT32)
self.fcast = ops.Cast(device="cpu", dtype=types.FLOAT)
self.crop = ops.Crop(device="cpu",
crop_d=params.data_size,
crop_h=params.data_size,
crop_w=params.data_size)
self.rotate1 = ops.Rotate(device="gpu",
axis=(1, 0, 0),
interp_type=types.INTERP_LINEAR)
self.rotate2 = ops.Rotate(device="gpu",
axis=(0, 1, 0),
interp_type=types.INTERP_LINEAR)
self.rotate3 = ops.Rotate(device="gpu",
axis=(0, 0, 1),
interp_type=types.INTERP_LINEAR)
self.transpose = ops.Transpose(device="gpu", perm=[3, 0, 1, 2])
def __init__(self, data_iterator, batch_size, num_threads, device_id):
super(ExternalSourcePipeline, self).__init__(batch_size,
num_threads,
device_id,
seed=12)
self.data_iterator = data_iterator
self.src = ops.ExternalSource()
# self.targ = ops.ExternalSource()
self.decode = ops.ImageDecoder(
device="mixed",
output_type=types.RGB,
)
self.cast = ops.Cast(device="gpu", dtype=types.INT32)
self.resize = ops.Resize(device="gpu",
resize_x=750,
resize_y=750,
interp_type=types.INTERP_TRIANGULAR)
self.cmnp = ops.CropMirrorNormalize(
device="gpu",
dtype=types.FLOAT,
output_layout=types.NCHW,
# image_type=types.RGB,
mean=[104.0, 117.0, 123.0],
std=[1, 1, 1])
self.cmnp2 = ops.CropMirrorNormalize(
device="gpu",
dtype=types.FLOAT,
output_layout=types.NCHW,
# image_type=types.RGB,
mean=[0, 0, 0],
std=[255, 255, 255])
def __init__(self,
p: float = .5,
ampl_x: float = 10.0,
ampl_y: float = 10.0,
freq_x: float = 0.049087,
freq_y: float = 0.049087,
phase_x: float = 0.0,
phase_y: float = 0.0,
fill_value: float = 0.0):
"""Initialization
Args:
p (float, optional): Probability to apply this transformation. Defaults to .5.
ampl_x (float, optional): Amplitude of the wave in x direction.. Defaults to 10.0.
ampl_y (float, optional): Amplitude of the wave in y direction.. Defaults to 10.0.
freq_x (float, optional): Frequency of the wave in x direction. Defaults to 0.049087.
freq_y (float, optional): Frequency of the wave in y direction. Defaults to 0.049087.
phase_x (float, optional): Phase of the wave in x direction.. Defaults to 0.0.
phase_y (float, optional): Phase of the wave in y direction.. Defaults to 0.0.
fill_value (float, optional): Color value used for padding pixels. Defaults to 0.0.
"""
self.water_aug = ops.Water(device='gpu',
ampl_x=ampl_x,
ampl_y=ampl_y,
freq_x=freq_x,
freq_y=freq_y,
phase_x=phase_x,
phase_y=phase_y,
fill_value=fill_value)
self.rng = ops.CoinFlip(probability=p)
self.bool = ops.Cast(dtype=types.DALIDataType.BOOL)
def __init__(self,
p: float = .5,
hue_limit: Union[List, float] = 20.,
saturation_limit: Union[List, float] = .5,
value_limit: Union[List, float] = .5):
"""Initialization
Args:
p (float, optional): Probability to apply this transformation. Defaults to .5.
hue_limit (Union[List,float], optional): Range for changing hue in [min,max] value format. If provided as a single float, the range will be (-limit, limit). Defaults to 20..
saturation_limit (Union[List,float], optional): Factor multiplier range for changing saturation in [min,max] value format. If provided as a single float, the range will be 1 + (-limit, limit). Defaults to 0.5.
value_limit (Union[List,float], optional): Factor multiplier range for changing value in [min,max] value format. If provided as a single float, the range will be 1 + (-limit, limit). Defaults to 0.5.
"""
hue_limit = _check_and_convert_limit_value(hue_limit, None, 0)
saturation_limit = _check_and_convert_limit_value(saturation_limit)
value_limit = _check_and_convert_limit_value(value_limit)
self.hsv = ops.Hsv(device='gpu')
self.hue_uniform = ops.Uniform(range=hue_limit)
self.saturation_uniform = ops.Uniform(range=saturation_limit)
self.value_uniform = ops.Uniform(range=value_limit)
self.rng = ops.CoinFlip(probability=p)
self.bool = ops.Cast(dtype=types.DALIDataType.BOOL)
def __init__(self,
file_root,
file_list,
batch_size,
resize_shorter,
crop,
interp,
mean,
std,
device_id,
shard_id=0,
num_shards=1,
random_shuffle=False,
num_threads=4,
seed=42):
super(HybridValPipe, self).__init__(
batch_size, num_threads, device_id, seed=seed)
self.input = ops.FileReader(
file_root=file_root,
file_list=file_list,
shard_id=shard_id,
num_shards=num_shards,
random_shuffle=random_shuffle)
self.decode = ops.ImageDecoder(device="mixed", output_type=types.RGB)
self.res = ops.Resize(
device="gpu", resize_shorter=resize_shorter, interp_type=interp)
self.cmnp = ops.CropMirrorNormalize(
device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NCHW,
crop=(crop, crop),
image_type=types.RGB,
mean=mean,
std=std)
self.to_int64 = ops.Cast(dtype=types.INT64, device="gpu")
def __init__(self,
device,
batch_size,
output_type,
input_type,
fixed_size=None,
num_threads=3,
device_id=0,
num_gpus=1):
super(RotatePipeline, self).__init__(batch_size,
num_threads,
device_id,
seed=7865,
exec_async=False,
exec_pipelined=False)
self.name = device
self.input = ops.CaffeReader(path=caffe_db_folder,
shard_id=device_id,
num_shards=num_gpus)
self.decode = ops.ImageDecoder(device="cpu", output_type=types.RGB)
if input_type != dali.types.UINT8:
self.cast = ops.Cast(device=device, dtype=input_type)
else:
self.cast = None
# TODO(michalz): When we move from Support to CPU operators, replace hardcoded angle
# with one taken from the distribution below
# self.uniform = ops.Uniform(range = (-180.0, 180.0), seed = 42);
self.rotate = ops.Rotate(device=device,
size=fixed_size,
angle=30,
fill_value=42,
output_dtype=output_type)
def __init__(self, args):
super(TFRecordDetectionPipeline,
self).__init__(args.batch_size, args.num_workers, 0, 0)
self.input = ops.TFRecordReader(
path=os.path.join(test_dummy_data_path, 'small_coco.tfrecord'),
index_path=os.path.join(test_dummy_data_path,
'small_coco_index.idx'),
features={
'image/encoded':
tfrec.FixedLenFeature((), tfrec.string, ""),
'image/object/class/label':
tfrec.VarLenFeature([1], tfrec.int64, 0),
'image/object/bbox':
tfrec.VarLenFeature([4], tfrec.float32, 0.0),
},
shard_id=0,
num_shards=1,
random_shuffle=False)
self.decode_gpu = ops.ImageDecoder(device="mixed",
output_type=types.RGB)
self.cast = ops.Cast(dtype=types.INT32)
self.box_encoder = ops.BoxEncoder(device="cpu",
criteria=0.5,
anchors=coco_anchors())
def __init__(self,
batch_size,
device,
data_dir,
mean,
std,
device_id=0,
shard_id=0,
num_shards=1,
num_threads=4,
seed=0):
super(DaliTransformsValPipeline,
self).__init__(batch_size, num_threads, device_id, seed)
# should we make this drive the device flags?
self.reader = ops.FileReader(file_root=data_dir,
shard_id=shard_id,
num_shards=num_shards,
random_shuffle=False)
self.decode = ops.ImageDecoder(device='mixed',
output_type=types.RGB,
memory_stats=True)
self.resize = ops.Resize(device=device,
size=[200, 300],
interp_type=types.INTERP_TRIANGULAR)
self.normalize = ops.CropMirrorNormalize(device=device,
dtype=types.FLOAT,
output_layout=types.NCHW) #,
#mean=mean*255, std=std*255)
#self.normalize = ops.Normalize(device=device, dtype=types.FLOAT)#, mean=mean, stddev=std)
self.to_int64 = ops.Cast(dtype=types.INT64, device=device)
def __init__(self, batch_size, num_threads, device_id = 0, seed = 0):
super(TestPipeline, self).__init__(batch_size, num_threads, device_id, seed)
self.input = ops.COCOReader(
file_root = file_root,
annotations_file = annotations_file,
shard_id = 0,
num_shards = 1,
ratio=False,
save_img_ids=True)
self.decode = ops.ImageDecoder(device = "cpu", output_type = types.RGB)
self.resize = ops.Resize(
device = "cpu",
image_type = types.RGB,
interp_type = types.INTERP_LINEAR)
self.cmn = ops.CropMirrorNormalize(
device = "cpu",
output_dtype = types.FLOAT,
crop = (224, 224),
image_type = types.RGB,
mean = [128., 128., 128.],
std = [1., 1., 1.])
self.res_uniform = ops.Uniform(range = (256.,480.))
self.uniform = ops.Uniform(range = (0.0, 1.0))
self.cast = ops.Cast(
device = "cpu",
dtype = types.FLOAT16)
def __init__(self, DATA_PATH, input_height, batch_size, num_threads,
device_id):
super(SimCLRTrainDataTransform, self).__init__(batch_size,
num_threads,
device_id,
seed=12)
self.COPIES = 3
self.input_height = input_height
self.input = ops.FileReader(file_root=DATA_PATH,
random_shuffle=True,
seed=12)
self.coin = ops.CoinFlip(probability=0.5)
self.uniform = ops.Uniform(range=[0.7, 1.3]) #-1 to 1
#read image (I think that has to be cpu, do a mixed operation to decode into gpu)
self.decode = ops.ImageDecoder(device='mixed', output_type=types.RGB)
self.crop = ops.RandomResizedCrop(size=self.input_height, device="gpu")
self.flip = ops.Flip(vertical=self.coin(),
horizontal=self.coin(),
device="gpu")
self.colorjit_gray = ops.ColorTwist(brightness=self.uniform(),
contrast=self.uniform(),
hue=self.uniform(),
saturation=self.uniform(),
device="gpu")
self.blur = ops.GaussianBlur(window_size=int(0.1 * self.input_height),
device="gpu")
self.swapaxes = ops.Transpose(perm=[2, 0, 1], device="gpu")
self.to_int64 = ops.Cast(dtype=types.INT64, device="gpu")
def __init__(self,
device,
batch_size,
output_type,
input_type,
fixed_size=None,
num_threads=3,
device_id=0,
num_gpus=1):
super(RotatePipeline, self).__init__(batch_size,
num_threads,
device_id,
seed=7865,
exec_async=False,
exec_pipelined=False)
self.name = device
self.input = ops.readers.Caffe(path=caffe_db_folder,
shard_id=device_id,
num_shards=num_gpus)
self.decode = ops.ImageDecoder(device="cpu", output_type=types.RGB)
if input_type != dali.types.UINT8:
self.cast = ops.Cast(device=device, dtype=input_type)
else:
self.cast = None
self.uniform = ops.random.Uniform(range=(-180.0, 180.0), seed=42)
self.rotate = ops.Rotate(device=device,
size=fixed_size,
fill_value=42,
dtype=output_type)
def __init__(self,
function,
batch_size,
iterator,
data_shape,
data_layout,
dtype,
num_threads=1,
device_id=0,
dictionary={},
default_value=0.0):
super(LookupTablePythonOpPipeline, self).__init__(batch_size,
num_threads,
device_id,
exec_async=False,
exec_pipelined=False)
self.iterator = iterator
self.inputs = ops.ExternalSource()
self.data_shape = data_shape
self.data_layout = data_layout
def lookup_table_func(input_data):
return function(input_data,
shape=data_shape,
dictionary=dictionary,
default_value=default_value)
self.lookup = ops.PythonFunction(function=lookup_table_func)
self.set_layout = ops.Reshape(layout=data_layout)
self.cast = ops.Cast(dtype=dtype)
def __init__(self,
batch_size,
num_threads,
device,
device_id=0,
shard_id=0,
num_shards=1,
seed=0):
super(TestPipeline, self).__init__(batch_size, num_threads, device_id,
seed)
self.device = device
self.input = ops.COCOReader(file_root=file_root,
annotations_file=annotations_file,
shard_id=shard_id,
num_shards=num_shards,
ratio=False,
save_img_ids=True)
self.decode = ops.ImageDecoder(
device='mixed' if device is 'gpu' else 'cpu',
output_type=types.RGB)
self.resize = ops.Resize(device=device,
image_type=types.RGB,
resize_x=224,
resize_y=224,
interp_type=types.INTERP_LINEAR)
self.cmn = ops.CropMirrorNormalize(device=device,
output_dtype=types.FLOAT,
image_type=types.RGB,
mean=[128., 128., 128.],
std=[1., 1., 1.])
self.cast = ops.Cast(device=device, dtype=types.INT16)
def __init__(self, tfrecord_files, idx_files,
batch_size, device_id=0, rank=0,
total_devices=1, num_threads=4):
super(TFRecordPipeline, self).__init__(batch_size,
num_threads,
device_id)
self.input = ops.TFRecordReader(path = tfrecord_files, index_path = idx_files,
shard_id = rank, num_shards = total_devices,
random_shuffle = True,
features = {"image/encoded" : tfrec.FixedLenFeature((), tfrec.string, ""),
'image/class/label': tfrec.FixedLenFeature([1], tfrec.int64, -1),
})
self.decode = ops.ImageDecoder(device = "mixed", output_type = types.RGB)
self.resize = ops.Resize(device = "gpu", resize_shorter = 256)
self.cmnp = ops.CropMirrorNormalize(device = "gpu",
output_dtype = types.FLOAT16,
crop = (224, 224),
image_type = types.RGB,
mean = [0, 0, 0],
std = [1., 1., 1.],
output_layout='HWC')
self.uniform = ops.Uniform(range = (0.0, 1.0))
self.flip = ops.CoinFlip()
self.brightness = ops.Uniform(range = (0.5, 1.5))
self.contrast = ops.Uniform(range = (0.8, 1.3))
self.cast = ops.Cast(device = "gpu", dtype = types.FLOAT16)
self.iter = 0
def __init__(self, device, batch_size, iterator, cast_dtypes, num_threads=1, device_id=0):
super(CastPipeline, self).__init__(batch_size, num_threads, device_id)
self.layout = "HWC"
self.device = device
self.iterator = iterator
self.inputs = ops.ExternalSource()
self.cast = [ops.Cast(device=device, dtype=dtype) for dtype in cast_dtypes]
def __init__(self, batch_size, num_threads, device_id, data_dir, crop,
size):
super(HybridValPipe, self).__init__(batch_size,
num_threads,
device_id,
seed=12 + device_id)
if torch.distributed.is_initialized():
local_rank = torch.distributed.get_rank()
world_size = torch.distributed.get_world_size()
else:
local_rank = 0
world_size = 1
self.input = ops.FileReader(file_root=data_dir,
shard_id=local_rank,
num_shards=world_size,
random_shuffle=False)
self.decode = ops.ImageDecoder(device="mixed", output_type=types.RGB)
self.res = ops.Resize(device="gpu",
resize_shorter=size,
interp_type=types.INTERP_TRIANGULAR)
self.cmnp = ops.CropMirrorNormalize(
device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NCHW,
crop=(crop, crop),
image_type=types.RGB,
# from https://github.com/Armour/pytorch-nn-practice/blob/master/utils/meanstd.py
mean=[0.50707516 * 255, 0.48654887 * 255, 0.44091784 * 255],
std=[0.26733429 * 255, 0.25643846 * 255, 0.27615047 * 255])
self.to_int64 = ops.Cast(dtype=types.INT64, device="gpu")
def __init__(self, file_list, batch_size, num_threads, device_id, external_data):
super(ExternalSourcePipeline, self).__init__(batch_size, num_threads, device_id)
self.input = ops.FileReader(file_list= file_list)
self.label = ops.ExternalSource()
self.crops = ops.ExternalSource()
self.decode = ops.ImageDecoder(device="mixed", output_type=types.RGB)
self.res = ops.Resize(device="gpu", resize_x=224, resize_y=224)
self.cast = ops.Cast(device="cpu", dtype=types.INT32)
self.external_data = external_data
self.iterator = iter(self.external_data)
def __init__(self,
file_root,
file_list,
batch_size,
resize_shorter,
crop,
min_area,
lower,
upper,
interp,
mean,
std,
device_id,
shard_id=0,
num_shards=1,
random_shuffle=True,
num_threads=4,
seed=42,
pad_output=False,
output_dtype=types.FLOAT):
super(HybridTrainPipe, self).__init__(batch_size,
num_threads,
device_id,
seed=seed)
self.input = ops.FileReader(file_root=file_root,
file_list=file_list,
shard_id=shard_id,
num_shards=num_shards,
random_shuffle=random_shuffle)
# set internal nvJPEG buffers size to handle full-sized ImageNet images
# without additional reallocations
device_memory_padding = 211025920
host_memory_padding = 140544512
self.decode = ops.ImageDecoderRandomCrop(
device='mixed',
output_type=types.RGB,
device_memory_padding=device_memory_padding,
host_memory_padding=host_memory_padding,
random_aspect_ratio=[lower, upper],
random_area=[min_area, 1.0],
num_attempts=100)
self.res = ops.Resize(device='gpu',
resize_x=crop,
resize_y=crop,
interp_type=interp)
self.cmnp = ops.CropMirrorNormalize(device="gpu",
output_dtype=output_dtype,
output_layout=types.NCHW,
crop=(crop, crop),
image_type=types.RGB,
mean=mean,
std=std,
pad_output=pad_output)
self.coin = ops.CoinFlip(probability=0.5)
self.to_int64 = ops.Cast(dtype=types.INT64, device="gpu")
def __init__(self, iterator, batch_size, num_threads, device_id):
super(ExternalSourcePipeline, self).__init__(batch_size,
num_threads,
device_id,
seed=12)
self.iterator = iterator
self.input = ops.ExternalSource()
self.input_label = ops.ExternalSource()
self.decode = ops.ImageDecoder(device = "mixed", output_type = types.RGB)
self.cast = ops.Cast(device = "gpu",
dtype = types.INT32)
def __init__(self,
device_id,
n_devices,
file_root,
file_list,
batch_size,
sample_rate=16000,
window_size=.02,
window_stride=.01,
nfeatures=64,
nfft=512,
frame_splicing_factor=3,
silence_threshold=-80,
dither=.00001,
preemph_coeff=.97,
lowfreq=0.0,
highfreq=0.0,
num_threads=1):
super().__init__(batch_size, num_threads, device_id, seed=42)
self.dither = dither
self.frame_splicing_factor = frame_splicing_factor
self.read = ops.readers.File(file_root=file_root, file_list=file_list, device="cpu",
shard_id=device_id, num_shards=n_devices)
self.decode = ops.AudioDecoder(device="cpu", dtype=types.FLOAT, downmix=True)
self.normal_distribution = ops.random.Normal(device="cpu")
self.preemph = ops.PreemphasisFilter(preemph_coeff=preemph_coeff)
self.spectrogram = ops.Spectrogram(device="cpu", nfft=nfft,
window_length=window_size * sample_rate,
window_step=window_stride * sample_rate)
self.mel_fbank = ops.MelFilterBank(device="cpu", sample_rate=sample_rate, nfilter=nfeatures,
normalize=True, freq_low=lowfreq, freq_high=highfreq)
self.log_features = ops.ToDecibels(device="cpu", multiplier=np.log(10), reference=1.0,
cutoff_db=-80)
self.get_shape = ops.Shapes(device="cpu")
self.normalize = ops.Normalize(axes=[0], device="cpu")
self.splicing_transpose = ops.Transpose(device="cpu", perm=[1, 0])
self.splicing_reshape = ops.Reshape(device="cpu", rel_shape=[-1, frame_splicing_factor])
self.splicing_pad = ops.Pad(axes=[0], fill_value=0, align=frame_splicing_factor, shape=[1],
device="cpu")
self.get_nonsilent_region = ops.NonsilentRegion(device="cpu", cutoff_db=silence_threshold)
self.trim_silence = ops.Slice(device="cpu", axes=[0])
self.to_float = ops.Cast(dtype=types.FLOAT)
def __init__(self, batch_size, num_threads, device_id, external_data):
super(ExternalSourcePipeline, self).__init__(batch_size,
num_threads,
device_id,
seed=12)
self.input = ops.ExternalSource()
self.input_label = ops.ExternalSource()
self.decode = ops.ImageDecoder(device="mixed", output_type=types.RGB)
self.res = ops.Resize(device="gpu", resize_x=240, resize_y=240)
self.cast = ops.Cast(device="gpu", dtype=types.UINT8)
self.external_data = external_data
self.iterator = iter(self.external_data)
def __init__(self, device, batch_size, num_threads=1, device_id=0, num_gpus=1, decoder_only=False):
super(NoCropPipeline, self).__init__(batch_size, num_threads, device_id)
self.decoder_only = decoder_only
self.device = device
self.input = ops.CaffeReader(path = caffe_db_folder, shard_id = device_id, num_shards = num_gpus)
self.decode = ops.ImageDecoder(device = "cpu", output_type = types.RGB)
if not self.decoder_only:
self.cast = ops.CropMirrorNormalize(device = self.device,
dtype = types.FLOAT,
output_layout = "HWC")
else:
self.cast = ops.Cast(device = self.device,
dtype = types.FLOAT)
def __init__(self,
device,
batch_size,
output_type,
input_type,
use_input,
num_threads=3,
device_id=0,
num_gpus=1,
inv_map=False):
super(WarpPipeline, self).__init__(batch_size,
num_threads,
device_id,
seed=7865,
exec_async=False,
exec_pipelined=False)
self.use_input = use_input
self.use_dynamic_size = use_input # avoid Cartesian product
self.name = device
self.input = ops.readers.Caffe(path=caffe_db_folder,
shard_id=device_id,
num_shards=num_gpus)
self.decode = ops.decoders.Image(device="cpu", output_type=types.RGB)
if input_type != dali.types.UINT8:
self.cast = ops.Cast(device=device, dtype=input_type)
else:
self.cast = None
static_size = None if self.use_dynamic_size else (240, 320)
fill = 12.5 if output_type == types.FLOAT else 42
output_type_arg = output_type if output_type != input_type else None
if use_input:
self.transform_source = ops.ExternalSource(
lambda: gen_transforms(self.max_batch_size, 10))
self.warp = ops.WarpAffine(device=device,
size=static_size,
fill_value=fill,
dtype=output_type_arg,
inverse_map=inv_map)
else:
warp_matrix = (0.1, 0.9, 10, 0.8, -0.2, -20)
self.warp = ops.WarpAffine(device=device,
size=static_size,
matrix=warp_matrix,
fill_value=fill,
dtype=output_type_arg,
inverse_map=inv_map)
self.iter = 0
def __init__(self, batch_size, num_threads, device_id, num_gpus,
external_data):
super(CXRImagePipeline, self).__init__(batch_size,
num_threads,
device_id,
seed=12)
self.input = ops.ExternalSource()
self.input_label = ops.ExternalSource()
self.decode = ops.ImageDecoder(device="mixed", output_type=types.GRAY)
#self.norm = ops.Normalize(device="cpu")
self.res = ops.Resize(device="gpu", resize_x=1024, resize_y=1024)
self.cast = ops.Cast(device="gpu", dtype=types.FLOAT)
self.external_data = external_data
self.iterator = iter(self.external_data)
def __init__(self, batch_size, num_threads, device_id, tfrecords,
idx_paths):
super(ResnetPipeline, self).__init__(batch_size, num_threads,
device_id)
# Transformation operations below.
# From https://docs.nvidia.com/deeplearning/sdk/dali-developer-guide/docs/supported_ops.html
self.input = ops.TFRecordReader(
path=tfrecords,
index_path=idx_paths,
features={
"image/encoded": tfrec.FixedLenFeature([], tfrec.string, ""),
"image/class/label": tfrec.FixedLenFeature([1], tfrec.float32,
0.0),
"image/class/text": tfrec.FixedLenFeature([], tfrec.string,
""),
"image/object/bbox/xmin":
tfrec.VarLenFeature(tfrec.float32, 0.0),
"image/object/bbox/ymin":
tfrec.VarLenFeature(tfrec.float32, 0.0),
"image/object/bbox/xmax":
tfrec.VarLenFeature(tfrec.float32, 0.0),
"image/object/bbox/ymax":
tfrec.VarLenFeature(tfrec.float32, 0.0)
})
self.decode = ops.nvJPEGDecoder(device="mixed",
cache_debug=True,
output_type=types.RGB)
self.resize = ops.Resize(device="gpu",
image_type=types.RGB,
interp_type=types.INTERP_LINEAR,
resize_shorter=256.)
self.cmn = ops.CropMirrorNormalize(device="gpu",
output_dtype=types.FLOAT,
crop=(224, 224),
image_type=types.RGB,
mean=[0., 0., 0.],
std=[1., 1., 1])
self.uniform = ops.Uniform(range=(0.0, 1.0))
self.transpose = ops.Transpose(device="gpu", perm=[0, 3, 1, 2])
self.cast = ops.Cast(device="gpu", dtype=types.INT32)
self.iter = 0
def __init__(self, batch_size, num_threads, device_id):
super(CMNvsCropPipe, self).__init__(batch_size, num_threads, device_id, seed = 12)
self.input = ops.CaffeReader(path = caffe_db_folder, shard_id = device_id, num_shards = 1)
self.decode = ops.nvJPEGDecoder(device = "mixed", output_type = types.RGB)
self.cmn = ops.CropMirrorNormalize(device = "gpu",
output_layout = types.NHWC,
output_dtype = types.FLOAT,
crop = (224, 224),
image_type = types.RGB,
mean = [0., 0., 0.],
std = [1., 1., 1.])
self.crop = ops.Crop(device = "gpu",
crop = (224, 224),
image_type = types.RGB)
self.uniform = ops.Uniform(range = (0.0, 1.0))
self.cast = ops.Cast(device = "gpu",
dtype = types.INT32)
