def __init__(self,
device,
batch_size,
num_threads=1,
device_id=0,
num_gpus=1,
should_perform_cast=False):
super(CropCastPipeline, self).__init__(batch_size, num_threads,
device_id)
self.should_perform_cast = should_perform_cast
self.device = device
self.input = ops.CaffeReader(path=caffe_db_folder,
shard_id=device_id,
num_shards=num_gpus)
self.decode = ops.HostDecoder(device="cpu", output_type=types.RGB)
if self.should_perform_cast:
self.crop = ops.Crop(device=self.device,
crop=(224, 224),
crop_pos_x=0.3,
crop_pos_y=0.2,
image_type=types.RGB,
output_dtype=types.FLOAT)
self.crop2 = ops.Crop(device=self.device,
crop=(224, 224),
crop_pos_x=0.0,
crop_pos_y=0.0,
image_type=types.RGB,
output_dtype=types.UINT8)
else:
self.crop = ops.Crop(device=self.device,
crop=(224, 224),
crop_pos_x=0.3,
crop_pos_y=0.2,
image_type=types.RGB)
def __init__(self, device, batch_size, layout, iterator, num_threads=1, device_id=0, tested_operator = None):
super(MultichannelSynthPipeline, self).__init__(batch_size,
num_threads,
device_id)
self.device = device
self.layout = layout
self.iterator = iterator
self.inputs = ops.ExternalSource()
self.tested_operator = tested_operator
if self.tested_operator == 'resize' or not self.tested_operator:
self.resize = ops.Resize(device = self.device,
resize_y = 900,
resize_x = 300,
min_filter=types.DALIInterpType.INTERP_LINEAR)
if self.tested_operator == 'crop' or not self.tested_operator:
self.crop = ops.Crop(device = self.device,
crop = (220, 224),
crop_pos_x = 0.3,
crop_pos_y = 0.2)
if self.tested_operator == 'transpose' or not self.tested_operator:
self.transpose = ops.Transpose(device = self.device,
perm = (1, 0, 2),
transpose_layout = False)
if self.tested_operator == 'normalize' or not self.tested_operator:
self.cmn = ops.CropMirrorNormalize(device = self.device,
std = 255.,
mean = 0.,
output_layout = "HWC",
dtype = types.FLOAT)
def __init__(self,
batch_size,
num_threads,
device_id,
data_dir,
crop=32,
dali_cpu=False,
local_rank=0,
world_size=1,
cutout=0):
super(HybridTrainPipe_CIFAR, self).__init__(batch_size,
num_threads,
device_id,
seed=12 + device_id)
self.iterator = iter(
CIFAR_INPUT_ITER(batch_size, 'train', root=data_dir))
dali_device = "gpu"
self.input = ops.ExternalSource()
self.input_label = ops.ExternalSource()
self.pad = ops.Paste(device=dali_device, ratio=1.25, fill_value=0)
self.uniform = ops.Uniform(range=(0., 1.))
self.crop = ops.Crop(device=dali_device, crop_h=crop, crop_w=crop)
self.cmnp = ops.CropMirrorNormalize(
device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NCHW,
image_type=types.RGB,
mean=[0.4914 * 255., 0.4822 * 255., 0.4465 * 255.],
std=[0.2023 * 255., 0.1994 * 255., 0.2010 * 255.])
self.coin = ops.CoinFlip(probability=0.5)
def __init__(self,
device,
batch_size,
iterator,
data_shape,
data_layout,
num_threads=1,
device_id=0):
super(Crop3dPipeline, self).__init__(batch_size, num_threads,
device_id)
self.device = device
self.iterator = iterator
self.inputs = ops.ExternalSource()
self.data_shape = data_shape
self.data_layout = data_layout
if self.data_layout == types.NDHWC:
D, H, W = self.data_shape[0], self.data_shape[1], self.data_shape[
2]
elif self.data_layout == types.NCDHW:
D, H, W = self.data_shape[1], self.data_shape[2], self.data_shape[
3]
else:
assert (False)
self.crop = ops.Crop(device=self.device,
crop_pos_z=0.1,
crop_pos_y=0.2,
crop_pos_x=0.3,
crop_d=D * 0.91,
crop_h=H * 0.85,
crop_w=W * 0.75,
image_type=types.RGB)
def __init__(self, root_dir, batch_size, num_threads, device_id,
use_shift_scale=False,
num_shards=None, shard_id=None):
super().__init__(batch_size, num_threads, device_id, seed=12)
self.random_angle = ops.Uniform(range=(0, 360.0))
self.random = ops.Uniform(range=(0.5, 1.5))
self.random_coin = ops.CoinFlip()
self.input = ops.FileReader(
file_root=root_dir, random_shuffle=True,
num_shards=num_shards, shard_id=shard_id,
)
self.decode = ops.ImageDecoder(device='mixed')
self.rotate = ops.Rotate(device='gpu', interp_type=types.INTERP_LINEAR)
self.crop = ops.Crop(device='gpu', crop=(224, 224))
self.use_shift_scale = use_shift_scale
if self.use_shift_scale:
self.shift_scale = ops.RandomResizedCrop(
device='gpu',
size=(224, 224),
interp_type=types.INTERP_LINEAR,
random_area=(0.3, 1.0),
)
self.flip = ops.Flip(device='gpu')
self.color_twist = ops.ColorTwist(device='gpu')
def __init__(self,
device,
batch_size,
iterator,
data_shape,
data_layout,
num_threads=1,
device_id=0,
crop_seq_as_depth=False):
super(Crop3dPipeline, self).__init__(batch_size, num_threads,
device_id)
self.device = device
self.iterator = iterator
self.inputs = ops.ExternalSource()
self.data_shape = data_shape
self.data_layout = data_layout
if self.data_layout == "DHWC":
D, H, W, _ = self.data_shape
elif self.data_layout == "CDHW":
_, D, H, W = self.data_shape
elif self.data_layout == "FHWC" and crop_seq_as_depth:
D, H, W, _ = self.data_shape
elif self.data_layout == "FCHW" and crop_seq_as_depth:
D, _, H, W = self.data_shape
else:
assert False
self.crop = ops.Crop(device=self.device,
crop_pos_z=0.1,
crop_pos_y=0.2,
crop_pos_x=0.3,
crop_d=D * 0.91,
crop_h=H * 0.85,
crop_w=W * 0.75)
def __init__(self,
dataset_dir,
batch_size,
num_threads,
device_id,
crop,
dali_cpu=False,
local_rank=0,
test=False):
super(HybridTrainPipe, self).__init__(batch_size,
num_threads,
device_id,
seed=666)
self.raw2rgbit = iter(
RAW2RGBInputIterator(dataset_dir, batch_size, test=test))
dali_device = "gpu"
self.input_data = ops.ExternalSource()
self.input_label = ops.ExternalSource()
self.data_decode = ops.ImageDecoder(device="mixed")
self.label_decode = ops.ImageDecoder(device="mixed",
output_type=types.RGB)
self.uniform = ops.Uniform(range=(0., 1.))
self.crop = ops.Crop(device=dali_device, crop_h=crop, crop_w=crop)
self.cmnp = ops.CropMirrorNormalize(device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NCHW)
self.coin = ops.CoinFlip(probability=0.5)
def __init__(self, params, device_id, files, labels):
super().__init__(params.batch_size,
params.num_gpus * 8,
device_id,
seed=params.seed)
# file_root有坑，并不是文件夹名字就是label，按照文件夹顺序(1, 10, 11, 2, 20, 21, ...)分别给与0，1，2，3，4...标签
self.input = ops.FileReader(files=files,
labels=labels,
random_shuffle=True)
self.decocer = ops.ImageDecoder(device='mixed', output_type=types.RGB)
self.resize = ops.Resize(device='gpu', resize_shorter=224)
self.pos_rng_x = ops.random.Uniform(range=(0.0, 1.0))
self.pos_rng_y = ops.random.Uniform(range=(0.0, 1.0))
self.crop = ops.Crop(device='gpu', crop_h=224, crop_w=224)
self.flip = ops.Flip(device='gpu')
self.coinflip = ops.random.CoinFlip(probability=0.5)
self.hsv = ops.Hsv(device='gpu')
self.saturation = ops.random.Uniform(range=(0.8, 1.0))
self.value = ops.random.Uniform(range=(0.8, 1.0))
mean = torch.Tensor(params.mean).unsqueeze(0).unsqueeze(0) * 255
std = torch.Tensor(params.std).unsqueeze(0).unsqueeze(0) * 255
self.normalize = ops.Normalize(axes=[0, 1],
mean=mean,
stddev=std,
device='gpu',
batch=False)
self.transpose = ops.Transpose(device='gpu', perm=[2, 0, 1])
def __init__(self,
device,
batch_size,
num_threads=1,
device_id=0,
num_gpus=1,
is_fused_decoder=False):
super(CropPipeline, self).__init__(batch_size, num_threads, device_id)
self.is_fused_decoder = is_fused_decoder
self.device = device
self.input = ops.CaffeReader(path=caffe_db_folder,
shard_id=device_id,
num_shards=num_gpus)
if self.is_fused_decoder:
self.decode = ops.HostDecoderCrop(device="cpu",
crop=(224, 224),
crop_pos_x=0.3,
crop_pos_y=0.2,
output_type=types.RGB)
else:
self.decode = ops.HostDecoder(device="cpu", output_type=types.RGB)
self.crop = ops.Crop(device=self.device,
crop=(224, 224),
crop_pos_x=0.3,
crop_pos_y=0.2,
image_type=types.RGB)
def __init__(self,
batch_size,
num_threads,
device_id,
data_dir,
crop,
size,
local_rank=0,
world_size=1):
super(HybridValPipe_CIFAR, self).__init__(batch_size,
num_threads,
device_id,
seed=12 + device_id)
self.iterator = iter(CIFAR_INPUT_ITER(batch_size, 'val',
root=data_dir))
self.input = ops.ExternalSource()
self.input_label = ops.ExternalSource()
self.pad = ops.Paste(device="gpu", ratio=1., fill_value=0)
self.uniform = ops.Uniform(range=(0., 1.))
self.crop = ops.Crop(device="gpu", crop_h=32, crop_w=32)
self.coin = ops.CoinFlip(probability=0.5)
self.flip = ops.Flip(device="gpu")
self.cmnp = ops.CropMirrorNormalize(device="gpu",
output_layout=types.NCHW,
mean=[125.31, 122.95, 113.87],
std=[63.0, 62.09, 66.70])
def __init__(self, batch_size, sequence_length, num_threads, device_id, file_root, crop_size, transforms=None):
super(VideoReaderPipeline, self).__init__(batch_size, num_threads, device_id, seed=12)
self.reader = ops.VideoReader(
device='gpu',
file_root=file_root,
sequence_length=sequence_length,
normalized=False,
random_shuffle=True,
image_type=types.RGB,
dtype=types.UINT8,
initial_fill=16
)
self.crop = ops.Crop(device="gpu", crop=crop_size, output_dtype=types.FLOAT)
self.transpose = ops.Transpose(device="gpu", perm=[3, 0, 1, 2])
self.uniform = ops.Uniform(range=(0.0, 1.0))
self.flip = ops.Flip(device="gpu", horizontal=1, vertical=0)
# self.normalize = ops.NormalizePermute(
# device="gpu",
# mean=[0.485, 0.456, 0.406],
# std=[0.229, 0.224, 0.225],
# width=224,
# height=224
# )
self.cmn = ops.CropMirrorNormalize(
device="gpu",
output_dtype=types.FLOAT,
# # output_layout=types.NCHW,
crop=(224, 224),
image_type=types.RGB,
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]
)
def __init__(self,
device,
batch_size,
num_threads=1,
device_id=0,
num_gpus=1,
is_old_crop=True):
super(CropSequencePipeline, self).__init__(batch_size, num_threads,
device_id)
self.device = device
VIDEO_FILES = [
test_data_video + '/' + file
for file in ['sintel_trailer_short.mp4']
]
self.input = ops.VideoReader(device='gpu',
filenames=VIDEO_FILES,
sequence_length=10,
shard_id=0,
num_shards=1,
random_shuffle=False,
normalized=True,
image_type=types.RGB,
dtype=types.UINT8)
if is_old_crop:
self.crop = ops.Crop(device=device,
crop=(224, 224),
crop_pos_x=0.3,
crop_pos_y=0.2,
image_type=types.RGB)
else:
self.crop = ops.NewCrop(device=device,
crop=(224, 224),
crop_pos_x=0.3,
crop_pos_y=0.2,
image_type=types.RGB)
def __init__(self,
device,
batch_size,
data_iterator,
num_threads=1,
device_id=0,
num_gpus=1,
crop_shape=(224, 224),
crop_x=0.3,
crop_y=0.2,
extra_outputs=False,
out_of_bounds_policy=None,
fill_values=None,
layout="HWC"):
super(CropSynthPipe, self).__init__(batch_size, num_threads, device_id)
self.device = device
self.extra_outputs = extra_outputs
self.inputs = ops.ExternalSource()
self.data_iterator = data_iterator
self.layout = layout
self.crop = ops.Crop(device=self.device,
crop=crop_shape,
crop_pos_x=crop_x,
crop_pos_y=crop_y,
out_of_bounds_policy=out_of_bounds_policy,
fill_values=fill_values)
def __init__(self,
root,
split,
batch_size,
device_id,
dali_cpu=False,
local_rank=0,
world_size=1,
num_workers=2,
augment=False,
resize=False,
resize_size=314,
crop=True,
crop_size=314,
shuffle=True,
flip=True,
rotate=False,
rotate_angle=10.0):
super(VOCDali, self).__init__(batch_size,
num_threads=num_workers,
device_id=device_id,
seed=12 + device_id)
self.iterator = iter(VOCIter(batch_size, split, root, shuffle))
self.split = split
assert self.split in ['train', 'val']
dali_device = "gpu" if not dali_cpu else "cpu"
self.input = ops.ExternalSource()
self.input_label = ops.ExternalSource()
self.is_flip = flip
self.is_rotate = rotate
self.is_augment = augment
self.is_crop = crop
self.is_resize = resize
self.decode = ops.ImageDecoder(device='mixed', output_type=types.RGB)
# self.cast = ops.Cast(device='gpu', dtype=types.INT32)
self.rng = ops.Uniform(range=(0., 1.))
self.coin = ops.CoinFlip(probability=0.5)
# 定义大小
self.resize = ops.Resize(device="gpu",
resize_x=resize_size,
resize_y=resize_size,
interp_type=types.INTERP_TRIANGULAR)
# 定义旋转
self.rotate = ops.Rotate(device="gpu", angle=rotate_angle)
# 定义翻转
self.flip = ops.Flip(device="gpu", vertical=1, horizontal=0)
# 定义剪裁
self.crop = ops.Crop(device=dali_device,
crop_h=crop_size,
crop_w=crop_size)
# 定义正则化
self.cmnp = ops.CropMirrorNormalize(
device=dali_device,
output_dtype=types.FLOAT,
output_layout=types.NCHW,
image_type=types.RGB,
mean=[0.45734706 * 255, 0.43338275 * 255, 0.40058118 * 255],
std=[0.23965294 * 255, 0.23532275 * 255, 0.2398498 * 255])
def __init__(self, device, batch_size, num_threads=1, device_id=0):
super(MultichannelPipeline, self).__init__(batch_size, num_threads,
device_id)
self.device = device
self.reader = ops.FileReader(file_root=multichannel_tiff_root)
decoder_device = 'mixed' if self.device == 'gpu' else 'cpu'
self.decoder = ops.ImageDecoder(device=decoder_device,
output_type=types.ANY_DATA)
self.resize = ops.Resize(device=self.device,
resize_y=900,
resize_x=300,
min_filter=types.DALIInterpType.INTERP_LINEAR)
self.crop = ops.Crop(device=self.device,
crop_h=220,
crop_w=224,
crop_pos_x=0.3,
crop_pos_y=0.2,
image_type=types.ANY_DATA)
self.transpose = ops.Transpose(device=self.device,
perm=(1, 0, 2),
transpose_layout=False)
self.cmn = ops.CropMirrorNormalize(device=self.device,
std=255.,
mean=0.,
output_layout="HWC",
output_dtype=types.FLOAT)
def __init__(self, size, fill_value):
self.rz_img = ops.Resize(device="gpu", resize_longer=size)
self.rz_ann = ops.Resize(device="gpu",
resize_longer=size,
interp_type=types.DALIInterpType.INTERP_NN)
self.cp_img = ops.Crop(device="gpu",
fill_values=fill_value,
out_of_bounds_policy='pad')
self.cp_ann = ops.Crop(device="gpu",
fill_values=0,
out_of_bounds_policy='pad')
self.size = ops.Constant(fdata=size)
self.pos = ops.Uniform(range=[0, 1])
self.scale = ops.Uniform(range=[0.667, 1.5])
self.shape = ops.Shapes(device="gpu")
def __init__(self, batch_size, num_threads, device_id, use_list):
super().__init__(batch_size, num_threads, device_id)
self.input = ops.ExternalSource(device="gpu")
self.crop = ops.Crop(device="gpu",
crop_h=32,
crop_w=32,
crop_pos_x=0.2,
crop_pos_y=0.2)
self.use_list = use_list
def __init__(self,
batch_size,
num_threads,
device_id,
data_dir,
crop,
size,
mean,
std,
local_rank=0,
world_size=1,
dali_cpu=False,
shuffle=False,
fp16=False):
# As we're recreating the Pipeline at every epoch, the seed must be -1 (random seed)
super(HybridValPipe, self).__init__(batch_size,
num_threads,
device_id,
seed=-1)
# Enabling read_ahead slowed down processing ~40%
# Note: initial_fill is for the shuffle buffer. As we only want to see every example once, this is set to 1
self.input = ops.FileReader(file_root=data_dir,
shard_id=local_rank,
num_shards=world_size,
random_shuffle=shuffle,
initial_fill=1)
if dali_cpu:
decode_device = "cpu"
self.dali_device = "cpu"
self.crop = ops.Crop(device="cpu", crop=(crop, crop))
else:
decode_device = "mixed"
self.dali_device = "gpu"
output_dtype = types.FLOAT
if fp16:
output_dtype = types.FLOAT16
self.cmnp = ops.CropMirrorNormalize(device="gpu",
output_dtype=output_dtype,
output_layout=types.NCHW,
crop=(crop, crop),
image_type=types.RGB,
mean=mean,
std=std)
self.decode = ops.ImageDecoder(device=decode_device,
output_type=types.RGB)
# Resize to desired size. To match torchvision dataloader, use triangular interpolation
self.res = ops.Resize(device=self.dali_device,
resize_shorter=size,
interp_type=types.INTERP_TRIANGULAR)
def __init__(self, device, batch_size, layout, iterator, num_threads=1, device_id=0):
super(CropSequencePipeline, self).__init__(batch_size,
num_threads,
device_id)
self.device = device
self.layout = layout
self.iterator = iterator
self.inputs = ops.ExternalSource()
self.crop = ops.Crop(device = self.device,
crop = (224, 224),
crop_pos_x = 0.3,
crop_pos_y = 0.2)
def __init__(self, params, num_threads, device_id):
super(DaliPipeline, self).__init__(params.batch_size,
num_threads,
device_id,
seed=12)
with h5py.File(params.data_path, 'r') as f:
# load hydro and clean up
Hydro = f['Hydro'][...]
self.Hydro = types.Constant(Hydro,
shape=Hydro.shape,
layout="DHWC",
device="cpu")
del Hydro
# load nbody and clean up
Nbody = f['Nbody'][...]
self.Nbody = types.Constant(Nbody,
shape=Nbody.shape,
layout="DHWC",
device="cpu")
del Nbody
#self.ndummy = np.zeros((20, 20, 20, 4), dtype=np.float32)
#self.hdummy = np.zeros((20, 20, 20, 5), dtype=np.float32)
#self.Nbody = types.Constant(self.ndummy, shape = self.ndummy.shape, layout = "DHWC", device="cpu")
#self.Hydro = types.Constant(self.hdummy, shape = self.hdummy.shape, layout = "DHWC", device="cpu")
#self.Nbody = ops.Constant(fdata = self.ndummy.flatten().tolist(), shape = self.ndummy.shape, layout = "DHWC", device = "cpu")
#self.Hydro = ops.Constant(fdata = self.hdummy.flatten().tolist(), shape = self.hdummy.shape, layout = "DHWC", device = "cpu")
self.do_rotate = True if params.rotate_input == 1 else False
print("Enable Rotation" if self.do_rotate else "Disable Rotation")
self.rng_angle = ops.Uniform(device="cpu", range=[-1.5, 2.5])
self.rng_pos = ops.Uniform(device="cpu", range=[0., 1.])
self.icast = ops.Cast(device="cpu", dtype=types.INT32)
self.fcast = ops.Cast(device="cpu", dtype=types.FLOAT)
self.crop = ops.Crop(device="cpu",
crop_d=params.data_size,
crop_h=params.data_size,
crop_w=params.data_size)
self.rotate1 = ops.Rotate(device="gpu",
axis=(1, 0, 0),
interp_type=types.INTERP_LINEAR)
self.rotate2 = ops.Rotate(device="gpu",
axis=(0, 1, 0),
interp_type=types.INTERP_LINEAR)
self.rotate3 = ops.Rotate(device="gpu",
axis=(0, 0, 1),
interp_type=types.INTERP_LINEAR)
self.transpose = ops.Transpose(device="gpu", perm=[3, 0, 1, 2])
def __init__(self, batch_size, num_threads, device_id):
super(TransposePipe, self).__init__(batch_size,
num_threads,
device_id,
seed=12)
self.input = ops.CaffeReader(path=caffe_db_folder,
shard_id=device_id,
num_shards=1)
self.decode = ops.nvJPEGDecoder(device="mixed",
output_type=types.RGB)
self.crop = ops.Crop(device="gpu",
crop=(224, 224),
image_type=types.RGB)
self.transpose = ops.Transpose(device="gpu", perm=[2, 0, 1])
def __init__(self, batch_size, num_threads, device_id):
super(CMNvsCropPipe, self).__init__(batch_size, num_threads, device_id, seed = 12)
self.input = ops.CaffeReader(path = caffe_db_folder, shard_id = device_id, num_shards = 1)
self.decode = ops.nvJPEGDecoder(device = "mixed", output_type = types.RGB)
self.cmn = ops.CropMirrorNormalize(device = "gpu",
output_layout = types.NHWC,
output_dtype = types.FLOAT,
crop = (224, 224),
image_type = types.RGB,
mean = [0., 0., 0.],
std = [1., 1., 1.])
self.crop = ops.Crop(device = "gpu",
crop = (224, 224),
image_type = types.RGB)
self.uniform = ops.Uniform(range = (0.0, 1.0))
self.cast = ops.Cast(device = "gpu",
dtype = types.INT32)
def __init__(self,
batch_size,
device,
data_dir,
mean,
std,
device_id=0,
shard_id=0,
num_shards=1,
num_threads=4,
seed=0):
super(DaliTransformsTrainPipeline,
self).__init__(batch_size, num_threads, device_id, seed)
# should we make this drive the device flags?
self.reader = ops.FileReader(file_root=data_dir,
shard_id=shard_id,
num_shards=num_shards,
random_shuffle=True)
self.decode = ops.ImageDecoder(device='mixed',
output_type=types.RGB,
memory_stats=True)
self.resize = ops.Resize(device=device,
size=[200, 300],
interp_type=types.INTERP_TRIANGULAR)
self.centrecrop = ops.Crop(device=device, crop=[100, 100])
self.randomcrop = ops.RandomResizedCrop(device=device, size=[80, 80])
self.hz_coin = ops.CoinFlip(probability=0.5)
self.horizontalflip = ops.Flip(device=device)
self.rotate_angle = ops.Uniform(range=[-90, 90])
self.rotate_coin = ops.CoinFlip(probability=0.5)
self.rotate = ops.Rotate(device=device, keep_size=True)
self.vt_coin = ops.CoinFlip(probability=0.5)
self.verticalflip = ops.Flip(device=device, horizontal=0)
self.normalize = ops.CropMirrorNormalize(device=device,
dtype=types.FLOAT,
output_layout=types.NCHW) #,
#mean=mean*255, std=std*255)
#self.normalize = ops.Normalize(device=device, dtype=types.FLOAT)#, mean=mean, stddev=std)
self.to_int64 = ops.Cast(dtype=types.INT64, device=device)
def __init__(self, files, labels):
super().__init__(256, 8, 0, seed=42)
self.input = ops.FileReader(files=files,
labels=labels,
random_shuffle=False)
self.decocer = ops.ImageDecoder(device='mixed', output_type=types.RGB)
self.resize = ops.Resize(device='gpu', resize_shorter=224)
self.crop = ops.Crop(device='gpu', crop_h=224, crop_w=224)
mean = torch.Tensor([0.485, 0.456, 0.406
]).unsqueeze(0).unsqueeze(0) * 255
std = torch.Tensor([0.229, 0.224, 0.225
]).unsqueeze(0).unsqueeze(0) * 255
self.normalize = ops.Normalize(axes=[0, 1],
mean=mean,
stddev=std,
device='gpu',
batch=False)
self.transpose = ops.Transpose(device='gpu', perm=[2, 0, 1])
def __init__(self, batch_size, sequence_length, num_threads, device_id,
files, crop_size):
super(VideoReaderPipeline, self).__init__(batch_size,
num_threads,
device_id,
seed=12)
self.reader = ops.VideoReader(device="gpu",
filenames=files,
sequence_length=sequence_length,
normalized=False,
random_shuffle=True,
image_type=types.RGB,
dtype=types.UINT8,
initial_fill=16,
pad_last_batch=True)
self.crop = ops.Crop(device="gpu", crop=crop_size, dtype=types.FLOAT)
self.uniform = ops.Uniform(range=(0.0, 1.0))
self.transpose = ops.Transpose(device="gpu", perm=[3, 0, 1, 2])
def __init__(self, params, device_id, files, labels):
super().__init__(params.batch_size,
params.num_gpus * 8,
device_id,
seed=params.seed)
self.input = ops.FileReader(files=files,
labels=labels,
random_shuffle=False)
self.decocer = ops.ImageDecoder(device='mixed', output_type=types.RGB)
self.resize = ops.Resize(device='gpu', resize_shorter=224)
self.crop = ops.Crop(device='gpu', crop_h=224, crop_w=224)
mean = torch.Tensor(params.mean).unsqueeze(0).unsqueeze(0) * 255
std = torch.Tensor(params.std).unsqueeze(0).unsqueeze(0) * 255
self.normalize = ops.Normalize(axes=[0, 1],
mean=mean,
stddev=std,
device='gpu',
batch=False)
self.transpose = ops.Transpose(device='gpu', perm=[2, 0, 1])
def __init__(self,
file_list,
batch_size,
sequence_length,
num_threads,
device_id,
crop_size,
step=-1,
stride=1,
random_shuffle=True):
super(VideoReaderPipeline, self).__init__(batch_size,
num_threads,
device_id,
seed=12)
# Define VideoReader
self.reader = ops.VideoReader(device="gpu",
file_list=file_list,
sequence_length=sequence_length,
normalized=False,
random_shuffle=random_shuffle,
image_type=types.RGB,
dtype=types.UINT8,
step=step,
stride=stride,
initial_fill=16)
# Define crop, mirror and normalisation operations to apply to every sequence
self.crop = ops.Crop(device="gpu",
crop=crop_size,
output_dtype=types.FLOAT)
self.transpose = ops.Transpose(device="gpu", perm=[0, 3, 1,
2]) # [N F C H W]
self.uniform = ops.Uniform(range=(0.2, 1.0)) # used for random crop
def __init__(self, params, num_threads, device_id):
super(DaliPipeline, self).__init__(params.batch_size,
num_threads,
device_id,
seed=12)
dii = DaliInputIterator(params)
self.no_copy = params.no_copy
if self.no_copy:
print("Use Zero Copy ES")
self.source = ops.ExternalSource(source = dii, num_outputs = 2, layout = ["DHWC", "DHWC"], no_copy = self.no_copy)
self.do_rotate = True if params.rotate_input==1 else False
print("Enable Rotation" if self.do_rotate else "Disable Rotation")
self.rng_angle = ops.Uniform(device = "cpu",
range = [0., 180.])
self.rng_axis = ops.Uniform(device = "cpu",
range = [-1., 1.],
shape=(3))
self.rotate = ops.Rotate(device = "gpu",
interp_type = types.INTERP_LINEAR,
keep_size=True)
self.transpose = ops.Transpose(device = "gpu",
perm=[3,0,1,2])
self.crop = ops.Crop(device = "gpu",
crop = (dii.size, dii.size, dii.size))
def __init__(self, batch_size, sequence_length, num_threads, files, gt_files, \
crop_size, random_shuffle=True, step=-1, device_id=-1, seed=12):
super(VideoReaderPipeline, self).__init__(batch_size,
num_threads,
device_id,
seed=seed)
#Define VideoReader
self.reader = ops.VideoReader(device="gpu", \
filenames=files, \
sequence_length=sequence_length, \
normalized=False, \
random_shuffle=random_shuffle, \
image_type=types.RGB, \
dtype=types.UINT8, \
step=step, \
initial_fill=16,
seed=seed)
self.gt_reader = ops.VideoReader(device="gpu", \
filenames=gt_files, \
sequence_length=sequence_length, \
normalized=False, \
random_shuffle=random_shuffle, \
image_type=types.RGB, \
dtype=types.UINT8, \
step=step, \
initial_fill=16,
seed=seed)
# Define crop and permute operations to apply to every sequence
self.crop = ops.Crop(device="gpu", \
crop=[crop_size, crop_size], \
output_dtype=types.FLOAT)
self.uniform = ops.Uniform(range=(0.0, 1.0)) # used for random crop
self.transpose = ops.Transpose(device="gpu", perm=[3, 0, 1, 2])
def __init__(self,
batch_size,
num_threads,
device_id,
image_size,
tfrecord_path,
index_path,
config,
shard_id=0):
super(CommonPipeline, self).__init__(batch_size, num_threads,
device_id)
self.image_size = image_size
self.input = self._input(tfrecord_path, index_path, shard_id=shard_id)
# The nvjpeg decoder throws an error for some unsupported jpegs.
# until this is fixed, we'll use the host decoder, which runs on the
# CPU.
# self.decode = ops.nvJPEGDecoder(device="mixed",
# output_type=types.RGB)
self.decode = ops.HostDecoder(device="cpu", output_type=types.RGB)
self.resize = ops.Resize(device="gpu",
image_type=types.RGB,
interp_type=types.INTERP_LINEAR,
resize_x=image_size,
resize_y=image_size)
self.resize_large = ops.Resize(device="gpu",
image_type=types.RGB,
interp_type=types.INTERP_LINEAR,
resize_x=image_size * config.zoom_scale,
resize_y=image_size * config.zoom_scale)
self.color_twist = ops.ColorTwist(device="gpu", )
self.crop_mirror_normalize = ops.CropMirrorNormalize(
device="gpu",
crop=image_size,
output_dtype=types.FLOAT,
image_type=types.RGB,
output_layout=types.DALITensorLayout.NHWC,
mean=122.5,
std=255.0)
self.crop = ops.Crop(
device="gpu",
crop=image_size,
)
self.cast = ops.Cast(device="gpu", dtype=types.DALIDataType.INT64)
self.rotate = ops.Rotate(device="gpu", fill_value=0)
self.flip = ops.Flip(device="gpu")
self.coin = ops.CoinFlip(probability=0.5)
self.rotate_rng = ops.Uniform(range=(config.rotate_angle_min,
config.rotate_angle_max))
self.crop_x_rng = ops.Uniform(range=(0.0, config.crop_x_max))
self.crop_y_rng = ops.Uniform(range=(0.0, config.crop_y_max))
self.hue_rng = ops.Uniform(range=(config.hue_min, config.hue_max))
self.contrast_rng = ops.Uniform(range=(config.contrast_min,
config.contrast_max))
self.saturation_rng = ops.Uniform(range=(config.saturation_min,
config.saturation_max))
self.brightness_rng = ops.Uniform(range=(config.brightness_min,
config.brightness_max))
self.iter = 0