def __init__(self,
batch_size,
num_threads,
device_id,
data_dir,
crop,
dali_cpu=False):
super(HybridTrainPipe, self).__init__(batch_size,
num_threads,
device_id,
seed=12 + device_id)
shard = int(args.node_rank * args.world_size / args.nnodes +
args.local_rank)
if args.dist_mint:
self.input = ops.FileReader(file_root=data_dir,
shard_id=shard,
num_shards=args.world_size,
shuffle_after_epoch=True,
num_nodes=args.nnodes,
node_id=args.node_rank,
cache_size=args.cache_size,
node_port_list=args.node_port_list,
node_ip_list=args.node_ip_list)
else:
self.input = ops.FileReader(file_root=data_dir,
shard_id=shard,
num_shards=args.world_size,
shuffle_after_epoch=True,
cache_size=args.cache_size)
#let user decide which pipeline works him bets for RN version he runs
dali_device = 'cpu' if dali_cpu else 'gpu'
decoder_device = 'cpu' if dali_cpu else 'mixed'
# This padding sets the size of the internal nvJPEG buffers to be able to handle all images from full-sized ImageNet
# without additional reallocations
device_memory_padding = 211025920 if decoder_device == 'mixed' else 0
host_memory_padding = 140544512 if decoder_device == 'mixed' else 0
self.decode = ops.ImageDecoderRandomCrop(
device=decoder_device,
output_type=types.RGB,
device_memory_padding=device_memory_padding,
host_memory_padding=host_memory_padding,
random_aspect_ratio=[0.8, 1.25],
random_area=[0.1, 1.0],
num_attempts=100)
self.res = ops.Resize(device=dali_device,
resize_x=crop,
resize_y=crop,
interp_type=types.INTERP_TRIANGULAR)
self.cmnp = ops.CropMirrorNormalize(
device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NCHW,
crop=(crop, crop),
image_type=types.RGB,
mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
self.coin = ops.CoinFlip(probability=0.5)
print('DALI "{0}" variant'.format(dali_device))
def __init__(self, batch_size, num_threads, device_id, data_dir, crop):
super(LoadImagePipeline, self).__init__(batch_size,
num_threads,
device_id,
crop,
exec_async=False,
exec_pipelined=False)
self.input1 = ops.FileReader(file_root=data_dir, random_shuffle=True)
self.input2 = ops.FileReader(file_root=data_dir, random_shuffle=True)
self.decode = ops.ImageDecoder(device='mixed', output_type=types.RGB)
def __init__(self, batch_size, num_threads, device_id, data_dir, crop, shard_id, num_shards):
super(LoadImagePipeline, self).__init__(batch_size, num_threads, device_id, crop,
exec_async=False, exec_pipelined=False)
self.input1 = ops.FileReader(file_root=data_dir, shard_id=shard_id, num_shards=num_shards, random_shuffle=True, pad_last_batch=True)
self.input2 = ops.FileReader(file_root=data_dir, shard_id=shard_id, num_shards=num_shards, random_shuffle=True, pad_last_batch=True)
# device_memory_padding = 211025920
# host_memory_padding = 140544512
device_memory_padding = 320000
host_memory_padding = 160000
self.decode = ops.ImageDecoder(device='mixed', output_type=types.RGB, device_memory_padding=device_memory_padding,
host_memory_padding=host_memory_padding)
def __init__(self,
img_dir,
img_size,
batch_size,
num_threads,
device_id,
num_gpus=1,
seed=1):
super(DataPipeline, self).__init__(batch_size,
num_threads,
device_id,
seed=seed)
self.input = ops.FileReader(file_root=img_dir,
random_shuffle=True,
num_shards=num_gpus,
shard_id=device_id)
self.decode = ops.ImageDecoder(device='mixed', output_type=types.RGB)
self.res = ops.Resize(device="gpu", resize_x=img_size)
self.cmn = ops.CropMirrorNormalize(
device='gpu',
image_type=types.RGB,
mean=[0.5 * 255, 0.5 * 255, 0.5 * 255],
std=[0.5 * 255, 0.5 * 255, 0.5 * 255],
)
self.rrc = ops.RandomResizedCrop(device='gpu',
size=(img_size, img_size))
def __init__(self,
batch_size,
num_threads,
device_id,
data_dir,
crop,
size,
local_rank=0,
world_size=1):
super(HybridValPipe, self).__init__(batch_size,
num_threads,
device_id,
seed=12 + device_id)
self.input = ops.FileReader(file_root=data_dir,
shard_id=local_rank,
num_shards=world_size,
random_shuffle=False)
self.decode = ops.ImageDecoder(device="mixed", output_type=types.RGB)
self.res = ops.Resize(device="gpu",
resize_shorter=size,
interp_type=types.INTERP_TRIANGULAR)
self.cmnp = ops.CropMirrorNormalize(
device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NCHW,
crop=(crop, crop),
image_type=types.RGB,
mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
def __init__(self, batch_size, num_threads, device_id, data_dir, crop,
size):
super(HybridValPipe, self).__init__(batch_size,
num_threads,
device_id,
seed=12 + device_id)
if torch.distributed.is_initialized():
local_rank = torch.distributed.get_rank()
world_size = torch.distributed.get_world_size()
else:
local_rank = 0
world_size = 1
self.input = ops.FileReader(file_root=data_dir,
shard_id=local_rank,
num_shards=world_size,
random_shuffle=False)
self.decode = ops.nvJPEGDecoder(device="mixed", output_type=types.RGB)
self.res = ops.Resize(device="gpu", resize_shorter=size)
self.cmnp = ops.CropMirrorNormalize(
device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NCHW,
crop=(crop, crop),
image_type=types.RGB,
mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
def __init__(self, params, device_id, files, labels):
super().__init__(params.batch_size,
params.num_gpus * 8,
device_id,
seed=params.seed)
# file_root有坑，并不是文件夹名字就是label，按照文件夹顺序(1, 10, 11, 2, 20, 21, ...)分别给与0，1，2，3，4...标签
self.input = ops.FileReader(files=files,
labels=labels,
random_shuffle=True)
self.decocer = ops.ImageDecoder(device='mixed', output_type=types.RGB)
self.resize = ops.Resize(device='gpu', resize_shorter=224)
self.pos_rng_x = ops.random.Uniform(range=(0.0, 1.0))
self.pos_rng_y = ops.random.Uniform(range=(0.0, 1.0))
self.crop = ops.Crop(device='gpu', crop_h=224, crop_w=224)
self.flip = ops.Flip(device='gpu')
self.coinflip = ops.random.CoinFlip(probability=0.5)
self.hsv = ops.Hsv(device='gpu')
self.saturation = ops.random.Uniform(range=(0.8, 1.0))
self.value = ops.random.Uniform(range=(0.8, 1.0))
mean = torch.Tensor(params.mean).unsqueeze(0).unsqueeze(0) * 255
std = torch.Tensor(params.std).unsqueeze(0).unsqueeze(0) * 255
self.normalize = ops.Normalize(axes=[0, 1],
mean=mean,
stddev=std,
device='gpu',
batch=False)
self.transpose = ops.Transpose(device='gpu', perm=[2, 0, 1])
def __init__(self, batch_size, num_threads, device_id, num_gpus,
data_paths):
super(FileReadPipeline, self).__init__(batch_size, num_threads,
device_id)
self.input = ops.FileReader(file_root=data_paths[0],
shard_id=device_id,
num_shards=num_gpus)
def __init__(self, device_id, shard_id, num_shards, size, crop, options):
super(ValPipeline, self).__init__(options.batchsize,
options.dataloader_workers,
device_id,
seed=12 + device_id)
self.input = ops.FileReader(file_root=options.val_dir,
shard_id=shard_id,
num_shards=num_shards,
random_shuffle=False,
pad_last_batch=True)
self.decode = ops.ImageDecoder(device="mixed", output_type=types.RGB)
self.res = ops.Resize(device="gpu",
resize_shorter=size,
interp_type=types.INTERP_TRIANGULAR)
dtype = types.FLOAT16 if options.fp16 else types.FLOAT
layout = types.NCHW
img_mean, img_std = get_mean_std(options)
self.cmnp = ops.CropMirrorNormalize(device="gpu",
output_dtype=dtype,
output_layout=layout,
crop=(crop, crop),
image_type=types.RGB,
mean=img_mean,
std=img_std,
pad_output=False)
def __init__(self,
batch_size,
num_threads,
device_id,
data_dir,
crop_size,
local_rank=0,
world_size=1):
super(HybridTrainPipe, self).__init__(batch_size,
num_threads,
device_id,
seed=12 + device_id)
dali_device = "gpu"
self.input = ops.FileReader(file_root=data_dir,
shard_id=local_rank,
num_shards=world_size,
random_shuffle=True)
self.decode = ops.ImageDecoder(device="mixed", output_type=types.RGB)
self.rand_resize_crop = ops.RandomResizedCrop(device=dali_device,
size=crop_size,
random_area=[0.08, 1.25])
self.crop_mirror_norm = ops.CropMirrorNormalize(
device=dali_device,
output_dtype=types.FLOAT,
output_layout=types.NCHW,
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
self.coin_flip = ops.CoinFlip(probability=0.5)
def __init__(self,
DATA_PATH,
input_height,
batch_size,
copies,
stage,
num_threads,
device_id,
seed=1729):
super(SimCLRTransform, self).__init__(batch_size,
num_threads,
device_id,
seed=seed)
#this lets our pytorch compat function find the length of our dataset
self.num_samples = len(ImageFolder(DATA_PATH))
self.copies = copies
self.input_height = input_height
self.stage = stage
self.input = ops.FileReader(file_root=DATA_PATH,
random_shuffle=True,
seed=seed)
self.to_int64 = ops.Cast(dtype=types.INT64, device="gpu")
self.to_int32_cpu = ops.Cast(dtype=types.INT32, device="cpu")
self.coin = ops.random.CoinFlip(probability=0.5)
self.uniform = ops.random.Uniform(range=[0.6, 0.9])
self.blur_amt = ops.random.Uniform(values=[
float(i) for i in range(1, int(0.1 * self.input_height), 2)
])
self.angles = ops.random.Uniform(range=[0, 360])
self.cast = ops.Cast(dtype=types.FLOAT, device='gpu')
self.decode = ops.ImageDecoder(device='mixed', output_type=types.RGB)
self.crop = ops.RandomResizedCrop(size=self.input_height,
minibatch_size=batch_size,
random_area=[0.75, 1.0],
device="gpu")
self.resize = ops.Resize(resize_x=self.input_height,
resize_y=self.input_height,
device="gpu")
self.flip = ops.Flip(vertical=self.coin(),
horizontal=self.coin(),
device="gpu")
self.colorjit_gray = ops.ColorTwist(brightness=self.uniform(),
contrast=self.uniform(),
hue=self.uniform(),
saturation=self.uniform(),
device="gpu")
self.blur = ops.GaussianBlur(window_size=self.to_int32_cpu(
self.blur_amt()),
device="gpu")
self.rotate = ops.Rotate(
angle=self.angles(),
keep_size=True,
interp_type=types.DALIInterpType.INTERP_LINEAR,
device="gpu")
self.swapaxes = ops.Transpose(perm=[2, 0, 1], device="gpu")
def __init__(self, split, args=None):
#batch_size, num_threads, data_dir, crop, shuffle=False, device_id=0, size=256, dali_cpu=False):
self.split = split
self.bs = args.batch_size if self.split == 'train' else args.val_batch_size
self.shuffle = self.split == 'train'
self.data_dir = os.path.join(args.root, split)
self.crop = args.input_size
super(HybridPipe, self).__init__(self.bs, args.workers, 0, seed=12)
self.input = ops.FileReader(file_root=self.data_dir, shard_id=0, num_shards=1, random_shuffle=self.shuffle)
dali_device = "gpu"
if split == 'train':
self.decode = ops.ImageDecoderRandomCrop(device="mixed", output_type=types.RGB,
device_memory_padding=211025920, host_memory_padding=140544512,
random_aspect_ratio=[0.75, 1.333],
random_area=[0.08, 1.0],
num_attempts=100)
#self.res = ops.Resize(device=dali_device, resize_x=args.input_size, resize_y=args.input_size, interp_type=types.INTERP_TRIANGULAR)
self.res = ops.Resize(device=dali_device, resize_shorter=args.input_size, interp_type=types.INTERP_TRIANGULAR)
else:
self.decode = ops.ImageDecoder(device="mixed", output_type=types.RGB)
self.res = ops.Resize(device=dali_device, resize_shorter=256, interp_type=types.INTERP_TRIANGULAR)
self.cmnp = ops.CropMirrorNormalize(device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NCHW,
crop=(self.crop, self.crop),
image_type=types.RGB,
mean=[0.485 * 255,0.456 * 255,0.406 * 255],
std=[0.229 * 255,0.224 * 255,0.225 * 255])
self.coin = ops.CoinFlip(probability=0.5)
def __init__(self,
batch_size,
num_threads,
data_dir,
crop,
shuffle,
scale,
ratio,
world_size=1,
seed=10,
da=False):
super(HybridTrainPipe,
self).__init__(batch_size, num_threads, 0,
seed=seed) # , RANDOM_SEED_TRAINING
self.input = ops.FileReader(file_root=data_dir,
num_shards=world_size,
pad_last_batch=False,
prefetch_queue_depth=2,
random_shuffle=shuffle)
self.decode = ops.ImageDecoder(device="mixed", output_type=types.RGB)
self.res = ops.RandomResizedCrop(device="gpu",
size=crop,
random_area=scale,
random_aspect_ratio=ratio)
self.cmnp = ops.CropMirrorNormalize(
device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NCHW,
image_type=types.RGB,
mean=[0.5 * 255, 0.5 * 255, 0.5 * 255],
std=[0.5 * 255, 0.5 * 255, 0.5 * 255])
self.da = da
if self.da:
self.coin = ops.CoinFlip(probability=0.5)
def __init__(self,
batch_size,
num_threads,
device_id,
data_dir,
crop,
seed=12,
local_rank=0,
world_size=1,
spos_pre=False):
super(HybridTrainPipe, self).__init__(batch_size,
num_threads,
device_id,
seed=seed + device_id)
color_space_type = types.BGR if spos_pre else types.RGB
self.input = ops.FileReader(file_root=data_dir,
shard_id=local_rank,
num_shards=world_size,
random_shuffle=True)
self.decode = ops.ImageDecoder(device="mixed",
output_type=color_space_type)
self.res = ops.RandomResizedCrop(device="gpu",
size=crop,
interp_type=types.INTERP_LINEAR if
spos_pre else types.INTERP_TRIANGULAR)
self.twist = ops.ColorTwist(device="gpu")
self.jitter_rng = ops.Uniform(range=[0.6, 1.4])
self.cmnp = ops.CropMirrorNormalize(
device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NCHW,
image_type=color_space_type,
mean=0. if spos_pre else [0.485 * 255, 0.456 * 255, 0.406 * 255],
std=1. if spos_pre else [0.229 * 255, 0.224 * 255, 0.225 * 255])
self.coin = ops.CoinFlip(probability=0.5)
def __init__(self,
batch_size,
num_threads,
device_id,
data_dir,
crop,
size,
dali_cpu=True,
args_new=None):
super(HybridValPipe, self).__init__(batch_size,
num_threads,
device_id,
seed=12 + device_id)
self.input = ops.FileReader(file_root=data_dir,
shard_id=args_new.local_rank_per_job,
num_shards=args_new.world_size,
random_shuffle=False)
#Do the pipeline fully on CPU
dev = 'cpu'
self.decode = ops.ImageDecoder(device=dev, output_type=types.RGB)
self.res = ops.Resize(device=dev,
resize_shorter=size,
interp_type=types.INTERP_TRIANGULAR)
self.cmnp = ops.CropMirrorNormalize(
device=dev,
output_dtype=types.FLOAT,
output_layout=types.NCHW,
crop=(crop, crop),
image_type=types.RGB,
mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
def __init__(self, DATA_PATH, input_height, batch_size, num_threads,
device_id):
super(SimCLRTrainDataTransform, self).__init__(batch_size,
num_threads,
device_id,
seed=12)
self.COPIES = 3
self.input_height = input_height
self.input = ops.FileReader(file_root=DATA_PATH,
random_shuffle=True,
seed=12)
self.coin = ops.CoinFlip(probability=0.5)
self.uniform = ops.Uniform(range=[0.7, 1.3]) #-1 to 1
#read image (I think that has to be cpu, do a mixed operation to decode into gpu)
self.decode = ops.ImageDecoder(device='mixed', output_type=types.RGB)
self.crop = ops.RandomResizedCrop(size=self.input_height, device="gpu")
self.flip = ops.Flip(vertical=self.coin(),
horizontal=self.coin(),
device="gpu")
self.colorjit_gray = ops.ColorTwist(brightness=self.uniform(),
contrast=self.uniform(),
hue=self.uniform(),
saturation=self.uniform(),
device="gpu")
self.blur = ops.GaussianBlur(window_size=int(0.1 * self.input_height),
device="gpu")
self.swapaxes = ops.Transpose(perm=[2, 0, 1], device="gpu")
self.to_int64 = ops.Cast(dtype=types.INT64, device="gpu")
def __init__(self, batch_size, num_threads, device_id, data_dir,
crop, size, dali_cpu=False):
super(DALIValPipe, self).__init__(
batch_size, num_threads, device_id, seed=12 + device_id
)
self.input = ops.FileReader(
file_root=data_dir, shard_id=device_id,
num_shards=1, random_shuffle=True
)
#let user decide which pipeline works him bets for RN version he runs
if dali_cpu:
dali_device = "cpu"
self.decode = ops.HostDecoder(
device=dali_device, output_type=types.RGB)
else:
dali_device = "gpu"
# This padding sets the size of the internal nvJPEG buffers to be
# able to handle all images from full-sized ImageNet
# without additional reallocations
self.decode = ops.nvJPEGDecoder(
device="mixed", output_type=types.RGB,
device_memory_padding=211025920, host_memory_padding=140544512
)
self.res = ops.Resize(
device=dali_device, resize_shorter=size)
self.cmnp = ops.CropMirrorNormalize(
device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NCHW,
crop=(crop, crop),
image_type=types.RGB,
mean=[0.485 * 255,0.456 * 255,0.406 * 255],
std=[0.229 * 255,0.224 * 255,0.225 * 255])
print('DALI "{0}" variant'.format(dali_device))
def __init__(self, root_dir, batch_size, num_threads, device_id,
use_shift_scale=False,
num_shards=None, shard_id=None):
super().__init__(batch_size, num_threads, device_id, seed=12)
self.random_angle = ops.Uniform(range=(0, 360.0))
self.random = ops.Uniform(range=(0.5, 1.5))
self.random_coin = ops.CoinFlip()
self.input = ops.FileReader(
file_root=root_dir, random_shuffle=True,
num_shards=num_shards, shard_id=shard_id,
)
self.decode = ops.ImageDecoder(device='mixed')
self.rotate = ops.Rotate(device='gpu', interp_type=types.INTERP_LINEAR)
self.crop = ops.Crop(device='gpu', crop=(224, 224))
self.use_shift_scale = use_shift_scale
if self.use_shift_scale:
self.shift_scale = ops.RandomResizedCrop(
device='gpu',
size=(224, 224),
interp_type=types.INTERP_LINEAR,
random_area=(0.3, 1.0),
)
self.flip = ops.Flip(device='gpu')
self.color_twist = ops.ColorTwist(device='gpu')
def __init__(self, batch_size, num_threads, device_id, _seed, image_dir):
super(CommonPipeline, self).__init__(batch_size, num_threads, device_id, seed=_seed, exec_async=False,
exec_pipelined=False)
self.input = ops.FileReader(file_root=image_dir)
self.decode = ops.ImageDecoder(device = 'cpu', output_type=types.RGB)
self.resize = ops.PythonFunction(function=resize)
self.set_layout = ops.Reshape(layout="HWC")
def __init__(self,
file_list,
file_root,
crop_size,
batch_size,
num_threads,
device_id,
random_shuffle=False,
seed=-1,
mean=None,
std=None,
num_samples=None):
super(DaliPipelineVal, self).__init__(batch_size,
num_threads,
device_id,
seed=seed)
crop_size = _pair(crop_size)
if mean is None:
mean = [0.485 * 255, 0.456 * 255, 0.406 * 255]
if std is None:
std = [0.229 * 255, 0.224 * 255, 0.225 * 255]
if num_samples is None:
initial_fill = 512
else:
initial_fill = min(512, num_samples)
self.loader = ops.FileReader(file_root=file_root,
file_list=file_list,
random_shuffle=random_shuffle,
initial_fill=initial_fill)
self.decode = ops.HostDecoder()
self.resize = ops.Resize(device='gpu', resize_x=256, resize_y=256)
self.cmnorm = ops.CropMirrorNormalize(device='gpu',
crop=list(crop_size),
mean=mean,
std=std)
def __init__(self, batch_size, num_threads, device_id, data_dir, crop, half=False):
super(HybridTrainPipe, self).__init__(batch_size, num_threads, device_id, seed=0, exec_async=True)
out_type = types.FLOAT
if half:
out_type = types.FLOAT16
print('Reading from {}'.format(data_dir))
file_name = make_dali_cached_file_list_which_is_also_a_file(data_dir)
self.input = ops.FileReader(
file_root=data_dir,
file_list=file_name,
shard_id=0,
num_shards=1,
random_shuffle=True)
self.decode = ops.nvJPEGDecoder(device="mixed", output_type=types.RGB)
self.rrc = ops.RandomResizedCrop(device="gpu", size =(crop, crop))
self.cmnp = ops.CropMirrorNormalize(
device="gpu",
output_dtype=out_type,
output_layout=types.NCHW,
crop=(crop, crop),
image_type=types.RGB,
mean=[0.485 * 255,0.456 * 255,0.406 * 255],
std=[0.229 * 255,0.224 * 255,0.225 * 255])
self.coin = ops.CoinFlip(probability=0.5)
self.jpegs = None
self.labels = None
def __init__(self, batch_size, num_threads, device_id):
print(image_dir)
super(SimplePipeline, self).__init__(batch_size, num_threads, device_id, seed = 12)
self.input = ops.FileReader(file_root = image_dir)
# instead of path to file directory file with pairs image_name image_label_value can be provided
# self.input = ops.FileReader(file_root = image_dir, file_list = image_dir + '/file_list.txt')
self.decode = ops.HostDecoder(output_type = types.RGB)
def __init__(self, batch_size, num_threads, device_id, data_dir, crop, dali_cpu=False, local_rank=0, world_size=1):
super(HybridTrainPipe, self).__init__(batch_size, num_threads, device_id, seed=12 + device_id)
dali_device = "cpu" if dali_cpu else "gpu"
decoder_device = 'cpu' if dali_cpu else 'mixed'
# This padding sets the size of the internal nvJPEG buffers to be able to handle all images from full-sized ImageNet
# without additional reallocations
device_memory_padding = 211025920 if decoder_device == 'mixed' else 0
host_memory_padding = 140544512 if decoder_device == 'mixed' else 0
self.input = ops.FileReader(file_root=data_dir, shard_id=local_rank, num_shards=world_size, random_shuffle=True)
self.decode = ops.ImageDecoderRandomCrop(device=decoder_device, output_type=types.RGB,
device_memory_padding=device_memory_padding,
host_memory_padding=host_memory_padding,
random_aspect_ratio=[0.8, 1.25],
random_area=[0.1, 1.0],
num_attempts=100)
#self.res = ops.RandomResizedCrop(device="gpu", size=crop, random_area=[0.08, 1.25])
self.res = ops.Resize(device=dali_device, resize_x=crop, resize_y=crop, interp_type=types.INTERP_TRIANGULAR)
self.cmnp = ops.CropMirrorNormalize(device=dali_device,
output_dtype=types.FLOAT,
output_layout=types.NCHW,
crop=(crop, crop),
image_type=types.RGB,
mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
self.coin = ops.CoinFlip(probability=0.5)
print('DALI "{0}" variant'.format(dali_device))
def __init__(self, batch_size, num_threads, device_id, data_dir, crop, local_rank=0, world_size=1, dali_cpu=False):
super(HybridTrainPipe, self).__init__(batch_size, num_threads, device_id, seed=12 + device_id)
self.input = ops.FileReader(file_root=data_dir, shard_id=local_rank, num_shards=world_size, random_shuffle=True)
#let user decide which pipeline works him bets for RN version he runs
dali_device = 'cpu' if dali_cpu else 'gpu'
decoder_device = 'cpu' if dali_cpu else 'mixed'
# This padding sets the size of the internal nvJPEG buffers to be able to handle all images from full-sized ImageNet
# without additional reallocations
device_memory_padding = 211025920 if decoder_device == 'mixed' else 0
host_memory_padding = 140544512 if decoder_device == 'mixed' else 0
# device_memory_padding = 0
# host_memory_padding = 0
self.decode = ops.ImageDecoderRandomCrop(device=decoder_device, output_type=types.RGB,
device_memory_padding=device_memory_padding,
host_memory_padding=host_memory_padding,
random_aspect_ratio=[0.8, 1.25],
random_area=[0.1, 1.0],
num_attempts=100)
self.resize = ops.Resize(device=dali_device,
resize_x=crop,
resize_y=crop,
interp_type=types.INTERP_TRIANGULAR)
self.crop_mirror_norm = ops.CropMirrorNormalize(device=dali_device,
dtype=types.FLOAT,
output_layout=types.NCHW,
crop=(crop, crop),
mean=[0.485 * 255,0.456 * 255,0.406 * 255],
std=[0.229 * 255,0.224 * 255,0.225 * 255])
self.coin_flip = ops.CoinFlip(probability=0.5)
def __init__(self,
batch_size,
num_threads,
device_id,
image_dir,
file_list,
seed=1,
num_gpu=1):
super(ValPipeline, self).__init__(batch_size,
num_threads,
device_id,
seed=seed)
self.input = ops.FileReader(file_root=image_dir,
file_list=file_list,
random_shuffle=False,
num_shards=num_gpu,
shard_id=0)
self.decode = ops.nvJPEGDecoder(device="mixed", output_type=types.RGB)
self.res = ops.Resize(device="gpu", resize_shorter=256)
self.cmnp = ops.CropMirrorNormalize(
device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NCHW,
crop=(224, 224),
image_type=types.RGB,
mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
def __init__(self, device, batch_size, num_threads=1, device_id=0):
super(MultichannelPipeline, self).__init__(batch_size, num_threads,
device_id)
self.device = device
self.reader = ops.FileReader(file_root=multichannel_tiff_root)
decoder_device = 'mixed' if self.device == 'gpu' else 'cpu'
self.decoder = ops.ImageDecoder(device=decoder_device,
output_type=types.ANY_DATA)
self.resize = ops.Resize(device=self.device,
resize_y=900,
resize_x=300,
min_filter=types.DALIInterpType.INTERP_LINEAR)
self.crop = ops.Crop(device=self.device,
crop_h=220,
crop_w=224,
crop_pos_x=0.3,
crop_pos_y=0.2,
image_type=types.ANY_DATA)
self.transpose = ops.Transpose(device=self.device,
perm=(1, 0, 2),
transpose_layout=False)
self.cmn = ops.CropMirrorNormalize(device=self.device,
std=255.,
mean=0.,
output_layout="HWC",
output_dtype=types.FLOAT)
def __init__(self, batch_size, num_threads, device_id, num_gpus,
data_paths, prefetch, fp16, nhwc):
super(RN50Pipeline, self).__init__(batch_size,
num_threads,
device_id,
prefetch_queue_depth=prefetch)
self.input = ops.FileReader(file_root=data_paths[0],
shard_id=device_id,
num_shards=num_gpus)
self.decode_gpu = ops.nvJPEGDecoder(device="mixed",
output_type=types.RGB)
self.res = ops.RandomResizedCrop(device="gpu", size=(224, 224))
layout = types.args.nhwc if nhwc else types.NCHW
out_type = types.FLOAT16 if fp16 else types.FLOAT
self.cmnp = ops.CropMirrorNormalize(
device="gpu",
output_dtype=out_type,
output_layout=layout,
crop=(224, 224),
image_type=types.RGB,
mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
self.coin = ops.CoinFlip(probability=0.5)
def __init__(self,
batch_size,
num_threads,
device_id,
data_dir="./",
dali_cpu=False):
super(DecodeAndCropPipeline, self).__init__(batch_size,
num_threads,
device_id,
seed=12)
self.input = ops.FileReader(file_root=data_dir, random_shuffle=True)
if dali_cpu:
decoder_device = "cpu"
else:
decoder_device = "mixed"
device_memory_padding = 211025920 if decoder_device == 'mixed' else 0
host_memory_padding = 140544512 if decoder_device == 'mixed' else 0
self.decode = ops.ImageDecoder(
device=decoder_device,
output_type=types.RGB,
device_memory_padding=device_memory_padding,
host_memory_padding=host_memory_padding)
dali_device = "cpu" if dali_cpu else "gpu"
self.res = ops.RandomResizedCrop(device=dali_device,
size=224,
interp_type=types.INTERP_TRIANGULAR,
random_aspect_ratio=[0.8, 1.25],
random_area=[0.1, 1.0],
num_attempts=100)
def __init__(self,
batch_size,
num_threads,
device_id,
data_dir,
crop,
dali_cpu=False,
local_rank=0,
world_size=1):
super(HybridTrainPipe, self).__init__(batch_size,
num_threads,
device_id,
seed=12 + device_id)
dali_device = "gpu"
self.input = ops.FileReader(file_root=data_dir,
shard_id=local_rank,
num_shards=world_size,
random_shuffle=True)
self.decode = ops.ImageDecoder(device="mixed", output_type=types.RGB)
self.res = ops.RandomResizedCrop(device="gpu",
size=crop,
random_area=[0.08, 1.25])
self.cmnp = ops.CropMirrorNormalize(
device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NCHW,
image_type=types.RGB,
mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
self.coin = ops.CoinFlip(probability=0.5)
print('DALI "{0}" variant'.format(dali_device))
def __init__(self,
batch_size,
num_threads,
shard_id,
image_dir,
file_list,
nvjpeg_padding,
seed=1,
num_shards=1,
channel_last=True,
dtype='half',
pad_output=False):
super(ValPipeline, self).__init__(batch_size,
num_threads,
shard_id,
seed=seed)
self.input = ops.FileReader(file_root=image_dir,
file_list=file_list,
random_shuffle=False,
num_shards=num_shards,
shard_id=shard_id)
self.decode = ops.ImageDecoder(device="mixed",
output_type=types.RGB,
device_memory_padding=nvjpeg_padding,
host_memory_padding=nvjpeg_padding)
self.res = ops.Resize(device="gpu", resize_shorter=256)
self.cmnp = ops.CropMirrorNormalize(
device="gpu",
output_dtype=types.FLOAT16 if dtype == "half" else types.FLOAT,
output_layout=types.NHWC if channel_last else types.NCHW,
crop=(224, 224),
image_type=types.RGB,
mean=_pixel_mean,
std=_pixel_std,
pad_output=pad_output)