def __init__(self,
device,
batch_size,
output_type,
input_type,
fixed_size=None,
num_threads=3,
device_id=0,
num_gpus=1):
super(RotatePipeline, self).__init__(batch_size,
num_threads,
device_id,
seed=7865,
exec_async=False,
exec_pipelined=False)
self.name = device
self.input = ops.CaffeReader(path=caffe_db_folder,
shard_id=device_id,
num_shards=num_gpus)
self.decode = ops.ImageDecoder(device="cpu", output_type=types.RGB)
if input_type != dali.types.UINT8:
self.cast = ops.Cast(device=device, dtype=input_type)
else:
self.cast = None
# TODO(michalz): When we move from Support to CPU operators, replace hardcoded angle
# with one taken from the distribution below
# self.uniform = ops.Uniform(range = (-180.0, 180.0), seed = 42);
self.rotate = ops.Rotate(device=device,
size=fixed_size,
angle=30,
fill_value=42,
output_dtype=output_type)
def __init__(self,
batch_size,
pos_size_iter,
num_threads=1,
device_id=0,
num_gpus=1,
axes=None,
axis_names=None,
normalized_anchor=True,
normalized_shape=True):
super(SlicePythonOp, self).__init__(batch_size,
num_threads,
device_id,
seed=12345,
exec_async=False,
exec_pipelined=False)
self.device = "cpu"
self.layout = "HWC"
self.pos_size_iter = pos_size_iter
self.input = ops.CaffeReader(path=caffe_db_folder,
random_shuffle=False)
self.decode = ops.ImageDecoder(device='cpu', output_type=types.RGB)
self.input_crop_pos = ops.ExternalSource()
self.input_crop_size = ops.ExternalSource()
function = partial(slice_func_helper, axes, axis_names, self.layout,
normalized_anchor, normalized_shape)
self.slice = ops.PythonFunction(function=function)
self.set_layout = ops.Reshape(layout="HWC")
def __init__(self,
batch_size,
output_type,
input_type,
fixed_size,
num_threads=3,
device_id=0,
num_gpus=1):
super(CVPipeline, self).__init__(batch_size,
num_threads,
device_id,
seed=7865,
exec_async=False,
exec_pipelined=False)
self.name = "cv"
self.input = ops.CaffeReader(path=caffe_db_folder,
shard_id=device_id,
num_shards=num_gpus)
self.decode = ops.ImageDecoder(device="cpu", output_type=types.RGB)
self.rotate = ops.PythonFunction(
function=CVRotate(output_type, input_type, 30, fixed_size))
# TODO(michalz): When we move from Support to CPU operators, replace hardcoded angle
# with one taken from the distribution below
# self.uniform = ops.Uniform(range = (-180.0, 180.0), seed = 42);
self.iter = 0
def __init__(self, batch_size, num_threads, device_id, num_gpus):
super(CaffeReadPipeline, self).__init__(batch_size, num_threads,
device_id)
self.input = ops.CaffeReader(path=lmdb_folder,
random_shuffle=True,
shard_id=device_id,
num_shards=num_gpus)
def __init__(self, device, batch_size, pos_size_iter,
num_threads=1, device_id=0, is_fused_decoder=False,
axes=None, axis_names=None, normalized_anchor=True, normalized_shape=True):
super(SlicePipeline, self).__init__(
batch_size, num_threads, device_id, seed=1234)
self.is_fused_decoder = is_fused_decoder
self.pos_size_iter = pos_size_iter
self.device = device
self.input = ops.CaffeReader(path = caffe_db_folder, random_shuffle=False)
self.input_crop_pos = ops.ExternalSource()
self.input_crop_size = ops.ExternalSource()
if self.is_fused_decoder:
self.decode = ops.ImageDecoderSlice(device = "cpu",
output_type = types.RGB,
normalized_anchor=normalized_anchor,
normalized_shape=normalized_shape,
axis_names = axis_names,
axes = axes)
else:
self.decode = ops.ImageDecoder(device = "cpu",
output_type = types.RGB)
self.slice = ops.Slice(device = self.device,
normalized_anchor=normalized_anchor,
normalized_shape=normalized_shape,
axis_names = axis_names,
axes = axes)
def __init__(self, device, batch_size, num_threads=1, device_id=0, num_gpus=1,
is_new_cmn = False, output_dtype = types.FLOAT, output_layout = types.NHWC,
mirror_probability = 0.0, mean=[0., 0., 0.], std=[1., 1., 1.], pad_output=False):
super(CropMirrorNormalizePipeline, self).__init__(batch_size, num_threads, device_id, seed=7865)
self.device = device
self.is_new_cmn = is_new_cmn
self.input = ops.CaffeReader(path = caffe_db_folder, shard_id = device_id, num_shards = num_gpus)
self.decode = ops.HostDecoder(device = "cpu", output_type = types.RGB)
if self.is_new_cmn:
self.cmn = ops.NewCropMirrorNormalize(device = self.device,
output_dtype = output_dtype,
output_layout = output_layout,
crop = (224, 224),
crop_pos_x = 0.3,
crop_pos_y = 0.2,
image_type = types.RGB,
mean = mean,
std = std,
pad_output = pad_output)
else:
self.cmn = ops.CropMirrorNormalize(device = self.device,
output_dtype = output_dtype,
output_layout = output_layout,
crop = (224, 224),
crop_pos_x = 0.3,
crop_pos_y = 0.2,
image_type = types.RGB,
mean = mean,
std = std,
pad_output = pad_output)
self.coin = ops.CoinFlip(probability=mirror_probability)
def __init__(self,
device,
batch_size,
relative,
use_wildcard,
num_threads=3,
device_id=0,
num_gpus=1):
super(ReshapePipeline, self).__init__(batch_size,
num_threads,
device_id,
seed=7865,
exec_async=True,
exec_pipelined=True)
self.device = device
self.input = ops.CaffeReader(path=caffe_db_folder,
shard_id=device_id,
num_shards=num_gpus)
self.decode = ops.ImageDecoder(device="cpu", output_type=types.RGB)
W = 320
H = 224
self.resize = ops.Resize(device="cpu", resize_x=W, resize_y=H)
WC = -1 if use_wildcard else W * 3
if relative:
rel_shape = (-1, 3) if use_wildcard else (1, 3)
self.reshape = ops.Reshape(device=device,
rel_shape=rel_shape,
layout="ab")
else:
self.reshape = ops.Reshape(device=device,
shape=(H, WC),
layout="ab")
def __init__(self,
device,
batch_size,
num_threads=1,
device_id=0,
num_gpus=1,
dtype=types.FLOAT,
output_layout="HWC",
mirror_probability=0.0,
mean=[0., 0., 0.],
std=[1., 1., 1.],
scale=None,
shift=None,
pad_output=False):
super(CropMirrorNormalizePipeline, self).__init__(batch_size,
num_threads,
device_id,
seed=7865)
self.device = device
self.input = ops.CaffeReader(path=caffe_db_folder,
shard_id=device_id,
num_shards=num_gpus)
self.decode = ops.ImageDecoder(device="cpu", output_type=types.RGB)
self.cmn = ops.CropMirrorNormalize(device=self.device,
dtype=dtype,
output_layout=output_layout,
crop=(224, 224),
crop_pos_x=0.3,
crop_pos_y=0.2,
mean=mean,
std=std,
scale=scale,
shift=shift,
pad_output=pad_output)
self.coin = ops.CoinFlip(probability=mirror_probability, seed=7865)
def __init__(self,
device,
batch_size,
num_threads=1,
device_id=0,
num_gpus=1,
is_fused_decoder=False):
super(CropPipeline, self).__init__(batch_size, num_threads, device_id)
self.is_fused_decoder = is_fused_decoder
self.device = device
self.input = ops.CaffeReader(path=caffe_db_folder,
shard_id=device_id,
num_shards=num_gpus)
if self.is_fused_decoder:
self.decode = ops.HostDecoderCrop(device="cpu",
crop=(224, 224),
crop_pos_x=0.3,
crop_pos_y=0.2,
output_type=types.RGB)
else:
self.decode = ops.HostDecoder(device="cpu", output_type=types.RGB)
self.crop = ops.Crop(device=self.device,
crop=(224, 224),
crop_pos_x=0.3,
crop_pos_y=0.2,
image_type=types.RGB)
def __init__(self, batch_size, num_threads, device_id, num_gpus,
data_paths):
super(CaffeReadPipeline, self).__init__(batch_size, num_threads,
device_id)
self.input = ops.CaffeReader(path=data_paths[0],
shard_id=device_id,
num_shards=num_gpus)
def __init__(self,
batch_size,
output_type,
input_type,
use_input,
num_threads=3,
device_id=0,
num_gpus=1):
super(CVPipeline, self).__init__(batch_size,
num_threads,
device_id,
seed=7865,
exec_async=False,
exec_pipelined=False)
self.use_input = use_input
self.name = "cv"
self.input = ops.CaffeReader(path=caffe_db_folder,
shard_id=device_id,
num_shards=num_gpus)
self.decode = ops.ImageDecoder(device="cpu", output_type=types.RGB)
if self.use_input:
self.transform_source = ops.ExternalSource()
self.warp = ops.PythonFunction(
function=CVWarp(output_type, input_type))
else:
self.warp = ops.PythonFunction(function=CVWarp(
output_type, input_type, [[0.1, 0.9, 10], [0.8, -0.2, -20]]))
self.iter = 0
def __init__(self,
device,
batch_size,
phase_y,
phase_x,
freq_x,
freq_y,
ampl_x,
ampl_y,
num_threads=3,
device_id=0,
num_gpus=1):
super(WaterPipeline, self).__init__(batch_size, num_threads, device_id)
self.device = device
self.input = ops.CaffeReader(path=caffe_db_folder,
shard_id=device_id,
num_shards=num_gpus)
self.decode = ops.ImageDecoder(device="cpu", output_type=types.RGB)
self.water = ops.Water(device=self.device,
ampl_x=ampl_x,
ampl_y=ampl_y,
phase_x=phase_x,
phase_y=phase_y,
freq_x=freq_x,
freq_y=freq_y,
interp_type=dali.types.INTERP_LINEAR)
def __init__(self,
device,
batch_size,
num_threads=1,
device_id=0,
num_gpus=1,
should_perform_cast=False):
super(CropCastPipeline, self).__init__(batch_size, num_threads,
device_id)
self.should_perform_cast = should_perform_cast
self.device = device
self.input = ops.CaffeReader(path=caffe_db_folder,
shard_id=device_id,
num_shards=num_gpus)
self.decode = ops.HostDecoder(device="cpu", output_type=types.RGB)
if self.should_perform_cast:
self.crop = ops.Crop(device=self.device,
crop=(224, 224),
crop_pos_x=0.3,
crop_pos_y=0.2,
image_type=types.RGB,
output_dtype=types.FLOAT)
self.crop2 = ops.Crop(device=self.device,
crop=(224, 224),
crop_pos_x=0.0,
crop_pos_y=0.0,
image_type=types.RGB,
output_dtype=types.UINT8)
else:
self.crop = ops.Crop(device=self.device,
crop=(224, 224),
crop_pos_x=0.3,
crop_pos_y=0.2,
image_type=types.RGB)
def __init__(self, batch_size, num_threads, device_id):
super(HybridPipe, self).__init__(batch_size, num_threads, device_id, seed = 12)
self.input = ops.CaffeReader(path = caffe_db_folder, random_shuffle = True)
self.decode = ops.nvJPEGDecoder(device = "mixed", output_type = types.RGB)
self.cmnp_all = ops.CropMirrorNormalize(device = "gpu",
output_dtype = types.FLOAT,
output_layout = types.NHWC,
crop = (224, 224),
image_type = types.RGB,
mean = [128., 128., 128.],
std = [1., 1., 1.])
self.cmnp_int = ops.CropMirrorNormalize(device = "gpu",
output_dtype = types.FLOAT,
output_layout = types.NHWC,
crop = (224, 224),
image_type = types.RGB,
mean = [128, 128, 128],
std = [1., 1, 1]) # Left 1 of the arguments as float to test whether mixing types works
self.cmnp_1arg = ops.CropMirrorNormalize(device = "gpu",
output_dtype = types.FLOAT,
output_layout = types.NHWC,
crop = (224, 224),
image_type = types.RGB,
mean = 128,
std = 1)
self.uniform = ops.Uniform(range = (0,1))
def __init__(self, setting, split_name, batch_size, num_threads, device_id, preprocess=None):
super(BaseDataset, self).__init__(batch_size, num_threads, device_id)
self._split_name = split_name
self._img_path = setting['img_root']
self._gt_path = setting['gt_root']
self._portion = setting['portion'] if 'portion' in setting else None
self._train_source = setting['train_source']
self._eval_source = setting['eval_source']
self._test_source = setting['test_source']
self._down_sampling = setting['down_sampling']
self.preprocess = preprocess
self._file_names = self._get_file_names(split_name)
self.input = ops.CaffeReader(path = lmdb_folder,
random_shuffle = True, shard_id = device_id, num_shards = num_gpus)
self.decode = ops.ImageDecoder(device = "mixed", output_type = types.RGB)
self.resize = ops.Resize(device = "gpu",
interp_type = types.INTERP_LINEAR)
self.cmn = ops.CropMirrorNormalize(device = "gpu",
dtype = types.FLOAT,
crop = (227, 227),
mean = [128., 128., 128.],
std = [1., 1., 1.])
self.uniform = ops.Uniform(range = (0.0, 1.0))
self.resize_rng = ops.Uniform(range = (256, 480))
def __init__(self,
device,
batch_size,
output_type,
input_type,
fixed_size=None,
num_threads=3,
device_id=0,
num_gpus=1):
super(RotatePipeline, self).__init__(batch_size,
num_threads,
device_id,
seed=7865,
exec_async=False,
exec_pipelined=False)
self.name = device
self.input = ops.CaffeReader(path=caffe_db_folder,
shard_id=device_id,
num_shards=num_gpus)
self.decode = ops.ImageDecoder(device="cpu", output_type=types.RGB)
if input_type != dali.types.UINT8:
self.cast = ops.Cast(device=device, dtype=input_type)
else:
self.cast = None
self.uniform = ops.Uniform(range=(-180.0, 180.0), seed=42)
self.rotate = ops.Rotate(device=device,
size=fixed_size,
fill_value=42,
dtype=output_type)
def __init__(self, device, batch_size, num_threads=1, device_id=0, num_gpus=1, is_vertical=0, is_horizontal=1 ):
super(FlipPipeline, self).__init__(batch_size,
num_threads,
device_id)
self.device = device
self.input = ops.CaffeReader(path = caffe_db_folder, shard_id = device_id, num_shards = num_gpus)
self.decode = ops.ImageDecoder(device = "cpu", output_type = types.RGB)
self.flip = ops.Flip(device = self.device, vertical=is_vertical, horizontal=is_horizontal)
def __init__(self, batch_size, output_type, input_type, fixed_size, num_threads=3, device_id=0, num_gpus=1):
super(CVPipeline, self).__init__(batch_size, num_threads, device_id, seed=7865, exec_async=False, exec_pipelined=False)
self.name = "cv"
self.input = ops.CaffeReader(path = caffe_db_folder, shard_id = device_id, num_shards = num_gpus)
self.decode = ops.ImageDecoder(device = "cpu", output_type = types.RGB)
self.rotate = ops.PythonFunction(function=CVRotate(output_type, input_type, fixed_size))
self.uniform = ops.Uniform(range = (-180.0, 180.0), seed = 42);
self.iter = 0
def __init__(self, device, batch_size, use_wildcard, num_threads=3, device_id=0, num_gpus=1):
super(ReshapeWithInput, self).__init__(batch_size, num_threads, device_id, seed=7865, exec_async=False, exec_pipelined=False)
self.device = device
self.input = ops.CaffeReader(path = caffe_db_folder, shard_id = device_id, num_shards = num_gpus)
self.decode = ops.ImageDecoder(device = "cpu", output_type = types.RGB)
fn = CollapseChannelsWildcard if use_wildcard else CollapseChannels
self.gen_shapes = ops.PythonFunction(function=fn)
self.reshape = ops.Reshape(device = device, layout = "ab");
def __init__(self, function, batch_size, num_threads=1, device_id=0):
super(PythonOperatorPipeline, self).__init__(batch_size, num_threads, device_id,
exec_async=False,
exec_pipelined=False,
seed=1234)
self.input = ops.CaffeReader(path = caffe_db_folder, random_shuffle=False)
self.decode = ops.HostDecoder(device = 'cpu', output_type = types.RGB)
self.python_function = ops.PythonFunction(function=function)
def __init__(self, batch_size, num_threads, device_id, num_gpus):
super(HybridPipe, self).__init__(batch_size,
num_threads,
device_id)
self.input = ops.CaffeReader(path = caffe_db_folder, shard_id = device_id, num_shards = num_gpus)
self.decode = ops.HostDecoder(device = "cpu", output_type = types.RGB)
self.dump_cpu = ops.DumpImage(device = "cpu", suffix = "cpu")
self.dump_gpu = ops.DumpImage(device = "gpu", suffix = "gpu")
def __init__(self, device, batch_size, relative, use_wildcard, num_threads=3, device_id=0, num_gpus=1):
super(ReshapeWithArgInput, self).__init__(batch_size, num_threads, device_id, seed=7865, exec_async=False, exec_pipelined=False)
self.device = device
self.input = ops.CaffeReader(path = caffe_db_folder, shard_id = device_id, num_shards = num_gpus)
self.resize = ops.Resize(device = "cpu");
self.decode = ops.ImageDecoder(device = "cpu", output_type = types.RGB)
self.gen_shapes = ops.PythonFunction(function=MakeTallFunc(relative, use_wildcard))
self.reshape = ops.Reshape(device = device);
self.relative = relative
def __init__(self, **kwargs):
super(CaffeReadPipeline, self).__init__(**kwargs)
cache_enabled = kwargs['decoder_cache_params']['cache_enabled']
self.input = ops.CaffeReader(path = kwargs['data_paths'][0],
shard_id = kwargs['device_id'],
num_shards = kwargs['num_gpus'],
stick_to_shard = cache_enabled,
#skip_cached_images = cache_enabled,
prefetch_queue_depth = kwargs['reader_queue_depth'])
def __init__(self, batch_size,function, num_threads=1, device_id=0, num_gpus=1 ):
super(WaterPythonPipeline, self).__init__(batch_size,
num_threads,
device_id,
exec_async=False,
exec_pipelined=False)
self.input = ops.CaffeReader(path = caffe_db_folder, shard_id = device_id, num_shards = num_gpus)
self.decode = ops.HostDecoder(device = "cpu", output_type = types.RGB)
self.water = ops.PythonFunction(function=function)
def __init__(self, batch_size, num_threads=1, device_id=0):
super(CommonPipeline, self).__init__(batch_size,
num_threads,
device_id,
exec_async=False,
exec_pipelined=False,
seed=1234)
self.input = ops.CaffeReader(path=caffe_db_folder,
random_shuffle=False)
self.decode = ops.HostDecoder(device='cpu', output_type=types.RGB)
def __init__(self, path, batch_size, num_threads=1, device_id=0, num_gpus=1):
super(CaffeReaderPipeline, self).__init__(batch_size,
num_threads,
device_id)
self.input = ops.CaffeReader(path = path, shard_id = device_id, num_shards = num_gpus)
self.decode = ops.ImageDecoderCrop(device = "cpu",
crop = (224, 224),
crop_pos_x = 0.3,
crop_pos_y = 0.2,
output_type = types.RGB)
def __init__(self, device, batch_size, num_threads=1, device_id=0, num_gpus=1, decoder_only=False):
super(NoCropPipeline, self).__init__(batch_size, num_threads, device_id)
self.decoder_only = decoder_only
self.device = device
self.input = ops.CaffeReader(path = caffe_db_folder, shard_id = device_id, num_shards = num_gpus)
self.decode = ops.HostDecoder(device = "cpu", output_type = types.RGB)
if not self.decoder_only:
self.cmn = ops.NewCropMirrorNormalize(device = self.device,
image_type = types.RGB,
output_dtype = types.UINT8,
output_layout = types.NHWC)
def __init__(self, device, batch_size, num_threads=1, device_id=0, num_gpus=1, decoder_only=False):
super(NoCropPipeline, self).__init__(batch_size, num_threads, device_id)
self.decoder_only = decoder_only
self.device = device
self.input = ops.CaffeReader(path = caffe_db_folder, shard_id = device_id, num_shards = num_gpus)
self.decode = ops.ImageDecoder(device = "cpu", output_type = types.RGB)
if not self.decoder_only:
self.cast = ops.CropMirrorNormalize(device = self.device,
dtype = types.FLOAT,
output_layout = "HWC")
else:
self.cast = ops.Cast(device = self.device,
dtype = types.FLOAT)
def __init__(self, batch_size, num_threads, device_id):
super(TransposePipe, self).__init__(batch_size,
num_threads,
device_id,
seed=12)
self.input = ops.CaffeReader(path=caffe_db_folder,
shard_id=device_id,
num_shards=1)
self.decode = ops.nvJPEGDecoder(device="mixed",
output_type=types.RGB)
self.crop = ops.Crop(device="gpu",
crop=(224, 224),
image_type=types.RGB)
self.transpose = ops.Transpose(device="gpu", perm=[2, 0, 1])
def __init__(self,
device,
batch_size,
output_type,
input_type,
use_input,
num_threads=3,
device_id=0,
num_gpus=1,
inv_map=False):
super(WarpPipeline, self).__init__(batch_size,
num_threads,
device_id,
seed=7865,
exec_async=False,
exec_pipelined=False)
self.use_input = use_input
self.use_dynamic_size = use_input # avoid Cartesian product
self.name = device
self.input = ops.CaffeReader(path=caffe_db_folder,
shard_id=device_id,
num_shards=num_gpus)
self.decode = ops.ImageDecoder(device="cpu", output_type=types.RGB)
if input_type != dali.types.UINT8:
self.cast = ops.Cast(device=device, dtype=input_type)
else:
self.cast = None
static_size = None if self.use_dynamic_size else (240, 320)
if use_input:
self.transform_source = ops.ExternalSource(
lambda: gen_transforms(self.batch_size, 10))
self.warp = ops.WarpAffine(device=device,
size=static_size,
fill_value=42,
dtype=output_type,
inverse_map=inv_map)
else:
warp_matrix = (0.1, 0.9, 10, 0.8, -0.2, -20)
self.warp = ops.WarpAffine(device=device,
size=static_size,
matrix=warp_matrix,
fill_value=42,
dtype=output_type,
inverse_map=inv_map)
self.iter = 0