def __init__(self,
batch_size,
num_threads,
device_id,
data_dir,
crop=32,
dali_cpu=False,
local_rank=0,
world_size=1,
cutout=0):
super(HybridTrainPipe_CIFAR, self).__init__(batch_size,
num_threads,
device_id,
seed=12 + device_id)
self.iterator = iter(
CIFAR_INPUT_ITER(batch_size, 'train', root=data_dir))
dali_device = "gpu"
self.input = ops.ExternalSource()
self.input_label = ops.ExternalSource()
self.pad = ops.Paste(device=dali_device, ratio=1.25, fill_value=0)
self.uniform = ops.Uniform(range=(0., 1.))
self.crop = ops.Crop(device=dali_device, crop_h=crop, crop_w=crop)
self.cmnp = ops.CropMirrorNormalize(
device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NCHW,
image_type=types.RGB,
mean=[0.49139968 * 255., 0.48215827 * 255., 0.44653124 * 255.],
std=[0.24703233 * 255., 0.24348505 * 255., 0.26158768 * 255.])
self.coin = ops.CoinFlip(probability=0.5)
def __init__(self,
batch_size,
pos_size_iter,
num_threads=1,
device_id=0,
num_gpus=1,
axes=None,
axis_names=None,
normalized_anchor=True,
normalized_shape=True):
super(SlicePythonOp, self).__init__(batch_size,
num_threads,
device_id,
seed=12345,
exec_async=False,
exec_pipelined=False)
self.device = "cpu"
self.layout = "HWC"
self.pos_size_iter = pos_size_iter
self.input = ops.CaffeReader(path=caffe_db_folder,
random_shuffle=False)
self.decode = ops.ImageDecoder(device='cpu', output_type=types.RGB)
self.input_crop_pos = ops.ExternalSource()
self.input_crop_size = ops.ExternalSource()
function = partial(slice_func_helper, axes, axis_names, self.layout,
normalized_anchor, normalized_shape)
self.slice = ops.PythonFunction(function=function,
output_layouts="HWC")
def __init__(self,
dataset_dir,
batch_size,
num_threads,
device_id,
crop,
dali_cpu=False,
local_rank=0,
test=False):
super(HybridTrainPipe, self).__init__(batch_size,
num_threads,
device_id,
seed=666)
self.raw2rgbit = iter(
RAW2RGBInputIterator(dataset_dir, batch_size, test=test))
dali_device = "gpu"
self.input_data = ops.ExternalSource()
self.input_label = ops.ExternalSource()
self.data_decode = ops.ImageDecoder(device="mixed")
self.label_decode = ops.ImageDecoder(device="mixed",
output_type=types.RGB)
self.uniform = ops.Uniform(range=(0., 1.))
self.crop = ops.Crop(device=dali_device, crop_h=crop, crop_w=crop)
self.cmnp = ops.CropMirrorNormalize(device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NCHW)
self.coin = ops.CoinFlip(probability=0.5)
def __init__(self, cfg, root_dir, batch_size, num_threads, device_id=0):
super(SiamesePipeline, self).__init__(batch_size,
num_threads,
device_id,
seed=12)
self.input_target_images = ops.ExternalSource()
self.input_target_labels = ops.ExternalSource()
self.input_cmp_images = ops.ExternalSource()
self.input_cmp_labels = ops.ExternalSource()
self.input_siamese_labels = ops.ExternalSource()
self.dataset = CustomSiameseIterator(batch_size, root_dir,
cfg.same_cate_prob,
cfg.random_shuffle)
self.iterator = iter(self.dataset)
self.decode = ops.ImageDecoder(device='cpu', output_type=types.BGR)
self.resize_op = ops.Resize(resize_longer=cfg.input_size)
self.paste_ratio = ops.Uniform(range=(22, 25))
self.paste = ops.Paste(device="gpu", fill_value=(255, 255, 255))
# self.crop = ops.Crop(device ='gpu', crop=[224, 224])
output_dtype = types.FLOAT16 if cfg.fp16_using else types.FLOAT
# output_dtype = types.FLOAT
self.normalize = ops.CropMirrorNormalize(
device="gpu",
crop=(cfg.input_size, cfg.input_size),
mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255],
mirror=0,
output_dtype=output_dtype,
output_layout=types.NCHW,
pad_output=False)
def __init__(self,
batch_size,
layout,
iterator,
pos_size_iter,
num_threads=1,
device_id=0,
num_gpus=1,
axes=None,
axis_names=None,
normalized_anchor=True,
normalized_shape=True,
input_type=types.FLOAT,
output_type=None):
super().__init__(batch_size,
num_threads,
device_id,
seed=12345,
exec_async=False,
exec_pipelined=False)
self.device = "cpu"
self.layout = layout
self.iterator = iterator
self.pos_size_iter = pos_size_iter
self.inputs = ops.ExternalSource()
self.input_crop_pos = ops.ExternalSource()
self.input_crop_size = ops.ExternalSource()
self.cast_in = ops.Cast(dtype=input_type)
function = partial(slice_func_helper, axes, axis_names, self.layout,
normalized_anchor, normalized_shape)
self.slice = ops.PythonFunction(function=function,
output_layouts=layout)
self.output_type = output_type
if self.output_type is not None:
self.cast_out = ops.Cast(dtype=output_type)
def __init__(self,
imageset_dir,
image_size=128,
random_shuffle=False,
batch_size=64,
num_threads=2,
device_id=0):
super(ImagePipeline, self).__init__(batch_size,
num_threads,
device_id,
seed=12)
eii = ExternalInputIterator(imageset_dir, batch_size, random_shuffle)
self.iterator = iter(eii)
self.num_inputs = len(eii.frontal_indices)
print(f'>>> {self.num_inputs}')
# The source for the inputs and targets
self.input = ops.ExternalSource()
self.target = ops.ExternalSource()
# nvJPEGDecoder below accepts CPU inputs, but returns GPU outputs (hence device = "mixed")
# self.decode = ops.nvJPEGDecoder(device = "mixed", output_type = types.RGB)
self.decode = ops.ImageDecoder(device="mixed", output_type=types.RGB)
# The rest of pre-processing is done on the GPU
self.res = ops.Resize(device="gpu",
resize_x=image_size,
resize_y=image_size)
# self.norm = ops.NormalizePermute(device="gpu", output_dtype=types.FLOAT,
self.norm = ops.CropMirrorNormalize(device="gpu",
output_dtype=types.FLOAT,
mean=[128., 128., 128.],
std=[128., 128., 128.],
crop=[image_size, image_size])
# height=image_size, width=image_size)
print('>>>> In iter done')
def __init__(self,
batch_size,
num_threads,
device_id,
data_dir,
crop,
size,
local_rank=0,
world_size=1):
super(HybridValPipe_CIFAR, self).__init__(batch_size,
num_threads,
device_id,
seed=12 + device_id)
self.bs = batch_size
self.iterator = iter(CIFAR_INPUT_ITER(batch_size, 'val',
root=data_dir))
self.input = ops.ExternalSource()
self.input_label = ops.ExternalSource()
self.cmnp = ops.CropMirrorNormalize(
device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NCHW,
image_type=types.RGB,
mean=[0.49139968 * 255., 0.48215827 * 255., 0.44653124 * 255.],
std=[0.24703233 * 255., 0.24348505 * 255., 0.26158768 * 255.])
def __init__(self, batch_size, num_threads, device_id, data_dir, crop_size, resize_size, full_sized=False, square=False):
self.eii = ExternalInputIterator(data_dir, batch_size, train=False, full=full_sized)
self.eii_iterator = iter(self.eii)
super(CaffeValPipe, self).__init__(batch_size, num_threads, device_id)
self.input = ops.ExternalSource()
self.input_label = ops.ExternalSource()
self.decode = ops.ImageDecoder(device="mixed", output_type=types.RGB)
self.full_sized = full_sized
resize_not_needed = (not square) and (not full_sized)
self.resize_needed = not resize_not_needed
if self.resize_needed:
print(f"Resize will happen for VAL dataset, full_sized={full_sized}, square={square}")
if square:
self.res = ops.Resize(device='gpu', resize_x=resize_size, resize_y=resize_size, interp_type=types.INTERP_TRIANGULAR)
else:
self.res = ops.Resize(device='gpu', resize_shorter=resize_size, interp_type=types.INTERP_TRIANGULAR)
else:
print(f"No resize will happen for VAL dataset, full_sized={full_sized}, square={square}")
self.cmnp = ops.CropMirrorNormalize(device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NCHW,
crop=(crop_size, crop_size),
image_type=types.RGB,
mean=[0.485 * 255,0.456 * 255,0.406 * 255],
std=[0.229 * 255,0.224 * 255,0.225 * 255])
def __init__(self,
device,
batch_size,
pos_size_iter,
num_threads=1,
device_id=0,
is_fused_decoder=False):
super(SlicePipeline, self).__init__(batch_size,
num_threads,
device_id,
seed=1234)
self.is_fused_decoder = is_fused_decoder
self.pos_size_iter = pos_size_iter
self.device = device
self.input = ops.CaffeReader(path=caffe_db_folder,
random_shuffle=False)
self.input_crop_pos = ops.ExternalSource()
self.input_crop_size = ops.ExternalSource()
if self.is_fused_decoder:
self.decode = ops.ImageDecoderSlice(device='cpu',
output_type=types.RGB)
else:
self.decode = ops.ImageDecoder(device="cpu", output_type=types.RGB)
self.slice = ops.Slice(device=device, image_type=types.RGB)
def __init__(self,
batch_size,
num_threads,
device_id,
data_dir,
crop,
size,
local_rank=0,
world_size=1):
super(HybridValPipe_CIFAR, self).__init__(batch_size,
num_threads,
device_id,
seed=12 + device_id)
self.iterator = iter(CIFAR_INPUT_ITER(batch_size, 'val',
root=data_dir))
self.input = ops.ExternalSource()
self.input_label = ops.ExternalSource()
self.pad = ops.Paste(device="gpu", ratio=1., fill_value=0)
self.uniform = ops.Uniform(range=(0., 1.))
self.crop = ops.Crop(device="gpu", crop_h=32, crop_w=32)
self.coin = ops.CoinFlip(probability=0.5)
self.flip = ops.Flip(device="gpu")
self.cmnp = ops.CropMirrorNormalize(device="gpu",
output_layout=types.NCHW,
mean=[125.31, 122.95, 113.87],
std=[63.0, 62.09, 66.70])
def __init__(self, device, batch_size, pos_size_iter,
num_threads=1, device_id=0, is_fused_decoder=False,
axes=None, axis_names=None, normalized_anchor=True, normalized_shape=True):
super(SlicePipeline, self).__init__(
batch_size, num_threads, device_id, seed=1234)
self.is_fused_decoder = is_fused_decoder
self.pos_size_iter = pos_size_iter
self.device = device
self.input = ops.readers.Caffe(path = caffe_db_folder, random_shuffle=False)
self.input_crop_pos = ops.ExternalSource()
self.input_crop_size = ops.ExternalSource()
if self.is_fused_decoder:
self.decode = ops.decoders.ImageSlice(device = "cpu",
output_type = types.RGB,
normalized_anchor=normalized_anchor,
normalized_shape=normalized_shape,
axis_names = axis_names,
axes = axes)
else:
self.decode = ops.decoders.Image(device = "cpu",
output_type = types.RGB)
self.slice = ops.Slice(device = self.device,
normalized_anchor=normalized_anchor,
normalized_shape=normalized_shape,
axis_names = axis_names,
axes = axes)
def __init__(self, dev, iterator, op, batch_size, num_threads, device_id):
super(ExprOpPipeline, self).__init__(batch_size, num_threads, device_id, seed=12)
self.left_source = ops.ExternalSource()
self.right_source = ops.ExternalSource()
self.dev = dev
self.iterator = iterator
self.op = op
def __init__(self,
device,
batch_size,
layout,
iterator,
pos_size_iter,
num_threads=1,
device_id=0,
num_gpus=1,
axes=None,
axis_names=None,
normalized_anchor=True,
normalized_shape=True,
extra_outputs=False,
out_of_bounds_policy=None,
fill_values=None):
super(SliceSynthDataPipeline, self).__init__(batch_size,
num_threads,
device_id,
seed=1234)
self.device = device
self.layout = layout
self.iterator = iterator
self.pos_size_iter = pos_size_iter
self.inputs = ops.ExternalSource()
self.input_crop_pos = ops.ExternalSource()
self.input_crop_size = ops.ExternalSource()
self.extra_outputs = extra_outputs
self.slice = ops.Slice(device=self.device,
normalized_anchor=normalized_anchor,
normalized_shape=normalized_shape,
axes=axes,
axis_names=axis_names,
out_of_bounds_policy=out_of_bounds_policy,
fill_values=fill_values)
def __init__(self,
batch_size,
layout,
iterator,
pos_size_iter,
num_threads=1,
device_id=0,
num_gpus=1,
axes=None,
axis_names=None,
normalized_anchor=True,
normalized_shape=True):
super(SliceSynthDataPipelinePythonOp,
self).__init__(batch_size,
num_threads,
device_id,
seed=12345,
exec_async=False,
exec_pipelined=False)
self.device = "cpu"
self.layout = layout
self.iterator = iterator
self.pos_size_iter = pos_size_iter
self.inputs = ops.ExternalSource()
self.input_crop_pos = ops.ExternalSource()
self.input_crop_size = ops.ExternalSource()
function = partial(slice_func_helper, axes, axis_names, self.layout,
normalized_anchor, normalized_shape)
self.slice = ops.PythonFunction(function=function)
def __init__(self, batch_size, external_s_size, num_threads,
device_id):
super(ExternalSourcePipeline,
self).__init__(batch_size, num_threads, device_id)
self.input = ops.ExternalSource()
self.input_2 = ops.ExternalSource()
self.batch_size_ = batch_size
self.external_s_size_ = external_s_size
def __init__(self,
imageset_dir,
image_size,
random_shuffle,
batch_size=4,
num_threads=2,
device_id=0):
super(ImagePipeline, self).__init__(batch_size,
num_threads,
device_id,
seed=12)
self.imageset_dir = imageset_dir
self.random_shuffle = random_shuffle
eii = ExternalInputIterator(root=self.imageset_dir,
batch_size=self.batch_size,
random_shuffle=self.random_shuffle)
self.iterator = iter(eii)
self.num_inputs = len(self.iterator.files)
self.input_image = ops.ExternalSource()
self.input_mask = ops.ExternalSource()
self.decode_image = ops.ImageDecoder(device="mixed",
output_type=types.RGB)
self.decode_mask = ops.ImageDecoder(device="mixed",
output_type=types.GRAY)
# The rest of pre-processing is done on the GPU
self.res = ops.Resize(device="gpu",
resize_x=image_size,
resize_y=image_size)
self.flip = ops.Flip(device="gpu", horizontal=1, vertical=0)
rotate_degree = random.random() * 2 * 10 - 10
self.rotate_image = ops.Rotate(
device="gpu",
angle=rotate_degree,
interp_type=types.DALIInterpType.INTERP_LINEAR)
self.rotate_mask = ops.Rotate(
device="gpu",
angle=rotate_degree,
interp_type=types.DALIInterpType.INTERP_NN)
self.cmnp_image = ops.CropMirrorNormalize(
device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NCHW,
image_type=types.RGB,
mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
self.cmnp_mask = ops.CropMirrorNormalize(device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NCHW,
image_type=types.GRAY,
mean=[0],
std=[255])
def define_graph(self):
if use_fn_api:
self.batch_1 = fn.external_source(device = self._device, name = "src1")
self.batch_2 = fn.external_source(device = self._device, name = "src2")
else:
input_1 = ops.ExternalSource(device = self._device)
input_2 = ops.ExternalSource(device = self._device)
self.batch_1 = input_1(name = "src1")
self.batch_2 = input_2(name = "src2")
return [self.batch_1, self.batch_2]
def __init__(self, file_list, batch_size, num_threads, device_id, external_data):
super(ExternalSourcePipeline, self).__init__(batch_size, num_threads, device_id)
self.input = ops.FileReader(file_list= file_list)
self.label = ops.ExternalSource()
self.crops = ops.ExternalSource()
self.decode = ops.ImageDecoder(device="mixed", output_type=types.RGB)
self.res = ops.Resize(device="gpu", resize_x=224, resize_y=224)
self.cast = ops.Cast(device="cpu", dtype=types.INT32)
self.external_data = external_data
self.iterator = iter(self.external_data)
def define_graph(self):
if use_fn_api:
# pass a Torch stream where data is generated
self.batch_1 = fn.external_source(device = self._device, name = "src1", cuda_stream=torch.cuda.default_stream())
self.batch_2 = fn.external_source(device = self._device, name = "src2", cuda_stream=torch.cuda.default_stream())
else:
input_1 = ops.ExternalSource(device = self._device)
input_2 = ops.ExternalSource(device = self._device)
self.batch_1 = input_1(name = "src1")
self.batch_2 = input_2(name = "src2")
return [self.batch_1, self.batch_2]
def __init__(self, iterator, batch_size, num_threads, device_id):
super(ExternalSourcePipeline, self).__init__(batch_size,
num_threads,
device_id,
seed=12)
self.iterator = iterator
self.input = ops.ExternalSource()
self.input_label = ops.ExternalSource()
self.decode = ops.ImageDecoder(device = "mixed", output_type = types.RGB)
self.cast = ops.Cast(device = "gpu",
dtype = types.INT32)
def __init__(self, batch_size, img_dir, json_dir, num_threads=2, device_id=0, num_gpus=1, resize=None,
augment=False, shuffle=True):
"""
Args:
batch_size: batch size for output at the first dim.
num_threads: int, number of cpu working threads.
device_id: int, the slice number of gpu.
num_gpus: int, number of multiple gpu.
img_dir: str, dir path where the images are stored.
json_dir: str, json path for coco dataset.
resize(optional): default int, if other format please modify function params in ops.Resize.
Output:
(images, captions) pair stacked by batch_size. The output shape of images will be NCHW with type of float.
Note that the output type of captions will be a list of numpy which is encoded from the original string
caption. To use it in the custom model, one needs to decode the numpy into string by .tostring() function
or .tobytes().decode() function. .tostring will get a bytes type result while .tobytes.decode will directly
get the string.
Notes:
param 'device' in ops functions instruct which device will process the data. optional in 'mixed'/'cpu'/'gpu',
for detail please see DALI documentation online.
"""
super(COCOCaptionPipeline, self).__init__(batch_size, num_threads, device_id, seed=15)
self.coco_itr = COCOCaptionInputIterator(batch_size, device_id, num_gpus, img_dir, json_dir, shuffle=shuffle)
self.iterator = iter(self.coco_itr)
self.input = ops.ExternalSource()
self.caption = ops.ExternalSource()
self.img_id = ops.ExternalSource()
self.decode = ops.ImageDecoder(device="mixed", output_type=types.RGB)
self.augment = augment
if resize is not None:
if isinstance(resize, tuple):
resx, resy = resize
elif isinstance(resize, int):
resx = resy = resize
else:
resx = resy = 0.
self.res = ops.Resize(device="gpu", resize_x=resx, resize_y=resy)
else:
self.res = None
if augment:
self.cmnp = ops.CropMirrorNormalize(device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NHWC,
image_type=types.RGB,
mean=MEAN,
std=STD)
self.cf = ops.CoinFlip()
self.rotate = ops.Rotate(device="gpu")
self.rng = ops.Uniform(range=(-10.0, 10.0))
def __init__(self, external, batch_size, num_threads, device_id):
super().__init__(batch_size, num_threads, device_id)
self.input_imgs = ops.ExternalSource()
self.input_anns = ops.ExternalSource()
self.decode_imgs = ops.ImageDecoder(device="mixed",
output_type=types.RGB)
self.decode_anns = ops.ImageDecoder(device="mixed",
output_type=types.GRAY)
self.augment = Augmentation()
self.external_data = external
self.iterator = iter(self.external_data)
def __init__(self, batch_size, num_threads, device_id, external_data):
super(ExternalSourcePipeline, self).__init__(batch_size,
num_threads,
device_id,
seed=12)
self.input = ops.ExternalSource()
self.input_label = ops.ExternalSource()
self.decode = ops.ImageDecoder(device="mixed", output_type=types.RGB)
self.res = ops.Resize(device="gpu", resize_x=240, resize_y=240)
self.cast = ops.Cast(device="gpu", dtype=types.UINT8)
self.external_data = external_data
self.iterator = iter(self.external_data)
def __init__(self,
batch_size,
num_threads,
device_id,
crop,
colorjitter=None,
dali_cpu=False):
super(ImageNetTrainPipe, self).__init__(batch_size,
num_threads,
device_id,
seed=12 + device_id)
self.data_input = ops.ExternalSource()
self.label_input = ops.ExternalSource()
self.colorjitter = colorjitter
# let user decide which pipeline works him bets for RN version he runs
if dali_cpu:
dali_device = "cpu"
self.decode = ops.HostDecoderRandomCrop(device=dali_device,
output_type=types.RGB)
self.res = ops.Resize(resize_x=crop, resize_y=crop)
else:
dali_device = "gpu"
# This padding sets the size of the internal nvJPEG buffers to be able to
# handle all images from full-sized ImageNet without additional reallocations
self.decode = ops.nvJPEGDecoder(device="mixed",
output_type=types.RGB,
device_memory_padding=211025920,
host_memory_padding=140544512)
self.res = ops.RandomResizedCrop(device=dali_device,
size=(crop, crop))
self.cmnp = ops.CropMirrorNormalize(
device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NCHW,
crop=(crop, crop),
image_type=types.RGB,
mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
self.coin = ops.CoinFlip(probability=0.5)
if self.colorjitter is not None:
self.colorjit = ops.ColorTwist(device="gpu")
self.rng_brightness = ops.Uniform(range=(1.0 - self.colorjitter[0],
1.0 +
self.colorjitter[0]))
self.rng_contrast = ops.Uniform(range=(1.0 - self.colorjitter[1],
1.0 + self.colorjitter[1]))
self.rng_saturation = ops.Uniform(range=(1.0 - self.colorjitter[2],
1.0 +
self.colorjitter[2]))
self.rng_hue = ops.Uniform(range=(-self.colorjitter[3],
self.colorjitter[3]))
def __init__(self, batch_size, num_threads, device_id, num_gpus,
external_data):
super(CXRImagePipeline, self).__init__(batch_size,
num_threads,
device_id,
seed=12)
self.input = ops.ExternalSource()
self.input_label = ops.ExternalSource()
self.decode = ops.ImageDecoder(device="mixed", output_type=types.GRAY)
#self.norm = ops.Normalize(device="cpu")
self.res = ops.Resize(device="gpu", resize_x=1024, resize_y=1024)
self.cast = ops.Cast(device="gpu", dtype=types.FLOAT)
self.external_data = external_data
self.iterator = iter(self.external_data)
def __init__(self, batch_size, num_threads, device_id, external_data):
super(ExternalSourcePipeline, self).__init__(batch_size, num_threads, device_id)
self.input = ops.ExternalSource()
self.input_label = ops.ExternalSource()
self.decode = ops.ImageDecoder(device='mixed', output_type=types.RGB)
self.resize = ops.Resize(device="gpu", resize_x=224, resize_y=224)
self.norm = ops.CropMirrorNormalize(device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NCHW,
image_type=types.RGB,
mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
self.external_data = external_data
self.iterator = iter(self.external_data)
def __init__(self,
device,
batch_size,
iterator,
nfilter,
sample_rate,
freq_low,
freq_high,
normalize,
mel_formula,
num_threads=1,
device_id=0,
func=mel_fbank_func):
super(MelFilterBankPythonPipeline, self).__init__(batch_size,
num_threads,
device_id,
seed=12345,
exec_async=False,
exec_pipelined=False)
self.device = "cpu"
self.iterator = iterator
self.inputs = ops.ExternalSource()
function = partial(func, nfilter, sample_rate, freq_low, freq_high,
normalize, mel_formula)
self.mel_fbank = ops.PythonFunction(function=function)
def __init__(self, batch_size, eii, num_threads, device_id):
super(ExternalSourcePipeline, self).__init__(batch_size,
num_threads,
device_id,
seed=12)
self.source = ops.ExternalSource(source=eii, num_outputs=2)
self.build()
def __init__(self,
device,
batch_size,
layout,
iterator,
anchor,
shape,
axis_names,
axes,
fill_value,
normalized_anchor=False,
normalized_shape=False,
num_threads=1,
device_id=0,
num_gpus=1):
super(ErasePipeline, self).__init__(batch_size, num_threads, device_id)
self.device = device
self.layout = layout
self.iterator = iterator
self.inputs = ops.ExternalSource()
self.erase = ops.Erase(device=self.device,
anchor=anchor,
shape=shape,
axis_names=axis_names,
axes=axes,
fill_value=fill_value,
normalized_anchor=normalized_anchor,
normalized_shape=normalized_shape)
def __init__(self,
function,
batch_size,
data_layout,
iterator,
anchor,
shape,
axis_names,
axes,
fill_value,
erase_func=erase_func,
num_threads=1,
device_id=0):
super(ErasePythonPipeline, self).__init__(batch_size,
num_threads,
device_id,
exec_async=False,
exec_pipelined=False)
self.iterator = iterator
self.inputs = ops.ExternalSource()
self.data_layout = data_layout
function = partial(erase_func, anchor, shape, axis_names, axes,
data_layout, fill_value)
self.erase = ops.PythonFunction(function=function,
output_layouts=data_layout)
