def __init__(self, batch_size, output_type, input_type, use_input, num_threads=3, device_id=0, num_gpus=1): super(CVPipeline, self).__init__(batch_size, num_threads, device_id, seed=7865, exec_async=False, exec_pipelined=False) self.use_input = use_input self.name = "cv" self.input = ops.CaffeReader(path=caffe_db_folder, shard_id=device_id, num_shards=num_gpus) self.decode = ops.ImageDecoder(device="cpu", output_type=types.RGB) if self.use_input: self.transform_source = ops.ExternalSource() self.warp = ops.PythonFunction( function=CVWarp(output_type, input_type)) else: self.warp = ops.PythonFunction(function=CVWarp( output_type, input_type, [[0.1, 0.9, 10], [0.8, -0.2, -20]])) self.iter = 0
def __init__(self, device, batch_size, dims, axes, axis_names, batch=False, out_type=None, in_type=None, shift=None, scale=None, num_threads=3, device_id=0, num_gpus=1): super(NormalizePipeline, self).__init__( batch_size, num_threads, device_id, seed=7865, exec_async=False, exec_pipelined=False) common_args = { "device": device, "axes": axes, "axis_names": axis_names, "batch": batch, "dtype": dali_type(out_type), "shift": shift, "scale": scale } self.in_type = in_type self.out_type = out_type self.device = device self.input = ops.ExternalSource() self.add_layout = None if axis_names is not None: layout = '' for i in range(dims): layout += chr(ord('a') + i) self.add_layout = ops.Reshape(layout=layout) self.batch = batch self.dims = dims self.has_axes = axes is not None or axis_names is not None self.scale = scale self.shift = shift self.is_integral = out_type is not None and out_type is not np.float32 if axis_names is not None: axes = [] for a in axis_names: axes.append(ord(a) - ord('a')) self.axes = axes self.axis_names = axis_names self.ddof = 2 if axes is not None and len(axes) > 0 else 0 self.eps = 0.25 self.mean = ops.PythonFunction(function=custom_mean(batch, axes), batch_processing=True) self.stddev = ops.PythonFunction(function=custom_stddev(batch, axes), batch_processing=True) self.normalize = ops.Normalize(**common_args, ddof=self.ddof) self.scalar_mean = ops.Normalize(**common_args, mean=1, ddof=self.ddof, epsilon=self.eps) self.scalar_stddev = ops.Normalize(**common_args, stddev=2, epsilon=self.eps) self.scalar_params = ops.Normalize(**common_args, mean=1, stddev=2)
def __init__(self, batch_size, num_threads, device_id, data_dir, crop): super(HybridTrainPipe, self).__init__(batch_size, num_threads, device_id, data_dir, crop) self.pad = ops.Paste(device="gpu", fill_value=0, ratio=1.1, min_canvas_size=crop) self.res = ops.RandomResizedCrop(device="gpu", size=crop, random_area=[0.9, 1.1], random_aspect_ratio=1.33333) self.cutmix = ops.PythonFunction(function=cut_mixe_image, num_outputs=2, device='gpu') self.cmnp = ops.CropMirrorNormalize( device="gpu", output_dtype=types.FLOAT, output_layout=types.NCHW, image_type=types.RGB, mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], std=[0.229 * 255, 0.224 * 255, 0.225 * 255]) self.coin = ops.CoinFlip(probability=0.5) self.rotated = ops.Rotate(device="gpu", keep_size=True) self.rotated_rng = ops.Uniform(range=(-5.0, 5.0)) self.brightness = ops.Brightness(device="gpu") self.brightness_rng = ops.Uniform(range=(0.8, 1.2)) self.reshape = ops.Reshape(device="gpu", layout="HWC") self.one_hot = ops.OneHot(num_classes=3, dtype=types.INT32, device="cpu") self.jitter_rng = ops.CoinFlip(probability=0.3) self.jittered = ops.Jitter(device="gpu")
def __init__(self, batch_size, layout, iterator, pos_size_iter, num_threads=1, device_id=0, num_gpus=1, axes=None, axis_names=None, normalized_anchor=True, normalized_shape=True): super(SliceSynthDataPipelinePythonOp, self).__init__(batch_size, num_threads, device_id, seed=12345, exec_async=False, exec_pipelined=False) self.device = "cpu" self.layout = layout self.iterator = iterator self.pos_size_iter = pos_size_iter self.inputs = ops.ExternalSource() self.input_crop_pos = ops.ExternalSource() self.input_crop_size = ops.ExternalSource() function = partial(slice_func_helper, axes, axis_names, self.layout, normalized_anchor, normalized_shape) self.slice = ops.PythonFunction(function=function)
def __init__(self, batch_size, layout, iterator, pos_size_iter, num_threads=1, device_id=0, num_gpus=1, axes=None, axis_names=None, normalized_anchor=True, normalized_shape=True, input_type=types.FLOAT, output_type=None): super().__init__(batch_size, num_threads, device_id, seed=12345, exec_async=False, exec_pipelined=False) self.device = "cpu" self.layout = layout self.iterator = iterator self.pos_size_iter = pos_size_iter self.inputs = ops.ExternalSource() self.input_crop_pos = ops.ExternalSource() self.input_crop_size = ops.ExternalSource() self.cast_in = ops.Cast(dtype=input_type) function = partial(slice_func_helper, axes, axis_names, self.layout, normalized_anchor, normalized_shape) self.slice = ops.PythonFunction(function=function, output_layouts=layout) self.output_type = output_type if self.output_type is not None: self.cast_out = ops.Cast(dtype=output_type)
def __init__(self, batch_size, pos_size_iter, num_threads=1, device_id=0, num_gpus=1, axes=None, axis_names=None, normalized_anchor=True, normalized_shape=True): super(SlicePythonOp, self).__init__(batch_size, num_threads, device_id, seed=12345, exec_async=False, exec_pipelined=False) self.device = "cpu" self.layout = "HWC" self.pos_size_iter = pos_size_iter self.input = ops.CaffeReader(path=caffe_db_folder, random_shuffle=False) self.decode = ops.ImageDecoder(device='cpu', output_type=types.RGB) self.input_crop_pos = ops.ExternalSource() self.input_crop_size = ops.ExternalSource() function = partial(slice_func_helper, axes, axis_names, self.layout, normalized_anchor, normalized_shape) self.slice = ops.PythonFunction(function=function) self.set_layout = ops.Reshape(layout="HWC")
def __init__(self, batch_size, output_type, input_type, fixed_size, num_threads=3, device_id=0, num_gpus=1): super(CVPipeline, self).__init__(batch_size, num_threads, device_id, seed=7865, exec_async=False, exec_pipelined=False) self.name = "cv" self.input = ops.CaffeReader(path=caffe_db_folder, shard_id=device_id, num_shards=num_gpus) self.decode = ops.ImageDecoder(device="cpu", output_type=types.RGB) self.rotate = ops.PythonFunction( function=CVRotate(output_type, input_type, 30, fixed_size)) # TODO(michalz): When we move from Support to CPU operators, replace hardcoded angle # with one taken from the distribution below # self.uniform = ops.Uniform(range = (-180.0, 180.0), seed = 42); self.iter = 0
def __init__(self, batch_size, num_threads, device_id, seed, image_dir, function): super(TwoOutputsPythonOperatorPipeline, self).__init__(batch_size, num_threads, device_id, seed, image_dir) self.python_function = ops.PythonFunction(function=function, num_outputs=2)
def __init__(self, function, batch_size, iterator, data_shape, data_layout, num_threads=1, device_id=0): super(Crop3dPythonOpPipeline, self).__init__(batch_size, num_threads, device_id, exec_async=False, exec_pipelined=False) self.iterator = iterator self.inputs = ops.ExternalSource() self.data_shape = data_shape self.data_layout = data_layout def crop_func(image): return function(image, layout=self.data_layout, shape=self.data_shape) self.crop = ops.PythonFunction(function=crop_func, output_layouts=data_layout)
def __init__(self, function, batch_size, data_layout, iterator, anchor, shape, axis_names, axes, fill_value, erase_func=erase_func, num_threads=1, device_id=0): super(ErasePythonPipeline, self).__init__(batch_size, num_threads, device_id, exec_async=False, exec_pipelined=False) self.iterator = iterator self.inputs = ops.ExternalSource() self.data_layout = data_layout function = partial(erase_func, anchor, shape, axis_names, axes, data_layout, fill_value) self.erase = ops.PythonFunction(function=function, output_layouts=data_layout)
def __init__(self, batch_size, num_threads, device_id, seed, image_dir, function): super(MultiInputMultiOutputPipeline, self).__init__(batch_size, num_threads, device_id, seed, image_dir) self.python_function = ops.PythonFunction(function=function, num_outputs=3)
def __init__(self, device, batch_size, iterator, nfft, window_length, window_step, window=None, center=None, num_threads=1, device_id=0, spectrogram_func=spectrogram_func_librosa): super(SpectrogramPythonPipeline, self).__init__(batch_size, num_threads, device_id, seed=12345, exec_async=False, exec_pipelined=False) self.device = "cpu" self.iterator = iterator self.inputs = ops.ExternalSource() function = partial(spectrogram_func, nfft, window_length, window_step, window, center) self.spectrogram = ops.PythonFunction(function=function, output_layouts=["ft"])
def __init__(self, batch_size, nfft, window_length, window_step, center, layout="ft", num_threads=1, device_id=0, spectrogram_func=spectrogram_func_librosa): super(AudioSpectrogramPythonPipeline, self).__init__(batch_size, num_threads, device_id, seed=12345, exec_async=False, exec_pipelined=False) self.input = ops.readers.File(device="cpu", files=audio_files) self.decode = ops.decoders.Audio(device="cpu", dtype=types.FLOAT, downmix=True) function = partial(spectrogram_func, nfft, window_length, window_step, None, center) self.spectrogram = ops.PythonFunction(function=function, output_layouts=["ft"]) self.layout = layout
def __init__(self, iterator, batch_size): super().__init__(iterator, batch_size, exec_async=False, exec_pipelined=False) function = partial(log_tensor) self.log = ops.PythonFunction(function=function)
def __init__(self, function, batch_size, iterator, data_shape, data_layout, num_threads=1, device_id=0, dictionary={}, default_value=0.0): super(LookupTablePythonOpPipeline, self).__init__(batch_size, num_threads, device_id, exec_async=False, exec_pipelined=False) self.iterator = iterator self.inputs = ops.ExternalSource() self.data_shape = data_shape self.data_layout = data_layout def lookup_table_func(input_data): return function(input_data, shape=data_shape, dictionary=dictionary, default_value=default_value) self.lookup = ops.PythonFunction(function=lookup_table_func) self.set_layout = ops.Reshape(layout=data_layout)
def __init__(self, device, batch_size, iterator, nfilter, sample_rate, freq_low, freq_high, normalize, mel_formula, num_threads=1, device_id=0, func=mel_fbank_func): super(MelFilterBankPythonPipeline, self).__init__(batch_size, num_threads, device_id, seed=12345, exec_async=False, exec_pipelined=False) self.device = "cpu" self.iterator = iterator self.inputs = ops.ExternalSource() function = partial(func, nfilter, sample_rate, freq_low, freq_high, normalize, mel_formula) self.mel_fbank = ops.PythonFunction(function=function)
def __init__(self, batch_size, num_threads, device_id, _seed, image_dir): super(CommonPipeline, self).__init__(batch_size, num_threads, device_id, seed=_seed, exec_async=False, exec_pipelined=False) self.input = ops.FileReader(file_root=image_dir) self.decode = ops.ImageDecoder(device = 'cpu', output_type=types.RGB) self.resize = ops.PythonFunction(function=resize) self.set_layout = ops.Reshape(layout="HWC")
def __init__(self, batch_size, cutoff_value, window_size, reference_power, reset_interval): super(NonsilenceRosaPipeline, self).__init__(batch_size, num_threads=1, exec_async=False, exec_pipelined=False) hop_length = 1 function = partial(trim_ref, cutoff_value, np.max if not reference_power else reference_power, window_size, hop_length) self.nonsilence = ops.PythonFunction(function=function, num_outputs=2)
def __init__(self, device, batch_size, use_wildcard, num_threads=3, device_id=0, num_gpus=1): super(ReshapeWithInput, self).__init__(batch_size, num_threads, device_id, seed=7865, exec_async=False, exec_pipelined=False) self.device = device self.input = ops.CaffeReader(path = caffe_db_folder, shard_id = device_id, num_shards = num_gpus) self.decode = ops.ImageDecoder(device = "cpu", output_type = types.RGB) fn = CollapseChannelsWildcard if use_wildcard else CollapseChannels self.gen_shapes = ops.PythonFunction(function=fn) self.reshape = ops.Reshape(device = device, layout = "ab");
def __init__(self, batch_size, output_type, input_type, fixed_size, num_threads=3, device_id=0, num_gpus=1): super(CVPipeline, self).__init__(batch_size, num_threads, device_id, seed=7865, exec_async=False, exec_pipelined=False) self.name = "cv" self.input = ops.CaffeReader(path = caffe_db_folder, shard_id = device_id, num_shards = num_gpus) self.decode = ops.ImageDecoder(device = "cpu", output_type = types.RGB) self.rotate = ops.PythonFunction(function=CVRotate(output_type, input_type, fixed_size)) self.uniform = ops.Uniform(range = (-180.0, 180.0), seed = 42); self.iter = 0
def __init__(self, function, batch_size, num_threads=1, device_id=0): super(PythonOperatorPipeline, self).__init__(batch_size, num_threads, device_id, exec_async=False, exec_pipelined=False, seed=1234) self.input = ops.CaffeReader(path = caffe_db_folder, random_shuffle=False) self.decode = ops.HostDecoder(device = 'cpu', output_type = types.RGB) self.python_function = ops.PythonFunction(function=function)
def __init__(self, batch_size, num_threads, device_id, _seed): super(AsyncPipeline, self).__init__(batch_size, num_threads, device_id, seed=_seed, exec_async=True, exec_pipelined=True) self.op = ops.PythonFunction(function=lambda: numpy.zeros([2, 2, 2]))
def __init__(self, function, batch_size, num_threads=1, device_id=0): super(MultichannelPythonOpPipeline, self).__init__(batch_size, num_threads, device_id, exec_async=False, exec_pipelined=False) self.reader = ops.readers.File(file_root=multichannel_tiff_root) self.decoder = ops.decoders.Image(device = 'cpu', output_type = types.ANY_DATA) self.oper = ops.PythonFunction(function=function, output_layouts="HWC")
def __init__(self, batch_size,function, num_threads=1, device_id=0, num_gpus=1 ): super(WaterPythonPipeline, self).__init__(batch_size, num_threads, device_id, exec_async=False, exec_pipelined=False) self.input = ops.CaffeReader(path = caffe_db_folder, shard_id = device_id, num_shards = num_gpus) self.decode = ops.HostDecoder(device = "cpu", output_type = types.RGB) self.water = ops.PythonFunction(function=function)
def __init__(self, device, batch_size, relative, use_wildcard, num_threads=3, device_id=0, num_gpus=1): super(ReshapeWithArgInput, self).__init__(batch_size, num_threads, device_id, seed=7865, exec_async=False, exec_pipelined=False) self.device = device self.input = ops.CaffeReader(path = caffe_db_folder, shard_id = device_id, num_shards = num_gpus) self.resize = ops.Resize(device = "cpu"); self.decode = ops.ImageDecoder(device = "cpu", output_type = types.RGB) self.gen_shapes = ops.PythonFunction(function=MakeTallFunc(relative, use_wildcard)) self.reshape = ops.Reshape(device = device); self.relative = relative
def __init__(self, function, batch_size, layout, iterator, num_threads=1, device_id=0): super(PythonOpPipeline, self).__init__(batch_size, num_threads, device_id, exec_async=False, exec_pipelined=False) self.layout = layout self.iterator = iterator self.inputs = ops.ExternalSource() self.oper = ops.PythonFunction(function=function)
def __init__(self, function, device, num_outputs=1): super(PythonFunctionPipeline, self).__init__(BATCH_SIZE, NUM_WORKERS, DEVICE_ID, seed=SEED, exec_async=False, exec_pipelined=False) self.device = device self.reader = ops.readers.File(file_root=images_dir) self.decode = ops.decoders.Image(device='cpu', output_type=types.RGB) self.norm = ops.CropMirrorNormalize(std=255., mean=0., device=device, output_layout="HWC") self.func = ops.PythonFunction(device=device, function=function, num_outputs=num_outputs)
def __init__(self, device, batch_size, iterator, axis=0, dct_type=2, lifter=1.0, n_mfcc=20, norm=None, num_threads=1, device_id=0, func=mfcc_func): super(MFCCPythonPipeline, self).__init__( batch_size, num_threads, device_id, seed=12345, exec_async=False, exec_pipelined=False) self.device = "cpu" self.iterator = iterator self.inputs = ops.ExternalSource() function = partial(func, axis, dct_type, lifter, n_mfcc, norm) self.mfcc = ops.PythonFunction(function=function)
def __init__(self, batch_size, nfft, window_length, window_step, num_threads=1, device_id=0, spectrogram_func=spectrogram_func_librosa): super(AudioSpectrogramPythonPipeline, self).__init__( batch_size, num_threads, device_id, seed=12345, exec_async=False, exec_pipelined=False) self.input = ops.FileReader(device="cpu", file_root=audio_files) self.decode = ops.AudioDecoder(device="cpu", dtype=types.FLOAT, downmix=True) function = partial(spectrogram_func, nfft, window_length, window_step) self.spectrogram = ops.PythonFunction(function=function)
def __init__(self, device, batch_size, iterator, multiplier, reference, cutoff_db, num_threads=1, device_id=0, func=to_db_func): super(ToDecibelsPythonPipeline, self).__init__( batch_size, num_threads, device_id, seed=12345, exec_async=False, exec_pipelined=False) self.device = "cpu" self.iterator = iterator self.inputs = ops.ExternalSource() function = partial(func, multiplier, reference, cutoff_db) self.dB = ops.PythonFunction(function=function)