def __init__(self, device, batch_size, output_type, input_type, fixed_size=None, num_threads=3, device_id=0, num_gpus=1): super(RotatePipeline, self).__init__(batch_size, num_threads, device_id, seed=7865, exec_async=False, exec_pipelined=False) self.name = device self.input = ops.CaffeReader(path=caffe_db_folder, shard_id=device_id, num_shards=num_gpus) self.decode = ops.ImageDecoder(device="cpu", output_type=types.RGB) if input_type != dali.types.UINT8: self.cast = ops.Cast(device=device, dtype=input_type) else: self.cast = None # TODO(michalz): When we move from Support to CPU operators, replace hardcoded angle # with one taken from the distribution below # self.uniform = ops.Uniform(range = (-180.0, 180.0), seed = 42); self.rotate = ops.Rotate(device=device, size=fixed_size, angle=30, fill_value=42, output_dtype=output_type)
def __init__(self, batch_size, pos_size_iter, num_threads=1, device_id=0, num_gpus=1, axes=None, axis_names=None, normalized_anchor=True, normalized_shape=True): super(SlicePythonOp, self).__init__(batch_size, num_threads, device_id, seed=12345, exec_async=False, exec_pipelined=False) self.device = "cpu" self.layout = "HWC" self.pos_size_iter = pos_size_iter self.input = ops.CaffeReader(path=caffe_db_folder, random_shuffle=False) self.decode = ops.ImageDecoder(device='cpu', output_type=types.RGB) self.input_crop_pos = ops.ExternalSource() self.input_crop_size = ops.ExternalSource() function = partial(slice_func_helper, axes, axis_names, self.layout, normalized_anchor, normalized_shape) self.slice = ops.PythonFunction(function=function) self.set_layout = ops.Reshape(layout="HWC")
def __init__(self, batch_size, output_type, input_type, fixed_size, num_threads=3, device_id=0, num_gpus=1): super(CVPipeline, self).__init__(batch_size, num_threads, device_id, seed=7865, exec_async=False, exec_pipelined=False) self.name = "cv" self.input = ops.CaffeReader(path=caffe_db_folder, shard_id=device_id, num_shards=num_gpus) self.decode = ops.ImageDecoder(device="cpu", output_type=types.RGB) self.rotate = ops.PythonFunction( function=CVRotate(output_type, input_type, 30, fixed_size)) # TODO(michalz): When we move from Support to CPU operators, replace hardcoded angle # with one taken from the distribution below # self.uniform = ops.Uniform(range = (-180.0, 180.0), seed = 42); self.iter = 0
def __init__(self, batch_size, num_threads, device_id, num_gpus): super(CaffeReadPipeline, self).__init__(batch_size, num_threads, device_id) self.input = ops.CaffeReader(path=lmdb_folder, random_shuffle=True, shard_id=device_id, num_shards=num_gpus)
def __init__(self, device, batch_size, pos_size_iter, num_threads=1, device_id=0, is_fused_decoder=False, axes=None, axis_names=None, normalized_anchor=True, normalized_shape=True): super(SlicePipeline, self).__init__( batch_size, num_threads, device_id, seed=1234) self.is_fused_decoder = is_fused_decoder self.pos_size_iter = pos_size_iter self.device = device self.input = ops.CaffeReader(path = caffe_db_folder, random_shuffle=False) self.input_crop_pos = ops.ExternalSource() self.input_crop_size = ops.ExternalSource() if self.is_fused_decoder: self.decode = ops.ImageDecoderSlice(device = "cpu", output_type = types.RGB, normalized_anchor=normalized_anchor, normalized_shape=normalized_shape, axis_names = axis_names, axes = axes) else: self.decode = ops.ImageDecoder(device = "cpu", output_type = types.RGB) self.slice = ops.Slice(device = self.device, normalized_anchor=normalized_anchor, normalized_shape=normalized_shape, axis_names = axis_names, axes = axes)
def __init__(self, device, batch_size, num_threads=1, device_id=0, num_gpus=1, is_new_cmn = False, output_dtype = types.FLOAT, output_layout = types.NHWC, mirror_probability = 0.0, mean=[0., 0., 0.], std=[1., 1., 1.], pad_output=False): super(CropMirrorNormalizePipeline, self).__init__(batch_size, num_threads, device_id, seed=7865) self.device = device self.is_new_cmn = is_new_cmn self.input = ops.CaffeReader(path = caffe_db_folder, shard_id = device_id, num_shards = num_gpus) self.decode = ops.HostDecoder(device = "cpu", output_type = types.RGB) if self.is_new_cmn: self.cmn = ops.NewCropMirrorNormalize(device = self.device, output_dtype = output_dtype, output_layout = output_layout, crop = (224, 224), crop_pos_x = 0.3, crop_pos_y = 0.2, image_type = types.RGB, mean = mean, std = std, pad_output = pad_output) else: self.cmn = ops.CropMirrorNormalize(device = self.device, output_dtype = output_dtype, output_layout = output_layout, crop = (224, 224), crop_pos_x = 0.3, crop_pos_y = 0.2, image_type = types.RGB, mean = mean, std = std, pad_output = pad_output) self.coin = ops.CoinFlip(probability=mirror_probability)
def __init__(self, device, batch_size, relative, use_wildcard, num_threads=3, device_id=0, num_gpus=1): super(ReshapePipeline, self).__init__(batch_size, num_threads, device_id, seed=7865, exec_async=True, exec_pipelined=True) self.device = device self.input = ops.CaffeReader(path=caffe_db_folder, shard_id=device_id, num_shards=num_gpus) self.decode = ops.ImageDecoder(device="cpu", output_type=types.RGB) W = 320 H = 224 self.resize = ops.Resize(device="cpu", resize_x=W, resize_y=H) WC = -1 if use_wildcard else W * 3 if relative: rel_shape = (-1, 3) if use_wildcard else (1, 3) self.reshape = ops.Reshape(device=device, rel_shape=rel_shape, layout="ab") else: self.reshape = ops.Reshape(device=device, shape=(H, WC), layout="ab")
def __init__(self, device, batch_size, num_threads=1, device_id=0, num_gpus=1, dtype=types.FLOAT, output_layout="HWC", mirror_probability=0.0, mean=[0., 0., 0.], std=[1., 1., 1.], scale=None, shift=None, pad_output=False): super(CropMirrorNormalizePipeline, self).__init__(batch_size, num_threads, device_id, seed=7865) self.device = device self.input = ops.CaffeReader(path=caffe_db_folder, shard_id=device_id, num_shards=num_gpus) self.decode = ops.ImageDecoder(device="cpu", output_type=types.RGB) self.cmn = ops.CropMirrorNormalize(device=self.device, dtype=dtype, output_layout=output_layout, crop=(224, 224), crop_pos_x=0.3, crop_pos_y=0.2, mean=mean, std=std, scale=scale, shift=shift, pad_output=pad_output) self.coin = ops.CoinFlip(probability=mirror_probability, seed=7865)
def __init__(self, device, batch_size, num_threads=1, device_id=0, num_gpus=1, is_fused_decoder=False): super(CropPipeline, self).__init__(batch_size, num_threads, device_id) self.is_fused_decoder = is_fused_decoder self.device = device self.input = ops.CaffeReader(path=caffe_db_folder, shard_id=device_id, num_shards=num_gpus) if self.is_fused_decoder: self.decode = ops.HostDecoderCrop(device="cpu", crop=(224, 224), crop_pos_x=0.3, crop_pos_y=0.2, output_type=types.RGB) else: self.decode = ops.HostDecoder(device="cpu", output_type=types.RGB) self.crop = ops.Crop(device=self.device, crop=(224, 224), crop_pos_x=0.3, crop_pos_y=0.2, image_type=types.RGB)
def __init__(self, batch_size, num_threads, device_id, num_gpus, data_paths): super(CaffeReadPipeline, self).__init__(batch_size, num_threads, device_id) self.input = ops.CaffeReader(path=data_paths[0], shard_id=device_id, num_shards=num_gpus)
def __init__(self, batch_size, output_type, input_type, use_input, num_threads=3, device_id=0, num_gpus=1): super(CVPipeline, self).__init__(batch_size, num_threads, device_id, seed=7865, exec_async=False, exec_pipelined=False) self.use_input = use_input self.name = "cv" self.input = ops.CaffeReader(path=caffe_db_folder, shard_id=device_id, num_shards=num_gpus) self.decode = ops.ImageDecoder(device="cpu", output_type=types.RGB) if self.use_input: self.transform_source = ops.ExternalSource() self.warp = ops.PythonFunction( function=CVWarp(output_type, input_type)) else: self.warp = ops.PythonFunction(function=CVWarp( output_type, input_type, [[0.1, 0.9, 10], [0.8, -0.2, -20]])) self.iter = 0
def __init__(self, device, batch_size, phase_y, phase_x, freq_x, freq_y, ampl_x, ampl_y, num_threads=3, device_id=0, num_gpus=1): super(WaterPipeline, self).__init__(batch_size, num_threads, device_id) self.device = device self.input = ops.CaffeReader(path=caffe_db_folder, shard_id=device_id, num_shards=num_gpus) self.decode = ops.ImageDecoder(device="cpu", output_type=types.RGB) self.water = ops.Water(device=self.device, ampl_x=ampl_x, ampl_y=ampl_y, phase_x=phase_x, phase_y=phase_y, freq_x=freq_x, freq_y=freq_y, interp_type=dali.types.INTERP_LINEAR)
def __init__(self, device, batch_size, num_threads=1, device_id=0, num_gpus=1, should_perform_cast=False): super(CropCastPipeline, self).__init__(batch_size, num_threads, device_id) self.should_perform_cast = should_perform_cast self.device = device self.input = ops.CaffeReader(path=caffe_db_folder, shard_id=device_id, num_shards=num_gpus) self.decode = ops.HostDecoder(device="cpu", output_type=types.RGB) if self.should_perform_cast: self.crop = ops.Crop(device=self.device, crop=(224, 224), crop_pos_x=0.3, crop_pos_y=0.2, image_type=types.RGB, output_dtype=types.FLOAT) self.crop2 = ops.Crop(device=self.device, crop=(224, 224), crop_pos_x=0.0, crop_pos_y=0.0, image_type=types.RGB, output_dtype=types.UINT8) else: self.crop = ops.Crop(device=self.device, crop=(224, 224), crop_pos_x=0.3, crop_pos_y=0.2, image_type=types.RGB)
def __init__(self, batch_size, num_threads, device_id): super(HybridPipe, self).__init__(batch_size, num_threads, device_id, seed = 12) self.input = ops.CaffeReader(path = caffe_db_folder, random_shuffle = True) self.decode = ops.nvJPEGDecoder(device = "mixed", output_type = types.RGB) self.cmnp_all = ops.CropMirrorNormalize(device = "gpu", output_dtype = types.FLOAT, output_layout = types.NHWC, crop = (224, 224), image_type = types.RGB, mean = [128., 128., 128.], std = [1., 1., 1.]) self.cmnp_int = ops.CropMirrorNormalize(device = "gpu", output_dtype = types.FLOAT, output_layout = types.NHWC, crop = (224, 224), image_type = types.RGB, mean = [128, 128, 128], std = [1., 1, 1]) # Left 1 of the arguments as float to test whether mixing types works self.cmnp_1arg = ops.CropMirrorNormalize(device = "gpu", output_dtype = types.FLOAT, output_layout = types.NHWC, crop = (224, 224), image_type = types.RGB, mean = 128, std = 1) self.uniform = ops.Uniform(range = (0,1))
def __init__(self, setting, split_name, batch_size, num_threads, device_id, preprocess=None): super(BaseDataset, self).__init__(batch_size, num_threads, device_id) self._split_name = split_name self._img_path = setting['img_root'] self._gt_path = setting['gt_root'] self._portion = setting['portion'] if 'portion' in setting else None self._train_source = setting['train_source'] self._eval_source = setting['eval_source'] self._test_source = setting['test_source'] self._down_sampling = setting['down_sampling'] self.preprocess = preprocess self._file_names = self._get_file_names(split_name) self.input = ops.CaffeReader(path = lmdb_folder, random_shuffle = True, shard_id = device_id, num_shards = num_gpus) self.decode = ops.ImageDecoder(device = "mixed", output_type = types.RGB) self.resize = ops.Resize(device = "gpu", interp_type = types.INTERP_LINEAR) self.cmn = ops.CropMirrorNormalize(device = "gpu", dtype = types.FLOAT, crop = (227, 227), mean = [128., 128., 128.], std = [1., 1., 1.]) self.uniform = ops.Uniform(range = (0.0, 1.0)) self.resize_rng = ops.Uniform(range = (256, 480))
def __init__(self, device, batch_size, output_type, input_type, fixed_size=None, num_threads=3, device_id=0, num_gpus=1): super(RotatePipeline, self).__init__(batch_size, num_threads, device_id, seed=7865, exec_async=False, exec_pipelined=False) self.name = device self.input = ops.CaffeReader(path=caffe_db_folder, shard_id=device_id, num_shards=num_gpus) self.decode = ops.ImageDecoder(device="cpu", output_type=types.RGB) if input_type != dali.types.UINT8: self.cast = ops.Cast(device=device, dtype=input_type) else: self.cast = None self.uniform = ops.Uniform(range=(-180.0, 180.0), seed=42) self.rotate = ops.Rotate(device=device, size=fixed_size, fill_value=42, dtype=output_type)
def __init__(self, device, batch_size, num_threads=1, device_id=0, num_gpus=1, is_vertical=0, is_horizontal=1 ): super(FlipPipeline, self).__init__(batch_size, num_threads, device_id) self.device = device self.input = ops.CaffeReader(path = caffe_db_folder, shard_id = device_id, num_shards = num_gpus) self.decode = ops.ImageDecoder(device = "cpu", output_type = types.RGB) self.flip = ops.Flip(device = self.device, vertical=is_vertical, horizontal=is_horizontal)
def __init__(self, batch_size, output_type, input_type, fixed_size, num_threads=3, device_id=0, num_gpus=1): super(CVPipeline, self).__init__(batch_size, num_threads, device_id, seed=7865, exec_async=False, exec_pipelined=False) self.name = "cv" self.input = ops.CaffeReader(path = caffe_db_folder, shard_id = device_id, num_shards = num_gpus) self.decode = ops.ImageDecoder(device = "cpu", output_type = types.RGB) self.rotate = ops.PythonFunction(function=CVRotate(output_type, input_type, fixed_size)) self.uniform = ops.Uniform(range = (-180.0, 180.0), seed = 42); self.iter = 0
def __init__(self, device, batch_size, use_wildcard, num_threads=3, device_id=0, num_gpus=1): super(ReshapeWithInput, self).__init__(batch_size, num_threads, device_id, seed=7865, exec_async=False, exec_pipelined=False) self.device = device self.input = ops.CaffeReader(path = caffe_db_folder, shard_id = device_id, num_shards = num_gpus) self.decode = ops.ImageDecoder(device = "cpu", output_type = types.RGB) fn = CollapseChannelsWildcard if use_wildcard else CollapseChannels self.gen_shapes = ops.PythonFunction(function=fn) self.reshape = ops.Reshape(device = device, layout = "ab");
def __init__(self, function, batch_size, num_threads=1, device_id=0): super(PythonOperatorPipeline, self).__init__(batch_size, num_threads, device_id, exec_async=False, exec_pipelined=False, seed=1234) self.input = ops.CaffeReader(path = caffe_db_folder, random_shuffle=False) self.decode = ops.HostDecoder(device = 'cpu', output_type = types.RGB) self.python_function = ops.PythonFunction(function=function)
def __init__(self, batch_size, num_threads, device_id, num_gpus): super(HybridPipe, self).__init__(batch_size, num_threads, device_id) self.input = ops.CaffeReader(path = caffe_db_folder, shard_id = device_id, num_shards = num_gpus) self.decode = ops.HostDecoder(device = "cpu", output_type = types.RGB) self.dump_cpu = ops.DumpImage(device = "cpu", suffix = "cpu") self.dump_gpu = ops.DumpImage(device = "gpu", suffix = "gpu")
def __init__(self, device, batch_size, relative, use_wildcard, num_threads=3, device_id=0, num_gpus=1): super(ReshapeWithArgInput, self).__init__(batch_size, num_threads, device_id, seed=7865, exec_async=False, exec_pipelined=False) self.device = device self.input = ops.CaffeReader(path = caffe_db_folder, shard_id = device_id, num_shards = num_gpus) self.resize = ops.Resize(device = "cpu"); self.decode = ops.ImageDecoder(device = "cpu", output_type = types.RGB) self.gen_shapes = ops.PythonFunction(function=MakeTallFunc(relative, use_wildcard)) self.reshape = ops.Reshape(device = device); self.relative = relative
def __init__(self, **kwargs): super(CaffeReadPipeline, self).__init__(**kwargs) cache_enabled = kwargs['decoder_cache_params']['cache_enabled'] self.input = ops.CaffeReader(path = kwargs['data_paths'][0], shard_id = kwargs['device_id'], num_shards = kwargs['num_gpus'], stick_to_shard = cache_enabled, #skip_cached_images = cache_enabled, prefetch_queue_depth = kwargs['reader_queue_depth'])
def __init__(self, batch_size,function, num_threads=1, device_id=0, num_gpus=1 ): super(WaterPythonPipeline, self).__init__(batch_size, num_threads, device_id, exec_async=False, exec_pipelined=False) self.input = ops.CaffeReader(path = caffe_db_folder, shard_id = device_id, num_shards = num_gpus) self.decode = ops.HostDecoder(device = "cpu", output_type = types.RGB) self.water = ops.PythonFunction(function=function)
def __init__(self, batch_size, num_threads=1, device_id=0): super(CommonPipeline, self).__init__(batch_size, num_threads, device_id, exec_async=False, exec_pipelined=False, seed=1234) self.input = ops.CaffeReader(path=caffe_db_folder, random_shuffle=False) self.decode = ops.HostDecoder(device='cpu', output_type=types.RGB)
def __init__(self, path, batch_size, num_threads=1, device_id=0, num_gpus=1): super(CaffeReaderPipeline, self).__init__(batch_size, num_threads, device_id) self.input = ops.CaffeReader(path = path, shard_id = device_id, num_shards = num_gpus) self.decode = ops.ImageDecoderCrop(device = "cpu", crop = (224, 224), crop_pos_x = 0.3, crop_pos_y = 0.2, output_type = types.RGB)
def __init__(self, device, batch_size, num_threads=1, device_id=0, num_gpus=1, decoder_only=False): super(NoCropPipeline, self).__init__(batch_size, num_threads, device_id) self.decoder_only = decoder_only self.device = device self.input = ops.CaffeReader(path = caffe_db_folder, shard_id = device_id, num_shards = num_gpus) self.decode = ops.HostDecoder(device = "cpu", output_type = types.RGB) if not self.decoder_only: self.cmn = ops.NewCropMirrorNormalize(device = self.device, image_type = types.RGB, output_dtype = types.UINT8, output_layout = types.NHWC)
def __init__(self, device, batch_size, num_threads=1, device_id=0, num_gpus=1, decoder_only=False): super(NoCropPipeline, self).__init__(batch_size, num_threads, device_id) self.decoder_only = decoder_only self.device = device self.input = ops.CaffeReader(path = caffe_db_folder, shard_id = device_id, num_shards = num_gpus) self.decode = ops.ImageDecoder(device = "cpu", output_type = types.RGB) if not self.decoder_only: self.cast = ops.CropMirrorNormalize(device = self.device, dtype = types.FLOAT, output_layout = "HWC") else: self.cast = ops.Cast(device = self.device, dtype = types.FLOAT)
def __init__(self, batch_size, num_threads, device_id): super(TransposePipe, self).__init__(batch_size, num_threads, device_id, seed=12) self.input = ops.CaffeReader(path=caffe_db_folder, shard_id=device_id, num_shards=1) self.decode = ops.nvJPEGDecoder(device="mixed", output_type=types.RGB) self.crop = ops.Crop(device="gpu", crop=(224, 224), image_type=types.RGB) self.transpose = ops.Transpose(device="gpu", perm=[2, 0, 1])
def __init__(self, device, batch_size, output_type, input_type, use_input, num_threads=3, device_id=0, num_gpus=1, inv_map=False): super(WarpPipeline, self).__init__(batch_size, num_threads, device_id, seed=7865, exec_async=False, exec_pipelined=False) self.use_input = use_input self.use_dynamic_size = use_input # avoid Cartesian product self.name = device self.input = ops.CaffeReader(path=caffe_db_folder, shard_id=device_id, num_shards=num_gpus) self.decode = ops.ImageDecoder(device="cpu", output_type=types.RGB) if input_type != dali.types.UINT8: self.cast = ops.Cast(device=device, dtype=input_type) else: self.cast = None static_size = None if self.use_dynamic_size else (240, 320) if use_input: self.transform_source = ops.ExternalSource( lambda: gen_transforms(self.batch_size, 10)) self.warp = ops.WarpAffine(device=device, size=static_size, fill_value=42, dtype=output_type, inverse_map=inv_map) else: warp_matrix = (0.1, 0.9, 10, 0.8, -0.2, -20) self.warp = ops.WarpAffine(device=device, size=static_size, matrix=warp_matrix, fill_value=42, dtype=output_type, inverse_map=inv_map) self.iter = 0