def __init__(self, params, num_threads, device_id): super(DaliPipeline, self).__init__(params.batch_size, num_threads, device_id, seed=12) dii = DaliInputIterator(params, device_id) self.no_copy = params.no_copy if self.no_copy: print("Use Zero Copy ES") self.source = ops.ExternalSource(device="gpu", source=dii, num_outputs=2, layout=["DHWC", "DHWC"], no_copy=self.no_copy) self.do_rotate = True if params.rotate_input == 1 else False print("Enable Rotation" if self.do_rotate else "Disable Rotation") self.rng_angle = ops.Uniform(device="cpu", range=[-1.5, 2.5]) self.icast = ops.Cast(device="cpu", dtype=types.INT32) self.fcast = ops.Cast(device="cpu", dtype=types.FLOAT) self.rotate1 = ops.Rotate(device="gpu", axis=(1, 0, 0), interp_type=types.INTERP_LINEAR) self.rotate2 = ops.Rotate(device="gpu", axis=(0, 1, 0), interp_type=types.INTERP_LINEAR) self.rotate3 = ops.Rotate(device="gpu", axis=(0, 0, 1), interp_type=types.INTERP_LINEAR) self.transpose = ops.Transpose(device="gpu", perm=[3, 0, 1, 2])
def __init__(self, batch_size, layout, iterator, pos_size_iter, num_threads=1, device_id=0, num_gpus=1, axes=None, axis_names=None, normalized_anchor=True, normalized_shape=True, input_type=types.FLOAT, output_type=None): super().__init__(batch_size, num_threads, device_id, seed=12345, exec_async=False, exec_pipelined=False) self.device = "cpu" self.layout = layout self.iterator = iterator self.pos_size_iter = pos_size_iter self.inputs = ops.ExternalSource() self.input_crop_pos = ops.ExternalSource() self.input_crop_size = ops.ExternalSource() self.cast_in = ops.Cast(dtype=input_type) function = partial(slice_func_helper, axes, axis_names, self.layout, normalized_anchor, normalized_shape) self.slice = ops.PythonFunction(function=function, output_layouts=layout) self.output_type = output_type if self.output_type is not None: self.cast_out = ops.Cast(dtype=output_type)
def __init__(self, DATA_PATH, input_height, batch_size, copies, stage, num_threads, device_id, seed=1729): super(SimCLRTransform, self).__init__(batch_size, num_threads, device_id, seed=seed) #this lets our pytorch compat function find the length of our dataset self.num_samples = len(ImageFolder(DATA_PATH)) self.copies = copies self.input_height = input_height self.stage = stage self.input = ops.FileReader(file_root=DATA_PATH, random_shuffle=True, seed=seed) self.to_int64 = ops.Cast(dtype=types.INT64, device="gpu") self.to_int32_cpu = ops.Cast(dtype=types.INT32, device="cpu") self.coin = ops.random.CoinFlip(probability=0.5) self.uniform = ops.random.Uniform(range=[0.6, 0.9]) self.blur_amt = ops.random.Uniform(values=[ float(i) for i in range(1, int(0.1 * self.input_height), 2) ]) self.angles = ops.random.Uniform(range=[0, 360]) self.cast = ops.Cast(dtype=types.FLOAT, device='gpu') self.decode = ops.ImageDecoder(device='mixed', output_type=types.RGB) self.crop = ops.RandomResizedCrop(size=self.input_height, minibatch_size=batch_size, random_area=[0.75, 1.0], device="gpu") self.resize = ops.Resize(resize_x=self.input_height, resize_y=self.input_height, device="gpu") self.flip = ops.Flip(vertical=self.coin(), horizontal=self.coin(), device="gpu") self.colorjit_gray = ops.ColorTwist(brightness=self.uniform(), contrast=self.uniform(), hue=self.uniform(), saturation=self.uniform(), device="gpu") self.blur = ops.GaussianBlur(window_size=self.to_int32_cpu( self.blur_amt()), device="gpu") self.rotate = ops.Rotate( angle=self.angles(), keep_size=True, interp_type=types.DALIInterpType.INTERP_LINEAR, device="gpu") self.swapaxes = ops.Transpose(perm=[2, 0, 1], device="gpu")
def __init__(self, batch_size, num_threads, device_id, file_list, sequence_length, seg_num, seg_length, resize_shorter_scale, crop_target_size, is_training=False, initial_prefetch_size=10, num_shards=1, shard_id=0, dali_mean=0., dali_std=1.0): super(VideoPipe, self).__init__(batch_size, num_threads, device_id) self.input = ops.VideoReader(device="gpu", file_list=file_list, sequence_length=sequence_length, seg_num=seg_num, seg_length=seg_length, is_training=is_training, num_shards=num_shards, shard_id=shard_id, random_shuffle=is_training, initial_fill=initial_prefetch_size) # the sequece data read by ops.VideoReader is of shape [F, H, W, C] # Because the ops.Resize does not support sequence data, # it will be transposed into [H, W, F, C], # then reshaped to [H, W, FC], and then resized like a 2-D image. self.transpose = ops.Transpose(device="gpu", perm=[1, 2, 0, 3]) self.reshape = ops.Reshape(device="gpu", rel_shape=[1.0, 1.0, -1], layout='HWC') self.resize = ops.Resize(device="gpu", resize_shorter=resize_shorter_scale) # crops and mirror are applied by ops.CropMirrorNormalize. # Normalization will be implemented in paddle due to the difficulty of dimension broadcast, # It is not sure whether dimension broadcast can be implemented correctly by dali, just take the Paddle Op instead. self.pos_rng_x = ops.Uniform(range=(0.0, 1.0)) self.pos_rng_y = ops.Uniform(range=(0.0, 1.0)) self.mirror_generator = ops.Uniform(range=(0.0, 1.0)) self.cast_mirror = ops.Cast(dtype=types.DALIDataType.INT32) self.crop_mirror_norm = ops.CropMirrorNormalize( device="gpu", crop=[crop_target_size, crop_target_size], mean=dali_mean, std=dali_std) self.reshape_back = ops.Reshape(device="gpu", shape=[ seg_num, seg_length * 3, crop_target_size, crop_target_size ], layout='FCHW') self.cast_label = ops.Cast(device="gpu", dtype=types.DALIDataType.INT64)
def __init__(self, params, num_threads, device_id): super(DaliPipeline, self).__init__(params.batch_size, num_threads, device_id, seed=12) with h5py.File(params.data_path, 'r') as f: # load hydro and clean up Hydro = f['Hydro'][...] self.Hydro = types.Constant(Hydro, shape=Hydro.shape, layout="DHWC", device="cpu") del Hydro # load nbody and clean up Nbody = f['Nbody'][...] self.Nbody = types.Constant(Nbody, shape=Nbody.shape, layout="DHWC", device="cpu") del Nbody #self.ndummy = np.zeros((20, 20, 20, 4), dtype=np.float32) #self.hdummy = np.zeros((20, 20, 20, 5), dtype=np.float32) #self.Nbody = types.Constant(self.ndummy, shape = self.ndummy.shape, layout = "DHWC", device="cpu") #self.Hydro = types.Constant(self.hdummy, shape = self.hdummy.shape, layout = "DHWC", device="cpu") #self.Nbody = ops.Constant(fdata = self.ndummy.flatten().tolist(), shape = self.ndummy.shape, layout = "DHWC", device = "cpu") #self.Hydro = ops.Constant(fdata = self.hdummy.flatten().tolist(), shape = self.hdummy.shape, layout = "DHWC", device = "cpu") self.do_rotate = True if params.rotate_input == 1 else False print("Enable Rotation" if self.do_rotate else "Disable Rotation") self.rng_angle = ops.Uniform(device="cpu", range=[-1.5, 2.5]) self.rng_pos = ops.Uniform(device="cpu", range=[0., 1.]) self.icast = ops.Cast(device="cpu", dtype=types.INT32) self.fcast = ops.Cast(device="cpu", dtype=types.FLOAT) self.crop = ops.Crop(device="cpu", crop_d=params.data_size, crop_h=params.data_size, crop_w=params.data_size) self.rotate1 = ops.Rotate(device="gpu", axis=(1, 0, 0), interp_type=types.INTERP_LINEAR) self.rotate2 = ops.Rotate(device="gpu", axis=(0, 1, 0), interp_type=types.INTERP_LINEAR) self.rotate3 = ops.Rotate(device="gpu", axis=(0, 0, 1), interp_type=types.INTERP_LINEAR) self.transpose = ops.Transpose(device="gpu", perm=[3, 0, 1, 2])
def __init__(self, data_iterator, batch_size, num_threads, device_id): super(ExternalSourcePipeline, self).__init__(batch_size, num_threads, device_id, seed=12) self.data_iterator = data_iterator self.src = ops.ExternalSource() # self.targ = ops.ExternalSource() self.decode = ops.ImageDecoder( device="mixed", output_type=types.RGB, ) self.cast = ops.Cast(device="gpu", dtype=types.INT32) self.resize = ops.Resize(device="gpu", resize_x=750, resize_y=750, interp_type=types.INTERP_TRIANGULAR) self.cmnp = ops.CropMirrorNormalize( device="gpu", dtype=types.FLOAT, output_layout=types.NCHW, # image_type=types.RGB, mean=[104.0, 117.0, 123.0], std=[1, 1, 1]) self.cmnp2 = ops.CropMirrorNormalize( device="gpu", dtype=types.FLOAT, output_layout=types.NCHW, # image_type=types.RGB, mean=[0, 0, 0], std=[255, 255, 255])
def __init__(self, p: float = .5, ampl_x: float = 10.0, ampl_y: float = 10.0, freq_x: float = 0.049087, freq_y: float = 0.049087, phase_x: float = 0.0, phase_y: float = 0.0, fill_value: float = 0.0): """Initialization Args: p (float, optional): Probability to apply this transformation. Defaults to .5. ampl_x (float, optional): Amplitude of the wave in x direction.. Defaults to 10.0. ampl_y (float, optional): Amplitude of the wave in y direction.. Defaults to 10.0. freq_x (float, optional): Frequency of the wave in x direction. Defaults to 0.049087. freq_y (float, optional): Frequency of the wave in y direction. Defaults to 0.049087. phase_x (float, optional): Phase of the wave in x direction.. Defaults to 0.0. phase_y (float, optional): Phase of the wave in y direction.. Defaults to 0.0. fill_value (float, optional): Color value used for padding pixels. Defaults to 0.0. """ self.water_aug = ops.Water(device='gpu', ampl_x=ampl_x, ampl_y=ampl_y, freq_x=freq_x, freq_y=freq_y, phase_x=phase_x, phase_y=phase_y, fill_value=fill_value) self.rng = ops.CoinFlip(probability=p) self.bool = ops.Cast(dtype=types.DALIDataType.BOOL)
def __init__(self, p: float = .5, hue_limit: Union[List, float] = 20., saturation_limit: Union[List, float] = .5, value_limit: Union[List, float] = .5): """Initialization Args: p (float, optional): Probability to apply this transformation. Defaults to .5. hue_limit (Union[List,float], optional): Range for changing hue in [min,max] value format. If provided as a single float, the range will be (-limit, limit). Defaults to 20.. saturation_limit (Union[List,float], optional): Factor multiplier range for changing saturation in [min,max] value format. If provided as a single float, the range will be 1 + (-limit, limit). Defaults to 0.5. value_limit (Union[List,float], optional): Factor multiplier range for changing value in [min,max] value format. If provided as a single float, the range will be 1 + (-limit, limit). Defaults to 0.5. """ hue_limit = _check_and_convert_limit_value(hue_limit, None, 0) saturation_limit = _check_and_convert_limit_value(saturation_limit) value_limit = _check_and_convert_limit_value(value_limit) self.hsv = ops.Hsv(device='gpu') self.hue_uniform = ops.Uniform(range=hue_limit) self.saturation_uniform = ops.Uniform(range=saturation_limit) self.value_uniform = ops.Uniform(range=value_limit) self.rng = ops.CoinFlip(probability=p) self.bool = ops.Cast(dtype=types.DALIDataType.BOOL)
def __init__(self, file_root, file_list, batch_size, resize_shorter, crop, interp, mean, std, device_id, shard_id=0, num_shards=1, random_shuffle=False, num_threads=4, seed=42): super(HybridValPipe, self).__init__( batch_size, num_threads, device_id, seed=seed) self.input = ops.FileReader( file_root=file_root, file_list=file_list, shard_id=shard_id, num_shards=num_shards, random_shuffle=random_shuffle) self.decode = ops.ImageDecoder(device="mixed", output_type=types.RGB) self.res = ops.Resize( device="gpu", resize_shorter=resize_shorter, interp_type=interp) self.cmnp = ops.CropMirrorNormalize( device="gpu", output_dtype=types.FLOAT, output_layout=types.NCHW, crop=(crop, crop), image_type=types.RGB, mean=mean, std=std) self.to_int64 = ops.Cast(dtype=types.INT64, device="gpu")
def __init__(self, device, batch_size, output_type, input_type, fixed_size=None, num_threads=3, device_id=0, num_gpus=1): super(RotatePipeline, self).__init__(batch_size, num_threads, device_id, seed=7865, exec_async=False, exec_pipelined=False) self.name = device self.input = ops.CaffeReader(path=caffe_db_folder, shard_id=device_id, num_shards=num_gpus) self.decode = ops.ImageDecoder(device="cpu", output_type=types.RGB) if input_type != dali.types.UINT8: self.cast = ops.Cast(device=device, dtype=input_type) else: self.cast = None # TODO(michalz): When we move from Support to CPU operators, replace hardcoded angle # with one taken from the distribution below # self.uniform = ops.Uniform(range = (-180.0, 180.0), seed = 42); self.rotate = ops.Rotate(device=device, size=fixed_size, angle=30, fill_value=42, output_dtype=output_type)
def __init__(self, args): super(TFRecordDetectionPipeline, self).__init__(args.batch_size, args.num_workers, 0, 0) self.input = ops.TFRecordReader( path=os.path.join(test_dummy_data_path, 'small_coco.tfrecord'), index_path=os.path.join(test_dummy_data_path, 'small_coco_index.idx'), features={ 'image/encoded': tfrec.FixedLenFeature((), tfrec.string, ""), 'image/object/class/label': tfrec.VarLenFeature([1], tfrec.int64, 0), 'image/object/bbox': tfrec.VarLenFeature([4], tfrec.float32, 0.0), }, shard_id=0, num_shards=1, random_shuffle=False) self.decode_gpu = ops.ImageDecoder(device="mixed", output_type=types.RGB) self.cast = ops.Cast(dtype=types.INT32) self.box_encoder = ops.BoxEncoder(device="cpu", criteria=0.5, anchors=coco_anchors())
def __init__(self, batch_size, device, data_dir, mean, std, device_id=0, shard_id=0, num_shards=1, num_threads=4, seed=0): super(DaliTransformsValPipeline, self).__init__(batch_size, num_threads, device_id, seed) # should we make this drive the device flags? self.reader = ops.FileReader(file_root=data_dir, shard_id=shard_id, num_shards=num_shards, random_shuffle=False) self.decode = ops.ImageDecoder(device='mixed', output_type=types.RGB, memory_stats=True) self.resize = ops.Resize(device=device, size=[200, 300], interp_type=types.INTERP_TRIANGULAR) self.normalize = ops.CropMirrorNormalize(device=device, dtype=types.FLOAT, output_layout=types.NCHW) #, #mean=mean*255, std=std*255) #self.normalize = ops.Normalize(device=device, dtype=types.FLOAT)#, mean=mean, stddev=std) self.to_int64 = ops.Cast(dtype=types.INT64, device=device)
def __init__(self, batch_size, num_threads, device_id = 0, seed = 0): super(TestPipeline, self).__init__(batch_size, num_threads, device_id, seed) self.input = ops.COCOReader( file_root = file_root, annotations_file = annotations_file, shard_id = 0, num_shards = 1, ratio=False, save_img_ids=True) self.decode = ops.ImageDecoder(device = "cpu", output_type = types.RGB) self.resize = ops.Resize( device = "cpu", image_type = types.RGB, interp_type = types.INTERP_LINEAR) self.cmn = ops.CropMirrorNormalize( device = "cpu", output_dtype = types.FLOAT, crop = (224, 224), image_type = types.RGB, mean = [128., 128., 128.], std = [1., 1., 1.]) self.res_uniform = ops.Uniform(range = (256.,480.)) self.uniform = ops.Uniform(range = (0.0, 1.0)) self.cast = ops.Cast( device = "cpu", dtype = types.FLOAT16)
def __init__(self, DATA_PATH, input_height, batch_size, num_threads, device_id): super(SimCLRTrainDataTransform, self).__init__(batch_size, num_threads, device_id, seed=12) self.COPIES = 3 self.input_height = input_height self.input = ops.FileReader(file_root=DATA_PATH, random_shuffle=True, seed=12) self.coin = ops.CoinFlip(probability=0.5) self.uniform = ops.Uniform(range=[0.7, 1.3]) #-1 to 1 #read image (I think that has to be cpu, do a mixed operation to decode into gpu) self.decode = ops.ImageDecoder(device='mixed', output_type=types.RGB) self.crop = ops.RandomResizedCrop(size=self.input_height, device="gpu") self.flip = ops.Flip(vertical=self.coin(), horizontal=self.coin(), device="gpu") self.colorjit_gray = ops.ColorTwist(brightness=self.uniform(), contrast=self.uniform(), hue=self.uniform(), saturation=self.uniform(), device="gpu") self.blur = ops.GaussianBlur(window_size=int(0.1 * self.input_height), device="gpu") self.swapaxes = ops.Transpose(perm=[2, 0, 1], device="gpu") self.to_int64 = ops.Cast(dtype=types.INT64, device="gpu")
def __init__(self, device, batch_size, output_type, input_type, fixed_size=None, num_threads=3, device_id=0, num_gpus=1): super(RotatePipeline, self).__init__(batch_size, num_threads, device_id, seed=7865, exec_async=False, exec_pipelined=False) self.name = device self.input = ops.readers.Caffe(path=caffe_db_folder, shard_id=device_id, num_shards=num_gpus) self.decode = ops.ImageDecoder(device="cpu", output_type=types.RGB) if input_type != dali.types.UINT8: self.cast = ops.Cast(device=device, dtype=input_type) else: self.cast = None self.uniform = ops.random.Uniform(range=(-180.0, 180.0), seed=42) self.rotate = ops.Rotate(device=device, size=fixed_size, fill_value=42, dtype=output_type)
def __init__(self, function, batch_size, iterator, data_shape, data_layout, dtype, num_threads=1, device_id=0, dictionary={}, default_value=0.0): super(LookupTablePythonOpPipeline, self).__init__(batch_size, num_threads, device_id, exec_async=False, exec_pipelined=False) self.iterator = iterator self.inputs = ops.ExternalSource() self.data_shape = data_shape self.data_layout = data_layout def lookup_table_func(input_data): return function(input_data, shape=data_shape, dictionary=dictionary, default_value=default_value) self.lookup = ops.PythonFunction(function=lookup_table_func) self.set_layout = ops.Reshape(layout=data_layout) self.cast = ops.Cast(dtype=dtype)
def __init__(self, batch_size, num_threads, device, device_id=0, shard_id=0, num_shards=1, seed=0): super(TestPipeline, self).__init__(batch_size, num_threads, device_id, seed) self.device = device self.input = ops.COCOReader(file_root=file_root, annotations_file=annotations_file, shard_id=shard_id, num_shards=num_shards, ratio=False, save_img_ids=True) self.decode = ops.ImageDecoder( device='mixed' if device is 'gpu' else 'cpu', output_type=types.RGB) self.resize = ops.Resize(device=device, image_type=types.RGB, resize_x=224, resize_y=224, interp_type=types.INTERP_LINEAR) self.cmn = ops.CropMirrorNormalize(device=device, output_dtype=types.FLOAT, image_type=types.RGB, mean=[128., 128., 128.], std=[1., 1., 1.]) self.cast = ops.Cast(device=device, dtype=types.INT16)
def __init__(self, tfrecord_files, idx_files, batch_size, device_id=0, rank=0, total_devices=1, num_threads=4): super(TFRecordPipeline, self).__init__(batch_size, num_threads, device_id) self.input = ops.TFRecordReader(path = tfrecord_files, index_path = idx_files, shard_id = rank, num_shards = total_devices, random_shuffle = True, features = {"image/encoded" : tfrec.FixedLenFeature((), tfrec.string, ""), 'image/class/label': tfrec.FixedLenFeature([1], tfrec.int64, -1), }) self.decode = ops.ImageDecoder(device = "mixed", output_type = types.RGB) self.resize = ops.Resize(device = "gpu", resize_shorter = 256) self.cmnp = ops.CropMirrorNormalize(device = "gpu", output_dtype = types.FLOAT16, crop = (224, 224), image_type = types.RGB, mean = [0, 0, 0], std = [1., 1., 1.], output_layout='HWC') self.uniform = ops.Uniform(range = (0.0, 1.0)) self.flip = ops.CoinFlip() self.brightness = ops.Uniform(range = (0.5, 1.5)) self.contrast = ops.Uniform(range = (0.8, 1.3)) self.cast = ops.Cast(device = "gpu", dtype = types.FLOAT16) self.iter = 0
def __init__(self, device, batch_size, iterator, cast_dtypes, num_threads=1, device_id=0): super(CastPipeline, self).__init__(batch_size, num_threads, device_id) self.layout = "HWC" self.device = device self.iterator = iterator self.inputs = ops.ExternalSource() self.cast = [ops.Cast(device=device, dtype=dtype) for dtype in cast_dtypes]
def __init__(self, batch_size, num_threads, device_id, data_dir, crop, size): super(HybridValPipe, self).__init__(batch_size, num_threads, device_id, seed=12 + device_id) if torch.distributed.is_initialized(): local_rank = torch.distributed.get_rank() world_size = torch.distributed.get_world_size() else: local_rank = 0 world_size = 1 self.input = ops.FileReader(file_root=data_dir, shard_id=local_rank, num_shards=world_size, random_shuffle=False) self.decode = ops.ImageDecoder(device="mixed", output_type=types.RGB) self.res = ops.Resize(device="gpu", resize_shorter=size, interp_type=types.INTERP_TRIANGULAR) self.cmnp = ops.CropMirrorNormalize( device="gpu", output_dtype=types.FLOAT, output_layout=types.NCHW, crop=(crop, crop), image_type=types.RGB, # from https://github.com/Armour/pytorch-nn-practice/blob/master/utils/meanstd.py mean=[0.50707516 * 255, 0.48654887 * 255, 0.44091784 * 255], std=[0.26733429 * 255, 0.25643846 * 255, 0.27615047 * 255]) self.to_int64 = ops.Cast(dtype=types.INT64, device="gpu")
def __init__(self, file_list, batch_size, num_threads, device_id, external_data): super(ExternalSourcePipeline, self).__init__(batch_size, num_threads, device_id) self.input = ops.FileReader(file_list= file_list) self.label = ops.ExternalSource() self.crops = ops.ExternalSource() self.decode = ops.ImageDecoder(device="mixed", output_type=types.RGB) self.res = ops.Resize(device="gpu", resize_x=224, resize_y=224) self.cast = ops.Cast(device="cpu", dtype=types.INT32) self.external_data = external_data self.iterator = iter(self.external_data)
def __init__(self, file_root, file_list, batch_size, resize_shorter, crop, min_area, lower, upper, interp, mean, std, device_id, shard_id=0, num_shards=1, random_shuffle=True, num_threads=4, seed=42, pad_output=False, output_dtype=types.FLOAT): super(HybridTrainPipe, self).__init__(batch_size, num_threads, device_id, seed=seed) self.input = ops.FileReader(file_root=file_root, file_list=file_list, shard_id=shard_id, num_shards=num_shards, random_shuffle=random_shuffle) # set internal nvJPEG buffers size to handle full-sized ImageNet images # without additional reallocations device_memory_padding = 211025920 host_memory_padding = 140544512 self.decode = ops.ImageDecoderRandomCrop( device='mixed', output_type=types.RGB, device_memory_padding=device_memory_padding, host_memory_padding=host_memory_padding, random_aspect_ratio=[lower, upper], random_area=[min_area, 1.0], num_attempts=100) self.res = ops.Resize(device='gpu', resize_x=crop, resize_y=crop, interp_type=interp) self.cmnp = ops.CropMirrorNormalize(device="gpu", output_dtype=output_dtype, output_layout=types.NCHW, crop=(crop, crop), image_type=types.RGB, mean=mean, std=std, pad_output=pad_output) self.coin = ops.CoinFlip(probability=0.5) self.to_int64 = ops.Cast(dtype=types.INT64, device="gpu")
def __init__(self, iterator, batch_size, num_threads, device_id): super(ExternalSourcePipeline, self).__init__(batch_size, num_threads, device_id, seed=12) self.iterator = iterator self.input = ops.ExternalSource() self.input_label = ops.ExternalSource() self.decode = ops.ImageDecoder(device = "mixed", output_type = types.RGB) self.cast = ops.Cast(device = "gpu", dtype = types.INT32)
def __init__(self, device_id, n_devices, file_root, file_list, batch_size, sample_rate=16000, window_size=.02, window_stride=.01, nfeatures=64, nfft=512, frame_splicing_factor=3, silence_threshold=-80, dither=.00001, preemph_coeff=.97, lowfreq=0.0, highfreq=0.0, num_threads=1): super().__init__(batch_size, num_threads, device_id, seed=42) self.dither = dither self.frame_splicing_factor = frame_splicing_factor self.read = ops.readers.File(file_root=file_root, file_list=file_list, device="cpu", shard_id=device_id, num_shards=n_devices) self.decode = ops.AudioDecoder(device="cpu", dtype=types.FLOAT, downmix=True) self.normal_distribution = ops.random.Normal(device="cpu") self.preemph = ops.PreemphasisFilter(preemph_coeff=preemph_coeff) self.spectrogram = ops.Spectrogram(device="cpu", nfft=nfft, window_length=window_size * sample_rate, window_step=window_stride * sample_rate) self.mel_fbank = ops.MelFilterBank(device="cpu", sample_rate=sample_rate, nfilter=nfeatures, normalize=True, freq_low=lowfreq, freq_high=highfreq) self.log_features = ops.ToDecibels(device="cpu", multiplier=np.log(10), reference=1.0, cutoff_db=-80) self.get_shape = ops.Shapes(device="cpu") self.normalize = ops.Normalize(axes=[0], device="cpu") self.splicing_transpose = ops.Transpose(device="cpu", perm=[1, 0]) self.splicing_reshape = ops.Reshape(device="cpu", rel_shape=[-1, frame_splicing_factor]) self.splicing_pad = ops.Pad(axes=[0], fill_value=0, align=frame_splicing_factor, shape=[1], device="cpu") self.get_nonsilent_region = ops.NonsilentRegion(device="cpu", cutoff_db=silence_threshold) self.trim_silence = ops.Slice(device="cpu", axes=[0]) self.to_float = ops.Cast(dtype=types.FLOAT)
def __init__(self, batch_size, num_threads, device_id, external_data): super(ExternalSourcePipeline, self).__init__(batch_size, num_threads, device_id, seed=12) self.input = ops.ExternalSource() self.input_label = ops.ExternalSource() self.decode = ops.ImageDecoder(device="mixed", output_type=types.RGB) self.res = ops.Resize(device="gpu", resize_x=240, resize_y=240) self.cast = ops.Cast(device="gpu", dtype=types.UINT8) self.external_data = external_data self.iterator = iter(self.external_data)
def __init__(self, device, batch_size, num_threads=1, device_id=0, num_gpus=1, decoder_only=False): super(NoCropPipeline, self).__init__(batch_size, num_threads, device_id) self.decoder_only = decoder_only self.device = device self.input = ops.CaffeReader(path = caffe_db_folder, shard_id = device_id, num_shards = num_gpus) self.decode = ops.ImageDecoder(device = "cpu", output_type = types.RGB) if not self.decoder_only: self.cast = ops.CropMirrorNormalize(device = self.device, dtype = types.FLOAT, output_layout = "HWC") else: self.cast = ops.Cast(device = self.device, dtype = types.FLOAT)
def __init__(self, device, batch_size, output_type, input_type, use_input, num_threads=3, device_id=0, num_gpus=1, inv_map=False): super(WarpPipeline, self).__init__(batch_size, num_threads, device_id, seed=7865, exec_async=False, exec_pipelined=False) self.use_input = use_input self.use_dynamic_size = use_input # avoid Cartesian product self.name = device self.input = ops.readers.Caffe(path=caffe_db_folder, shard_id=device_id, num_shards=num_gpus) self.decode = ops.decoders.Image(device="cpu", output_type=types.RGB) if input_type != dali.types.UINT8: self.cast = ops.Cast(device=device, dtype=input_type) else: self.cast = None static_size = None if self.use_dynamic_size else (240, 320) fill = 12.5 if output_type == types.FLOAT else 42 output_type_arg = output_type if output_type != input_type else None if use_input: self.transform_source = ops.ExternalSource( lambda: gen_transforms(self.max_batch_size, 10)) self.warp = ops.WarpAffine(device=device, size=static_size, fill_value=fill, dtype=output_type_arg, inverse_map=inv_map) else: warp_matrix = (0.1, 0.9, 10, 0.8, -0.2, -20) self.warp = ops.WarpAffine(device=device, size=static_size, matrix=warp_matrix, fill_value=fill, dtype=output_type_arg, inverse_map=inv_map) self.iter = 0
def __init__(self, batch_size, num_threads, device_id, num_gpus, external_data): super(CXRImagePipeline, self).__init__(batch_size, num_threads, device_id, seed=12) self.input = ops.ExternalSource() self.input_label = ops.ExternalSource() self.decode = ops.ImageDecoder(device="mixed", output_type=types.GRAY) #self.norm = ops.Normalize(device="cpu") self.res = ops.Resize(device="gpu", resize_x=1024, resize_y=1024) self.cast = ops.Cast(device="gpu", dtype=types.FLOAT) self.external_data = external_data self.iterator = iter(self.external_data)
def __init__(self, batch_size, num_threads, device_id, tfrecords, idx_paths): super(ResnetPipeline, self).__init__(batch_size, num_threads, device_id) # Transformation operations below. # From https://docs.nvidia.com/deeplearning/sdk/dali-developer-guide/docs/supported_ops.html self.input = ops.TFRecordReader( path=tfrecords, index_path=idx_paths, features={ "image/encoded": tfrec.FixedLenFeature([], tfrec.string, ""), "image/class/label": tfrec.FixedLenFeature([1], tfrec.float32, 0.0), "image/class/text": tfrec.FixedLenFeature([], tfrec.string, ""), "image/object/bbox/xmin": tfrec.VarLenFeature(tfrec.float32, 0.0), "image/object/bbox/ymin": tfrec.VarLenFeature(tfrec.float32, 0.0), "image/object/bbox/xmax": tfrec.VarLenFeature(tfrec.float32, 0.0), "image/object/bbox/ymax": tfrec.VarLenFeature(tfrec.float32, 0.0) }) self.decode = ops.nvJPEGDecoder(device="mixed", cache_debug=True, output_type=types.RGB) self.resize = ops.Resize(device="gpu", image_type=types.RGB, interp_type=types.INTERP_LINEAR, resize_shorter=256.) self.cmn = ops.CropMirrorNormalize(device="gpu", output_dtype=types.FLOAT, crop=(224, 224), image_type=types.RGB, mean=[0., 0., 0.], std=[1., 1., 1]) self.uniform = ops.Uniform(range=(0.0, 1.0)) self.transpose = ops.Transpose(device="gpu", perm=[0, 3, 1, 2]) self.cast = ops.Cast(device="gpu", dtype=types.INT32) self.iter = 0
def __init__(self, batch_size, num_threads, device_id): super(CMNvsCropPipe, self).__init__(batch_size, num_threads, device_id, seed = 12) self.input = ops.CaffeReader(path = caffe_db_folder, shard_id = device_id, num_shards = 1) self.decode = ops.nvJPEGDecoder(device = "mixed", output_type = types.RGB) self.cmn = ops.CropMirrorNormalize(device = "gpu", output_layout = types.NHWC, output_dtype = types.FLOAT, crop = (224, 224), image_type = types.RGB, mean = [0., 0., 0.], std = [1., 1., 1.]) self.crop = ops.Crop(device = "gpu", crop = (224, 224), image_type = types.RGB) self.uniform = ops.Uniform(range = (0.0, 1.0)) self.cast = ops.Cast(device = "gpu", dtype = types.INT32)