def __init__(self, data_paths, num_gpus, batch_size, num_threads, device_id, prefetch, fp16, nhwc, decoder_type, decoder_cache_params, reader_queue_depth): super(CommonPipeline, self).__init__(batch_size, num_threads, device_id, prefetch_queue_depth=prefetch) if decoder_type == 'roi': print('Using nvJPEG with ROI decoding') self.decode_gpu = ops.nvJPEGDecoderRandomCrop( device="mixed", output_type=types.RGB) self.res = ops.Resize(device="gpu", resize_x=224, resize_y=224) elif decoder_type == 'roi_split': print('Using nvJPEG with ROI decoding and split CPU/GPU stages') self.decode_gpu = ops.nvJPEGDecoderRandomCrop( device="mixed", output_type=types.RGB, split_stages=True) self.res = ops.Resize(device="gpu", resize_x=224, resize_y=224) elif decoder_type == 'cached': assert decoder_cache_params['cache_enabled'] == True cache_size = decoder_cache_params['cache_size'] cache_threshold = decoder_cache_params['cache_threshold'] cache_type = decoder_cache_params['cache_type'] print( 'Using nvJPEG with cache (size : {} threshold: {}, type: {})'. format(cache_size, cache_threshold, cache_type)) self.decode_gpu = ops.nvJPEGDecoder( device="mixed", output_type=types.RGB, cache_size=cache_size, cache_threshold=cache_threshold, cache_type=cache_type, cache_debug=False) self.res = ops.RandomResizedCrop(device="gpu", size=(224, 224)) elif decoder_type == 'split': print('Using nvJPEG with split CPU/GPU stages') self.decode_gpu = ops.nvJPEGDecoder(device="mixed", output_type=types.RGB, split_stages=True) self.res = ops.RandomResizedCrop(device="gpu", size=(224, 224)) else: print('Using nvJPEG') self.decode_gpu = ops.nvJPEGDecoder(device="mixed", output_type=types.RGB) self.res = ops.RandomResizedCrop(device="gpu", size=(224, 224)) layout = types.NHWC if nhwc else types.NCHW out_type = types.FLOAT16 if fp16 else types.FLOAT self.cmnp = ops.CropMirrorNormalize( device="gpu", output_dtype=out_type, output_layout=layout, crop=(224, 224), image_type=types.RGB, mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], std=[0.229 * 255, 0.224 * 255, 0.225 * 255]) self.coin = ops.CoinFlip(probability=0.5)
def __init__(self, batch_size, num_threads, shard_id, image_dir, file_list, nvjpeg_padding, seed=1, num_shards=1, channel_last=True, dtype='half'): super(ValPipeline, self).__init__(batch_size, num_threads, shard_id, seed=seed) self.input = ops.FileReader(file_root=image_dir, file_list=file_list, random_shuffle=False, num_shards=num_shards, shard_id=shard_id) self.decode = ops.nvJPEGDecoder(device="mixed", output_type=types.RGB, device_memory_padding=nvjpeg_padding, host_memory_padding=nvjpeg_padding) self.res = ops.Resize(device="gpu", resize_shorter=256) self.cmnp = ops.CropMirrorNormalize( device="gpu", output_dtype=types.FLOAT16 if dtype == "half" else types.FLOAT, output_layout=types.NHWC if channel_last else types.NCHW, crop=(224, 224), image_type=types.RGB, mean=_pixel_mean, std=_pixel_std, pad_output=True)
def __init__(self, batch_size, num_threads, device_id, image_dir, file_list, seed=1, num_gpu=1, random_area=[0.08, 1.0]): super(TrainPipeline, self).__init__(batch_size, num_threads, device_id, seed=seed) self.input = ops.FileReader(file_root=image_dir, file_list=file_list, random_shuffle=True, num_shards=num_gpu, shard_id=device_id) self.decode = ops.nvJPEGDecoder(device="mixed", output_type=types.RGB) self.rrc = ops.RandomResizedCrop(device="gpu", size=(224, 224), random_area=random_area) self.cmnp = ops.CropMirrorNormalize( device="gpu", output_dtype=types.FLOAT, output_layout=types.NCHW, crop=(224, 224), image_type=types.RGB, mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], std=[0.229 * 255, 0.224 * 255, 0.225 * 255]) self.coin = ops.CoinFlip(probability=0.5)
def __init__(self, batch_size, num_threads, device_id): super(HybridPipe, self).__init__(batch_size, num_threads, device_id, seed = 12) self.input = ops.CaffeReader(path = caffe_db_folder, random_shuffle = True) self.decode = ops.nvJPEGDecoder(device = "mixed", output_type = types.RGB) self.cmnp_all = ops.CropMirrorNormalize(device = "gpu", output_dtype = types.FLOAT, output_layout = types.NHWC, crop = (224, 224), image_type = types.RGB, mean = [128., 128., 128.], std = [1., 1., 1.]) self.cmnp_int = ops.CropMirrorNormalize(device = "gpu", output_dtype = types.FLOAT, output_layout = types.NHWC, crop = (224, 224), image_type = types.RGB, mean = [128, 128, 128], std = [1., 1, 1]) # Left 1 of the arguments as float to test whether mixing types works self.cmnp_1arg = ops.CropMirrorNormalize(device = "gpu", output_dtype = types.FLOAT, output_layout = types.NHWC, crop = (224, 224), image_type = types.RGB, mean = 128, std = 1) self.uniform = ops.Uniform(range = (0,1))
def __init__(self, batch_size, num_threads, device_id, image_dir, file_list, seed=1, num_gpu=1): super(ValPipeline, self).__init__(batch_size, num_threads, device_id, seed=seed) self.input = ops.FileReader(file_root=image_dir, file_list=file_list, random_shuffle=False, num_shards=num_gpu, shard_id=0) self.decode = ops.nvJPEGDecoder(device="mixed", output_type=types.RGB) self.res = ops.Resize(device="gpu", resize_shorter=256) self.cmnp = ops.CropMirrorNormalize( device="gpu", output_dtype=types.FLOAT, output_layout=types.NCHW, crop=(224, 224), image_type=types.RGB, mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
def __init__(self, batch_size, num_threads, device_id, data_dir, crop, size, dali_cpu=False): super(DALIValPipe, self).__init__( batch_size, num_threads, device_id, seed=12 + device_id ) self.input = ops.FileReader( file_root=data_dir, shard_id=device_id, num_shards=1, random_shuffle=True ) #let user decide which pipeline works him bets for RN version he runs if dali_cpu: dali_device = "cpu" self.decode = ops.HostDecoder( device=dali_device, output_type=types.RGB) else: dali_device = "gpu" # This padding sets the size of the internal nvJPEG buffers to be # able to handle all images from full-sized ImageNet # without additional reallocations self.decode = ops.nvJPEGDecoder( device="mixed", output_type=types.RGB, device_memory_padding=211025920, host_memory_padding=140544512 ) self.res = ops.Resize( device=dali_device, resize_shorter=size) self.cmnp = ops.CropMirrorNormalize( device="gpu", output_dtype=types.FLOAT, output_layout=types.NCHW, crop=(crop, crop), image_type=types.RGB, mean=[0.485 * 255,0.456 * 255,0.406 * 255], std=[0.229 * 255,0.224 * 255,0.225 * 255]) print('DALI "{0}" variant'.format(dali_device))
def __init__(self, batch_size, num_threads, device_id, rec_path, idx_path, shard_id, num_shards, crop_shape, min_random_area, max_random_area, min_random_aspect_ratio, max_random_aspect_ratio, nvjpeg_padding, prefetch_queue=3, seed=12, output_layout=types.NCHW, pad_output=True, dtype='float16', mlperf_print=True): super(HybridTrainPipe, self).__init__( batch_size, num_threads, device_id, seed = seed + device_id, prefetch_queue_depth = prefetch_queue) if mlperf_print: # Shuffiling is done inside ops.MXNetReader mx_resnet_print(key=mlperf_log.INPUT_ORDER) self.input = ops.MXNetReader(path = [rec_path], index_path=[idx_path], random_shuffle=True, shard_id=shard_id, num_shards=num_shards) self.decode = ops.nvJPEGDecoder(device = "mixed", output_type = types.RGB, device_memory_padding = nvjpeg_padding, host_memory_padding = nvjpeg_padding) self.rrc = ops.RandomResizedCrop(device = "gpu", random_area = [ min_random_area, max_random_area], random_aspect_ratio = [ min_random_aspect_ratio, max_random_aspect_ratio], size = crop_shape) self.cmnp = ops.CropMirrorNormalize(device = "gpu", output_dtype = types.FLOAT16 if dtype == 'float16' else types.FLOAT, output_layout = output_layout, crop = crop_shape, pad_output = pad_output, image_type = types.RGB, mean = _mean_pixel, std = _std_pixel) self.coin = ops.CoinFlip(probability = 0.5) if mlperf_print: mx_resnet_print( key=mlperf_log.INPUT_CROP_USES_BBOXES, val=False) mx_resnet_print( key=mlperf_log.INPUT_DISTORTED_CROP_RATIO_RANGE, val=(min_random_aspect_ratio, max_random_aspect_ratio)) mx_resnet_print( key=mlperf_log.INPUT_DISTORTED_CROP_AREA_RANGE, val=(min_random_area, max_random_area)) mx_resnet_print( key=mlperf_log.INPUT_MEAN_SUBTRACTION, val=_mean_pixel) mx_resnet_print( key=mlperf_log.INPUT_RANDOM_FLIP)
def __init__(self, batch_size, num_threads, device_id, num_gpus, data_paths, prefetch, fp16, nhwc): super(RN50Pipeline, self).__init__(batch_size, num_threads, device_id, prefetch_queue_depth=prefetch) self.input = ops.FileReader(file_root=data_paths[0], shard_id=device_id, num_shards=num_gpus) self.decode_gpu = ops.nvJPEGDecoder(device="mixed", output_type=types.RGB) self.res = ops.RandomResizedCrop(device="gpu", size=(224, 224)) layout = types.args.nhwc if nhwc else types.NCHW out_type = types.FLOAT16 if fp16 else types.FLOAT self.cmnp = ops.CropMirrorNormalize( device="gpu", output_dtype=out_type, output_layout=layout, crop=(224, 224), image_type=types.RGB, mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], std=[0.229 * 255, 0.224 * 255, 0.225 * 255]) self.coin = ops.CoinFlip(probability=0.5)
def __init__(self, imageset_dir, image_size=128, random_shuffle=False, batch_size=64, num_threads=2, device_id=0): super(ImagePipeline, self).__init__(batch_size, num_threads, device_id, seed=12) eii = ExternalInputIterator(imageset_dir, batch_size, random_shuffle) self.iterator = iter(eii) self.num_inputs = len(eii.frontal_indices) # The source for the inputs and targets self.input = ops.ExternalSource() self.target = ops.ExternalSource() # nvJPEGDecoder below accepts CPU inputs, but returns GPU outputs (hence device = "mixed") self.decode = ops.nvJPEGDecoder(device="mixed", output_type=types.RGB) # The rest of pre-processing is done on the GPU self.res = ops.Resize(device="gpu", resize_x=image_size, resize_y=image_size) self.norm = ops.NormalizePermute(device="gpu", output_dtype=types.FLOAT, mean=[128., 128., 128.], std=[128., 128., 128.], height=image_size, width=image_size)
def __init__(self, batch_size, num_threads, device_id, data_dir, crop, size, local_rank=0, world_size=1): super(HybridValPipe, self).__init__(batch_size, num_threads, device_id, seed=12 + device_id) self.input = ops.FileReader(file_root=data_dir, shard_id=local_rank, num_shards=world_size, random_shuffle=False) self.decode = ops.nvJPEGDecoder(device="mixed", output_type=types.RGB) self.res = ops.Resize(device="gpu", resize_shorter=size, interp_type=types.INTERP_TRIANGULAR) self.cmnp = ops.CropMirrorNormalize( device="gpu", output_dtype=types.FLOAT, output_layout=types.NCHW, crop=(crop, crop), image_type=types.RGB, mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
def __init__(self, batch_size, num_threads, device_id, tfrecord, tfrecord_idx): super(TFRecordPipeline, self).__init__(batch_size, num_threads, device_id) self.input = ops.TFRecordReader(path=tfrecord, index_path=tfrecord_idx, features={ "image/encoded": tfrec.FixedLenFeature( (), tfrec.string, ""), 'image/class/label': tfrec.FixedLenFeature([1], tfrec.int64, -1), 'image/class/text': tfrec.FixedLenFeature([], tfrec.string, ''), }) self.decode = ops.nvJPEGDecoder(device="mixed", output_type=types.RGB) self.resize = ops.Resize(device="gpu", resize_a=256, resize_b=256) self.cmnp = ops.CropMirrorNormalize(device="gpu", output_dtype=types.FLOAT, crop=(224, 224), image_type=types.RGB, mean=[0., 0., 0.], std=[1., 1., 1.], output_layout=types.NHWC) self.uniform = ops.Uniform(range=(0.0, 1.0)) self.iter = 0
class HybridPipe(Pipeline): def __init__(self, batch_size, num_threads, device_id, pipelined=True, async=True): super(HybridPipe, self).__init__(batch_size, num_threads, device_id, exec_pipelined=pipelined, exec_async=async) self.input = ops.ExternalSource() self.decode = ops.nvJPEGDecoder(device="mixed", output_type=types.RGB) self.resize = ops.Resize(device="gpu", image_type=types.RGB, interp_type=types.INTERP_LINEAR) self.cmnp = ops.CropMirrorNormalize(device="gpu", output_dtype=types.FLOAT16, crop=(224, 224), image_type=types.RGB, mean=[128., 128., 128.], std=[1., 1., 1.]) self.uniform = ops.Uniform(range=(0., 1.)) self.resize_uniform = ops.Uniform(range=(256., 480.)) self.mirror = ops.CoinFlip(probability=0.5) self.iter = 0
def __init__(self, name, batch_size, num_workers, device_id, num_gpu, root=os.path.expanduser('~/.mxnet/datasets/face')): super().__init__(batch_size, num_workers, device_id, seed=12 + device_id) idx_files = [os.path.join(root, name, "train.idx")] rec_files = [os.path.join(root, name, "train.rec")] prop = open(os.path.join(root, name, "property"), "r").read().strip().split(',') assert len(prop) == 3 self.num_classes = int(prop[0]) self.image_size = [int(prop[1]), int(prop[2])] self._input = ops.MXNetReader(path=rec_files, index_path=idx_files, random_shuffle=True, num_shards=num_gpu, tensor_init_bytes=self.image_size[0] * self.image_size[1] * 8) self._decode = ops.nvJPEGDecoder(device="mixed", output_type=types.RGB) self._cmnp = ops.CropMirrorNormalize(device="gpu", output_dtype=types.FLOAT, output_layout=types.NCHW, crop=self.image_size, image_type=types.RGB, mean=[127.5, 127.5, 127.5], std=[127.5, 127.5, 127.5]) self._contrast = ops.Contrast(device="gpu", ) self._saturation = ops.Saturation(device="gpu", ) self._brightness = ops.Brightness(device="gpu", ) self._uniform = ops.Uniform(range=(0.7, 1.3)) self._coin = ops.CoinFlip(probability=0.5) self.iter = 0
def __init__(self, batch_size, num_threads, device_id): super(CommonPipeline, self).__init__(batch_size, num_threads, device_id) self.decode_gpu = ops.nvJPEGDecoder(device="mixed", output_type=types.RGB) self.decode_host = ops.HostDecoder(device="cpu", output_type=types.RGB)
def __init__(self, batch_size, num_threads, device_id, prefetch, fp16, nhwc): super(CommonPipeline, self).__init__(batch_size, num_threads, device_id, prefetch_queue_depth=prefetch) self.decode_gpu = ops.nvJPEGDecoder(device="mixed", output_type=types.RGB) self.res = ops.RandomResizedCrop(device="gpu", size=(224, 224)) if nhwc: layout = types.NHWC else: layout = types.NCHW if fp16: out_type = types.FLOAT16 else: out_type = types.FLOAT self.cmnp = ops.CropMirrorNormalize( device="gpu", output_dtype=out_type, output_layout=layout, crop=(224, 224), image_type=types.RGB, mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], std=[0.229 * 255, 0.224 * 255, 0.225 * 255]) self.coin = ops.CoinFlip(probability=0.5)
def __init__(self, batch_size, num_threads, device_id, data_dir, crop, half=False): super(HybridTrainPipe, self).__init__(batch_size, num_threads, device_id, seed=0, exec_async=True) out_type = types.FLOAT if half: out_type = types.FLOAT16 print('Reading from {}'.format(data_dir)) file_name = make_dali_cached_file_list_which_is_also_a_file(data_dir) self.input = ops.FileReader( file_root=data_dir, file_list=file_name, shard_id=0, num_shards=1, random_shuffle=True) self.decode = ops.nvJPEGDecoder(device="mixed", output_type=types.RGB) self.rrc = ops.RandomResizedCrop(device="gpu", size =(crop, crop)) self.cmnp = ops.CropMirrorNormalize( device="gpu", output_dtype=out_type, output_layout=types.NCHW, crop=(crop, crop), image_type=types.RGB, mean=[0.485 * 255,0.456 * 255,0.406 * 255], std=[0.229 * 255,0.224 * 255,0.225 * 255]) self.coin = ops.CoinFlip(probability=0.5) self.jpegs = None self.labels = None
def __init__(self, batch_size, num_threads, device_id, data_dir, crop, size): super(HybridValPipe, self).__init__(batch_size, num_threads, device_id, seed=12 + device_id) if torch.distributed.is_initialized(): local_rank = torch.distributed.get_rank() world_size = torch.distributed.get_world_size() else: local_rank = 0 world_size = 1 self.input = ops.FileReader(file_root=data_dir, shard_id=local_rank, num_shards=world_size, random_shuffle=False) self.decode = ops.nvJPEGDecoder(device="mixed", output_type=types.RGB) self.res = ops.Resize(device="gpu", resize_shorter=size) self.cmnp = ops.CropMirrorNormalize( device="gpu", output_dtype=types.FLOAT, output_layout=types.NCHW, crop=(crop, crop), image_type=types.RGB, mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
def __init__(self, batch_size, num_threads, device_id): super(nvJPEGPipeline, self).__init__(batch_size, num_threads, device_id, seed=12) self.input = ops.FileReader(file_root=FLAGS.image_dir, random_shuffle=True, initial_fill=21) self.decode = ops.nvJPEGDecoder(device="mixed", output_type=types.RGB)
def __init__(self, batch_size, num_threads, device_id, rec_path, idx_path, shard_id, num_shards, crop_shape, nvjpeg_padding, prefetch_queue=3, seed=12, resize_shp=None, output_layout=types.NCHW, pad_output=True, dtype='float16', mlperf_print=True): super(HybridValPipe, self).__init__(batch_size, num_threads, device_id, seed=seed + device_id, prefetch_queue_depth=prefetch_queue) self.input = ops.MXNetReader(path=[rec_path], index_path=[idx_path], random_shuffle=False, shard_id=shard_id, num_shards=num_shards) self.decode = ops.nvJPEGDecoder(device="mixed", output_type=types.RGB, device_memory_padding=nvjpeg_padding, host_memory_padding=nvjpeg_padding) self.resize = ops.Resize( device="gpu", resize_shorter=resize_shp) if resize_shp else None self.cmnp = ops.CropMirrorNormalize( device="gpu", output_dtype=types.FLOAT16 if dtype == 'float16' else types.FLOAT, output_layout=output_layout, crop=crop_shape, pad_output=pad_output, image_type=types.RGB, mean=_mean_pixel, std=_std_pixel) if mlperf_print: mx_resnet_print(key=mlperf_log.INPUT_MEAN_SUBTRACTION, val=_mean_pixel) mx_resnet_print(key=mlperf_log.INPUT_RESIZE_ASPECT_PRESERVING) mx_resnet_print(key=mlperf_log.INPUT_CENTRAL_CROP)
def __init__(self, batch_size, num_threads, device_id, size=1024): super(CommonPipeline, self).__init__(batch_size, num_threads, device_id) self.decode = ops.nvJPEGDecoder(device="mixed", output_type=types.RGB) self.cmn = ops.NormalizePermute(device="gpu", height=size, width=size, output_dtype=types.FLOAT, image_type=types.RGB, mean=[0.5 * 255, 0.5 * 255, 0.5 * 255], std=[0.5 * 255, 0.5 * 255, 0.5 * 255])
def __init__(self, batch_size, num_threads, device_id, crop, colorjitter=None, dali_cpu=False): super(ImageNetTrainPipe, self).__init__(batch_size, num_threads, device_id, seed=12 + device_id) self.data_input = ops.ExternalSource() self.label_input = ops.ExternalSource() self.colorjitter = colorjitter # let user decide which pipeline works him bets for RN version he runs if dali_cpu: dali_device = "cpu" self.decode = ops.HostDecoderRandomCrop(device=dali_device, output_type=types.RGB) self.res = ops.Resize(resize_x=crop, resize_y=crop) else: dali_device = "gpu" # This padding sets the size of the internal nvJPEG buffers to be able to # handle all images from full-sized ImageNet without additional reallocations self.decode = ops.nvJPEGDecoder(device="mixed", output_type=types.RGB, device_memory_padding=211025920, host_memory_padding=140544512) self.res = ops.RandomResizedCrop(device=dali_device, size=(crop, crop)) self.cmnp = ops.CropMirrorNormalize( device="gpu", output_dtype=types.FLOAT, output_layout=types.NCHW, crop=(crop, crop), image_type=types.RGB, mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], std=[0.229 * 255, 0.224 * 255, 0.225 * 255]) self.coin = ops.CoinFlip(probability=0.5) if self.colorjitter is not None: self.colorjit = ops.ColorTwist(device="gpu") self.rng_brightness = ops.Uniform(range=(1.0 - self.colorjitter[0], 1.0 + self.colorjitter[0])) self.rng_contrast = ops.Uniform(range=(1.0 - self.colorjitter[1], 1.0 + self.colorjitter[1])) self.rng_saturation = ops.Uniform(range=(1.0 - self.colorjitter[2], 1.0 + self.colorjitter[2])) self.rng_hue = ops.Uniform(range=(-self.colorjitter[3], self.colorjitter[3]))
def __init__(self, args, batch_size, num_threads, device_id, rec_path, idx_path, shard_id, num_shards, crop_shape, nvjpeg_padding, prefetch_queue=3, resize_shp=None, output_layout=types.NCHW, pad_output=True, dtype='float16', dali_cpu=False): super(HybridValPipe, self).__init__(batch_size, num_threads, device_id, seed=12 + device_id, prefetch_queue_depth=prefetch_queue) self.input = ops.MXNetReader(path=[rec_path], index_path=[idx_path], random_shuffle=False, shard_id=shard_id, num_shards=num_shards) if dali_cpu: dali_device = "cpu" self.decode = ops.HostDecoder(device=dali_device, output_type=types.RGB) else: dali_device = "gpu" self.decode = ops.nvJPEGDecoder( device="mixed", output_type=types.RGB, device_memory_padding=nvjpeg_padding, host_memory_padding=nvjpeg_padding) self.resize = ops.Resize( device=dali_device, resize_shorter=resize_shp) if resize_shp else None self.cmnp = ops.CropMirrorNormalize( device="gpu", output_dtype=types.FLOAT16 if dtype == 'float16' else types.FLOAT, output_layout=output_layout, crop=crop_shape, pad_output=pad_output, image_type=types.RGB, mean=args.rgb_mean, std=args.rgb_std)
def __init__(self, data_path, batch_size, num_threads, device_id, decoder): super(DecoderPipeline, self).__init__(batch_size, num_threads, device_id, prefetch_queue_depth=1) self.input = ops.FileReader(file_root=data_path, shard_id=0, num_shards=1) if decoder == 'nvJPEGDecoder': self.decode = ops.nvJPEGDecoder(device="mixed", output_type=types.RGB) elif decoder == 'HostDecoder': self.decode = ops.HostDecoder(device="cpu", output_type=types.RGB)
def __init__(self, batch_size, num_threads, device_id, data_dir, crop): super(HybridTrainPipe, self).__init__(batch_size, num_threads, device_id, seed = 12 + device_id) self.input = ops.FileReader(file_root = data_dir, shard_id = args.local_rank, num_shards = args.world_size, random_shuffle = True) self.decode = ops.nvJPEGDecoder(device = "mixed", output_type = types.RGB) self.rrc = ops.RandomResizedCrop(device = "gpu", size = (crop, crop)) self.cmnp = ops.CropMirrorNormalize(device = "gpu", output_dtype = types.FLOAT, output_layout = types.NCHW, crop = (crop, crop), image_type = types.RGB, mean = [0.485 * 255,0.456 * 255,0.406 * 255], std = [0.229 * 255,0.224 * 255,0.225 * 255]) self.coin = ops.CoinFlip(probability = 0.5)
def __init__(self, batch_size, num_threads, device_id): super(TransposePipe, self).__init__(batch_size, num_threads, device_id, seed=12) self.input = ops.CaffeReader(path=caffe_db_folder, shard_id=device_id, num_shards=1) self.decode = ops.nvJPEGDecoder(device="mixed", output_type=types.RGB) self.crop = ops.Crop(device="gpu", crop=(224, 224), image_type=types.RGB) self.transpose = ops.Transpose(device="gpu", perm=[2, 0, 1])
def __init__(self, batch_size, num_threads, device_id, tfrecords, idx_paths): super(ResnetPipeline, self).__init__(batch_size, num_threads, device_id) # Transformation operations below. # From https://docs.nvidia.com/deeplearning/sdk/dali-developer-guide/docs/supported_ops.html self.input = ops.TFRecordReader( path=tfrecords, index_path=idx_paths, features={ "image/encoded": tfrec.FixedLenFeature([], tfrec.string, ""), "image/class/label": tfrec.FixedLenFeature([1], tfrec.float32, 0.0), "image/class/text": tfrec.FixedLenFeature([], tfrec.string, ""), "image/object/bbox/xmin": tfrec.VarLenFeature(tfrec.float32, 0.0), "image/object/bbox/ymin": tfrec.VarLenFeature(tfrec.float32, 0.0), "image/object/bbox/xmax": tfrec.VarLenFeature(tfrec.float32, 0.0), "image/object/bbox/ymax": tfrec.VarLenFeature(tfrec.float32, 0.0) }) self.decode = ops.nvJPEGDecoder(device="mixed", cache_debug=True, output_type=types.RGB) self.resize = ops.Resize(device="gpu", image_type=types.RGB, interp_type=types.INTERP_LINEAR, resize_shorter=256.) self.cmn = ops.CropMirrorNormalize(device="gpu", output_dtype=types.FLOAT, crop=(224, 224), image_type=types.RGB, mean=[0., 0., 0.], std=[1., 1., 1]) self.uniform = ops.Uniform(range=(0.0, 1.0)) self.transpose = ops.Transpose(device="gpu", perm=[0, 3, 1, 2]) self.cast = ops.Cast(device="gpu", dtype=types.INT32) self.iter = 0
def __init__(self, batch_size, num_threads, device_id, cache_size): super(nvJPEGDecoderPipeline, self).__init__(batch_size, num_threads, device_id, seed=seed) self.input = ops.FileReader(file_root=image_dir) policy = None if cache_size > 0: policy = "threshold" self.decode = ops.nvJPEGDecoder(device='mixed', output_type=types.RGB, cache_debug=False, cache_size=cache_size, cache_type=policy, cache_batch_copy=True)
def __init__(self, batch_size, file_root, annotations_file, default_boxes, seed, device_id=0, num_threads=4): super(COCOPipeline, self).__init__(batch_size=batch_size, device_id=device_id, num_threads=num_threads, seed=seed) self.input = ops.COCOReader(file_root=file_root, annotations_file=annotations_file, ratio=True, ltrb=True, random_shuffle=True) self.decode = ops.nvJPEGDecoder(device="mixed", output_type=types.RGB) # Augumentation techniques self.crop = ops.RandomBBoxCrop(device="cpu", aspect_ratio=[0.5, 2.0], thresholds=[0.1, 0.3, 0.5, 0.7, 0.9], scaling=[0.8, 1.0], ltrb=True) self.slice = ops.Slice(device="gpu") self.twist = ops.ColorTwist(device="gpu") self.resize = ops.Resize(device="gpu", resize_x=300, resize_y=300) self.normalize = ops.CropMirrorNormalize( device="gpu", crop=(300, 300), mean=[0.485 * 255., 0.456 * 255., 0.406 * 255.], std=[0.229 * 255., 0.224 * 255., 0.225 * 255.]) # Random variables self.rng1 = ops.Uniform(range=[0.5, 1.5]) self.rng2 = ops.Uniform(range=[0.875, 1.125]) self.rng3 = ops.Uniform(range=[-0.5, 0.5]) self.flip = ops.Flip(device="gpu") self.bbflip = ops.BbFlip(device="cpu", ltrb=True) self.flip_coin = ops.CoinFlip(probability=0.5) self.box_encoder = ops.BoxEncoder(device="cpu", criteria=0.5, anchors=default_boxes.as_ltrb_list())
def __init__(self, batch_size, num_threads, device_id, data_dir): super(HybridPipe, self).__init__(batch_size, num_threads, device_id) self.input = ops.Caffe2Reader(path = data_dir, shard_id = args.rank, num_shards = args.world_size) self.decode= ops.nvJPEGDecoder(device = "mixed", output_type = types.RGB) self.rrc = ops.RandomResizedCrop(device = "gpu", size = (224, 224)) self.cmnp = ops.CropMirrorNormalize(device = "gpu", output_dtype = types.FLOAT, output_layout = types.NCHW, image_type = types.RGB, crop = (224, 224), mean = [0.485 * 255, 0.456 * 255, 0.406 * 255], std = [0.229 * 255, 0.224 * 255, 0.225 * 255]) self.coin = ops.CoinFlip(probability = 0.5)
def __init__(self, batch_size, num_threads, device_id): super(HybridPipe, self).__init__(batch_size, num_threads, device_id, seed = 12) self.input = ops.CaffeReader(path = caffe_db_folder, random_shuffle = True) self.decode = ops.nvJPEGDecoder(device = "mixed", output_type = types.RGB) self.cmnp = ops.CropMirrorNormalize(device = "gpu", output_dtype = types.FLOAT, output_layout = types.NHWC, crop = (224, 224), image_type = types.RGB, mean = [128., 128., 128.], std = [1., 1., 1.]) self.rotate = ops.Rotate(device = "gpu", angle = 45.0, fill_value = 128, interp_type=types.INTERP_LINEAR) self.uniform = ops.Uniform(range = (0.0,1.0)) self.iter = 0
def __init__(self, batch_size, num_threads, device_id): super(HybridPipe, self).__init__(batch_size, num_threads, device_id, seed = 12) self.input = ops.CaffeReader(path = caffe_db_folder, random_shuffle = True) self.decode = ops.nvJPEGDecoder(device = "mixed", output_type = types.RGB) self.cmnp = ops.CropMirrorNormalize(device = "gpu", output_dtype = types.FLOAT, crop = (224, 224), image_type = types.RGB, mean = [128., 128., 128.], std = [1., 1., 1.]) self.coin = ops.CoinFlip() self.uniform = ops.Uniform(range = (0.0,1.0)) self.iter = 0