Esempio n. 1
0
    def __init__(self, batch_size, num_threads, device_id):
        super(CommonPipeline, self).__init__(batch_size, num_threads,
                                             device_id)

        self.decode = ops.nvJPEGDecoder(device="mixed", output_type=types.RGB)
        self.resize = ops.Resize(device="gpu",
                                 image_type=types.RGB,
                                 interp_type=types.INTERP_LINEAR)
        self.cmn = ops.CropMirrorNormalize(device="gpu",
                                           output_dtype=types.FLOAT,
                                           crop=(227, 227),
                                           image_type=types.RGB,
                                           mean=[128., 128., 128.],
                                           std=[1., 1., 1.])
        self.uniform = ops.Uniform(range=(0.0, 1.0))
        self.resize_rng = ops.Uniform(range=(256, 480))
Esempio n. 2
0
 def __init__(self, batch_size, num_threads, device_id):
     super(HybridPipe, self).__init__(batch_size, num_threads, device_id, seed = 12)
     self.input = ops.CaffeReader(path = caffe_db_folder, random_shuffle = True)
     self.decode = ops.nvJPEGDecoder(device = "mixed", output_type = types.RGB)
     self.cmnp = ops.CropMirrorNormalize(device = "gpu",
                                         output_dtype = types.FLOAT,
                                         output_layout = types.NHWC,
                                         crop = (224, 224),
                                         image_type = types.RGB,
                                         mean = [128., 128., 128.],
                                         std = [1., 1., 1.])
     self.rotate = ops.Rotate(device = "gpu", angle = 45.0,
                              fill_value = 128,
                              interp_type=types.INTERP_LINEAR)
     self.uniform = ops.Uniform(range = (0.0,1.0))
     self.iter = 0
    def __init__(self, batch_size, num_threads, device_id, num_gpus, data_paths, prefetch, fp16, nhwc):
        super(RN50Pipeline, self).__init__(batch_size, num_threads, device_id, prefetch_queue_depth=prefetch)
        self.input = ops.FileReader(file_root = data_paths[0], shard_id = device_id, num_shards = num_gpus)
        self.decode_gpu = ops.ImageDecoder(device = "mixed", output_type = types.RGB)
        self.res = ops.RandomResizedCrop(device="gpu", size =(224,224))

        layout = types.args.nhwc if nhwc else types.NCHW
        out_type = types.FLOAT16 if fp16 else types.FLOAT

        self.cmnp = ops.CropMirrorNormalize(device="gpu",
                                            dtype=out_type,
                                            output_layout=layout,
                                            crop=(224, 224),
                                            mean=[0.485 * 255,0.456 * 255,0.406 * 255],
                                            std=[0.229 * 255,0.224 * 255,0.225 * 255])
        self.coin = ops.random.CoinFlip(probability=0.5)
Esempio n. 4
0
 def __init__(self, image_dir, batch_size, num_threads, device_id, num_gpus=1, seed=1, train=True):
     super(DataPipeline, self).__init__(
         batch_size, num_threads, device_id, seed=seed)
     self.input = ops.FileReader(file_root=image_dir,
                                 random_shuffle=True, num_shards=num_gpus, shard_id=device_id)
     self.decode = ops.ImageDecoder(device='mixed', output_type=types.RGB)
     self.rrc = ops.RandomResizedCrop(device='gpu', size=(128, 128))
     self.cmn = ops.CropMirrorNormalize(device='gpu',
                                        crop=(128, 128),
                                        image_type=types.RGB,
                                        mean=[0.5*256, 0.5*256, 0.5*256],
                                        std=[0.5*256, 0.5*256, 0.5*256],
                                        )
     self.coin = ops.CoinFlip(probability=0.5)
     self.res = ops.Resize(device="gpu", resize_shorter=256)
     self.train = train
Esempio n. 5
0
 def __init__(self, batch_size, num_threads, device_id, pipelined = True, exec_async = True):
     super(C2Pipe, self).__init__(batch_size,
                                  num_threads,
                                  device_id,
                                  exec_pipelined=pipelined,
                                  exec_async=exec_async)
     self.input = ops.ExternalSource()
     self.decode = ops.ImageDecoder(device = 'cpu', output_type = types.RGB)
     self.rcm = ops.FastResizeCropMirror(crop = (224, 224))
     self.np = ops.CropMirrorNormalize(device = "gpu",
                                       dtype = types.FLOAT16,
                                       mean = [128., 128., 128.],
                                       std = [1., 1., 1.])
     self.uniform = ops.Uniform(range = (0., 1.))
     self.resize_uniform = ops.Uniform(range = (256., 480.))
     self.mirror = ops.CoinFlip(probability = 0.5)
Esempio n. 6
0
    def __init__(self, file_tfrecord, batch_size, num_workers, device_id=0):
        super().__init__(batch_size, num_workers, device_id)
        self.input = ops.TFRecordReader(
            path=file_tfrecord.as_posix(),
            index_path=(file_tfrecord.parent /
                        (file_tfrecord.stem + '.idx')).as_posix(),
            features={'encoded': tfrec.FixedLenFeature((), tfrec.string, "")})

        self.decode = ops.ImageDecoder(device='mixed', output_type=types.RGB)

        self.cmnp = ops.CropMirrorNormalize(device='gpu',
                                            output_dtype=types.FLOAT,
                                            output_layout=types.NCHW,
                                            image_type=types.RGB,
                                            mean=[124, 116, 104],
                                            std=[58, 57, 57])
    def __init__(self, data_root, data_list, sampler, crop, size):
        super(ImageNetValPipeV2, self).__init__()

        self.mc_input = ops.McReader(file_root=data_root,
                                     file_list=data_list,
                                     sampler_index=list(sampler))

        self.decode = ops.ImageDecoder(device="mixed", output_type=types.RGB)
        self.res = ops.Resize(device="gpu", resize_shorter=size)
        self.cmnp = ops.CropMirrorNormalize(device="gpu",
                                            output_dtype=types.FLOAT,
                                            output_layout=types.NCHW,
                                            crop=(crop, crop),
                                            image_type=types.RGB,
                                            mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
                                            std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
Esempio n. 8
0
 def __init__(self, batch_size, num_threads, device_id):
     super(HybridPipe, self).__init__(batch_size,
                                      num_threads,
                                      device_id,
                                      seed = 12)
     self.input = ops.CaffeReader(path = caffe_db_folder, random_shuffle = True)
     self.decode = ops.nvJPEGDecoder(device = "mixed", output_type = types.RGB)
     self.cmnp = ops.CropMirrorNormalize(device = "gpu",
                                         output_dtype = types.FLOAT,
                                         crop = (224, 224),
                                         image_type = types.RGB,
                                         mean = [128., 128., 128.],
                                         std = [1., 1., 1.])
     self.coin = ops.CoinFlip()
     self.uniform = ops.Uniform(range = (0.0,1.0))
     self.iter = 0
Esempio n. 9
0
    def __init__(
            self,
            name,
            batch_size,
            num_threads,
            device_id,
            num_shards,
            shard_id,
            root=os.path.expanduser('~/.mxnet/datasets/face'),
    ):
        super().__init__(batch_size, num_threads, device_id, seed=12)

        idx_files = [os.path.join(root, name, "train.idx")]
        rec_files = [os.path.join(root, name, "train.rec")]
        prop = open(os.path.join(root, name, "property"),
                    "r").read().strip().split(',')
        assert len(prop) == 3
        self.num_classes = int(prop[0])
        self.image_size = [int(prop[1]), int(prop[2])]
        self.size = 0
        for idx_file in idx_files:
            with open(idx_file, "r") as f:
                self.size += len(list(f.readlines()))

        self._input = ops.MXNetReader(path=rec_files,
                                      index_path=idx_files,
                                      random_shuffle=True,
                                      num_shards=num_shards,
                                      shard_id=shard_id,
                                      seed=12,
                                      tensor_init_bytes=self.image_size[0] *
                                      self.image_size[1] * 8)
        self._decode = ops.ImageDecoder(device="mixed", output_type=types.RGB)

        self._cmnp = ops.CropMirrorNormalize(device="gpu",
                                             output_dtype=types.FLOAT,
                                             output_layout=types.NCHW,
                                             crop=self.image_size,
                                             image_type=types.RGB,
                                             mean=[0., 0., 0.],
                                             std=[255., 255., 255.])
        self._contrast = ops.Contrast(device="gpu")
        self._saturation = ops.Saturation(device="gpu")
        self._brightness = ops.Brightness(device="gpu")

        self._uniform = ops.Uniform(range=(0.7, 1.3))
        self._coin = ops.CoinFlip(probability=0.5)
Esempio n. 10
0
    def __init__(self, data_paths, num_shards, batch_size, num_threads,
                 device_id, prefetch, fp16, random_shuffle, nhwc,
                 dont_use_mmap, decoder_type, decoder_cache_params,
                 reader_queue_depth, shard_id):
        super(CommonPipeline, self).__init__(batch_size,
                                             num_threads,
                                             device_id,
                                             random_shuffle,
                                             prefetch_queue_depth=prefetch)
        if decoder_type == 'roi':
            print('Using nvJPEG with ROI decoding')
            self.decode_gpu = ops.decoders.ImageRandomCrop(
                device="mixed", output_type=types.RGB)
            self.res = ops.Resize(device="gpu", resize_x=224, resize_y=224)
        elif decoder_type == 'cached':
            assert decoder_cache_params['cache_enabled'] == True
            cache_size = decoder_cache_params['cache_size']
            cache_threshold = decoder_cache_params['cache_threshold']
            cache_type = decoder_cache_params['cache_type']
            print(
                'Using nvJPEG with cache (size : {} threshold: {}, type: {})'.
                format(cache_size, cache_threshold, cache_type))
            self.decode_gpu = ops.decoders.Image(
                device="mixed",
                output_type=types.RGB,
                cache_size=cache_size,
                cache_threshold=cache_threshold,
                cache_type=cache_type,
                cache_debug=False)
            self.res = ops.RandomResizedCrop(device="gpu", size=(224, 224))
        else:
            print('Using nvJPEG')
            self.decode_gpu = ops.decoders.Image(device="mixed",
                                                 output_type=types.RGB)
            self.res = ops.RandomResizedCrop(device="gpu", size=(224, 224))

        layout = types.NHWC if nhwc else types.NCHW
        out_type = types.FLOAT16 if fp16 else types.FLOAT

        self.cmnp = ops.CropMirrorNormalize(
            device="gpu",
            dtype=out_type,
            output_layout=layout,
            crop=(224, 224),
            mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
            std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
        self.coin = ops.random.CoinFlip(probability=0.5)
Esempio n. 11
0
    def __new__(
        cls,
        crop=None,
        mirror=None,
        mean=0.,
        std=1.,
        dtype='float32',
        output_layout='NCHW',
        **kwargs
    ):
        """Create a ``CropMirrorNormalize`` operator.

        Parameters
        ----------
        crop : Sequence[int], optional
            The cropped spatial dimensions for output.
        mirror : {0, 1}, optional
            Whether to apply the horizontal flip.
        mean : Union[float, Sequence[float]], optional
            The values to subtract.
        std : Union[float, Sequence[float]], optional
            The values to divide after subtraction.
        dtype : {'float16', 'float32'}, optional
            The data type of output.
        output_layout : {'NCHW', 'NHWC'}, optional
            The data format of output.

        Returns
        -------
        nvidia.dali.ops.CropMirrorNormalize
            The operator.

        """
        if isinstance(dtype, six.string_types):
            dtype = getattr(types, dtype.upper())
        if isinstance(output_layout, six.string_types):
            output_layout = getattr(types, output_layout.upper())
        return ops.CropMirrorNormalize(
            crop=crop,
            mirror=mirror,
            mean=mean,
            std=std,
            dtype=dtype,
            output_layout=output_layout,
            device=context.get_device_type(),
            **kwargs
        )
Esempio n. 12
0
 def __init__(self,
              batch_size,
              num_threads,
              device_id,
              data_dir,
              crop,
              dali_cpu=False):
     super(HybridTrainPipe, self).__init__(batch_size,
                                           num_threads,
                                           device_id,
                                           seed=12 + device_id)
     # MXnet rec reader
     self.input = ops.MXNetReader(path=join(data_dir, "train_label.rec"),
                                  index_path=join(data_dir,
                                                  "train_label.idx"),
                                  random_shuffle=True,
                                  shard_id=args.local_rank,
                                  num_shards=args.world_size)
     #let user decide which pipeline works him bets for RN version he runs
     dali_device = 'cpu' if dali_cpu else 'gpu'
     decoder_device = 'cpu' if dali_cpu else 'mixed'
     # This padding sets the size of the internal nvJPEG buffers to be able to handle all images from full-sized ImageNet
     # without additional reallocations
     device_memory_padding = 211025920 if decoder_device == 'mixed' else 0
     host_memory_padding = 140544512 if decoder_device == 'mixed' else 0
     self.decode = ops.ImageDecoderRandomCrop(
         device=decoder_device,
         output_type=types.RGB,
         device_memory_padding=device_memory_padding,
         host_memory_padding=host_memory_padding,
         random_aspect_ratio=[0.8, 1.25],
         random_area=[0.1, 1.0],
         num_attempts=100)
     self.res = ops.Resize(device=dali_device,
                           resize_x=crop,
                           resize_y=crop,
                           interp_type=types.INTERP_TRIANGULAR)
     self.cmnp = ops.CropMirrorNormalize(
         device="gpu",
         output_dtype=types.FLOAT,
         output_layout=types.NCHW,
         crop=(crop, crop),
         image_type=types.RGB,
         mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
         std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
     self.coin = ops.CoinFlip(probability=0.5)
     print('DALI "{0}" variant'.format(dali_device))
Esempio n. 13
0
 def __init__(self, batch_size, num_threads, device_id, num_gpus, cfg):
     super(CommonPipeline, self).__init__(batch_size, num_threads,
                                          device_id)
     self.decode = ops.ImageDecoder(device="mixed", output_type=types.RGB)
     self.resize = ops.Resize(device="gpu",
                              image_type=types.RGB,
                              interp_type=types.INTERP_LINEAR)
     self.cmn = ops.CropMirrorNormalize(
         device="gpu",
         output_dtype=types.FLOAT,
         # crop=(227, 227),
         image_type=types.RGB,
         mean=cfg.mean,
         std=cfg.std)
     self.uniform = ops.Uniform(range=(0.0, 1.0))
     self.resize_rng = ops.Uniform(range=(512, 512))
     self.cast = ops.Cast(device="gpu", dtype=types.UINT8)
Esempio n. 14
0
 def __init__(self,
              batch_size,
              num_threads,
              device_id,
              data_dir,
              crop,
              dali_cpu=False,
              args_new=None):
     super(HybridUnifiedTrainPipe, self).__init__(batch_size,
                                                  num_threads,
                                                  device_id,
                                                  seed=12 + device_id)
     self.input = ops.FileReader(file_root=data_dir,
                                 shard_id=args_new.local_rank,
                                 num_shards=args_new.num_procs,
                                 shuffle_after_epoch=True,
                                 cache_size=args.cache_size)
     dali_device = 'cpu' if dali_cpu else 'gpu'
     decoder_device = 'cpu' if dali_cpu else 'mixed'
     # This padding sets the size of the internal nvJPEG buffers to be able to handle all images from full-sized ImageNet
     # without additional reallocations
     device_memory_padding = 211025920 if decoder_device == 'mixed' else 0
     host_memory_padding = 140544512 if decoder_device == 'mixed' else 0
     self.decode = ops.ImageDecoderRandomCrop(
         device=decoder_device,
         output_type=types.RGB,
         device_memory_padding=device_memory_padding,
         host_memory_padding=host_memory_padding,
         random_aspect_ratio=[0.8, 1.25],
         random_area=[0.1, 1.0],
         num_attempts=100)
     self.res = ops.Resize(device=dali_device,
                           resize_x=crop,
                           resize_y=crop,
                           interp_type=types.INTERP_TRIANGULAR)
     self.cmnp = ops.CropMirrorNormalize(
         device=dali_device,
         output_dtype=types.FLOAT,
         output_layout=types.NCHW,
         crop=(crop, crop),
         image_type=types.RGB,
         mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
         std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
     self.coin = ops.CoinFlip(probability=0.5)
     print('DALI "{0}" variant, shards={1}'.format(dali_device,
                                                   args_new.num_procs))
Esempio n. 15
0
 def __init__(self, function, device, num_outputs=1):
     super(PythonFunctionPipeline, self).__init__(BATCH_SIZE,
                                                  NUM_WORKERS,
                                                  DEVICE_ID,
                                                  seed=SEED,
                                                  exec_async=False,
                                                  exec_pipelined=False)
     self.device = device
     self.reader = ops.readers.File(file_root=images_dir)
     self.decode = ops.decoders.Image(device='cpu', output_type=types.RGB)
     self.norm = ops.CropMirrorNormalize(std=255.,
                                         mean=0.,
                                         device=device,
                                         output_layout="HWC")
     self.func = ops.PythonFunction(device=device,
                                    function=function,
                                    num_outputs=num_outputs)
Esempio n. 16
0
 def __init__(self, batch_size, num_threads, device_id, crop, size):
     super(ImageNetValPipe, self).__init__(batch_size,
                                           num_threads,
                                           device_id,
                                           seed=12 + device_id)
     self.data_input = ops.ExternalSource()
     self.label_input = ops.ExternalSource()
     self.decode = ops.nvJPEGDecoder(device="mixed", output_type=types.RGB)
     self.res = ops.Resize(device="gpu", resize_shorter=size)
     self.cmnp = ops.CropMirrorNormalize(
         device="gpu",
         output_dtype=types.FLOAT,
         output_layout=types.NCHW,
         crop=(crop, crop),
         image_type=types.RGB,
         mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
         std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
 def __init__(self, device, batch_size, num_threads=1, device_id=0, num_gpus=1,
              dtype = types.FLOAT, output_layout = "HWC",
              mirror_probability = 0.0, mean=[0., 0., 0.], std=[1., 1., 1.], pad_output=False):
     super(CropMirrorNormalizePipeline, self).__init__(batch_size, num_threads, device_id, seed=7865)
     self.device = device
     self.input = ops.CaffeReader(path = caffe_db_folder, shard_id = device_id, num_shards = num_gpus)
     self.decode = ops.ImageDecoder(device = "cpu", output_type = types.RGB)
     self.cmn = ops.CropMirrorNormalize(device = self.device,
                                        dtype = dtype,
                                        output_layout = output_layout,
                                        crop = (224, 224),
                                        crop_pos_x = 0.3,
                                        crop_pos_y = 0.2,
                                        mean = mean,
                                        std = std,
                                        pad_output = pad_output)
     self.coin = ops.CoinFlip(probability=mirror_probability, seed=7865)
Esempio n. 18
0
    def __init__(self,
                 batch_size,
                 num_threads,
                 device_id,
                 data_dir,
                 crop,
                 dali_cpu=False,
                 fp16=False):
        super(HybridTrainPipe, self).__init__(batch_size,
                                              num_threads,
                                              device_id,
                                              seed=12 + device_id)
        self.input = ops.FileReader(file_root=data_dir,
                                    shard_id=args.local_rank,
                                    num_shards=args.world_size,
                                    random_shuffle=True)
        #let user decide which pipeline works him bets for RN version he runs
        if dali_cpu:
            dali_device = "cpu"
            self.decode = ops.HostDecoder(device=dali_device,
                                          output_type=types.RGB)
        else:
            dali_device = "gpu"
            # This padding sets the size of the internal nvJPEG buffers to be able to handle all images from full-sized ImageNet
            # without additional reallocations
            self.decode = ops.nvJPEGDecoder(device="mixed",
                                            output_type=types.RGB,
                                            device_memory_padding=211025920,
                                            host_memory_padding=140544512)

        self.rrc = ops.RandomResizedCrop(device=dali_device, size=(crop, crop))

        output_dtype = types.FLOAT
        if fp16:
            output_dtype = types.FLOAT16

        self.cmnp = ops.CropMirrorNormalize(
            device=dali_device,  # 1: changed from "cpu" to dali_device
            output_dtype=output_dtype,  # 2: output to fp16 when fp16 training
            output_layout=types.NCHW,
            crop=(crop, crop),
            image_type=types.RGB,
            mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
            std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
        self.coin = ops.CoinFlip(probability=0.5)
        print('DALI "{0}" variant'.format(dali_device))
Esempio n. 19
0
    def __init__(self, batch_size, num_threads, device_id, num_gpus):
        super(CaffeReadPipeline, self).__init__(batch_size, num_threads, device_id)

        self.input = ops.CaffeReader(path = lmdb_folder,
                                     random_shuffle = True, shard_id = device_id, num_shards = num_gpus)
        self.decode = ops.ImageDecoder(device = "mixed", output_type = types.RGB)
        self.resize = ops.Resize(device = "gpu",
                                 image_type = types.RGB,
                                 interp_type = types.INTERP_LINEAR)
        self.cmn = ops.CropMirrorNormalize(device = "gpu",
                                            output_dtype = types.FLOAT,
                                            crop = (227, 227),
                                            image_type = types.RGB,
                                            mean = [128., 128., 128.],
                                            std = [1., 1., 1.])
        self.uniform = ops.Uniform(range = (0.0, 1.0))
        self.resize_rng = ops.Uniform(range = (256, 480))
Esempio n. 20
0
 def __init__(self, batch_size, num_threads, device_id):
     super(CMNvsCropPipe, self).__init__(batch_size, num_threads, device_id, seed = 12)
     self.input = ops.CaffeReader(path = caffe_db_folder, shard_id = device_id, num_shards = 1)
     self.decode = ops.nvJPEGDecoder(device = "mixed", output_type = types.RGB)
     self.cmn = ops.CropMirrorNormalize(device = "gpu",
                                         output_layout = types.NHWC,
                                         output_dtype = types.FLOAT,
                                         crop = (224, 224),
                                         image_type = types.RGB,
                                         mean = [0., 0., 0.],
                                         std = [1., 1., 1.])
     self.crop = ops.Crop(device = "gpu",
                          crop = (224, 224),
                          image_type = types.RGB)
     self.uniform = ops.Uniform(range = (0.0, 1.0))
     self.cast = ops.Cast(device = "gpu",
                          dtype = types.INT32)
Esempio n. 21
0
    def __init__(self,
                 batch_size,
                 device,
                 data_dir,
                 mean,
                 std,
                 device_id=0,
                 shard_id=0,
                 num_shards=1,
                 num_threads=4,
                 seed=0):
        super(DaliTransformsTrainPipeline,
              self).__init__(batch_size, num_threads, device_id, seed)

        # should we make this drive the device flags?
        self.reader = ops.FileReader(file_root=data_dir,
                                     shard_id=shard_id,
                                     num_shards=num_shards,
                                     random_shuffle=True)

        self.decode = ops.ImageDecoder(device='mixed',
                                       output_type=types.RGB,
                                       memory_stats=True)
        self.resize = ops.Resize(device=device,
                                 size=[200, 300],
                                 interp_type=types.INTERP_TRIANGULAR)
        self.centrecrop = ops.Crop(device=device, crop=[100, 100])
        self.randomcrop = ops.RandomResizedCrop(device=device, size=[80, 80])

        self.hz_coin = ops.CoinFlip(probability=0.5)
        self.horizontalflip = ops.Flip(device=device)

        self.rotate_angle = ops.Uniform(range=[-90, 90])
        self.rotate_coin = ops.CoinFlip(probability=0.5)
        self.rotate = ops.Rotate(device=device, keep_size=True)

        self.vt_coin = ops.CoinFlip(probability=0.5)
        self.verticalflip = ops.Flip(device=device, horizontal=0)

        self.normalize = ops.CropMirrorNormalize(device=device,
                                                 dtype=types.FLOAT,
                                                 output_layout=types.NCHW)  #,
        #mean=mean*255, std=std*255)

        #self.normalize = ops.Normalize(device=device, dtype=types.FLOAT)#, mean=mean, stddev=std)
        self.to_int64 = ops.Cast(dtype=types.INT64, device=device)
Esempio n. 22
0
    def __init__(self, batch_size, num_threads, device_id, data_dir):
        super(HybridPipe, self).__init__(batch_size, num_threads, device_id)
        self.input = ops.Caffe2Reader(path=data_dir,
                                      shard_id=args.rank,
                                      num_shards=args.world_size)
        self.decode = ops.nvJPEGDecoder(device="mixed", output_type=types.RGB)
        self.rrc = ops.RandomResizedCrop(device="gpu", size=(224, 224))
        self.cmnp = ops.CropMirrorNormalize(
            device="gpu",
            output_dtype=types.FLOAT,
            output_layout=types.NCHW,
            image_type=types.RGB,
            crop=(224, 224),
            mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
            std=[0.229 * 255, 0.224 * 255, 0.225 * 255])

        self.coin = ops.CoinFlip(probability=0.5)
    def __init__(self, device_id, crop, size, rank, seed_rank, options):
        super(TrainPipeline, self).__init__(options.batchsize,
                                            4,
                                            device_id,
                                            prefetch_queue_depth=3,
                                            set_affinity=True,
                                            seed=options.seed + seed_rank)
        self.input = ops.FileReader(file_root=options.train_dir,
                                    shard_id=rank,
                                    num_shards=size,
                                    random_shuffle=True)

        # This padding sets the size of the internal nvJPEG buffers to be able to handle all images from full-sized ImageNet
        # without additional reallocations
        random_aspect_ratio = [0.75, 4. / 3.]
        random_area = [0.08, 1.0]
        self.decode = ops.ImageDecoderRandomCrop(
            device="mixed",
            output_type=types.RGB,
            device_memory_padding=211025920,
            host_memory_padding=140544512,
            random_aspect_ratio=random_aspect_ratio,
            random_area=random_area,
            num_attempts=100,
            seed=options.seed + seed_rank + 1641)

        self.res = ops.Resize(device="gpu",
                              resize_x=crop,
                              resize_y=crop,
                              interp_type=types.INTERP_TRIANGULAR)
        dtype = types.FLOAT16 if options.fp16 else types.FLOAT
        layout = types.NCHW
        padding = False
        img_mean, img_std = get_mean_std(options)
        self.cmnp = ops.CropMirrorNormalize(device="gpu",
                                            output_dtype=dtype,
                                            output_layout=layout,
                                            crop=(crop, crop),
                                            image_type=types.RGB,
                                            mean=img_mean,
                                            std=img_std,
                                            pad_output=padding,
                                            seed=options.seed + seed_rank +
                                            1223)
        self.coin = ops.CoinFlip(probability=0.5,
                                 seed=options.seed + seed_rank + 412)
Esempio n. 24
0
    def __init__(self,
                 batch_size,
                 num_threads,
                 device_id,
                 data_dir,
                 crop,
                 seed,
                 dali_cpu=False):
        super(TinyImageNetHybridTrainPipe,
              self).__init__(batch_size, num_threads, device_id, seed)

        if torch.distributed.is_initialized():
            local_rank = torch.distributed.get_rank()
            world_size = torch.distributed.get_world_size()
        else:
            local_rank = 0
            world_size = 1
        self.input = ops.FileReader(file_root=data_dir,
                                    shard_id=local_rank,
                                    num_shards=world_size,
                                    pad_last_batch=True,
                                    random_shuffle=False,
                                    shuffle_after_epoch=True)

        # decide to work on cpu or gpu
        dali_device = 'cpu' if dali_cpu else 'gpu'
        decoder_device = 'cpu' if dali_cpu else 'mixed'

        self.decode = ops.ImageDecoder(device=decoder_device,
                                       output_type=types.RGB)
        self.res = ops.RandomResizedCrop(device=dali_device,
                                         size=crop,
                                         random_aspect_ratio=[0.75, 4. / 3],
                                         random_area=[0.08, 1.0],
                                         num_attempts=100,
                                         interp_type=types.INTERP_TRIANGULAR)
        self.cmnp = ops.CropMirrorNormalize(
            device='gpu',
            output_dtype=types.FLOAT,
            output_layout=types.NCHW,
            crop=(crop, crop),
            image_type=types.RGB,
            mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
            std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
        self.coin = ops.CoinFlip(probability=0.5)
Esempio n. 25
0
    def __init__(self,
                 args,
                 batch_size,
                 num_threads,
                 device_id,
                 rec_path,
                 idx_path,
                 shard_id,
                 num_shards,
                 crop_shape,
                 nvjpeg_padding,
                 prefetch_queue=3,
                 resize_shp=None,
                 output_layout=types.NCHW,
                 pad_output=True,
                 dtype='float16',
                 dali_cpu=False):
        super(HybridValPipe,
              self).__init__(batch_size,
                             num_threads,
                             device_id,
                             seed=12 + device_id,
                             prefetch_queue_depth=prefetch_queue)
        self.input = ops.MXNetReader(path=[rec_path],
                                     index_path=[idx_path],
                                     random_shuffle=False,
                                     shard_id=shard_id,
                                     num_shards=num_shards)

        dali_device, decoder_device = get_device_names(dali_cpu)
        self.decode = ops.ImageDecoder(device=decoder_device,
                                       output_type=types.RGB,
                                       device_memory_padding=nvjpeg_padding,
                                       host_memory_padding=nvjpeg_padding)
        self.resize = ops.Resize(
            device=dali_device,
            resize_shorter=resize_shp) if resize_shp else None
        self.cmnp = ops.CropMirrorNormalize(
            device="gpu",
            dtype=types.FLOAT16 if dtype == 'float16' else types.FLOAT,
            output_layout=output_layout,
            crop=crop_shape,
            pad_output=pad_output,
            mean=args.rgb_mean,
            std=args.rgb_std)
Esempio n. 26
0
    def __init__(self, db_folder, batch_size, num_threads, device_id,
                 num_gpus):
        super(MXNetReaderPipeline, self).__init__(batch_size, num_threads,
                                                  device_id)
        self.input = ops.MXNetReader(path=[db_folder + "train.rec"],
                                     index_path=[db_folder + "train.idx"],
                                     random_shuffle=True,
                                     shard_id=device_id,
                                     num_shards=num_gpus)

        dali_device = "gpu"
        self.decode = ops.ImageDecoder(device="mixed", output_type=types.RGB)
        self.cmn = ops.CropMirrorNormalize(device="gpu",
                                           output_dtype=types.FLOAT,
                                           image_type=types.RGB,
                                           mean=[128., 128., 128.],
                                           std=[1., 1., 1.])
        self.coin = ops.CoinFlip(probability=0.5)
Esempio n. 27
0
 def __init__(self, batch_size, num_threads, device_id, num_gpus, db_dir):
     super(TrainPipe, self).__init__(batch_size, num_threads, device_id)
     self.input = ops.MXNetReader(path=[db_dir + "train.rec"],
                                  index_path=[db_dir + "train.idx"],
                                  random_shuffle=False,
                                  shard_id=device_id,
                                  num_shards=num_gpus)
     self.decode = ops.ImageDecoder(device="mixed", output_type=types.RGB)
     # self.blur = ops.GaussianBlur(device="cpu", sigma=5.0, window_size=5)
     # self.reshape = ops.Reshape(device="cpu", layout="HWC")
     self.normalize = ops.CropMirrorNormalize(device="gpu",
                                              output_dtype=types.FLOAT,
                                              output_layout=types.NCHW,
                                              mean=[123.0, 116.0, 103.0],
                                              std=[100.0, 100.0, 100.0])
     self.transform_source = ops.ExternalSource()
     self.warp_gpu = ops.WarpAffine(device="gpu",
                                    interp_type=types.INTERP_LINEAR)
Esempio n. 28
0
 def __init__(self, file_root, annotations_file,
              batch_size, num_threads, device_id=0, num_gpus=1, 
              mean=(123.675, 116.28, 103.53), stddev=(1., 1., 1.),
              random_shuffle=True):
     super(COCOPipeline, self).__init__(batch_size, num_threads, device_id, seed = 15)
     self.input = ops.COCOReader(file_root=file_root, annotations_file=annotations_file,
                                 shard_id=device_id, num_shards=num_gpus, ratio=True, 
                                 skip_empty=True, prefetch_queue_depth=32, random_shuffle=True)
     self.decode = ops.ImageDecoder(device='mixed', output_type=types.BGR)
     self.resize = ops.Resize(device='gpu', max_size=1216, resize_shorter=800)
     self.flip = ops.CoinFlip(device='cpu')
     self.bbox_flip = ops.BbFlip(device='gpu')
     self.CMN = ops.CropMirrorNormalize(device='gpu', mean=mean, std=stddev, output_layout='HWC')
     self.image_pad = ops.Pad(device='gpu', fill_value=0, axes=(0,1), shape=(1216, 1216))
     self.bbox_pad = ops.Pad(device='gpu', fill_value=0, axes=(0,), shape=(100,))
     self.label_pad = ops.Pad(device='gpu', fill_value=-1, axes=(0,), shape=(100,))
     self.get_shape = ops.Shapes(device='gpu')
     self.float_cast = ops.Cast(device='gpu', dtype=types.FLOAT)
 def __init__(self, batch_size, num_threads, device_id, data_dir, crop, size, seed=12, local_rank=0, world_size=1,
              spos_pre=False, shuffle=False):
     super(HybridValPipe, self).__init__(batch_size,
                                         num_threads, device_id, seed=seed + device_id)
     color_space_type = types.BGR if spos_pre else types.RGB
     self.input = ops.FileReader(file_root=data_dir, shard_id=local_rank, num_shards=world_size,
                                 random_shuffle=shuffle)
     self.decode = ops.ImageDecoder(
         device="mixed", output_type=color_space_type)
     self.res = ops.Resize(device="gpu", resize_shorter=size,
                           interp_type=types.INTERP_LINEAR if spos_pre else types.INTERP_TRIANGULAR)
     self.cmnp = ops.CropMirrorNormalize(device="gpu",
                                         dtype=types.FLOAT,
                                         output_layout=types.NCHW,
                                         crop=(crop, crop),
                                         mean=0. if spos_pre else [
                                             0.485 * 255, 0.456 * 255, 0.406 * 255],
                                         std=1. if spos_pre else [0.229 * 255, 0.224 * 255, 0.225 * 255])
Esempio n. 30
0
    def __init__(self,
                 batch_size,
                 num_threads,
                 device_id,
                 data_dir,
                 crop,
                 num_shards,
                 dali_cpu=False):
        super(HybridTrainPipe, self).__init__(batch_size,
                                              num_threads,
                                              device_id,
                                              seed=12 + device_id)
        self.input = ops.MXNetReader(
            path=[os.path.join(data_dir, "_train.rec")],
            index_path=[os.path.join(data_dir, "_train.idx")],
            random_shuffle=True,
            shard_id=device_id,
            num_shards=num_shards)

        # self.input = ops.FileReader(file_root=data_dir, shard_id=0, num_shards=4, random_shuffle=True)
        # let user decide which pipeline works him bets for RN version he runs

        if dali_cpu:
            dali_device = "cpu"
            self.decode = ops.HostDecoder(device=dali_device,
                                          output_type=types.RGB)
        else:
            dali_device = "gpu"
            # This padding sets the size of the internal nvJPEG buffers to be able to handle all images from full-sized ImageNet
            # without additional reallocations
            self.decode = ops.nvJPEGDecoder(device="mixed",
                                            output_type=types.RGB)

        self.rrc = ops.RandomResizedCrop(device=dali_device, size=(crop, crop))
        self.cmnp = ops.CropMirrorNormalize(
            device="gpu",
            output_dtype=types.FLOAT,
            output_layout=types.NCHW,
            crop=(crop, crop),
            image_type=types.RGB,
            mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
            std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
        self.coin = ops.CoinFlip(probability=0.5)
        print('DALI "{0}" variant'.format(dali_device))