def __init__(self, batch_size, num_threads, device_id):
super(TFRecordPipeline, self).__init__(batch_size,
num_threads,
device_id)
self.input = ops.TFRecordReader(path = tfrecord,
index_path = tfrecord_idx,
features = {"image/encoded" : tfrec.FixedLenFeature((), tfrec.string, ""),
'image/filename': tfrec.FixedLenFeature([ ], tfrec.string, ''),
'image/height': tfrec.FixedLenFeature([1], tfrec.int64, -1),
'image/width': tfrec.FixedLenFeature([1], tfrec.int64, -1),
'image/colorspace': tfrec.FixedLenFeature([ ], tfrec.string, ''),
'image/channels': tfrec.FixedLenFeature([1], tfrec.int64, -1),
'image/format': tfrec.FixedLenFeature([ ], tfrec.string, ''),
'image/class/label': tfrec.FixedLenFeature([1], tfrec.int64, -1),
'image/class/synset': tfrec.FixedLenFeature([ ], tfrec.string, ''),
'image/class/text': tfrec.FixedLenFeature([ ], tfrec.string, ''),
'image/object/bbox/xmin': tfrec.VarLenFeature(tfrec.float32, 0.0),
'image/object/bbox/ymin': tfrec.VarLenFeature(tfrec.float32, 0.0),
'image/object/bbox/xmax': tfrec.VarLenFeature(tfrec.float32, 0.0),
'image/object/bbox/ymax': tfrec.VarLenFeature(tfrec.float32, 0.0),
'image/object/bbox/label': tfrec.FixedLenFeature([1], tfrec.int64, -1)})
self.decode = ops.ImageDecoder(device = "mixed", output_type = types.RGB)
self.resize = ops.Resize(device = "gpu", resize_x = 1920.,resize_y=1920.)
self.vert_flip = ops.Flip(device = "gpu", horizontal=0)
self.vert_coin = ops.CoinFlip(probability=0.5)
self.cmnp = ops.CropMirrorNormalize(device = "gpu",
output_dtype = types.FLOAT,
crop = (1920, 1920),
image_type = types.RGB,
mean = [0., 0., 0.],
std = [1., 1., 1.])
self.uniform = ops.Uniform(range = (0.0, 1.0))
self.iter = 0
def __init__(self, batch_size, num_threads, device_id):
super(TFRecordPipeline, self).__init__(batch_size, num_threads,
device_id)
self.input = ops.TFRecordReader(
path=tfrecord,
index_path=tfrecord_idx,
features={
"image/encoded": tfrec.FixedLenFeature((), tfrec.string, ""),
'image/class/label': tfrec.FixedLenFeature([1], tfrec.int64,
-1),
'image/class/text': tfrec.FixedLenFeature([], tfrec.string,
''),
'image/object/bbox/xmin':
tfrec.VarLenFeature(tfrec.float32, 0.0),
'image/object/bbox/ymin':
tfrec.VarLenFeature(tfrec.float32, 0.0),
'image/object/bbox/xmax':
tfrec.VarLenFeature(tfrec.float32, 0.0),
'image/object/bbox/ymax':
tfrec.VarLenFeature(tfrec.float32, 0.0)
})
self.decode = ops.ImageDecoder(device="mixed", output_type=types.RGB)
self.resize = ops.Resize(device="gpu", resize_shorter=256.)
self.cmnp = ops.CropMirrorNormalize(device="gpu",
output_dtype=types.FLOAT,
crop=(224, 224),
image_type=types.RGB,
mean=[0., 0., 0.],
std=[1., 1., 1.])
self.uniform = ops.Uniform(range=(0.0, 1.0))
self.iter = 0
def __init__(self, batch_size, num_threads, device_id, **kwargs):
super(TFRecordTrain, self).__init__(batch_size, num_threads, device_id)
self.dim = kwargs["dim"]
self.seed = kwargs["seed"]
self.oversampling = kwargs["oversampling"]
self.input = ops.TFRecordReader(
path=kwargs["tfrecords"],
index_path=kwargs["tfrecords_idx"],
features={
"X_shape": tfrec.FixedLenFeature([self.dim + 1], tfrec.int64, 0),
"Y_shape": tfrec.FixedLenFeature([self.dim + 1], tfrec.int64, 0),
"X": tfrec.VarLenFeature([], tfrec.float32, 0.0),
"Y": tfrec.FixedLenFeature([], tfrec.string, ""),
"fname": tfrec.FixedLenFeature([], tfrec.string, ""),
},
num_shards=kwargs["gpus"],
shard_id=device_id,
random_shuffle=True,
pad_last_batch=True,
read_ahead=True,
seed=self.seed,
)
self.patch_size = kwargs["patch_size"]
self.crop_shape = types.Constant(np.array(self.patch_size), dtype=types.INT64)
self.crop_shape_float = types.Constant(np.array(self.patch_size), dtype=types.FLOAT)
self.layout = "CDHW" if self.dim == 3 else "CHW"
self.axis_name = "DHW" if self.dim == 3 else "HW"
def __init__(self, batch_size, num_threads, device_id, data_dir, crop, size):
super(HybridValPipe, self).__init__(batch_size, num_threads, device_id, seed=12 + device_id)
# self.input = ops.FileReader(file_root=data_dir, shard_id=args.local_rank, num_shards=args.world_size, random_shuffle=False)
index_path = []
for path in os.listdir("/home/guojia/idx_files/val"):
index_path.append(os.path.join("/home/guojia/idx_files/val", path))
index_path = sorted(index_path)
self.input = ops.TFRecordReader(path=data_dir, index_path=index_path, shard_id=args.local_rank,
num_shards=args.world_size, random_shuffle=True,
features={
'image/height': tfrec.FixedLenFeature([1], tfrec.int64, -1),
'image/width': tfrec.FixedLenFeature([1], tfrec.int64, -1),
'image/colorspace': tfrec.FixedLenFeature([], tfrec.string, ''),
'image/channels': tfrec.FixedLenFeature([], tfrec.int64, -1),
'image/class/label': tfrec.FixedLenFeature([1], tfrec.int64, -1),
'image/class/synset': tfrec.FixedLenFeature([], tfrec.string, ''),
'image/format': tfrec.FixedLenFeature((), tfrec.string, ""),
'image/filename': tfrec.FixedLenFeature((), tfrec.string, ""),
'image/encoded': tfrec.FixedLenFeature((), tfrec.string, "")
})
self.decode = ops.ImageDecoder(device="mixed", output_type=types.RGB)
self.res = ops.Resize(device="gpu", resize_shorter=size, interp_type=types.INTERP_TRIANGULAR)
self.cmnp = ops.CropMirrorNormalize(device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NCHW,
crop=(crop, crop),
image_type=types.RGB,
mean=[0.485 * 255,0.456 * 255,0.406 * 255],
std=[0.229 * 255,0.224 * 255,0.225 * 255])
def __init__(self, batch_size, num_threads, device_id, num_gpus, data, data_idx):
super(TFRecordPipeline, self).__init__(batch_size, num_threads, device_id)
self.input = ops.TFRecordReader(path = data,
index_path = data_idx,
features = {"image/encoded" : tfrec.FixedLenFeature((), tfrec.string, ""),
"image/class/label": tfrec.FixedLenFeature([1], tfrec.int64, -1)
})
def __init__(self,
batch_size,
num_threads,
device_id,
data_dir,
crop,
dali_cpu=False):
super(HybridTrainPipe, self).__init__(batch_size,
num_threads,
device_id,
seed=12 + device_id)
self.input = ops.TFRecordReader(
path=data_dir + ".tfrecord",
index_path=data_dir + ".index",
features={
"image": tfrec.FixedLenFeature([], tfrec.string, ""),
'label': tfrec.FixedLenFeature([], tfrec.float32, 0.0),
'index': tfrec.FixedLenFeature([], tfrec.int64, 0),
})
self.decode = ops.ImageDecoder(
device="cpu",
output_type=types.RGB,
split_stages=True,
)
self.res = ops.RandomResizedCrop(device="gpu", size=(224, 224))
self.cmnp = ops.CropMirrorNormalize(
device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NCHW,
crop=(crop, crop),
image_type=types.RGB,
mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
self.coin = ops.CoinFlip(probability=0.5)
print('DALI "{0}" variant'.format('gpu'))
def __init__(self, args):
super(TFRecordDetectionPipeline,
self).__init__(args.batch_size, args.num_workers, 0, 0)
self.input = ops.TFRecordReader(
path=os.path.join(test_dummy_data_path, 'small_coco.tfrecord'),
index_path=os.path.join(test_dummy_data_path,
'small_coco_index.idx'),
features={
'image/encoded':
tfrec.FixedLenFeature((), tfrec.string, ""),
'image/object/class/label':
tfrec.VarLenFeature([1], tfrec.int64, 0),
'image/object/bbox':
tfrec.VarLenFeature([4], tfrec.float32, 0.0),
},
shard_id=0,
num_shards=1,
random_shuffle=False)
self.decode_gpu = ops.ImageDecoder(device="mixed",
output_type=types.RGB)
self.cast = ops.Cast(dtype=types.INT32)
self.box_encoder = ops.BoxEncoder(device="cpu",
criteria=0.5,
anchors=coco_anchors())
def __init__(self, batch_size, num_threads, device_id, tfrecord,
tfrecord_idx):
super(TFRecordPipeline, self).__init__(batch_size, num_threads,
device_id)
self.input = ops.TFRecordReader(path=tfrecord,
index_path=tfrecord_idx,
features={
"image/encoded":
tfrec.FixedLenFeature(
(), tfrec.string, ""),
'image/class/label':
tfrec.FixedLenFeature([1],
tfrec.int64,
-1),
'image/class/text':
tfrec.FixedLenFeature([],
tfrec.string,
''),
})
self.decode = ops.nvJPEGDecoder(device="mixed", output_type=types.RGB)
self.resize = ops.Resize(device="gpu", resize_a=256, resize_b=256)
self.cmnp = ops.CropMirrorNormalize(device="gpu",
output_dtype=types.FLOAT,
crop=(224, 224),
image_type=types.RGB,
mean=[0., 0., 0.],
std=[1., 1., 1.],
output_layout=types.NHWC)
self.uniform = ops.Uniform(range=(0.0, 1.0))
self.iter = 0
def __init__(self, tfrecord_files, idx_files,
batch_size, device_id=0, rank=0,
total_devices=1, num_threads=4):
super(TFRecordPipeline, self).__init__(batch_size,
num_threads,
device_id)
self.input = ops.TFRecordReader(path = tfrecord_files, index_path = idx_files,
shard_id = rank, num_shards = total_devices,
random_shuffle = True,
features = {"image/encoded" : tfrec.FixedLenFeature((), tfrec.string, ""),
'image/class/label': tfrec.FixedLenFeature([1], tfrec.int64, -1),
})
self.decode = ops.ImageDecoder(device = "mixed", output_type = types.RGB)
self.resize = ops.Resize(device = "gpu", resize_shorter = 256)
self.cmnp = ops.CropMirrorNormalize(device = "gpu",
output_dtype = types.FLOAT16,
crop = (224, 224),
image_type = types.RGB,
mean = [0, 0, 0],
std = [1., 1., 1.],
output_layout='HWC')
self.uniform = ops.Uniform(range = (0.0, 1.0))
self.flip = ops.CoinFlip()
self.brightness = ops.Uniform(range = (0.5, 1.5))
self.contrast = ops.Uniform(range = (0.8, 1.3))
self.cast = ops.Cast(device = "gpu", dtype = types.FLOAT16)
self.iter = 0
def __init__(self, tfrec_filenames, tfrec_idx_filenames, batch_size,
num_threads, device_id, set_affinity, prefetch_queue_depth):
super(Dali_CPU_Pipe,
self).__init__(batch_size,
num_threads,
device_id,
set_affinity=set_affinity,
prefetch_queue_depth=prefetch_queue_depth)
self.input = ops.TFRecordReader(path=tfrec_filenames,
index_path=tfrec_idx_filenames,
initial_fill=10000,
features={
"image/encoded":
tfrec.FixedLenFeature(
(), tfrec.string, ""),
'image/class/label':
tfrec.FixedLenFeature([1],
tfrec.int64,
-1)
})
self.decode = ops.HostDecoder(output_type=types.RGB)
self.resize = ops.Resize(device="cpu", resize_shorter=_RESIZE_MIN)
self.cmnp = ops.CropMirrorNormalize(device="cpu",
output_dtype=types.FLOAT,
crop=(INPUT_SIZE, INPUT_SIZE),
image_type=types.RGB,
mean=_CHANNEL_MEANS,
std=[58.395, 57.120, 57.375],
output_layout=dali.types.NCHW)
self.iter = 0
def __init__(self, batch_size, num_threads, device_id, num_gpus, data_paths):
super(TFRecordPipeline, self).__init__(batch_size, num_threads, device_id)
tfrecord = sorted(glob.glob(data_paths[0]))
tfrecord_idx = sorted(glob.glob(data_paths[1]))
self.input = ops.TFRecordReader(path = tfrecord,
index_path = tfrecord_idx,
shard_id = device_id,
num_shards = num_gpus,
features = {"image/encoded" : tfrec.FixedLenFeature((), tfrec.string, ""),
"image/class/label": tfrec.FixedLenFeature([1], tfrec.int64, -1)
})
def __init__(self, **kwargs):
super(TFRecordPipeline, self).__init__(**kwargs)
tfrecord = sorted(glob.glob(kwargs['data_paths'][0]))
tfrecord_idx = sorted(glob.glob(kwargs['data_paths'][1]))
cache_enabled = kwargs['decoder_cache_params']['cache_enabled']
self.input = ops.TFRecordReader(path = tfrecord,
index_path = tfrecord_idx,
shard_id = kwargs['device_id'],
num_shards = kwargs['num_gpus'],
stick_to_shard = cache_enabled,
#skip_cached_images = cache_enabled,
features = {"image/encoded" : tfrec.FixedLenFeature((), tfrec.string, ""),
"image/class/label": tfrec.FixedLenFeature([1], tfrec.int64, -1)
})
def __init__(self, batch_size, num_threads, device_id, data_dir, crop, dali_cpu=False):
super(HybridTrainPipe, self).__init__(batch_size, num_threads, device_id, seed=12 + device_id)
# self.input = ops.FileReader(file_root=data_dir, shard_id=args.local_rank, num_shards=args.world_size, random_shuffle=True)
index_path = []
for path in os.listdir("/home/guojia/idx_files/train"):
index_path.append(os.path.join("/home/guojia/idx_files/train", path))
index_path = sorted(index_path)
self.input = ops.TFRecordReader(path=data_dir, index_path=index_path, shard_id=args.local_rank,
num_shards=args.world_size, random_shuffle=True,
features={
'image/height': tfrec.FixedLenFeature([1], tfrec.int64, -1),
'image/width': tfrec.FixedLenFeature([1], tfrec.int64, -1),
'image/colorspace': tfrec.FixedLenFeature([ ], tfrec.string, ''),
'image/channels': tfrec.FixedLenFeature([], tfrec.int64, -1),
'image/class/label': tfrec.FixedLenFeature([1], tfrec.int64, -1),
'image/class/synset': tfrec.FixedLenFeature([ ], tfrec.string, ''),
# 'image/class/text': tfrec.FixedLenFeature([ ], tfrec.string, ''),
# 'image/object/bbox/xmin': tfrec.VarLenFeature(tfrec.float32, 0.0),
# 'image/object/bbox/xmax': tfrec.VarLenFeature(tfrec.float32, 0.0),
# 'image/object/bbox/ymin': tfrec.VarLenFeature(tfrec.float32, 0.0),
# 'image/object/bbox/ymax': tfrec.VarLenFeature(tfrec.float32, 0.0),
# 'image/object/bbox/label': tfrec.FixedLenFeature([1], tfrec.int64,-1),
'image/format': tfrec.FixedLenFeature((), tfrec.string, ""),
'image/filename': tfrec.FixedLenFeature((), tfrec.string, ""),
'image/encoded': tfrec.FixedLenFeature((), tfrec.string, "")
})
#let user decide which pipeline works him bets for RN version he runs
dali_device = 'cpu' if dali_cpu else 'gpu'
decoder_device = 'cpu' if dali_cpu else 'mixed'
# This padding sets the size of the internal nvJPEG buffers to be able to handle all images from full-sized ImageNet
# without additional reallocations
device_memory_padding = 211025920 if decoder_device == 'mixed' else 0
host_memory_padding = 140544512 if decoder_device == 'mixed' else 0
self.decode = ops.ImageDecoderRandomCrop(device=decoder_device, output_type=types.RGB,
device_memory_padding=device_memory_padding,
host_memory_padding=host_memory_padding,
random_aspect_ratio=[0.8, 1.25],
random_area=[0.1, 1.0],
num_attempts=100)
self.res = ops.Resize(device=dali_device, resize_x=crop, resize_y=crop, interp_type=types.INTERP_TRIANGULAR)
self.cmnp = ops.CropMirrorNormalize(device="gpu",
output_dtype=types.FLOAT,
output_layout=types.NCHW,
crop=(crop, crop),
image_type=types.RGB,
mean=[0.485 * 255,0.456 * 255,0.406 * 255],
std=[0.229 * 255,0.224 * 255,0.225 * 255])
self.coin = ops.CoinFlip(probability=0.5)
print('DALI "{0}" variant'.format(dali_device))
def __init__(self, batch_size, num_threads, device_id, tfrecords,
idx_paths):
super(ResnetPipeline, self).__init__(batch_size, num_threads,
device_id)
# Transformation operations below.
# From https://docs.nvidia.com/deeplearning/sdk/dali-developer-guide/docs/supported_ops.html
self.input = ops.TFRecordReader(
path=tfrecords,
index_path=idx_paths,
features={
"image/encoded": tfrec.FixedLenFeature([], tfrec.string, ""),
"image/class/label": tfrec.FixedLenFeature([1], tfrec.float32,
0.0),
"image/class/text": tfrec.FixedLenFeature([], tfrec.string,
""),
"image/object/bbox/xmin":
tfrec.VarLenFeature(tfrec.float32, 0.0),
"image/object/bbox/ymin":
tfrec.VarLenFeature(tfrec.float32, 0.0),
"image/object/bbox/xmax":
tfrec.VarLenFeature(tfrec.float32, 0.0),
"image/object/bbox/ymax":
tfrec.VarLenFeature(tfrec.float32, 0.0)
})
self.decode = ops.nvJPEGDecoder(device="mixed",
cache_debug=True,
output_type=types.RGB)
self.resize = ops.Resize(device="gpu",
image_type=types.RGB,
interp_type=types.INTERP_LINEAR,
resize_shorter=256.)
self.cmn = ops.CropMirrorNormalize(device="gpu",
output_dtype=types.FLOAT,
crop=(224, 224),
image_type=types.RGB,
mean=[0., 0., 0.],
std=[1., 1., 1])
self.uniform = ops.Uniform(range=(0.0, 1.0))
self.transpose = ops.Transpose(device="gpu", perm=[0, 3, 1, 2])
self.cast = ops.Cast(device="gpu", dtype=types.INT32)
self.iter = 0
def __init__(self, batch_size, num_threads, device_id, **kwargs):
super(TFRecordTest, self).__init__(batch_size, num_threads, device_id)
self.input = ops.TFRecordReader(
path=kwargs["tfrecords"],
index_path=kwargs["tfrecords_idx"],
features={
"X_shape": tfrec.FixedLenFeature([4], tfrec.int64, 0),
"X": tfrec.VarLenFeature([], tfrec.float32, 0.0),
"fname": tfrec.FixedLenFeature([], tfrec.string, ""),
},
shard_id=device_id,
num_shards=kwargs["gpus"],
read_ahead=True,
random_shuffle=False,
pad_last_batch=True,
)
def __init__(self, file_tfrecord, batch_size, num_workers, device_id=0):
super().__init__(batch_size, num_workers, device_id)
self.input = ops.TFRecordReader(
path=file_tfrecord.as_posix(),
index_path=(file_tfrecord.parent /
(file_tfrecord.stem + '.idx')).as_posix(),
features={'encoded': tfrec.FixedLenFeature((), tfrec.string, "")})
self.decode = ops.ImageDecoder(device='mixed', output_type=types.RGB)
self.cmnp = ops.CropMirrorNormalize(device='gpu',
output_dtype=types.FLOAT,
output_layout=types.NCHW,
image_type=types.RGB,
mean=[124, 116, 104],
std=[58, 57, 57])
def __init__(
self,
batch_size,
num_threads,
device_id,
size=1024,
path="/home/guyuchao/ssd/dataset/cityscape/leftImg8bit/dalirecord/dataset-r10.tfrecords",
index_path="/home/guyuchao/ssd/dataset/cityscape/leftImg8bit/dalirecord/dataset-r10.idx"
):
super(TFRecordPipeline, self).__init__(batch_size, num_threads,
device_id, size)
self.input = ops.TFRecordReader(path=path,
index_path=index_path,
features={
"image/encoded":
tfrec.FixedLenFeature(
(), tfrec.string, "")
})
def __init__(self, batch_size, num_threads, device_id, **kwargs):
super(TFRecordBenchmark, self).__init__(batch_size, num_threads, device_id)
self.dim = kwargs["dim"]
self.input = ops.TFRecordReader(
path=kwargs["tfrecords"],
index_path=kwargs["tfrecords_idx"],
features={
"X_shape": tfrec.FixedLenFeature([self.dim + 1], tfrec.int64, 0),
"Y_shape": tfrec.FixedLenFeature([self.dim + 1], tfrec.int64, 0),
"X": tfrec.VarLenFeature([], tfrec.float32, 0.0),
"Y": tfrec.FixedLenFeature([], tfrec.string, ""),
"fname": tfrec.FixedLenFeature([], tfrec.string, ""),
},
shard_id=device_id,
num_shards=kwargs["gpus"],
read_ahead=True,
)
self.patch_size = kwargs["patch_size"]
self.layout = "CDHW" if self.dim == 3 else "CHW"
def _input(self, tfrecord_path, index_path, shard_id=0):
return ops.TFRecordReader(
path=tfrecord_path,
index_path=index_path,
random_shuffle=True,
features={
'image/encoded':
tfrec.FixedLenFeature((), tfrec.string, ""),
'image/filename':
tfrec.FixedLenFeature((), tfrec.string, ""),
'image/format':
tfrec.FixedLenFeature((), tfrec.string, ""),
'image/height':
tfrec.FixedLenFeature([1], tfrec.int64, -1),
'image/width':
tfrec.FixedLenFeature([1], tfrec.int64, -1),
'image/channels':
tfrec.FixedLenFeature([1], tfrec.int64, -1),
'image/segmentation/class/encoded': (tfrec.FixedLenFeature(
(), tfrec.string, "")),
'image/segmentation/class/format': (tfrec.FixedLenFeature(
(), tfrec.string, ""))
})
def __new__(cls,
path,
shard_id=0,
num_shards=1,
random_shuffle=False,
initial_fill=1024,
**kwargs):
"""Create a ``TFRecordReader``.
Parameters
----------
path : str
The folder of record files.
shard_id : int, optional, default=0
The index of partition to read.
num_shards : int, optional, default=1
The total number of partitions over dataset.
random_shuffle : bool, optional, default=False
Whether to shuffle the data.
initial_fill : int, optional, default=1024
The length of sampling sequence for shuffle.
Returns
-------
nvidia.dali.ops.TFRecordReader
The reader instance.
"""
path, index_path, features = cls.check_files(path)
return ops.TFRecordReader(path=path,
index_path=index_path,
shard_id=shard_id,
num_shards=num_shards,
features=features,
random_shuffle=random_shuffle,
initial_fill=initial_fill,
**kwargs)
def __init__(self,
tfrecords,
batch_size,
target_size,
preproc_param,
num_threads,
num_shards,
device_ids,
training=False):
Pipeline.__init__(self,
batch_size=batch_size,
num_threads=num_threads,
device_id=device_ids,
prefetch_queue_depth=num_threads,
seed=42,
exec_async=False,
exec_pipelined=False)
DaliPipeline.__init__(self,
target_size=target_size,
preproc_param=preproc_param,
training=training)
tfrecords_idx = [tfrecord + "_idx" for tfrecord in tfrecords]
for tfrecord, tfrecord_idx in zip(tfrecords, tfrecords_idx):
if os.path.exists(tfrecord_idx):
continue
call(["tfrecord2idx", tfrecord, tfrecord + "_idx"])
self.length = sum([len(open(f).readlines()) for f in tfrecords_idx])
self.input = ops.TFRecordReader(
path=tfrecords,
index_path=tfrecords_idx,
features={
'image/height':
tfrec.FixedLenFeature([1], tfrec.int64, -1),
'image/width':
tfrec.FixedLenFeature([1], tfrec.int64, -1),
'image/encoded':
tfrec.FixedLenFeature((), tfrec.string, ""),
'image/format':
tfrec.FixedLenFeature((), tfrec.string, ""),
'image/object/bbox/xmin':
tfrec.VarLenFeature(tfrec.float32, 0.0),
'image/object/bbox/ymin':
tfrec.VarLenFeature(tfrec.float32, 0.0),
'image/object/bbox/xmax':
tfrec.VarLenFeature(tfrec.float32, 0.0),
'image/object/bbox/ymax':
tfrec.VarLenFeature(tfrec.float32, 0.0),
'image/object/class/text':
tfrec.FixedLenFeature([], tfrec.string, ''),
'image/object/class/label':
tfrec.VarLenFeature(tfrec.int64, -1)
},
num_shards=num_shards,
random_shuffle=training)
self.training = training
self.cat = dalitorch.TorchPythonFunction(
function=lambda l, t, r, b: torch.cat([l, t, r, b]).view(
4, -1).permute(1, 0)) #[l*w,t*h,r*w,b*h], [l,t,r,b]
self.cast = ops.Cast(dtype=types.DALIDataType.INT32)
def test_python_operator_and_custom_plugin(self):
plugin_manager.load_library(test_bin_dir + "/libcustomdummyplugin.so")
ops.TFRecordReader(path="dummy", index_path="dummy", features={})
