def test_tensorflow_imagenet_dataset(self):
import tensorflow as tf
tf.compat.v1.disable_eager_execution()
random_array = np.random.random_sample([100, 100, 3]) * 255
random_array = random_array.astype(np.uint8)
im = Image.fromarray(random_array)
im.save('test.jpeg')
dataloader_args = {
'dataset': {
"ImageRecord": {
'root': './'
}
},
'transform': None,
'filter': None
}
self.assertRaises(ValueError, create_dataloader, 'tensorflow',
dataloader_args)
image = tf.compat.v1.gfile.FastGFile('test.jpeg', 'rb').read()
example = tf.train.Example(features=tf.train.Features(
feature={
'image/encoded':
tf.train.Feature(bytes_list=tf.train.BytesList(value=[image])),
'image/class/label':
tf.train.Feature(int64_list=tf.train.Int64List(value=[1])),
}))
with tf.io.TFRecordWriter('validation-00000-of-00000') as writer:
writer.write(example.SerializeToString())
eval_dataset = create_dataset('tensorflow',
{'ImageRecord': {
'root': './'
}}, {'ParseDecodeImagenet': {}}, None)
dataloader = DATALOADERS['tensorflow'](dataset=eval_dataset,
batch_size=1)
for (inputs, labels) in dataloader:
self.assertEqual(inputs.shape, (1, 100, 100, 3))
self.assertEqual(labels.shape, (1, 1))
# test old api
eval_dataset = create_dataset('tensorflow',
{'Imagenet': {
'root': './'
}}, {'ParseDecodeImagenet': {}}, None)
dataloader = DataLoader('tensorflow',
dataset=eval_dataset,
batch_size=1)
for (inputs, labels) in dataloader:
self.assertEqual(inputs.shape, (1, 100, 100, 3))
self.assertEqual(labels.shape, (1, 1))
os.remove('validation-00000-of-00000')
os.remove('test.jpeg')
def testParseDecodeImagenet(self):
random_array = np.random.random_sample([100,100,3]) * 255
random_array = random_array.astype(np.uint8)
im = Image.fromarray(random_array)
im.save('test.jpeg')
image = tf.compat.v1.gfile.FastGFile('test.jpeg','rb').read()
label = 10
example = tf.train.Example(features=tf.train.Features(feature={
'image/encoded': tf.train.Feature(
bytes_list=tf.train.BytesList(value=[image])),
'image/class/label': tf.train.Feature(
int64_list=tf.train.Int64List(value=[label])),
'image/object/bbox/xmin': tf.train.Feature(
float_list=tf.train.FloatList(value=[10])),
'image/object/bbox/ymin': tf.train.Feature(
float_list=tf.train.FloatList(value=[20])),
'image/object/bbox/xmax': tf.train.Feature(
float_list=tf.train.FloatList(value=[100])),
'image/object/bbox/ymax': tf.train.Feature(
float_list=tf.train.FloatList(value=[200])),
}))
with tf.io.TFRecordWriter('test.record') as writer:
writer.write(example.SerializeToString())
eval_dataset = create_dataset(
'tensorflow', {'TFRecordDataset':{'root':'test.record'}}, {'ParseDecodeImagenet':{}}, None)
dataloader = DATALOADERS['tensorflow'](dataset=eval_dataset, batch_size=1)
for (inputs, labels) in dataloader:
self.assertEqual(inputs.shape, (1,100,100,3))
self.assertEqual(labels[0][0], 10)
break
os.remove('test.record')
os.remove('test.jpeg')
def test_coco_record(self):
import tensorflow as tf
tf.compat.v1.disable_eager_execution()
random_array = np.random.random_sample([100, 100, 3]) * 255
random_array = random_array.astype(np.uint8)
im = Image.fromarray(random_array)
im.save('test.jpeg')
image = tf.compat.v1.gfile.FastGFile('test.jpeg', 'rb').read()
source_id = '000000397133.jpg'.encode('utf-8')
label = 'person'.encode('utf-8')
example = tf.train.Example(features=tf.train.Features(
feature={
'image/encoded':
tf.train.Feature(bytes_list=tf.train.BytesList(value=[image])),
'image/object/class/text':
tf.train.Feature(bytes_list=tf.train.BytesList(value=[label])),
'image/source_id':
tf.train.Feature(bytes_list=tf.train.BytesList(
value=[source_id])),
'image/object/bbox/xmin':
tf.train.Feature(float_list=tf.train.FloatList(value=[10])),
'image/object/bbox/ymin':
tf.train.Feature(float_list=tf.train.FloatList(value=[10])),
'image/object/bbox/xmax':
tf.train.Feature(float_list=tf.train.FloatList(value=[100])),
'image/object/bbox/ymax':
tf.train.Feature(float_list=tf.train.FloatList(value=[100])),
}))
with tf.io.TFRecordWriter('test.record') as writer:
writer.write(example.SerializeToString())
eval_dataset = create_dataset('tensorflow',
{'COCORecord': {
'root': 'test.record'
}}, {
'ParseDecodeCoco': {},
'RandomVerticalFlip': {},
'RandomHorizontalFlip': {},
'CropResize': {
'x': 0,
'y': 0,
'width': 10,
'height': 10,
'size': [5, 5]
},
'Transpose': {
'perm': [2, 0, 1]
}
}, None)
dataloader = DATALOADERS['tensorflow'](dataset=eval_dataset,
batch_size=1)
for inputs, labels in dataloader:
self.assertEqual(inputs.shape, (1, 3, 5, 5))
self.assertEqual(labels[0].shape, (1, 1, 4))
os.remove('test.record')
os.remove('test.jpeg')
def test_pytorch_bert_dataset(self):
dataset = [[[101, 2043, 2001], [1, 1, 1],
[[0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0]], [1, 1, 1], [1, 1, 1],
[[0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0]]]]
with self.assertRaises(AssertionError):
create_dataset('pytorch',
{'bert': {
'dataset': dataset,
'task': 'test'
}}, None, None)
ds = create_dataset(
'pytorch', {
'bert': {
'dataset': dataset,
'task': 'classifier',
'model_type': 'distilbert'
}
}, None, None)
self.assertEqual(len(ds), 1)
self.assertEqual(3, len(ds[0][0]))
ds = create_dataset(
'pytorch', {
'bert': {
'dataset': dataset,
'task': 'classifier',
'model_type': 'bert'
}
}, None, None)
self.assertEqual(4, len(ds[0][0]))
ds = create_dataset('pytorch',
{'bert': {
'dataset': dataset,
'task': 'squad'
}}, None, None)
self.assertEqual(3, len(ds[0][0]))
ds = create_dataset(
'pytorch', {
'bert': {
'dataset': dataset,
'task': 'squad',
'model_type': 'distilbert'
}
}, None, None)
self.assertEqual(2, len(ds[0][0]))
ds = create_dataset('pytorch', {
'bert': {
'dataset': dataset,
'task': 'squad',
'model_type': 'xlnet'
}
}, None, None)
self.assertEqual(5, len(ds[0][0]))
def test_onnx_imagenet(self):
os.makedirs('val', exist_ok=True)
os.makedirs('val/0', exist_ok=True)
random_array = np.random.random_sample([100, 100, 3]) * 255
random_array = random_array.astype(np.uint8)
random_array = random_array.astype(np.uint8)
im = Image.fromarray(random_array)
im.save('val/test.jpg')
args = {'ImageFolder': {'root': './val'}}
ds = create_dataset('onnxrt_qlinearops', args, None, None)
dataloader = DATALOADERS['onnxrt_qlinearops'](ds)
for image, label in dataloader:
self.assertEqual(image[0].size, (100, 100))
shutil.rmtree('val')
def test_onnx_imagenet(self):
import shutil
os.makedirs('val')
os.makedirs('val/0')
random_array = np.random.random_sample([100, 100, 3]) * 255
random_array = random_array.astype(np.uint8)
random_array = random_array.astype(np.uint8)
im = Image.fromarray(random_array)
im.save('val/0000000397133.jpg')
args = {'Imagenet': {'root': './'}}
ds = create_dataset('onnxrt_qlinearops', args, None, None)
dataloader = DataLoader('onnxrt_qlinearops', ds)
for image, label in dataloader:
self.assertEqual(image[0].size, (100, 100))
shutil.rmtree('val')
def testCOCODecode(self):
from lpot.data.transforms.coco_transform import ParseDecodeCocoTransform
tf.compat.v1.disable_eager_execution()
random_array = np.random.random_sample([100,100,3]) * 255
random_array = random_array.astype(np.uint8)
im = Image.fromarray(random_array)
im.save('test.jpeg')
image = tf.compat.v1.gfile.FastGFile('test.jpeg','rb').read()
source_id = '000000397133.jpg'.encode('utf-8')
label = 'person'.encode('utf-8')
example = tf.train.Example(features=tf.train.Features(feature={
'image/encoded':tf.train.Feature(
bytes_list=tf.train.BytesList(value=[image])),
'image/object/class/text':tf.train.Feature(
bytes_list=tf.train.BytesList(value=[label])),
'image/source_id':tf.train.Feature(
bytes_list=tf.train.BytesList(value=[source_id])),
'image/object/bbox/xmin':tf.train.Feature(
float_list=tf.train.FloatList(value=[10])),
'image/object/bbox/ymin':tf.train.Feature(
float_list=tf.train.FloatList(value=[10])),
'image/object/bbox/xmax':tf.train.Feature(
float_list=tf.train.FloatList(value=[100])),
'image/object/bbox/ymax':tf.train.Feature(
float_list=tf.train.FloatList(value=[100])),
}))
with tf.io.TFRecordWriter('test.record') as writer:
writer.write(example.SerializeToString())
eval_dataset = create_dataset(
'tensorflow', {'COCORecord':{'root':'test.record'}}, {'ParseDecodeCoco':{}}, None)
dataloader = DATALOADERS['tensorflow'](dataset=eval_dataset, batch_size=1)
for (inputs, labels) in dataloader:
self.assertEqual(inputs.shape, (1,100,100,3))
self.assertEqual(labels[0].shape, (1,1,4))
func = ParseDecodeCocoTransform()
out = func(example.SerializeToString())
self.assertEqual(out[0].eval(session=tf.compat.v1.Session()).shape, (100,100,3))
example = tf.train.Example(features=tf.train.Features(feature={
'image/encoded':tf.train.Feature(
bytes_list=tf.train.BytesList(value=[image])),
'image/source_id':tf.train.Feature(
bytes_list=tf.train.BytesList(value=[source_id])),
'image/object/bbox/xmin':tf.train.Feature(
float_list=tf.train.FloatList(value=[10])),
'image/object/bbox/ymin':tf.train.Feature(
float_list=tf.train.FloatList(value=[10])),
'image/object/bbox/xmax':tf.train.Feature(
float_list=tf.train.FloatList(value=[100])),
'image/object/bbox/ymax':tf.train.Feature(
float_list=tf.train.FloatList(value=[100])),
}))
with tf.io.TFRecordWriter('test2.record') as writer:
writer.write(example.SerializeToString())
self.assertRaises(ValueError, create_dataset,
'tensorflow', {'COCORecord':{'root':'test2.record'}}, None, None)
os.remove('test2.record')
os.remove('test.record')
os.remove('test.jpeg')
def testLabelBalanceCOCORecord(self):
from PIL import Image
tf.compat.v1.disable_eager_execution()
random_array = np.random.random_sample([100, 100, 3]) * 255
random_array = random_array.astype(np.uint8)
im = Image.fromarray(random_array)
im.save('test.jpeg')
image = tf.compat.v1.gfile.FastGFile('test.jpeg', 'rb').read()
source_id = '000000397133.jpg'.encode('utf-8')
label = 'person'.encode('utf-8')
example1 = tf.train.Example(features=tf.train.Features(
feature={
'image/encoded':
tf.train.Feature(bytes_list=tf.train.BytesList(value=[image])),
'image/object/class/text':
tf.train.Feature(bytes_list=tf.train.BytesList(value=[label])),
'image/source_id':
tf.train.Feature(bytes_list=tf.train.BytesList(
value=[source_id])),
'image/object/bbox/xmin':
tf.train.Feature(float_list=tf.train.FloatList(value=[10])),
'image/object/bbox/ymin':
tf.train.Feature(float_list=tf.train.FloatList(value=[10])),
'image/object/bbox/xmax':
tf.train.Feature(float_list=tf.train.FloatList(value=[100])),
'image/object/bbox/ymax':
tf.train.Feature(float_list=tf.train.FloatList(value=[100])),
}))
example2 = tf.train.Example(features=tf.train.Features(
feature={
'image/encoded':
tf.train.Feature(bytes_list=tf.train.BytesList(value=[image])),
'image/object/class/text':
tf.train.Feature(bytes_list=tf.train.BytesList(value=[label])),
'image/source_id':
tf.train.Feature(bytes_list=tf.train.BytesList(
value=[source_id])),
'image/object/bbox/xmin':
tf.train.Feature(float_list=tf.train.FloatList(
value=[10, 20])),
'image/object/bbox/ymin':
tf.train.Feature(float_list=tf.train.FloatList(
value=[10, 20])),
'image/object/bbox/xmax':
tf.train.Feature(float_list=tf.train.FloatList(
value=[100, 200])),
'image/object/bbox/ymax':
tf.train.Feature(float_list=tf.train.FloatList(
value=[100, 200])),
}))
with tf.io.TFRecordWriter('test.record') as writer:
writer.write(example1.SerializeToString())
writer.write(example2.SerializeToString())
preprocesses = TRANSFORMS('tensorflow', 'preprocess')
preprocess = get_preprocess(preprocesses, {'ParseDecodeCoco': {}})
filters = FILTERS('tensorflow')
filter = filters['LabelBalanceCOCORecord'](2)
datasets = DATASETS('tensorflow')
dataset = datasets['COCORecord']('test.record', \
transform=preprocess, filter=filter)
dataloader = DATALOADERS['tensorflow'](dataset=dataset, batch_size=1)
for (inputs, labels) in dataloader:
self.assertEqual(inputs.shape, (1, 100, 100, 3))
self.assertEqual(labels[0].shape, (1, 2, 4))
dataset2 = create_dataset('tensorflow',
{'COCORecord': {
'root': 'test.record'
}}, {'ParseDecodeCoco': {}},
{'LabelBalance': {
'size': 2
}})
dataloader2 = DATALOADERS['tensorflow'](dataset=dataset2, batch_size=1)
for (inputs, labels) in dataloader2:
self.assertEqual(inputs.shape, (1, 100, 100, 3))
self.assertEqual(labels[0].shape, (1, 2, 4))
dataloader3 = create_dataloader('tensorflow', {'batch_size':1, 'dataset':{'COCORecord':{'root':'test.record'}},\
'filter':{'LabelBalance':{'size':2}}, 'transform':{'ParseDecodeCoco':{}}})
for (inputs, labels) in dataloader3:
self.assertEqual(inputs.shape, (1, 100, 100, 3))
self.assertEqual(labels[0].shape, (1, 2, 4))
os.remove('test.record')
os.remove('test.jpeg')
def testVOCDecode(self):
import shutil
tf.compat.v1.disable_eager_execution()
def _bytes_list_feature(values):
import six
def norm2bytes(value):
return value.encode() if isinstance(value, str) and six.PY3 else value
return tf.train.Feature(
bytes_list=tf.train.BytesList(value=[norm2bytes(values)]))
def _int64_list_feature(values):
import collections
if not isinstance(values, collections.Iterable):
values = [values]
return tf.train.Feature(int64_list=tf.train.Int64List(value=values))
random_array = np.random.random_sample([100,100,3]) * 255
random_array = random_array.astype(np.uint8)
im = Image.fromarray(random_array)
im.save('test.jpg')
random_array = np.random.random_sample([100,100,3]) * 0
random_array = random_array.astype(np.uint8)
im = Image.fromarray(random_array)
im.save('test.png')
image_data = tf.compat.v1.gfile.GFile('test.jpg', 'rb').read()
seg_data = tf.compat.v1.gfile.GFile('test.png', 'rb').read()
filename = 'test'
example = tf.train.Example(features=tf.train.Features(feature={
'image/encoded': _bytes_list_feature(image_data),
'image/filename': _bytes_list_feature(filename),
'image/format': _bytes_list_feature('png'),
'image/height': _int64_list_feature(100),
'image/width': _int64_list_feature(100),
'image/channels': _int64_list_feature(3),
'image/segmentation/class/encoded': (
_bytes_list_feature(seg_data)),
'image/segmentation/class/format': _bytes_list_feature('png'),
}))
if not os.path.exists('./test_record'):
os.mkdir('./test_record')
with tf.io.TFRecordWriter('./test_record/val-test.record') as writer:
writer.write(example.SerializeToString())
eval_dataset = create_dataset(
'tensorflow', {'VOCRecord':{'root':'./test_record'}}, {'ParseDecodeVoc':{}}, None)
dataloader = DATALOADERS['tensorflow'](dataset=eval_dataset, batch_size=1)
for (inputs, labels) in dataloader:
self.assertEqual(inputs.shape, (1,100,100,3))
self.assertEqual(labels[0].shape, (100,100,1))
from lpot.experimental.data.transforms.transform import ParseDecodeVocTransform
func = ParseDecodeVocTransform()
out = func(example.SerializeToString())
self.assertEqual(out[0].eval(session=tf.compat.v1.Session()).shape, (100,100,3))
os.remove('./test_record/val-test.record')
os.remove('test.jpg')
os.remove('test.png')
shutil.rmtree('./test_record')
def testCOCODecode(self):
tf.compat.v1.disable_eager_execution()
random_array = np.random.random_sample([100,100,3]) * 255
random_array = random_array.astype(np.uint8)
im = Image.fromarray(random_array)
im.save('test.jpeg')
image = tf.compat.v1.gfile.FastGFile('test.jpeg','rb').read()
source_id = '000000397133.jpg'.encode('utf-8')
label = 'person'.encode('utf-8')
example = tf.train.Example(features=tf.train.Features(feature={
'image/encoded':tf.train.Feature(
bytes_list=tf.train.BytesList(value=[image])),
'image/object/class/text':tf.train.Feature(
bytes_list=tf.train.BytesList(value=[label])),
'image/source_id':tf.train.Feature(
bytes_list=tf.train.BytesList(value=[source_id])),
'image/object/bbox/xmin':tf.train.Feature(
float_list=tf.train.FloatList(value=[10])),
'image/object/bbox/ymin':tf.train.Feature(
float_list=tf.train.FloatList(value=[10])),
'image/object/bbox/xmax':tf.train.Feature(
float_list=tf.train.FloatList(value=[100])),
'image/object/bbox/ymax':tf.train.Feature(
float_list=tf.train.FloatList(value=[100])),
}))
with tf.io.TFRecordWriter('test.record') as writer:
writer.write(example.SerializeToString())
eval_dataset = create_dataset(
'tensorflow', {'COCORecord':{'root':'test.record'}},
{'ParseDecodeCoco':{}, 'Resize': {'size': 50}, 'Cast':{'dtype':'int64'},
'CropToBoundingBox':{'offset_height':2, 'offset_width':2, 'target_height':5, 'target_width':5},
'CenterCrop':{'size':[4,4]},
'RandomResizedCrop':{'size':[4,5]},
}, None)
dataloader = DATALOADERS['tensorflow'](dataset=eval_dataset, batch_size=1)
for (inputs, labels) in dataloader:
self.assertEqual(inputs.shape, (1,4,5,3))
self.assertEqual(labels[0].shape, (1,1,4))
from lpot.experimental.data.transforms.transform import TensorflowResizeWithRatio
from lpot.experimental.data.datasets.coco_dataset import ParseDecodeCoco
func = ParseDecodeCoco()
out = func(example.SerializeToString())
self.assertEqual(out[0].eval(session=tf.compat.v1.Session()).shape, (100,100,3))
func = ParseDecodeCoco()
out = func(example.SerializeToString())
self.assertEqual(out[0].eval(session=tf.compat.v1.Session()).shape, (100,100,3))
func = TensorflowResizeWithRatio(**{'padding':True})
out = func(out)
self.assertEqual(out[0].eval(session=tf.compat.v1.Session()).shape, (1365,1365,3))
example = tf.train.Example(features=tf.train.Features(feature={
'image/encoded':tf.train.Feature(
bytes_list=tf.train.BytesList(value=[image])),
'image/source_id':tf.train.Feature(
bytes_list=tf.train.BytesList(value=[source_id])),
'image/object/bbox/xmin':tf.train.Feature(
float_list=tf.train.FloatList(value=[10])),
'image/object/bbox/ymin':tf.train.Feature(
float_list=tf.train.FloatList(value=[10])),
'image/object/bbox/xmax':tf.train.Feature(
float_list=tf.train.FloatList(value=[100])),
'image/object/bbox/ymax':tf.train.Feature(
float_list=tf.train.FloatList(value=[100])),
}))
with tf.io.TFRecordWriter('test2.record') as writer:
writer.write(example.SerializeToString())
self.assertRaises(ValueError, create_dataset,
'tensorflow', {'COCORecord':{'root':'test2.record'}}, None, None)
os.remove('test2.record')
os.remove('test.record')
os.remove('test.jpeg')
def test_coco_raw(self):
import json
import collections
from lpot.data import TRANSFORMS
import mxnet as mx
random_array = np.random.random_sample([100, 100, 3]) * 255
random_array = random_array.astype(np.uint8)
im = Image.fromarray(random_array)
im.save('test_0.jpg')
im.save('test_1.jpg')
fake_dict = {
'info': {
'description': 'COCO 2017 Dataset',
'url': 'http://cocodataset.org',
'version': '1.0',
'year': 2017,
'contributor': 'COCO Consortium',
'date_created': '2017/09/01'
},
'licenses': {},
'images': [{
'file_name': 'test_0.jpg',
'height': 100,
'width': 100,
'id': 0
}, {
'file_name': 'test_1.jpg',
'height': 100,
'width': 100,
'id': 1
}, {
'file_name': 'test_2.jpg',
'height': 100,
'width': 100,
'id': 2
}],
'annotations': [{
'category_id': 18,
'id': 1767,
'iscrowd': 0,
'image_id': 0,
'bbox': [473.07, 395.93, 38.65, 28.67],
}, {
'category_id': 18,
'id': 1768,
'iscrowd': 0,
'image_id': 1,
'bbox': [473.07, 395.93, 38.65, 28.67],
}, {
'category_id': 18,
'id': 1769,
'iscrowd': 0,
'image_id': 2,
'bbox': [],
}],
'categories': [{
'supercategory': 'animal',
'id': 18,
'name': 'dog'
}]
}
fake_json = json.dumps(fake_dict)
with open('anno.json', 'w') as f:
f.write(fake_json)
args = {
'COCORaw': {
'root': './',
'img_dir': '',
'anno_dir': 'anno.json'
}
}
ds = create_dataset('tensorflow', args, None, None)
dataloader = DATALOADERS['tensorflow'](ds)
for image, label in dataloader:
self.assertEqual(image[0].shape, (100, 100, 3))
trans_args = {'Transpose': {'perm': [2, 0, 1]}}
ds = create_dataset('tensorflow', args, trans_args, None)
dataloader = DATALOADERS['tensorflow'](ds)
for image, label in dataloader:
self.assertEqual(image[0].shape, (3, 100, 100))
args = {
'COCORaw': {
'root': './',
'img_dir': '',
'anno_dir': 'anno.json'
}
}
ds = create_dataset('onnxrt_qlinearops', args, None, None)
dataloader = DATALOADERS['onnxrt_qlinearops'](ds)
for image, label in dataloader:
self.assertEqual(image[0].shape, (100, 100, 3))
args = {
'COCORaw': {
'root': './',
'img_dir': '',
'anno_dir': 'anno.json'
}
}
ds = create_dataset('mxnet', args, None, None)
def collate(batch):
elem = batch[0]
if isinstance(elem, mx.ndarray.NDArray):
return mx.nd.stack(*batch)
elif isinstance(elem, collections.abc.Sequence):
batch = zip(*batch)
return [collate(samples) for samples in batch]
elif isinstance(elem, collections.abc.Mapping):
return {key: collate([d[key] for d in batch]) for key in elem}
elif isinstance(elem, np.ndarray):
return np.stack(batch)
else:
return batch
dataloader = DATALOADERS['mxnet'](ds, collate_fn=collate)
for image, label in dataloader:
self.assertEqual(image[0].shape, (100, 100, 3))
args = {
'COCORaw': {
'root': './',
'img_dir': '',
'anno_dir': 'anno.json'
}
}
ds = create_dataset('pytorch', args, None, None)
def collate(batch):
elem = batch[0]
if isinstance(elem, collections.abc.Mapping):
return {key: collate([d[key] for d in batch]) for key in elem}
elif isinstance(elem, collections.abc.Sequence):
batch = zip(*batch)
return [collate(samples) for samples in batch]
elif isinstance(elem, np.ndarray):
return np.stack(batch)
else:
return batch
dataloader = DATALOADERS['pytorch'](dataset=ds, collate_fn=collate)
for image, label in dataloader:
self.assertEqual(image[0].shape, (100, 100, 3))
os.remove('test_0.jpg')
os.remove('test_1.jpg')
os.remove('anno.json')
def test_onnx(self):
dataloader_args = {
'dataset': {
"ImagenetRaw": {
'data_path': 'val',
'image_list': None
}
},
'transform': {
'Resize': {
'size': 24
}
},
'filter': None
}
dataloader = create_dataloader('onnxrt_integerops', dataloader_args)
for data in dataloader:
self.assertEqual(data[0][0].shape, (24, 24, 3))
break
dataloader_args = {
'dataset': {
"ImagenetRaw": {
'data_path': 'val',
'image_list': 'val/val.txt'
}
},
'transform': {
'Resize': {
'size': 24
}
},
'filter': None
}
dataloader = create_dataloader('onnxrt_integerops', dataloader_args)
for data in dataloader:
self.assertEqual(data[0][0].shape, (24, 24, 3))
break
# test old api
eval_dataset = create_dataset('onnxrt_integerops',
{'Imagenet': {
'root': './'
}}, None, None)
dataloader = DataLoader('onnxrt_integerops',
dataset=eval_dataset,
batch_size=1)
for data in dataloader:
self.assertEqual(data[0][0].shape, (100, 100, 3))
break
with open('val/fake_map.txt', 'w') as f:
f.write('test.jpg 0 \n')
f.write('test2.jpg 1')
dataset_args = {
"ImagenetRaw": {
'data_path': 'val',
'image_list': 'val/fake_map.txt'
},
}
dataset = create_dataset('onnxrt_integerops', dataset_args, None, None)
self.assertEqual(len(dataset), 1)
with open('val/fake_map.txt', 'w') as f:
f.write('test2.jpg 1')
dataloader_args = {
'dataset': {
"ImagenetRaw": {
'data_path': 'val',
'image_list': 'val/fake_map.txt'
}
},
'transform': None,
'filter': None
}
self.assertRaises(ValueError, create_dataloader, 'onnxrt_integerops',
dataloader_args)
with open('val/not_found_map.txt', 'w') as f:
f.write('test.jpg 0' + '\n')
f.write('not_found.jpg 1')
dataloader_args = {
'dataset': {
"ImagenetRaw": {
'data_path': 'val',
'image_list': 'val/not_found_map.txt'
}
},
'transform': {
'Resize': {
'size': 24
}
},
'filter': None
}
dataloader = create_dataloader('onnxrt_integerops', dataloader_args)
for data in dataloader:
self.assertEqual(data[0][0].shape, (24, 24, 3))
break
with open('val/blank.txt', 'w') as f:
f.write('blank.jpg 0')
dataloader_args = {
'dataset': {
"ImagenetRaw": {
'data_path': 'val',
'image_list': 'val/blank.txt'
}
},
'transform': None,
'filter': None
}
self.assertRaises(ValueError, create_dataloader, 'onnxrt_qlinearops',
dataloader_args)
def test_coco_raw(self):
import json
from lpot.data import TRANSFORMS
random_array = np.random.random_sample([100, 100, 3]) * 255
random_array = random_array.astype(np.uint8)
im = Image.fromarray(random_array)
im.save('000000397133.jpg')
im.save('000000397134.jpg')
fake_dict = {
'info': {
'description': 'COCO 2017 Dataset',
'url': 'http://cocodataset.org',
'version': '1.0',
'year': 2017,
'contributor': 'COCO Consortium',
'date_created': '2017/09/01'
},
'licenses': {},
'images': [{
'file_name': '000000397133.jpg',
'height': 100,
'width': 100,
'id': 397133
}, {
'file_name': '000000397134.jpg',
'height': 100,
'width': 100,
'id': 397134
}, {
'file_name': '000000397135.jpg',
'height': 100,
'width': 100,
'id': 397135
}],
'annotations': [{
'category_id': 18,
'id': 1768,
'iscrowd': 0,
'image_id': 397133,
'bbox': [473.07, 395.93, 38.65, 28.67],
}, {
'category_id': 18,
'id': 1768,
'iscrowd': 0,
'image_id': 397134,
'bbox': [473.07, 395.93, 38.65, 28.67],
}, {
'category_id': 18,
'id': 1768,
'iscrowd': 0,
'image_id': 397135,
'bbox': [],
}],
'categories': [{
'supercategory': 'animal',
'id': 18,
'name': 'dog'
}]
}
fake_json = json.dumps(fake_dict)
with open('anno.json', 'w') as f:
f.write(fake_json)
args = {
'COCORaw': {
'root': './',
'img_dir': '',
'anno_dir': 'anno.json'
}
}
ds = create_dataset('tensorflow', args, None, None)
dataloader = DataLoader('tensorflow', ds)
for image, label in dataloader:
self.assertEqual(image[0].size, (100, 100))
trans_args = {'Rescale': {}}
ds = create_dataset('tensorflow', args, trans_args, None)
dataloader = DataLoader('tensorflow', ds)
for image, label in dataloader:
self.assertEqual(image[0].shape, (100, 100, 3))
os.remove('000000397133.jpg')
os.remove('000000397134.jpg')
os.remove('anno.json')
