def test_pytorch():
print('testing pytorch')
# Create arrays
images = hub.array((100, 100, 100),
name='test/dataloaders:images',
dtype='uint8')
labels = hub.array((100, 1), name='test/dataloaders:labels', dtype='uint8')
# Create dataset
ds = hub.dataset({
'images': images,
'labels': labels
},
name='test/loaders:dataset')
# Transform to Pytorch
train_dataset = ds.to_pytorch()
# Create data loader
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=32,
num_workers=1,
pin_memory=True)
# Loop over attributes
for i, (images, labels) in enumerate(train_loader):
print(images.shape, labels.shape)
assert (images.shape == (32, 100, 100))
assert (labels.shape == (32, 1))
break
print('pass')
def create_datasets():
hubs = {}
for name in [
'RADAR_FRONT', 'RADAR_BACK_RIGHT', 'RADAR_FRONT_LEFT',
'RADAR_FRONT_RIGHT', 'RADAR_BACK_LEFT'
]:
hubs[name] = hub.array((length, 18, 200),
dtype=np.float32,
name='aptiv/nutonomy3:{}'.format(name),
chunk_size=(1000, 18, 200))
for name in ['LIDAR_TOP']:
hubs[name] = hub.array((length, 4, 30000),
dtype=np.float32,
name='aptiv/nutonomy3:{}'.format(name),
chunk_size=(100, 4, 30000))
for name in [
'CAM_FRONT', 'CAM_FRONT_RIGHT', 'CAM_BACK_RIGHT', 'CAM_BACK',
'CAM_BACK_LEFT', 'CAM_FRONT_LEFT'
]:
hubs[name] = hub.array((length, 900, 1600, 3),
dtype=np.uint8,
name='aptiv/nutonomy3:{}'.format(name),
chunk_size=(5, 900, 1600, 3))
dataset = hub.dataset(hubs, name='aptiv/nutonomy3:v1.0-trainval')
return dataset
def test_dataset():
x = hub.array((10000, 512, 512), name='test/example:input', dtype='uint8')
y = hub.array((10000, 4), name='test/example:label', dtype='uint8')
ds = hub.dataset({'input': x, 'label': y}, name='test/dataset:train3')
assert ds['input'].shape[0] == 10000 # return single array
assert ds['label', 0].mean() == 0 # equivalent ds['train'][0]
# return pair of arrays as long as dimensions requested are broadcastable
assert ds[0][0].mean() == 0
def test_tensorflow():
print('testing tensorflow')
# Create arrays
images = hub.array((10, 100, 100), name='name1', dtype='uint8')
labels = hub.array((10, 1), name='name2', dtype='uint8')
# Create dataset
ds = hub.dataset({
'images': images,
'labels': labels
},
name='test/loaders:dataset')
# Transform to Pytorch
train_dataset = ds.to_tensorflow()
assert len(train_dataset) == 10
for image, label in train_dataset:
assert len(image) == 100
assert len(image[0]) == 100
assert len(label) == 1
train_dataset = train_dataset.batch(32, drop_remainder=True)
# Loop over attributes
for _, (images, labels) in train_dataset.enumerate():
# assert len(images) == 32
# assert len(labels) == 32
assert len(images[0]) == 100
assert len(images[0][0]) == 100
assert len(labels[0]) == 1
break
print('pass')
def test_load_dataset():
ds = hub.dataset(name='test/dataset:train3') # return the dataset object
assert ds.chunk_shape == [1] # returns a dict of shapes
assert ds.shape == [10000]