def test_pytorch(): print('testing pytorch') # Create arrays images = hub.array((100, 100, 100), name='test/dataloaders:images', dtype='uint8') labels = hub.array((100, 1), name='test/dataloaders:labels', dtype='uint8') # Create dataset ds = hub.dataset({ 'images': images, 'labels': labels }, name='test/loaders:dataset') # Transform to Pytorch train_dataset = ds.to_pytorch() # Create data loader train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=32, num_workers=1, pin_memory=True) # Loop over attributes for i, (images, labels) in enumerate(train_loader): print(images.shape, labels.shape) assert (images.shape == (32, 100, 100)) assert (labels.shape == (32, 1)) break print('pass')
def create_datasets(): hubs = {} for name in [ 'RADAR_FRONT', 'RADAR_BACK_RIGHT', 'RADAR_FRONT_LEFT', 'RADAR_FRONT_RIGHT', 'RADAR_BACK_LEFT' ]: hubs[name] = hub.array((length, 18, 200), dtype=np.float32, name='aptiv/nutonomy3:{}'.format(name), chunk_size=(1000, 18, 200)) for name in ['LIDAR_TOP']: hubs[name] = hub.array((length, 4, 30000), dtype=np.float32, name='aptiv/nutonomy3:{}'.format(name), chunk_size=(100, 4, 30000)) for name in [ 'CAM_FRONT', 'CAM_FRONT_RIGHT', 'CAM_BACK_RIGHT', 'CAM_BACK', 'CAM_BACK_LEFT', 'CAM_FRONT_LEFT' ]: hubs[name] = hub.array((length, 900, 1600, 3), dtype=np.uint8, name='aptiv/nutonomy3:{}'.format(name), chunk_size=(5, 900, 1600, 3)) dataset = hub.dataset(hubs, name='aptiv/nutonomy3:v1.0-trainval') return dataset
def test_dataset(): x = hub.array((10000, 512, 512), name='test/example:input', dtype='uint8') y = hub.array((10000, 4), name='test/example:label', dtype='uint8') ds = hub.dataset({'input': x, 'label': y}, name='test/dataset:train3') assert ds['input'].shape[0] == 10000 # return single array assert ds['label', 0].mean() == 0 # equivalent ds['train'][0] # return pair of arrays as long as dimensions requested are broadcastable assert ds[0][0].mean() == 0
def test_tensorflow(): print('testing tensorflow') # Create arrays images = hub.array((10, 100, 100), name='name1', dtype='uint8') labels = hub.array((10, 1), name='name2', dtype='uint8') # Create dataset ds = hub.dataset({ 'images': images, 'labels': labels }, name='test/loaders:dataset') # Transform to Pytorch train_dataset = ds.to_tensorflow() assert len(train_dataset) == 10 for image, label in train_dataset: assert len(image) == 100 assert len(image[0]) == 100 assert len(label) == 1 train_dataset = train_dataset.batch(32, drop_remainder=True) # Loop over attributes for _, (images, labels) in train_dataset.enumerate(): # assert len(images) == 32 # assert len(labels) == 32 assert len(images[0]) == 100 assert len(images[0][0]) == 100 assert len(labels[0]) == 1 break print('pass')
def test_load_dataset(): ds = hub.dataset(name='test/dataset:train3') # return the dataset object assert ds.chunk_shape == [1] # returns a dict of shapes assert ds.shape == [10000]