def test_lots_of_data_with_multiple_backend(self, repo_300_filled_samples):
repo = repo_300_filled_samples
co = repo.checkout()
aset = co.columns['aset']
torch_dset = make_torch_dataset([aset], as_dict=True)
loader = DataLoader(torch_dset, batch_size=10, drop_last=True)
for data in loader:
assert isinstance(data, dict)
assert data['aset'].shape == (10, 5, 7)
co.close()
def test_lots_of_data_with_multiple_backend_multiple_worker_dataloader(
self, repo_300_filled_samples):
repo = repo_300_filled_samples
co = repo.checkout()
aset = co.columns['aset']
torch_dset = make_torch_dataset([aset])
loader = DataLoader(torch_dset,
batch_size=10,
drop_last=True,
num_workers=2)
for data in loader:
assert data.shape == (10, 5, 7)
co.close()
def test_return_as_dict(self, repo_20_filled_samples):
repo = repo_20_filled_samples
co = repo.checkout()
first_aset = co.columns['writtenaset']
second_aset = co.columns['second_aset']
torch_dset = make_torch_dataset([first_aset, second_aset],
as_dict=True)
assert len(torch_dset) == 20
loader = DataLoader(torch_dset, batch_size=5)
for sample in loader:
assert 'writtenaset' in sample.keys()
assert 'second_aset' in sample.keys()
co.close()
def test_multiple_dataset_loader(self, repo_20_filled_samples):
repo = repo_20_filled_samples
co = repo.checkout()
first_aset = co.columns['writtenaset']
second_aset = co.columns['second_aset']
torch_dset = make_torch_dataset([first_aset, second_aset])
loader = DataLoader(torch_dset, batch_size=6, drop_last=True)
total_samples = 0
for dset1, dset2 in loader:
total_samples += dset1.shape[0]
assert dset1.shape == (6, 5, 7)
assert dset2.shape == (6, 5, 7)
assert total_samples == 18 # drop last is True
co.close()
def test_two_aset_loader_two_worker_dataloader(self,
repo_20_filled_samples):
repo = repo_20_filled_samples
co = repo.checkout()
first_aset = co.columns['writtenaset']
second_aset = co.columns['second_aset']
torch_dset = make_torch_dataset([first_aset, second_aset])
loader = DataLoader(torch_dset,
batch_size=2,
drop_last=True,
num_workers=2)
count = 0
for asets_batch in loader:
assert isinstance(asets_batch, list)
assert len(asets_batch) == 2
assert asets_batch[0].shape == (2, 5, 7)
assert asets_batch[1].shape == (2, 5, 7)
assert np.allclose(asets_batch[0], -asets_batch[1])
count += 1
assert count == 10
co.close()