def test_MultiWebLoader_torch_pipe():
import torch
def f(source):
for sample in source:
assert isinstance(sample, (list, tuple))
assert len(sample) == 2
yield sample
wl = loader.MultiWebLoader("testdata/sample.tgz",
90,
multi_pipe=f,
fields="ppm;jpg;jpeg;png cls".split(),
batch_size=32,
converters=loader.totorch())
for sample in wl:
break
assert len(sample) == 2
assert isinstance(sample[0], torch.Tensor), sample[0]
assert isinstance(sample[1], torch.Tensor), sample[1]
assert sample[0].dtype == torch.float32, sample[0]
assert sample[1].dtype == torch.int64, sample[1]
assert len(sample[0].shape) == 4, sample[0].shape
assert len(sample[1].shape) == 1, sample[1].shape
assert sample[0].shape[0] == 32, sample[0].shape
assert sample[1].shape[0] == 32, sample[1].shape
assert sample[0].shape[1] == 3, sample[0].size()
def FIXME_test_MultiWebLoader_torch_sync_gpu2():
# transfering torch tensors does not currently work;
# use converters="torch_np" and then convert in the parent process
import torch
wl = loader.MultiWebLoader("testdata/sample.tgz",
90,
multi_pipe="sync_gpu_transfer",
fields="png cls".split(),
use_torch_mp=True,
converters="torch",
batch_size=32)
for sample in wl:
break
assert len(sample) == 2
assert isinstance(sample[0], torch.Tensor), sample[0]
assert isinstance(sample[1], torch.Tensor), sample[1]
assert sample[0].dtype == torch.float32, sample[0]
assert sample[1].dtype == torch.int64, sample[1]
assert len(sample[0].shape) == 4, sample[0].shape
assert len(sample[1].shape) == 1, sample[1].shape
assert sample[0].shape[0] == 32, sample[0].shape
assert sample[1].shape[0] == 32, sample[1].shape
assert sample[0].shape[1] == 3, sample[0].size()
assert sample[0].device.type == "cuda", sample[0].device
assert sample[1].device.type == "cuda", sample[1].device
wl.terminate()
def test_MultiWebLoader_torch_1():
wl = loader.MultiWebLoader("testdata/sample.tgz",
2,
multi_pipe="sync_gpu_transfer",
fields="png cls".split(),
batch_size=32)
total = sum(len(sample[0]) for sample in wl)
assert total == 32 * 2, total
wl.terminate()
def test_MultiWebLoader_batching_epochs1():
nbatches = 3
bs = 8
wl = loader.MultiWebLoader(["testdata/imagenet-000000.tgz"] * 4,
nbatches,
fields="__key__ ppm;jpg;jpeg;png cls".split(),
batch_size=bs,
tensor_batches=False,
processes=4)
total = count_samples(wl)
assert total == nbatches * bs, total
def test_MultiWebLoader_batching_split():
repeats = 4
wl = loader.MultiWebLoader(["testdata/imagenet-000000.tgz"] * repeats,
2000,
split=True,
epochs=1,
fields="__key__ ppm;jpg;jpeg;png cls".split(),
tensor_batches=False,
processes=4)
total = count_samples(wl, verbose=1, max=1000, batched=False)
assert total == 47 * repeats, total
def test_MultiWebLoader_torch():
import torch
wl = loader.MultiWebLoader("testdata/sample.tgz",
90,
fields="png cls".split(),
batch_size=32,
converters=loader.totorch())
for sample in wl:
break
assert len(sample) == 2
assert isinstance(sample[0], torch.Tensor), sample[0]
assert isinstance(sample[1], torch.Tensor), sample[1]
assert sample[0].dtype == torch.float32, sample[0]
assert sample[1].dtype == torch.int64, sample[1]
assert len(sample[0].shape) == 4, sample[0].shape
assert len(sample[1].shape) == 1, sample[1].shape
assert sample[0].shape[0] == 32, sample[0].shape
assert sample[1].shape[0] == 32, sample[1].shape
assert sample[0].shape[1] == 3, sample[0].size()
def test_MultiWebLoader_torch_sync_gpu():
import torch
wl = loader.MultiWebLoader("testdata/sample.tgz",
90,
multi_pipe="sync_gpu_transfer",
fields="png cls".split(),
converters="torch_np",
batch_size=32)
for sample in wl:
break
assert len(sample) == 2
assert isinstance(sample[0], torch.Tensor), sample[0]
assert isinstance(sample[1], torch.Tensor), sample[1]
assert sample[0].dtype == torch.float32, sample[0]
assert sample[1].dtype == torch.int64, sample[1]
assert len(sample[0].shape) == 4, sample[0].shape
assert len(sample[1].shape) == 1, sample[1].shape
assert sample[0].shape[0] == 32, sample[0].shape
assert sample[1].shape[0] == 32, sample[1].shape
assert sample[0].shape[1] == 3, sample[0].size()
assert sample[0].device.type == "cuda", sample[0].device
assert sample[1].device.type == "cuda", sample[1].device
wl.terminate()
def FIXME_test_MultiWebLoader_torch_gpu():
import torch
wl = loader.MultiWebLoader("testdata/sample.tgz",
90,
fields="png cls".split(),
use_torch_mp=True,
batch_size=32,
converters="torch_cuda")
for sample in wl:
break
assert len(sample) == 2
assert isinstance(sample[0], torch.Tensor), sample[0]
assert isinstance(sample[1], torch.Tensor), sample[1]
assert sample[0].dtype == torch.float32, sample[0]
assert sample[1].dtype == torch.int64, sample[1]
assert len(sample[0].shape) == 4, sample[0].shape
assert len(sample[1].shape) == 1, sample[1].shape
assert sample[0].shape[0] == 32, sample[0].shape
assert sample[1].shape[0] == 32, sample[1].shape
assert sample[0].shape[1] == 3, sample[0].size()
assert sample[0].device.type == "cuda", sample[0].device
assert sample[1].device.type == "cuda", sample[1].device
wl.terminate()