def test_d1():
channels_in = 3
channels_out = 3
samples = 4
device = "cuda"
batches = 10
batch_size = 32
data_shape = (batches * batch_size, channels_in, 512, 512)
model = torch.nn.Sequential(
torch.nn.Conv2d(channels_in, channels_out, (3, 3)),
torch.nn.ReLU(),
torch.nn.Conv2d(channels_out, channels_out, (3, 3)),
torch.nn.ReLU(),
torch.nn.Conv2d(channels_out, channels_out, (3, 3)),
torch.nn.ReLU(),
).to("cuda")
for _ in range(samples):
s1 = time.time()
for _, a in zip(
range(batches),
to_tensor_generator(
batched_recycle(numpy.random.sample(data_shape),
batch_size),
device=device,
preload_next=False,
),
):
model(a)
s2 = time.time()
for _, a in zip(
range(batches),
to_tensor_generator(
batched_recycle(numpy.random.sample(data_shape),
batch_size),
device=device,
),
):
model(a)
s3 = time.time()
print(s2 - s1)
print(s3 - s2)
def test_d7():
import numpy
channels_in = 3
samples = 10
device = "cuda"
batches = 3
batch_size = 32
data_shape = (batches * batch_size, 256, 256, channels_in)
batch_shape = torch.Size([batch_size, 256, 256, channels_in])
dtype = torch.float
class RandomDataset(Dataset):
""" """
def __init__(self):
self.d = numpy.random.sample(data_shape)
def __len__(self):
return len(self.d)
def __getitem__(self, item):
return self.d[item]
dataloader = torch.utils.data.DataLoader(
RandomDataset(),
batch_size=batch_size,
shuffle=True,
num_workers=1,
pin_memory=True,
)
generator = to_tensor_generator(
batched_recycle(numpy.random.sample(data_shape), batch_size),
device=device,
preload_next=True,
dtype=dtype,
)
for _ in range(samples):
s1 = time.time()
for _, a in zip(range(batches), generator):
assert batch_shape == a.shape, a.shape
s2 = time.time()
for _, a in zip(range(batches), dataloader):
a = a.to(device, dtype=dtype)
assert batch_shape == a.shape, a.shape
s3 = time.time()
print(f"generator: {s2 - s1}")
print(f"dataloader: {s3 - s2}")
def test_d1():
channels_in = 3
channels_out = 3
samples = 10
device = "cuda"
batches = 3
batch_size = 32
data_shape = (batches * batch_size, channels_in, 512, 512)
model = torch.nn.Sequential(
torch.nn.Conv2d(channels_in, channels_out, (3, 3)),
torch.nn.ReLU(),
torch.nn.Conv2d(channels_out, channels_out, (3, 3)),
torch.nn.ReLU(),
torch.nn.Conv2d(channels_out, channels_out, (3, 3)),
torch.nn.ReLU(),
).to(device)
for _ in range(samples):
s1 = time.time()
for _, a in zip(
range(batches),
to_tensor_generator(
batched_recycle(numpy.random.sample(data_shape),
batch_size),
device=device,
preload_next=False,
),
):
model(a)
s2 = time.time()
for _, a in zip(
range(batches),
torch.utils.data.DataLoader(
numpy.random.sample(data_shape),
batch_size=batch_size,
shuffle=True,
num_workers=1,
pin_memory=False,
),
):
model(a.to(device, dtype=torch.float))
s3 = time.time()
print(f"generator: {s2 - s1}")
print(f"dataloader: {s3 - s2}")
def test_batch_with_label():
import numpy
channels_in = 3
batches = 3
batch_size = 32
data_shape = (batches * batch_size, 256, 256, channels_in)
generator = batched_recycle(
zip(numpy.random.sample(data_shape),
numpy.random.sample(data_shape[0])),
batch_size,
)
for i, a in enumerate(generator):
print(a)
break
def test_batch_generator1():
a = range(9)
batch_size = 3
for i, b in zip(range(18), batched_recycle(a, batch_size)):
assert [b_ in a for b_ in b]
assert i == 17
def test_d6():
from torchvision.transforms import transforms
import numpy
from draugr import inner_map
a_transform = transforms.Compose([
transforms.ToPILImage("RGB"),
transforms.Resize(224),
transforms.CenterCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
])
channels_in = 3
channels_out = 3
samples = 10
device = "cuda"
batches = 3
batch_size = 32
data_shape = (batches * batch_size, 256, 256, channels_in)
batch_shape = torch.Size([batch_size, channels_in, 224, 224])
class RandomDataset(Dataset):
""" """
def __init__(self):
self.d = numpy.random.sample(data_shape)
def __len__(self):
return len(self.d)
def __getitem__(self, item):
return a_transform(self.d[item])
dataloader = torch.utils.data.DataLoader(
RandomDataset(),
batch_size=batch_size,
shuffle=True,
num_workers=1,
pin_memory=True,
)
generator = to_tensor_generator(
inner_map(a_transform,
batched_recycle(numpy.random.sample(data_shape),
batch_size)),
device=device,
preload_next=True,
)
for _ in range(samples):
s1 = time.time()
for _, a in zip(range(batches), generator):
assert batch_shape == a.shape, a.shape
s2 = time.time()
for _, a in zip(range(batches), dataloader):
assert batch_shape == a.shape, a.shape
s3 = time.time()
print(f"generator: {s2 - s1}")
print(f"dataloader: {s3 - s2}")