def test_padded_tensor(self):
x = torch.randn(3, 5, 7, 9, requires_grad=True)
xs = torch.tensor([[1, 1], [3, 9], [7, 5]])
layer = MaskImageFromSize(mask_value=-99)
y = layer(PaddedTensor(x, xs))
# Expected output
expected_y = x.clone()
expected_y[0, :, 1:, :] = -99
expected_y[0, :, :, 1:] = -99
expected_y[1, :, 3:, :] = -99
expected_y[2, :, :, 5:] = -99
# Check expected output
torch.testing.assert_allclose(y.data, expected_y)
# Test backward pass (mask with 0, so that we can sum without masking)
layer = MaskImageFromSize(mask_value=0)
y = layer(PaddedTensor(x, xs))
dx, = torch.autograd.grad([torch.sum(y.data)], [x])
# Expected output
expected_dx = torch.ones(3, 5, 7, 9)
expected_dx[0, :, 1:, :] = 0
expected_dx[0, :, :, 1:] = 0
expected_dx[1, :, 3:, :] = 0
expected_dx[2, :, :, 5:] = 0
# Check expected output
torch.testing.assert_allclose(dx, expected_dx)
def test_padded_tensor_repr():
sizes = torch.randint(3, size=(2, 3))
t = PaddedTensor.build(torch.empty(2, 1, 5, 5), sizes)
assert (
repr(t) ==
f"PaddedTensor(data.size()=[2, 1, 5, 5], sizes={sizes.tolist()}, device=cpu)"
)
def test_backprop(dtype, device, kwargs):
# Note: this only checks that the gradient w.r.t. all layers is different from zero.
m = ConvBlock(**kwargs).to(device, dtype=dtype).train()
# Convert batch input and batch sizes to appropriate type
x = torch.randn(2,
kwargs["in_channels"],
17,
19,
device=device,
dtype=dtype)
xs = torch.tensor([[13, 19], [17, 13]], device=device)
# Check model for normal tensor inputs
m.zero_grad()
cost = m(x).sum()
cost.backward()
for n, p in m.named_parameters():
assert p.grad is not None, f"Parameter {n} does not have a gradient"
sp = torch.abs(p.grad).sum()
assert not torch.allclose(
sp, torch.tensor(0, dtype=dtype)
), f"Gradients for parameter {n} are close to 0 ({sp:g})"
# Check model for padded tensor inputs
m.zero_grad()
cost = padded_cost_function(m(PaddedTensor(x, xs)))
cost.backward()
for n, p in m.named_parameters():
assert p.grad is not None, f"Parameter {n} does not have a gradient"
sp = torch.abs(p.grad).sum()
assert not torch.allclose(
sp, torch.tensor(0, dtype=dtype)
), f"Gradients for parameter {n} are close to 0 ({sp:g})"
def test_output_size_padded_tensor(self):
m = ConvBlock(4, 5, kernel_size=3, stride=1, dilation=1, poolsize=2)
x = torch.randn(3, 4, 11, 13)
y = m(PaddedTensor(x, torch.tensor([[11, 13], [10, 12], [3, 2]])))
self.assertEqual(
[[11 // 2, 13 // 2], [10 // 2, 12 // 2], [3 // 2, 2 // 2]],
y.sizes.tolist())
def test_output_size_dilation(self):
# Note: padding should be added automatically to have the same output size
m = ConvBlock(4, 5, dilation=3)
x = torch.randn(1, 4, 11, 13)
y = m(PaddedTensor(x, torch.tensor([[11, 13]])))
self.assertEqual([[11, 13]], y.sizes.tolist())
self.assertEqual([11, 13], list(y.data.size())[2:])
def forward(self, x: Union[Tensor,
PaddedTensor]) -> Union[Tensor, PaddedTensor]:
x, xs = (x.data, x.sizes) if isinstance(x, PaddedTensor) else (x, None)
assert x.size(1) == self.in_channels, (
f"Input image depth ({x.size(1)}) does not match the "
f"expected ({self.in_channels})")
if self.dropout and 0.0 < self.dropout < 1.0:
x = F.dropout(x, p=self.dropout, training=self.training)
x = self.conv(x)
if self.use_masks:
x = mask_image_from_size(x, batch_sizes=xs, mask_value=0)
if self.batchnorm:
x = self.batchnorm(x)
if self.activation:
x = self.activation(x)
if self.use_masks:
x = mask_image_from_size(x, batch_sizes=xs, mask_value=0)
if self.pool:
x = self.pool(x)
return (x if xs is None else PaddedTensor.build(
x, self.get_batch_output_size(xs)))
def test_padded_tensor(self):
x = torch.randn(4, 16, 48, 32, requires_grad=True)
xs = torch.tensor([[30, 30], [40, 32], [10, 32], [48, 10]])
x = PaddedTensor(x, xs)
layer = ResnetConv2dBlock(16, 32)
y = layer(x)
self.assertEqual((4, 32, 48, 32), y.data.size())
self.assertTrue(torch.equal(xs, y.sizes))
def forward(self, x):
x, xs = (x.data, x.sizes) if isinstance(x, PaddedTensor) else (x, None)
y = self._func(batch_input=x,
output_sizes=self.output_sizes,
batch_sizes=xs)
if xs is None or self._fixed_size:
return y
ys = xs.clone()
dim = int(self.output_sizes[0] is None)
ys[:, dim] = self.output_sizes[dim]
return PaddedTensor.build(y, ys)
def test_forward_padded_tensor(self, name, block, output_dim,
extra_groups):
del name # not used
options = resnet.ResnetOptions(
block=block,
input_channels=1,
layers=(1, 1, 1, 1), # Fewer layers/block to be faster.
stride=(1, 2, 1, 1),
width_per_group=4, # Fewer units/group to be faster.
norm_layer=None, # No normalization to be faster.
)
net = resnet.ResnetConv(options)
input_sizes = torch.tensor([[19, 37], [19, 17], [11, 9], [17, 37]],
dtype=torch.int64)
y = net(PaddedTensor(data=torch.randn(4, 1, 19, 37),
sizes=input_sizes))
self.assertIsInstance(y, PaddedTensor)
self.assertEqual(y.data.size(), (4, output_dim, 3, 5))
expected_sizes = np.asarray([[3, 5], [3, 3], [2, 2], [3, 5]],
dtype=np.int64)
np.testing.assert_array_equal(y.sizes.numpy(), expected_sizes)
options = resnet.ResnetOptions(
block=block,
input_channels=1,
root_kernel=3,
layers=(1, 2, 1, 1), # Fewer layers/block to be faster.
stride=(1, 1, 1, 1),
width_per_group=4, # Fewer units/group to be faster.
zero_init_residual=True,
groups=extra_groups,
norm_layer=torch.nn.BatchNorm2d,
)
net = resnet.ResnetConv(options)
input_sizes = torch.tensor([[17, 19], [5, 3], [17, 11]],
dtype=torch.int32)
y = net(PaddedTensor(data=torch.randn(3, 1, 17, 19),
sizes=input_sizes))
self.assertEqual(y.data.size(), (3, extra_groups * output_dim, 5, 5))
expected_sizes = np.asarray([[5, 5], [2, 1], [5, 3]], dtype=np.int32)
np.testing.assert_array_equal(y.sizes.numpy(), expected_sizes)
def forward(self, x):
if isinstance(x, PaddedTensor):
x, xs = x.data, x.sizes
y = mask_image_from_size(
batch_input=x,
batch_sizes=xs,
mask_value=self.mask_value,
inplace=self.inplace,
)
return PaddedTensor(y, xs)
else:
return x
def test_forward_with_size(self):
x = torch.tensor(
[[[[1, 2, 3], [4, 5, 6]]], [[[7, 8, 0], [10, 11, 0]]]],
dtype=torch.float)
xs = torch.tensor([[2, 3], [2, 2]])
m = ImageToSequence(columnwise=True)
y, ys = m(PaddedTensor(x, xs))
expected_y = torch.tensor(
[[[1, 4], [7, 10]], [[2, 5], [8, 11]], [[3, 6], [0, 0]]],
dtype=torch.float)
torch.testing.assert_allclose(y, expected_y)
self.assertEqual(ys, [3, 2])
def forward(self, x):
# type: (Union[torch.Tensor, PaddedTensor]) -> torch.Tensor
x, xs = (x.data, x.sizes) if isinstance(x, PaddedTensor) else (x, None)
x = self.conv(x)
if xs is not None:
xs = size_after_conv(xs)
x = PaddedTensor(x, xs)
if self.tpp and self.spp:
x = torch.cat((self.tpp(x), self.spp(x)), dim=1)
else:
x = self.tpp(x) if self.tpp else self.spp(x)
return self.fc(x)
def _feed(self, x):
if isinstance(x, torch.Tensor):
return x.requires_grad(self._requires_grad).to(self._device)
elif isinstance(x, PaddedTensor):
xs = x.sizes.to(self._device)
x = x.data.requires_grad_(self._requires_grad).to(self._device)
return PaddedTensor(x, xs)
elif isinstance(x, PackedSequence):
xs = x.batch_sizes.to(self._device)
x = x.data.requires_grad_(self._requires_grad).to(self._device)
return PackedSequence(x, xs)
else:
raise ValueError("Type {!r} is not supported".format(type(x)))
def test_padded_tensor(use_nnutils):
x = torch.tensor(
[
[
[
[1, 2, 3, 4, 5, 6, 7, 8],
[9, 10, 11, 12, 13, 14, 15, 16],
[17, 18, 19, 20, 21, 22, 23, 24],
[25, 26, 27, 28, 29, 30, 31, 32],
]
]
],
dtype=torch.double,
requires_grad=True,
)
xs = torch.tensor([[3, 4]])
layer = PyramidMaxPool2d(levels=[1, 2], use_nnutils=use_nnutils)
y = layer(PaddedTensor(x, xs))
torch.testing.assert_allclose(
y, torch.tensor([[20, 10, 12, 18, 20]], dtype=x.dtype)
)
torch.autograd.gradcheck(lambda x: layer(PaddedTensor(x, xs)), x)
def forward(self, x):
if isinstance(x, PaddedTensor):
x, xs = x.data, x.sizes
else:
xs = None
y = F.relu(self.bn1(self.conv1(x)))
y = self.bn2(self.conv2(y))
y += self.shortcut(x)
y = F.relu(y)
if xs is not None:
ys = (xs + self.stride - 1) / self.stride
y = PaddedTensor(y, ys)
return y
def _test(self):
m = ImagePoolingSequencer(sequencer=f"{sequencer}-{poolsize}",
columnwise=columnwise).to(x.device)
x.requires_grad_()
y = m(PaddedTensor(x, xs))
(dx1, ) = torch.autograd.grad(y.data.sum(), (x, ))
for i, (xk, xsk) in enumerate(zip(x, xs.tolist())):
xk = xk[:, :xsk[0], :xsk[1]].unsqueeze(0).to(x.device)
xk.requires_grad_()
yk = fn(xk,
output_size=(poolsize, xsk[1]) if columnwise else
(xsk[0], poolsize))
(dxk, ) = torch.autograd.grad(yk.sum(), (xk, ))
torch.testing.assert_allclose(dxk, (dx1[i, :, :xsk[0], :xsk[1]]))
def test_backward_with_size(self):
x = torch.tensor(
[[[[1, 2, 3], [4, 5, 6]]], [[[7, 8, 0], [10, 11, 0]]]],
dtype=torch.float,
requires_grad=True,
)
xs = torch.tensor([[2, 3], [2, 2]])
m = ImageToSequence(columnwise=True)
y, ys = m(PaddedTensor(x, xs))
dx, = torch.autograd.grad(
[y[0, :, :].sum() + y[1, :, :].sum() + y[2, 0, :].sum()], [x])
expected_dx = torch.tensor(
[[[[1, 1, 1], [1, 1, 1]]], [[[1, 1, 0], [1, 1, 0]]]],
dtype=torch.float)
torch.testing.assert_allclose(dx, expected_dx)
def feed(self, x):
if isinstance(x, PaddedTensor):
x, xs = x.data, x.sizes
elif isinstance(x, tuple) and len(x) == 2:
# required because of: https://github.com/pytorch/pytorch/issues/44009
# can be deleted when we no longer support torch<=1.6.0
x, xs = x
else:
x, xs = x, None
x = self.view_as_4d(x) # N x C x H x W
if xs is not None and self._keep_padded_tensors:
if xs.size(1) == 3 and not self._keep_channels_in_size:
xs = xs[:, 1:]
return PaddedTensor.build(x, xs)
return x
def _feed(self, batch):
batch = super(ImageFeeder, self)._feed(batch)
# View image batch as a N-C-H-W
batch = self._view_as_4d(
batch.data if isinstance(batch, PaddedTensor) else batch)
batch.requires_grad_(self._requires_grad)
if isinstance(batch, PaddedTensor) and self._keep_padded_tensors:
xs = batch.sizes
# Ensure that the size tensor is the expected
if xs.dim() != 2 or (xs.size(1) != 2 and xs.size(1) != 3):
raise ValueError("Size tensor in PaddedTensor has not an "
"expected shape: {!r}".format(xs.size()))
if xs.size(1) == 3 and not self._keep_channels_in_size:
xs = xs[:, 1:]
return PaddedTensor(batch, xs)
return batch
def _run_test_padded_tensor(self, use_nnutils):
x = torch.tensor(
[[[
[1, 2, 3, 4, 5, 6, 7, 8],
[9, 10, 11, 12, 13, 14, 15, 16],
[17, 18, 19, 20, 21, 22, 23, 24],
[25, 26, 27, 28, 29, 30, 31, 32],
]]],
dtype=torch.float,
requires_grad=True,
)
xs = torch.tensor([[3, 4]])
layer = PyramidMaxPool2d(levels=[1, 2], use_nnutils=use_nnutils)
y = layer(PaddedTensor(x, xs))
torch.testing.assert_allclose(
y, torch.tensor([[20, 10, 12, 18, 20]], dtype=torch.float))
def _feed(self, x):
x = super(ImageFeeder, self)._feed(x)
# View image batch as a N-C-H-W
x, xs = (x.data, x.sizes) if isinstance(x, PaddedTensor) else (x, None)
x = self._view_as_4d(x)
x.requires_grad_(self._requires_grad)
if xs is not None and self._keep_padded_tensors:
# Ensure that the size tensor is the expected
if xs.dim() != 2 or (xs.size(1) != 2 and xs.size(1) != 3):
raise ValueError(
"Size tensor in PaddedTensor has not an "
"expected shape: {!r}".format(xs.size())
)
if xs.size(1) == 3 and not self._keep_channels_in_size:
xs = xs[:, 1:]
return PaddedTensor(x, xs)
else:
return x
def forward(self, x):
x, xs = (x.data, x.sizes) if isinstance(x, PaddedTensor) else (x, None)
assert x.size(1) == self._options.input_channels, (
f"Input image depth ({x.size(1)}) does not match "
f"the expected ({self._options.input_channels})")
x = self.conv1(x)
if self.bn1 is not None:
x = self.bn1(x)
x = self.relu(x)
x = self.maxpool(x)
x = self.layer1(x)
x = self.layer2(x)
x = self.layer3(x)
x = self.layer4(x)
if xs is None:
return x
return PaddedTensor.build(x, self.get_output_batch_size(xs))
def test_forward_backward_packed(self):
x = torch.tensor(
[[[[1, 2, 3], [4, 5, 6]]], [[[7, 8, 0], [10, 11, 0]]]],
dtype=torch.float,
requires_grad=True,
)
xs = torch.tensor([[2, 3], [2, 2]])
m = ImageToSequence(columnwise=True, return_packed=True)
# Test forward
y = m(PaddedTensor(x, xs))
expected_y = torch.tensor([[1, 4], [7, 10], [2, 5], [8, 11], [3, 6]],
dtype=torch.float)
torch.testing.assert_allclose(y.data, expected_y)
self.assertTrue(torch.equal(torch.tensor([2, 2, 1]), y.batch_sizes))
# Test backward pass
dx, = torch.autograd.grad([y.data.sum()], [x])
expected_dx = torch.tensor(
[[[[1, 1, 1], [1, 1, 1]]], [[[1, 1, 0], [1, 1, 0]]]],
dtype=torch.float)
torch.testing.assert_allclose(dx, expected_dx)
def test_forward_padded_tensor(self):
m = AdaptiveAvgPool2d(output_size=(1, 2))
x = PaddedTensor(self.x, torch.tensor([[2, 2], [1, 3]]))
y = m(x)
expected_y = torch.tensor([
# n = 0
[
# c = 0
[[3 / 2, 1 / 2]],
# c = 1
[[9 / 2, 6 / 2]],
],
# n = 1
[
# c = 0
[[12 / 2, 7 / 2]],
# c = 1
[[2 / 2, 6 / 2]],
],
])
torch.testing.assert_allclose(y, expected_y)
def test_masking(self):
m = ConvBlock(1, 1, activation=None, use_masks=True)
# Reset parameters so that the operation does nothing
for name, param in m.named_parameters():
param.data.zero_()
if name == "conv.weight":
param[:, :, 1, 1] = 1
x = torch.randn(3, 1, 11, 13)
y = m(PaddedTensor(x, torch.tensor([[11, 13], [10, 12], [3, 2]]))).data
# Check sample 1
torch.testing.assert_allclose(x[0, :, :, :], y[0, :, :, :])
# Check sample 2
torch.testing.assert_allclose(x[1, :, :10, :12], y[1, :, :10, :12])
torch.testing.assert_allclose(torch.zeros(1, 1, 13), y[1, :, 10:, :])
torch.testing.assert_allclose(torch.zeros(1, 11, 1), y[1, :, :, 12:])
# Check sample 3
torch.testing.assert_allclose(x[2, :, :3, :2], y[2, :, :3, :2])
torch.testing.assert_allclose(torch.zeros(1, 8, 13), y[2, :, 3:, :])
torch.testing.assert_allclose(torch.zeros(1, 11, 11), y[2, :, :, 2:])
def forward(self, x):
# type: (Union[Tensor, PaddedTensor]) -> Union[Tensor, PaddedTensor]
if isinstance(x, PaddedTensor):
x, xs = x.data, x.sizes
assert xs.dim() == 2, "PaddedTensor.sizes must be a matrix"
assert xs.size(1) == 2, (
"PaddedTensor.sizes must have 2 columns: Height and Width, "
"{} columns given instead.".format(xs.size(1)))
assert x.size(0) == xs.size(0), (
"Number of batch sizes ({}) does not match the number of "
"samples in the batch {}".format(xs.size(0), x.size(0)))
else:
xs = None
assert x.size(1) == self.in_channels, (
"Input image depth ({}) does not match the "
"expected ({})".format(x.size(1), self.in_channels))
if self.dropout and 0.0 < self.dropout < 1.0:
x = F.dropout(x, p=self.dropout, training=self.training)
x = self.conv(x)
if self.use_masks:
x = mask_image_from_size(x, batch_sizes=xs, mask_value=0)
if self.batchnorm:
x = self.batchnorm(x)
if self.activation:
x = self.activation(x)
if self.use_masks:
x = mask_image_from_size(x, batch_sizes=xs, mask_value=0)
if self.pool:
x = self.pool(x)
return x if xs is None else PaddedTensor(
x, self.get_output_batch_size(xs))
def test_padded_tensor(use_nnutils):
x = torch.tensor(
[[[
[1, 2, 3, 4, 5, 6, 7, 8],
[9, 10, 11, 12, 13, 14, 15, 16],
[17, 18, 19, 20, 21, 22, 23, 24],
[25, 26, 27, 28, 29, 30, 31, 32],
]]],
requires_grad=True,
dtype=torch.float,
)
layer = TemporalPyramidMaxPool2d(levels=[1, 2], use_nnutils=use_nnutils)
y = layer(PaddedTensor(x, torch.tensor([[3, 4]])))
(dx, ) = torch.autograd.grad([torch.sum(y)], [x])
# Expected gradient w.r.t. inputs
expected_dx = torch.zeros(1, 1, 4, 8)
expected_dx[0, 0, 2, 3] = 2
expected_dx[0, 0, 2, 1] = 1
# Check output and gradient w.r.t input
torch.testing.assert_allclose(y, torch.tensor([[20.0, 18.0, 20.0]]))
torch.testing.assert_allclose(dx, expected_dx)
def test_avgpool16(self):
m = LaiaCRNN(
3,
30,
cnn_num_features=[16, 32, 48, 64],
cnn_kernel_size=[3, 3, 3, 3],
cnn_stride=[1, 1, 1, 1],
cnn_dilation=[1, 1, 1, 1],
cnn_activation=[torch.nn.ReLU] * 4,
cnn_poolsize=[2, 2, 2, 0],
cnn_dropout=[0, 0, 0.2, 0.1],
cnn_batchnorm=[False, False, True, True],
image_sequencer="avgpool-16",
rnn_units=128,
rnn_layers=4,
rnn_dropout=0.5,
lin_dropout=0.5,
rnn_type=torch.nn.LSTM,
)
x = torch.randn(5, 3, 150, 300, requires_grad=True)
y = m(
PaddedTensor(
data=x,
sizes=torch.tensor(
[[128, 300], [150, 290], [70, 200], [122, 200], [16, 20]]
),
)
)
y, ys = pad_packed_sequence(y)
# Check output size
self.assertEqual(ys.tolist(), [300 // 8, 290 // 8, 200 // 8, 200 // 8, 20 // 8])
# Check number of parameters
self.assertEqual(2421982, sum(p.numel() for p in m.parameters()))
# Check gradient
dx, = torch.autograd.grad([y.sum()], [x])
self.assertNotAlmostEqual(0.0, torch.sum(dx).item())
def test_forward_padded_tensor(self):
m = AdaptiveMaxPool2d(output_size=(1, 2))
x = PaddedTensor(self.x, torch.tensor([[2, 2], [1, 3]]))
y = m(x)
expected_y = torch.tensor(
[
# n = 0
[
# c = 0
[[2, 1]],
# c = 1
[[6, 5]],
],
# n = 1
[
# c = 0
[[8, 8]],
# c = 1
[[4, 4]],
],
],
dtype=self.x.dtype,
)
torch.testing.assert_allclose(y, expected_y)
def test_output_size_stride(self):
m = ConvBlock(4, 5, stride=2)
x = torch.randn(1, 4, 11, 13)
y = m(PaddedTensor(x, torch.tensor([[11, 13]])))
self.assertEqual([[11 // 2 + 1, 13 // 2 + 1]], y.sizes.tolist())
self.assertEqual([11 // 2 + 1, 13 // 2 + 1], list(y.data.size())[2:])