def forward(self, data):
self.weight_tensor = utils.from_numpy(self.weight.data.swapaxes(0, 1))
self.bias_tensor = utils.from_numpy(self.bias.data)
self.data = utils.from_numpy(data)
self.output = F.conv2d(self.data, self.weight_tensor, self.bias_tensor,
self.stride, self.padding)
return utils.to_numpy(self.output)
def _test_backward(input_shape, reduction, axis):
layer = SoftmaxCrossEntropyLossLayer(reduction=reduction)
data = np.random.random(input_shape) * 2 - 1
labels_shape = list(data.shape)
labels_shape.pop(axis)
labels = np.random.randint(0, data.shape[axis],
labels_shape).astype(np.int64)
loss = layer(data, labels, axis=axis)
if axis == 1:
torch_input = utils.from_numpy(data).requires_grad_(True)
else:
torch_input = utils.from_numpy(np.moveaxis(data, axis,
1)).requires_grad_(True)
pytorch_loss = F.cross_entropy(torch_input,
utils.from_numpy(labels),
reduction=reduction)
if len(pytorch_loss.shape) > 0:
pytorch_loss.sum().backward()
else:
pytorch_loss.backward()
utils.assert_close(loss, utils.to_numpy(pytorch_loss))
grad = layer.backward()
torch_grad = utils.to_numpy(torch_input.grad)
if axis != 1:
torch_grad = np.moveaxis(torch_grad, 1, axis)
utils.assert_close(grad, torch_grad, atol=0.001)
def _test_conv_backward(input_shape, out_channels, kernel_size, stride):
np.random.seed(0)
torch.manual_seed(0)
in_channels = input_shape[1]
#print('test ksize',kernel_size)
#print('strid',stride)
padding = (kernel_size - 1) // 2
#print('test pad',padding)
input = np.random.random(input_shape).astype(np.float32) * 20
layer = ConvLayer(in_channels, out_channels, kernel_size, stride)
torch_layer = nn.Conv2d(in_channels,
out_channels,
kernel_size,
stride,
padding,
bias=True)
utils.assign_conv_layer_weights(layer, torch_layer)
output = layer.forward(input)
out_grad = layer.backward(2 * np.ones_like(output) / output.size)
torch_input = utils.from_numpy(input).requires_grad_(True)
torch_out = torch_layer(torch_input)
(2 * torch_out.mean()).backward()
utils.assert_close(out_grad, torch_input.grad, atol=TOLERANCE)
utils.check_conv_grad_match(layer, torch_layer)
def _test_conv_forward(input_shape, out_channels, kernel_size, stride):
np.random.seed(0)
torch.manual_seed(0)
in_channels = input_shape[1]
padding = (kernel_size - 1) // 2
input = np.random.random(input_shape).astype(np.float32) * 20
original_input = input.copy()
layer = ConvLayer(in_channels, out_channels, kernel_size, stride)
torch_layer = nn.Conv2d(in_channels,
out_channels,
kernel_size,
stride,
padding,
bias=True)
utils.assign_conv_layer_weights(layer, torch_layer)
output = layer.forward(input)
torch_data = utils.from_numpy(input)
torch_out = torch_layer(torch_data)
assert np.all(input == original_input)
assert output.shape == torch_out.shape
utils.assert_close(output, torch_out, atol=TOLERANCE)
def backward(self, previous_partial_gradient):
gradients = utils.from_numpy(previous_partial_gradient)
new_gradients = torch.autograd.grad(self.output,
self.data,
gradients,
retain_graph=False)[0]
return utils.to_numpy(new_gradients)
def _test_forward(input_shape, reduction, axis):
layer = SoftmaxCrossEntropyLossLayer(reduction=reduction)
data = np.random.random(input_shape) * 2 - 1
labels_shape = list(data.shape)
labels_shape.pop(axis)
labels = np.random.randint(0, data.shape[axis],
labels_shape).astype(np.int64)
loss = layer(data, labels, axis=axis)
if axis == 1:
pytorch_loss = F.cross_entropy(utils.from_numpy(data),
utils.from_numpy(labels),
reduction=reduction)
else:
pytorch_loss = F.cross_entropy(utils.from_numpy(data.swapaxes(1,
axis)),
utils.from_numpy(labels),
reduction=reduction)
pytorch_loss = utils.to_numpy(pytorch_loss)
utils.assert_close(loss, pytorch_loss, atol=0.001)
def backward(self, previous_partial_gradient):
gradients = utils.from_numpy(previous_partial_gradient)
input_grad = grad.conv2d_input(self.data.shape, self.weight_tensor,
gradients, self.stride, self.padding)
weight_grad = grad.conv2d_weight(self.data, self.weight_tensor.shape,
gradients, self.stride, self.padding)
bias_grad = gradients.sum((0, 2, 3))
self.weight.grad = utils.to_numpy(weight_grad.transpose(1, 0))
self.bias.grad = utils.to_numpy(bias_grad)
data_gradient = utils.to_numpy(input_grad)
return data_gradient
def _test_linear_backward(input_shape, out_channels):
in_channels = input_shape[1]
input = np.random.random(input_shape).astype(np.float32) * 20
layer = LinearLayer(in_channels, out_channels)
torch_layer = nn.Linear(in_channels, out_channels, bias=True)
utils.assign_linear_layer_weights(layer, torch_layer)
output = layer.forward(input)
out_grad = layer.backward(np.ones_like(output) * 2)
torch_input = utils.from_numpy(input).requires_grad_(True)
torch_out = torch_layer(torch_input)
(2 * torch_out).sum().backward()
utils.assert_close(out_grad, torch_input.grad, atol=TOLERANCE)
utils.check_linear_grad_match(layer, torch_layer, tolerance=TOLERANCE)
def _test_linear_forward(input_shape, out_channels):
in_channels = input_shape[1]
input = np.random.random(input_shape).astype(np.float32) * 20
original_input = input.copy()
layer = LinearLayer(in_channels, out_channels)
torch_layer = nn.Linear(in_channels, out_channels, bias=True)
utils.assign_linear_layer_weights(layer, torch_layer)
output = layer.forward(input)
torch_data = utils.from_numpy(input)
torch_out = torch_layer(torch_data)
assert np.all(input == original_input)
assert output.shape == torch_out.shape
utils.assert_close(output, torch_out, atol=TOLERANCE)
def _test_max_pool_backward(input_shape, kernel_size, stride):
np.random.seed(0)
torch.manual_seed(0)
padding = (kernel_size - 1) // 2
input = np.random.random(input_shape).astype(np.float32) * 20
layer = MaxPoolLayer(kernel_size, stride)
torch_layer = nn.MaxPool2d(kernel_size, stride, padding)
output = layer.forward(input)
out_grad = layer.backward(2 * np.ones_like(output) / output.size)
torch_input = utils.from_numpy(input).requires_grad_(True)
torch_out = torch_layer(torch_input)
(2 * torch_out.mean()).backward()
torch_out_grad = utils.to_numpy(torch_input.grad)
utils.assert_close(out_grad, torch_out_grad, atol=TOLERANCE)
def _test_max_pool_forward(input_shape, kernel_size, stride):
np.random.seed(0)
torch.manual_seed(0)
padding = (kernel_size - 1) // 2
input = np.random.random(input_shape).astype(np.float32) * 20
original_input = input.copy()
layer = MaxPoolLayer(kernel_size, stride)
torch_layer = nn.MaxPool2d(kernel_size, stride, padding)
output = layer.forward(input)
torch_data = utils.from_numpy(input)
torch_out = utils.to_numpy(torch_layer(torch_data))
output[np.abs(output) < 1e-4] = 0
torch_out[np.abs(torch_out) < 1e-4] = 0
assert np.all(input == original_input)
assert output.shape == torch_out.shape
utils.assert_close(output, torch_out, atol=TOLERANCE)
def test_networks():
np.random.seed(0)
torch.manual_seed(0)
data = np.random.random((100, 1, 28, 28)).astype(np.float32) * 10 - 5
labels = np.random.randint(0, 10, 100).astype(np.int64)
net = MNISTResNetwork()
torch_net = TorchMNISTResNetwork()
utils.assign_conv_layer_weights(net.layers[0], torch_net.layers[0])
utils.assign_conv_layer_weights(net.layers[3], torch_net.layers[3])
utils.assign_conv_layer_weights(net.layers[4].conv_layers[0],
torch_net.layers[4].conv1)
utils.assign_conv_layer_weights(net.layers[4].conv_layers[2],
torch_net.layers[4].conv2)
utils.assign_conv_layer_weights(net.layers[5].conv_layers[0],
torch_net.layers[5].conv1)
utils.assign_conv_layer_weights(net.layers[5].conv_layers[2],
torch_net.layers[5].conv2)
utils.assign_linear_layer_weights(net.layers[9], torch_net.layers[9])
utils.assign_linear_layer_weights(net.layers[11], torch_net.layers[11])
utils.assign_linear_layer_weights(net.layers[13], torch_net.layers[13])
forward = net(data)
data_torch = utils.from_numpy(data).requires_grad_(True)
forward_torch = torch_net(data_torch)
utils.assert_close(forward, forward_torch)
loss = net.loss(forward, labels)
torch_loss = torch_net.loss(forward_torch, utils.from_numpy(labels))
utils.assert_close(loss, torch_loss)
out_grad = net.backward()
torch_loss.backward()
utils.assert_close(out_grad, data_torch.grad, atol=0.01)
tolerance = 1e-4
utils.check_linear_grad_match(net.layers[13],
torch_net.layers[13],
tolerance=tolerance)
utils.check_linear_grad_match(net.layers[11],
torch_net.layers[11],
tolerance=tolerance)
utils.check_linear_grad_match(net.layers[9],
torch_net.layers[9],
tolerance=tolerance)
utils.check_conv_grad_match(net.layers[5].conv_layers[2],
torch_net.layers[5].conv2,
tolerance=tolerance)
utils.check_conv_grad_match(net.layers[5].conv_layers[0],
torch_net.layers[5].conv1,
tolerance=tolerance)
utils.check_conv_grad_match(net.layers[4].conv_layers[2],
torch_net.layers[4].conv2,
tolerance=tolerance)
utils.check_conv_grad_match(net.layers[4].conv_layers[0],
torch_net.layers[4].conv1,
tolerance=tolerance)
utils.check_conv_grad_match(net.layers[3],
torch_net.layers[3],
tolerance=tolerance)
utils.check_conv_grad_match(net.layers[0],
torch_net.layers[0],
tolerance=tolerance)
def forward(self, data):
self.data = utils.from_numpy(data)
self.data.requires_grad_(True)
self.output = F.max_pool2d(self.data, self.kernel_size, self.stride,
self.padding)
return utils.to_numpy(self.output)
