def forward(self, data): self.weight_tensor = utils.from_numpy(self.weight.data.swapaxes(0, 1)) self.bias_tensor = utils.from_numpy(self.bias.data) self.data = utils.from_numpy(data) self.output = F.conv2d(self.data, self.weight_tensor, self.bias_tensor, self.stride, self.padding) return utils.to_numpy(self.output)
def _test_backward(input_shape, reduction, axis): layer = SoftmaxCrossEntropyLossLayer(reduction=reduction) data = np.random.random(input_shape) * 2 - 1 labels_shape = list(data.shape) labels_shape.pop(axis) labels = np.random.randint(0, data.shape[axis], labels_shape).astype(np.int64) loss = layer(data, labels, axis=axis) if axis == 1: torch_input = utils.from_numpy(data).requires_grad_(True) else: torch_input = utils.from_numpy(np.moveaxis(data, axis, 1)).requires_grad_(True) pytorch_loss = F.cross_entropy(torch_input, utils.from_numpy(labels), reduction=reduction) if len(pytorch_loss.shape) > 0: pytorch_loss.sum().backward() else: pytorch_loss.backward() utils.assert_close(loss, utils.to_numpy(pytorch_loss)) grad = layer.backward() torch_grad = utils.to_numpy(torch_input.grad) if axis != 1: torch_grad = np.moveaxis(torch_grad, 1, axis) utils.assert_close(grad, torch_grad, atol=0.001)
def _test_conv_backward(input_shape, out_channels, kernel_size, stride): np.random.seed(0) torch.manual_seed(0) in_channels = input_shape[1] #print('test ksize',kernel_size) #print('strid',stride) padding = (kernel_size - 1) // 2 #print('test pad',padding) input = np.random.random(input_shape).astype(np.float32) * 20 layer = ConvLayer(in_channels, out_channels, kernel_size, stride) torch_layer = nn.Conv2d(in_channels, out_channels, kernel_size, stride, padding, bias=True) utils.assign_conv_layer_weights(layer, torch_layer) output = layer.forward(input) out_grad = layer.backward(2 * np.ones_like(output) / output.size) torch_input = utils.from_numpy(input).requires_grad_(True) torch_out = torch_layer(torch_input) (2 * torch_out.mean()).backward() utils.assert_close(out_grad, torch_input.grad, atol=TOLERANCE) utils.check_conv_grad_match(layer, torch_layer)
def _test_conv_forward(input_shape, out_channels, kernel_size, stride): np.random.seed(0) torch.manual_seed(0) in_channels = input_shape[1] padding = (kernel_size - 1) // 2 input = np.random.random(input_shape).astype(np.float32) * 20 original_input = input.copy() layer = ConvLayer(in_channels, out_channels, kernel_size, stride) torch_layer = nn.Conv2d(in_channels, out_channels, kernel_size, stride, padding, bias=True) utils.assign_conv_layer_weights(layer, torch_layer) output = layer.forward(input) torch_data = utils.from_numpy(input) torch_out = torch_layer(torch_data) assert np.all(input == original_input) assert output.shape == torch_out.shape utils.assert_close(output, torch_out, atol=TOLERANCE)
def backward(self, previous_partial_gradient): gradients = utils.from_numpy(previous_partial_gradient) new_gradients = torch.autograd.grad(self.output, self.data, gradients, retain_graph=False)[0] return utils.to_numpy(new_gradients)
def _test_forward(input_shape, reduction, axis): layer = SoftmaxCrossEntropyLossLayer(reduction=reduction) data = np.random.random(input_shape) * 2 - 1 labels_shape = list(data.shape) labels_shape.pop(axis) labels = np.random.randint(0, data.shape[axis], labels_shape).astype(np.int64) loss = layer(data, labels, axis=axis) if axis == 1: pytorch_loss = F.cross_entropy(utils.from_numpy(data), utils.from_numpy(labels), reduction=reduction) else: pytorch_loss = F.cross_entropy(utils.from_numpy(data.swapaxes(1, axis)), utils.from_numpy(labels), reduction=reduction) pytorch_loss = utils.to_numpy(pytorch_loss) utils.assert_close(loss, pytorch_loss, atol=0.001)
def backward(self, previous_partial_gradient): gradients = utils.from_numpy(previous_partial_gradient) input_grad = grad.conv2d_input(self.data.shape, self.weight_tensor, gradients, self.stride, self.padding) weight_grad = grad.conv2d_weight(self.data, self.weight_tensor.shape, gradients, self.stride, self.padding) bias_grad = gradients.sum((0, 2, 3)) self.weight.grad = utils.to_numpy(weight_grad.transpose(1, 0)) self.bias.grad = utils.to_numpy(bias_grad) data_gradient = utils.to_numpy(input_grad) return data_gradient
def _test_linear_backward(input_shape, out_channels): in_channels = input_shape[1] input = np.random.random(input_shape).astype(np.float32) * 20 layer = LinearLayer(in_channels, out_channels) torch_layer = nn.Linear(in_channels, out_channels, bias=True) utils.assign_linear_layer_weights(layer, torch_layer) output = layer.forward(input) out_grad = layer.backward(np.ones_like(output) * 2) torch_input = utils.from_numpy(input).requires_grad_(True) torch_out = torch_layer(torch_input) (2 * torch_out).sum().backward() utils.assert_close(out_grad, torch_input.grad, atol=TOLERANCE) utils.check_linear_grad_match(layer, torch_layer, tolerance=TOLERANCE)
def _test_linear_forward(input_shape, out_channels): in_channels = input_shape[1] input = np.random.random(input_shape).astype(np.float32) * 20 original_input = input.copy() layer = LinearLayer(in_channels, out_channels) torch_layer = nn.Linear(in_channels, out_channels, bias=True) utils.assign_linear_layer_weights(layer, torch_layer) output = layer.forward(input) torch_data = utils.from_numpy(input) torch_out = torch_layer(torch_data) assert np.all(input == original_input) assert output.shape == torch_out.shape utils.assert_close(output, torch_out, atol=TOLERANCE)
def _test_max_pool_backward(input_shape, kernel_size, stride): np.random.seed(0) torch.manual_seed(0) padding = (kernel_size - 1) // 2 input = np.random.random(input_shape).astype(np.float32) * 20 layer = MaxPoolLayer(kernel_size, stride) torch_layer = nn.MaxPool2d(kernel_size, stride, padding) output = layer.forward(input) out_grad = layer.backward(2 * np.ones_like(output) / output.size) torch_input = utils.from_numpy(input).requires_grad_(True) torch_out = torch_layer(torch_input) (2 * torch_out.mean()).backward() torch_out_grad = utils.to_numpy(torch_input.grad) utils.assert_close(out_grad, torch_out_grad, atol=TOLERANCE)
def _test_max_pool_forward(input_shape, kernel_size, stride): np.random.seed(0) torch.manual_seed(0) padding = (kernel_size - 1) // 2 input = np.random.random(input_shape).astype(np.float32) * 20 original_input = input.copy() layer = MaxPoolLayer(kernel_size, stride) torch_layer = nn.MaxPool2d(kernel_size, stride, padding) output = layer.forward(input) torch_data = utils.from_numpy(input) torch_out = utils.to_numpy(torch_layer(torch_data)) output[np.abs(output) < 1e-4] = 0 torch_out[np.abs(torch_out) < 1e-4] = 0 assert np.all(input == original_input) assert output.shape == torch_out.shape utils.assert_close(output, torch_out, atol=TOLERANCE)
def test_networks(): np.random.seed(0) torch.manual_seed(0) data = np.random.random((100, 1, 28, 28)).astype(np.float32) * 10 - 5 labels = np.random.randint(0, 10, 100).astype(np.int64) net = MNISTResNetwork() torch_net = TorchMNISTResNetwork() utils.assign_conv_layer_weights(net.layers[0], torch_net.layers[0]) utils.assign_conv_layer_weights(net.layers[3], torch_net.layers[3]) utils.assign_conv_layer_weights(net.layers[4].conv_layers[0], torch_net.layers[4].conv1) utils.assign_conv_layer_weights(net.layers[4].conv_layers[2], torch_net.layers[4].conv2) utils.assign_conv_layer_weights(net.layers[5].conv_layers[0], torch_net.layers[5].conv1) utils.assign_conv_layer_weights(net.layers[5].conv_layers[2], torch_net.layers[5].conv2) utils.assign_linear_layer_weights(net.layers[9], torch_net.layers[9]) utils.assign_linear_layer_weights(net.layers[11], torch_net.layers[11]) utils.assign_linear_layer_weights(net.layers[13], torch_net.layers[13]) forward = net(data) data_torch = utils.from_numpy(data).requires_grad_(True) forward_torch = torch_net(data_torch) utils.assert_close(forward, forward_torch) loss = net.loss(forward, labels) torch_loss = torch_net.loss(forward_torch, utils.from_numpy(labels)) utils.assert_close(loss, torch_loss) out_grad = net.backward() torch_loss.backward() utils.assert_close(out_grad, data_torch.grad, atol=0.01) tolerance = 1e-4 utils.check_linear_grad_match(net.layers[13], torch_net.layers[13], tolerance=tolerance) utils.check_linear_grad_match(net.layers[11], torch_net.layers[11], tolerance=tolerance) utils.check_linear_grad_match(net.layers[9], torch_net.layers[9], tolerance=tolerance) utils.check_conv_grad_match(net.layers[5].conv_layers[2], torch_net.layers[5].conv2, tolerance=tolerance) utils.check_conv_grad_match(net.layers[5].conv_layers[0], torch_net.layers[5].conv1, tolerance=tolerance) utils.check_conv_grad_match(net.layers[4].conv_layers[2], torch_net.layers[4].conv2, tolerance=tolerance) utils.check_conv_grad_match(net.layers[4].conv_layers[0], torch_net.layers[4].conv1, tolerance=tolerance) utils.check_conv_grad_match(net.layers[3], torch_net.layers[3], tolerance=tolerance) utils.check_conv_grad_match(net.layers[0], torch_net.layers[0], tolerance=tolerance)
def forward(self, data): self.data = utils.from_numpy(data) self.data.requires_grad_(True) self.output = F.max_pool2d(self.data, self.kernel_size, self.stride, self.padding) return utils.to_numpy(self.output)