def test_strided_conv2d(self): bs = 4 cin = 3 H,W = 3,3 helper_test_op([(bs,cin,11,28), (4,cin,H,W)], lambda x,w: torch.nn.functional.conv2d(x,w,stride=2).relu(), lambda x,w: Tensor.conv2d(x,w,stride=2).relu(), gpu=self.gpu, forward_only=self.gpu) helper_test_op([(bs,cin,11,28), (4,cin,H,W)], lambda x,w: torch.nn.functional.conv2d(x,w,stride=(2,1)).relu(), lambda x,w: Tensor.conv2d(x,w,stride=(2,1)).relu(), gpu=self.gpu, forward_only=self.gpu)
def test_strided_conv2d(self): bs = 4 cin = 3 H,W = 3,3 helper_test_op([(bs,cin,11,28), (4,cin,H,W)], lambda x,w: torch.nn.functional.conv2d(x,w,stride=2).relu(), lambda x,w: Tensor.conv2d(x,w,stride=2).relu(), atol=2e-5, grad_atol=2e-6) helper_test_op([(bs,cin,11,28), (4,cin,H,W)], lambda x,w: torch.nn.functional.conv2d(x,w,stride=(2,1)).relu(), lambda x,w: Tensor.conv2d(x,w,stride=(2,1)).relu(), atol=2e-5, grad_atol=2e-6)
def test_conv2d(self): for bs in [1,8]: for cin in [1,3]: for groups in [1,3] if cin == 3 else [1]: for H in [1,2,5]: for W in [1,2,3,5]: helper_test_op([(bs,cin,11,28), (6,cin//groups,H,W)], lambda x,w: torch.nn.functional.conv2d(x,w,groups=groups).relu(), lambda x,w: Tensor.conv2d(x,w,groups=groups).relu(), gpu=self.gpu, grad_rtol=1e-5, forward_only=self.gpu)
def test_grouped_conv2d(self): groups = 2 helper_test_op([(1, 2, 5, 5), (groups, 1, 3, 3)], lambda x, w: torch.nn.functional.conv2d( x, w, groups=groups).relu(), lambda x, w: Tensor.conv2d(x, w, groups=groups).relu(), atol=1e-4, grad_rtol=1e-5, forward_only=True)
def test_fancy_conv2d(self): bs = 2 cin = 3 cout = 1 groups = 3 H,W = 3,3 helper_test_op([(bs,cin,11,28), (groups*cout,cin//groups,H,W)], lambda x,w: torch.nn.functional.conv2d(x,w,groups=groups).relu(), lambda x,w: Tensor.conv2d(x,w,groups=groups).relu(), atol=1e-4, grad_rtol=1e-5, forward_only=True)
def test_conv2d(self): for bs in [1,8]: for cin in [1,3]: for groups in [1,3] if cin == 3 else [1]: for H in [1,2,5]: for W in [1,2,3,5]: with self.subTest(batch_size=bs, channels=cin, groups=groups, height=H, width=W): helper_test_op([(bs,cin,11,28), (6,cin//groups,H,W)], lambda x,w: torch.nn.functional.conv2d(x,w,groups=groups).relu(), lambda x,w: Tensor.conv2d(x,w,groups=groups).relu(), gpu=self.gpu, grad_rtol=1e-5, forward_only=self.gpu)
def test_conv2d(self): for bs in [1, 8]: for cin in [1, 3]: for H in [2, 5]: for W in [2, 3, 5]: helper_test_op([(bs, cin, 11, 28), (4, cin, H, W)], lambda x, w: torch.nn.functional.conv2d( x, w).relu(), lambda x, w: Tensor.conv2d(x, w).relu(), atol=2e-5, grad_atol=2e-6)
def test_large_input_conv2d(self): bs = 4 cin = 16 groups = 1 H = 5 W = 2 helper_test_op([(bs, cin, 64, 64), (6, cin // groups, H, W)], lambda x, w: torch.nn.functional.conv2d( x, w, groups=groups).relu(), lambda x, w: Tensor.conv2d(x, w, groups=groups).relu(), atol=1e-4, grad_rtol=1e-5)
def test_conv2d(self): x = torch.randn((5, 2, 10, 7), requires_grad=True) w = torch.randn((4, 2, 3, 3), requires_grad=True) xt = Tensor(x.detach().numpy()) wt = Tensor(w.detach().numpy()) out = torch.nn.functional.conv2d(x, w) ret = Tensor.conv2d(xt, wt) np.testing.assert_allclose(ret.data, out.detach().numpy(), atol=1e-5) out.mean().backward() ret.mean().backward() np.testing.assert_allclose(w.grad, wt.grad, atol=1e-5) np.testing.assert_allclose(x.grad, xt.grad, atol=1e-5)
for groups in [1,3] if cin == 3 else [1]: for H in [1,2,5]: for W in [1,2,3,5]: with self.subTest(batch_size=bs, channels=cin, groups=groups, height=H, width=W): helper_test_op([(bs,cin,11,28), (6,cin//groups,H,W)], lambda x,w: torch.nn.functional.conv2d(x,w,groups=groups).relu(), lambda x,w: Tensor.conv2d(x,w,groups=groups).relu(), gpu=self.gpu, grad_rtol=1e-5, forward_only=self.gpu) def test_strided_conv2d(self): bs = 4 cin = 3 H,W = 3,3 with self.subTest(stride := 2): helper_test_op([(bs,cin,11,28), (4,cin,H,W)], lambda x,w: torch.nn.functional.conv2d(x,w,stride=2).relu(), lambda x,w: Tensor.conv2d(x,w,stride=stride).relu(), gpu=self.gpu, forward_only=self.gpu) with self.subTest(stride := (2,1)): helper_test_op([(bs,cin,11,28), (4,cin,H,W)], lambda x,w: torch.nn.functional.conv2d(x,w,stride=stride).relu(), lambda x,w: Tensor.conv2d(x,w,stride=(2,1)).relu(), gpu=self.gpu, forward_only=self.gpu) def test_maxpool2d(self): for ksz in [(2,2), (3,3), (3,2), (5,5), (5,1)]: with self.subTest(kernel_size=ksz): helper_test_op([(32,2,110,28)], lambda x: torch.nn.functional.max_pool2d(x, kernel_size=ksz), lambda x: Tensor.max_pool2d(x, kernel_size=ksz), gpu=self.gpu, forward_only=self.gpu) def test_avgpool2d(self): shape = (32,2,111,28) for ksz in [(2,2), (3,3), (3,2), (5,5), (5,1), shape[2:]]:
lambda x, w: torch.nn.functional.conv2d( x, w, groups=groups).relu(), lambda x, w: Tensor.conv2d( x, w, groups=groups).relu(), device=self.device, grad_rtol=1e-5) def test_strided_conv2d(self): bs = 4 cin = 3 H, W = 3, 3 with self.subTest(stride := 2): helper_test_op( [(bs, cin, 11, 28), (4, cin, H, W)], lambda x, w: torch.nn.functional.conv2d(x, w, stride=2).relu(), lambda x, w: Tensor.conv2d(x, w, stride=stride).relu(), device=self.device) with self.subTest(stride := (2, 1)): helper_test_op( [(bs, cin, 11, 28), (4, cin, H, W)], lambda x, w: torch.nn.functional.conv2d(x, w, stride=stride ).relu(), lambda x, w: Tensor.conv2d(x, w, stride=(2, 1)).relu(), device=self.device) def test_maxpool2d(self): for ksz in [(2, 2), (3, 3), (3, 2), (5, 5), (5, 1)]: with self.subTest(kernel_size=ksz): helper_test_op([(32, 2, 110, 28)], lambda x: torch.nn.functional.max_pool2d( x, kernel_size=ksz),
for groups in [1,3] if cin == 3 else [1]: for H in [1,2,5]: for W in [1,2,3,5]: with self.subTest(batch_size=bs, channels=cin, groups=groups, height=H, width=W): helper_test_op([(bs,cin,11,28), (6,cin//groups,H,W)], lambda x,w: torch.nn.functional.conv2d(x,w,groups=groups).relu(), lambda x,w: Tensor.conv2d(x,w,groups=groups).relu(), device=self.device, grad_rtol=1e-5) def test_strided_conv2d(self): bs = 4 cin = 3 H,W = 3,3 with self.subTest(stride := 2): helper_test_op([(bs,cin,11,28), (4,cin,H,W)], lambda x,w: torch.nn.functional.conv2d(x,w,stride=2).relu(), lambda x,w: Tensor.conv2d(x,w,stride=stride).relu(), device=self.device) with self.subTest(stride := (2,1)): helper_test_op([(bs,cin,11,28), (4,cin,H,W)], lambda x,w: torch.nn.functional.conv2d(x,w,stride=stride).relu(), lambda x,w: Tensor.conv2d(x,w,stride=(2,1)).relu(), device=self.device) def test_maxpool2d(self): for ksz in [(2,2), (3,3), (3,2), (5,5), (5,1)]: with self.subTest(kernel_size=ksz): helper_test_op([(32,2,110,28)], lambda x: torch.nn.functional.max_pool2d(x, kernel_size=ksz), lambda x: Tensor.max_pool2d(x, kernel_size=ksz), device=self.device) def test_avgpool2d(self): shape = (32,2,111,28) for ksz in [(2,2), (3,3), (3,2), (5,5), (5,1), shape[2:]]:
cin = 3 cout = 1 groups = 3 H,W = 3,3 helper_test_op([(bs,cin,11,28), (groups*cout,cin//groups,H,W)], lambda x,w: torch.nn.functional.conv2d(x,w,groups=groups).relu(), lambda x,w: Tensor.conv2d(x,w,groups=groups).relu(), atol=1e-4, grad_rtol=1e-5, forward_only=True) def test_strided_conv2d(self): bs = 4 cin = 3 H,W = 3,3 with self.subTest(stride := 2): helper_test_op([(bs,cin,11,28), (4,cin,H,W)], lambda x,w: torch.nn.functional.conv2d(x,w,stride=2).relu(), lambda x,w: Tensor.conv2d(x,w,stride=stride).relu(), atol=1e-4) with self.subTest(stride := (2,1)): helper_test_op([(bs,cin,11,28), (4,cin,H,W)], lambda x,w: torch.nn.functional.conv2d(x,w,stride=stride).relu(), lambda x,w: Tensor.conv2d(x,w,stride=(2,1)).relu(), atol=1e-4) def test_maxpool2d(self): for ksz in [(2,2), (3,3), (3,2), (5,5), (5,1)]: with self.subTest(kernel_size=ksz): helper_test_op([(32,2,110,28)], lambda x: torch.nn.functional.max_pool2d(x, kernel_size=ksz), # TODO: why is this tolerance so high? lambda x: Tensor.max_pool2d(x, kernel_size=ksz), grad_atol=1e-4) def test_avgpool2d(self): shape = (32,2,111,28)