def test_matmul_mat_with_two_matrices(): mat1 = make_random_mat(20, 5) mat2 = make_random_mat(20, 5) vec = Variable(torch.randn(20, 7), requires_grad=True) mat1_copy = Variable(mat1.data, requires_grad=True) mat2_copy = Variable(mat2.data, requires_grad=True) vec_copy = Variable(vec.data, requires_grad=True) # Forward res = MulLazyVariable(RootLazyVariable(mat1), RootLazyVariable(mat2)).matmul(vec) actual = prod([ mat1_copy.matmul(mat1_copy.transpose(-1, -2)), mat2_copy.matmul(mat2_copy.transpose(-1, -2)), ]).matmul(vec_copy) assert torch.max(((res.data - actual.data) / actual.data).abs()) < 0.01 # Backward res.sum().backward() actual.sum().backward() assert torch.max(((mat1.grad.data - mat1_copy.grad.data) / mat1_copy.grad.data).abs()) < 0.01 assert torch.max(((mat2.grad.data - mat2_copy.grad.data) / mat2_copy.grad.data).abs()) < 0.01 assert torch.max(((vec.grad.data - vec_copy.grad.data) / vec_copy.grad.data).abs()) < 0.01
def test_batch_matmul_mat_with_two_matrices(self): mat1 = make_random_mat(20, rank=4, batch_size=5) mat2 = make_random_mat(20, rank=4, batch_size=5) vec = Variable(torch.randn(5, 20, 7), requires_grad=True) mat1_copy = Variable(mat1.data, requires_grad=True) mat2_copy = Variable(mat2.data, requires_grad=True) vec_copy = Variable(vec.data, requires_grad=True) # Forward res = MulLazyVariable(RootLazyVariable(mat1), RootLazyVariable(mat2)).matmul(vec) actual = prod([ mat1_copy.matmul(mat1_copy.transpose(-1, -2)), mat2_copy.matmul(mat2_copy.transpose(-1, -2)) ]).matmul(vec_copy) self.assertLess( torch.max(((res.data - actual.data) / actual.data).abs()), 0.01) # Backward res.sum().backward() actual.sum().backward() self.assertLess( torch.max(((mat1.grad.data - mat1_copy.grad.data) / mat1_copy.grad.data).abs()), 0.01) self.assertLess( torch.max(((mat2.grad.data - mat2_copy.grad.data) / mat2_copy.grad.data).abs()), 0.01) self.assertLess( torch.max(((vec.grad.data - vec_copy.grad.data) / vec_copy.grad.data).abs()), 0.01)
def test_matmul_vec_with_five_matrices(self): mat1 = make_random_mat(20, 5) mat2 = make_random_mat(20, 5) mat3 = make_random_mat(20, 5) mat4 = make_random_mat(20, 5) mat5 = make_random_mat(20, 5) vec = Variable(torch.randn(20), requires_grad=True) mat1_copy = Variable(mat1.data, requires_grad=True) mat2_copy = Variable(mat2.data, requires_grad=True) mat3_copy = Variable(mat3.data, requires_grad=True) mat4_copy = Variable(mat4.data, requires_grad=True) mat5_copy = Variable(mat5.data, requires_grad=True) vec_copy = Variable(vec.data, requires_grad=True) # Forward res = MulLazyVariable( RootLazyVariable(mat1), RootLazyVariable(mat2), RootLazyVariable(mat3), RootLazyVariable(mat4), RootLazyVariable(mat5), ).matmul(vec) actual = prod([ mat1_copy.matmul(mat1_copy.transpose(-1, -2)), mat2_copy.matmul(mat2_copy.transpose(-1, -2)), mat3_copy.matmul(mat3_copy.transpose(-1, -2)), mat4_copy.matmul(mat4_copy.transpose(-1, -2)), mat5_copy.matmul(mat5_copy.transpose(-1, -2)), ]).matmul(vec_copy) self.assertLess( torch.max(((res.data - actual.data) / actual.data).abs()), 0.01) # Backward res.sum().backward() actual.sum().backward() self.assertLess( torch.max(((mat1.grad.data - mat1_copy.grad.data) / mat1_copy.grad.data).abs()), 0.01) self.assertLess( torch.max(((mat2.grad.data - mat2_copy.grad.data) / mat2_copy.grad.data).abs()), 0.01) self.assertLess( torch.max(((mat3.grad.data - mat3_copy.grad.data) / mat3_copy.grad.data).abs()), 0.01) self.assertLess( torch.max(((mat4.grad.data - mat4_copy.grad.data) / mat4_copy.grad.data).abs()), 0.01) self.assertLess( torch.max(((mat5.grad.data - mat5_copy.grad.data) / mat5_copy.grad.data).abs()), 0.01) self.assertLess( torch.max(((vec.grad.data - vec_copy.grad.data) / vec_copy.grad.data).abs()), 0.01)
def test_batch_matmul_mat_with_five_matrices(): mat1 = make_random_mat(20, rank=4, batch_size=5) mat2 = make_random_mat(20, rank=4, batch_size=5) mat3 = make_random_mat(20, rank=4, batch_size=5) mat4 = make_random_mat(20, rank=4, batch_size=5) mat5 = make_random_mat(20, rank=4, batch_size=5) vec = Variable(torch.randn(5, 20, 7), requires_grad=True) mat1_copy = Variable(mat1.data, requires_grad=True) mat2_copy = Variable(mat2.data, requires_grad=True) mat3_copy = Variable(mat3.data, requires_grad=True) mat4_copy = Variable(mat4.data, requires_grad=True) mat5_copy = Variable(mat5.data, requires_grad=True) vec_copy = Variable(vec.data, requires_grad=True) # Forward res = MulLazyVariable(RootLazyVariable(mat1), RootLazyVariable(mat2), RootLazyVariable(mat3), RootLazyVariable(mat4), RootLazyVariable(mat5)).matmul(vec) actual = prod([ mat1_copy.matmul(mat1_copy.transpose(-1, -2)), mat2_copy.matmul(mat2_copy.transpose(-1, -2)), mat3_copy.matmul(mat3_copy.transpose(-1, -2)), mat4_copy.matmul(mat4_copy.transpose(-1, -2)), mat5_copy.matmul(mat5_copy.transpose(-1, -2)), ]).matmul(vec_copy) assert torch.max(((res.data - actual.data) / actual.data).abs()) < 0.01 # Backward res.sum().backward() actual.sum().backward() assert torch.max(((mat1.grad.data - mat1_copy.grad.data) / mat1_copy.grad.data).abs()) < 0.01 assert torch.max(((mat2.grad.data - mat2_copy.grad.data) / mat2_copy.grad.data).abs()) < 0.01 assert torch.max(((mat3.grad.data - mat3_copy.grad.data) / mat3_copy.grad.data).abs()) < 0.01 assert torch.max(((mat4.grad.data - mat4_copy.grad.data) / mat4_copy.grad.data).abs()) < 0.01 assert torch.max(((mat5.grad.data - mat5_copy.grad.data) / mat5_copy.grad.data).abs()) < 0.01 assert torch.max(((vec.grad.data - vec_copy.grad.data) / vec_copy.grad.data).abs()) < 0.01