def _check_helper(self, device, dtype, op, variant, check): if variant is None: self.skipTest("Skipped! Variant not implemented.") if not op.supports_dtype(dtype, torch.device(device).type): self.skipTest( f"Skipped! {op.name} does not support dtype {str(dtype)}") samples = op.sample_inputs(device, dtype, requires_grad=True) for sample in samples: partial_fn = partial(variant, **sample.kwargs) if check == 'gradcheck': self.assertTrue( gradcheck(partial_fn, (sample.input, ) + sample.args, check_grad_dtypes=True)) elif check == 'gradgradcheck': self.assertTrue( gradgradcheck(partial_fn, (sample.input, ) + sample.args, gen_non_contig_grad_outputs=False, check_grad_dtypes=True)) self.assertTrue( gradgradcheck(partial_fn, (sample.input, ) + sample.args, gen_non_contig_grad_outputs=True, check_grad_dtypes=True)) else: self.assertTrue(False, msg="Unknown check requested!")
def _check_helper(self, device, dtype, op, variant, check): if variant is None: self.skipTest("Skipped! Variant not implemented.") if not op.supports_dtype(dtype, torch.device(device).type): self.skipTest(f"Skipped! {op.name} does not support dtype {str(dtype)}") samples = op.sample_inputs(device, dtype, requires_grad=True) for sample in samples: if sample.output_process_fn_grad is not None: out_fn = sample.output_process_fn_grad def variant_out_fn(*args, **kwargs): return out_fn(variant(*args, **kwargs)) else: variant_out_fn = variant def fn(*inputs): output = variant_out_fn(*inputs, **sample.kwargs) return op.output_func(output) if check == 'gradcheck': self.assertTrue(gradcheck(fn, (*sample.input,) + sample.args, check_batched_grad=op.check_batched_grad, check_grad_dtypes=True)) elif check == 'gradgradcheck': self.assertTrue(gradgradcheck(fn, (*sample.input,) + sample.args, gen_non_contig_grad_outputs=False, check_batched_grad=op.check_batched_gradgrad, check_grad_dtypes=True)) self.assertTrue(gradgradcheck(fn, (*sample.input,) + sample.args, gen_non_contig_grad_outputs=True, check_batched_grad=op.check_batched_gradgrad, check_grad_dtypes=True)) else: self.assertTrue(False, msg="Unknown check requested!")
def _check_helper(self, device, dtype, op, variant, check): if variant is None: self.skipTest("Skipped! Variant not implemented.") if not op.supports_dtype(dtype, torch.device(device).type): self.skipTest( f"Skipped! {op.name} does not support dtype {str(dtype)}") def is_inplace(variant): if hasattr(variant, "__wrapped__"): return variant.__wrapped__ is op.get_inplace() return variant is op.get_inplace() samples = op.sample_inputs(device, dtype, requires_grad=True) for sample in samples: if sample.broadcasts_input and is_inplace(variant): continue # Note on TensorList inputs # # gradcheck does not support TensorList inputs so here we pass TensorList # inputs of size n as n single Tensor inputs to gradcheck and wrap the op # in a function that puts the n Tensor inputs back into a TensorList def fn(*inputs): # Put tensors back into TensorList since we splat them when passing to gradcheck if is_iterable_of_tensors(sample.input): n = len(sample.input) inputs = (inputs[:n], *inputs[n:]) output = op.gradcheck_wrapper(variant, *inputs, **sample.kwargs) if sample.output_process_fn_grad is not None: return sample.output_process_fn_grad(output) return output # Splat TensorList inputs into single Tensor inputs gradcheck_args = (sample.input, ) if isinstance( sample.input, torch.Tensor) else tuple(sample.input) gradcheck_args += sample.args if check == 'gradcheck': self.assertTrue( gradcheck(fn, gradcheck_args, check_batched_grad=op.check_batched_grad, check_grad_dtypes=True)) elif check == 'gradgradcheck': self.assertTrue( gradgradcheck(fn, gradcheck_args, gen_non_contig_grad_outputs=False, check_batched_grad=op.check_batched_gradgrad, check_grad_dtypes=True)) self.assertTrue( gradgradcheck(fn, gradcheck_args, gen_non_contig_grad_outputs=True, check_batched_grad=op.check_batched_gradgrad, check_grad_dtypes=True)) else: self.assertTrue(False, msg="Unknown check requested!")
def test_grad_grad_viterbi(operator): states, emissions, theta = make_data(10) theta = torch.from_numpy(theta) theta = theta[:, None, :, :] theta.requires_grad_() viterbi = Viterbi(operator) gradgradcheck(viterbi, (theta, ))
def _check_helper(self, device, dtype, op, variant, check): if variant is None: self.skipTest("Skipped! Variant not implemented.") if not op.supports_dtype(dtype, torch.device(device).type): self.skipTest( f"Skipped! {op.name} does not support dtype {str(dtype)}") samples = op.sample_inputs(device, dtype, requires_grad=True) for sample in samples: if sample.output_process_fn_grad is not None: out_fn = sample.output_process_fn_grad def variant_out_fn(*args, **kwargs): return out_fn(variant(*args, **kwargs)) else: variant_out_fn = variant def fn(*inputs): # Pack input back into TensorList since we splat it when passing to gradcheck if is_iterable_of_tensors(sample.input): n = len(sample.input) inputs = (inputs[:n], *inputs[n:]) output = variant_out_fn(*inputs, **sample.kwargs) return op.output_func(output) # Gradcheck does not support TensorList so we splat it with the remaining args gradcheck_args = (sample.input, ) if isinstance( sample.input, torch.Tensor) else tuple(sample.input) gradcheck_args += sample.args if check == 'gradcheck': self.assertTrue( gradcheck(fn, gradcheck_args, check_batched_grad=op.check_batched_grad, check_grad_dtypes=True)) elif check == 'gradgradcheck': self.assertTrue( gradgradcheck(fn, gradcheck_args, gen_non_contig_grad_outputs=False, check_batched_grad=op.check_batched_gradgrad, check_grad_dtypes=True)) self.assertTrue( gradgradcheck(fn, gradcheck_args, gen_non_contig_grad_outputs=True, check_batched_grad=op.check_batched_gradgrad, check_grad_dtypes=True)) else: self.assertTrue(False, msg="Unknown check requested!")
def test_grad_hessian_viterbi_two_samples(operator): states1, emissions1, theta1 = make_data(10) states2, emissions2, theta2 = make_data(5) lengths = torch.LongTensor([10, 5]) theta1 = torch.from_numpy(theta1) theta2 = torch.from_numpy(theta2) theta1.requires_grad_() theta2.requires_grad_() viterbi = Viterbi(operator) def func(theta1_, theta2_): W = pad_sequence([theta1_, theta2_]) return viterbi(W, lengths) gradcheck(func, (theta1, theta2)) gradgradcheck(func, (theta1, theta2))
def test_autograd_to_mkldnn(self): # MKLDNN only supports float32 root = torch.randn(4, 5, dtype=torch.float32, requires_grad=True) def func(root): return root.to_mkldnn().to_dense() # because MKLDNN only supports float32, we need to lessen the precision. # these numbers are just empirical results that seem to work. self.assertWarnsRegex(lambda: gradcheck(func, [root], atol=4e-2, rtol=1e-2), 'double precision floating point') self.assertWarnsRegex(lambda: gradgradcheck(func, [root], atol=4e-2, rtol=1e-2), 'double precision floating point')
def run_conv_double_back_test(self, kern, stride, padding, chan_in, chan_out, batch_size, inp_size, dilation, no_weight, groups=1, use_cuda=False, use_bias=True, dtype=torch.double): device = torch.device("dpcpp:0") x = torch.randn(batch_size, chan_in, inp_size, inp_size, device=device, dtype=dtype, requires_grad=True) weight = torch.randn(chan_out, chan_in // groups, kern, kern, device=device, dtype=dtype, requires_grad=not no_weight) if use_bias: bias = torch.randn(chan_out, device=device, dtype=dtype, requires_grad=True) else: bias = None def func(*inputs): if use_bias: lx, lweight, lbias = inputs else: lx, lweight = inputs lbias = None # We disable cudnn during forward to avoid finite difference imprecision issues with cudnn.flags(enabled=False): out = F.conv2d(lx, lweight, lbias, stride, padding, dilation, groups) return out if use_bias: inputs = x, weight, bias else: inputs = x, weight dummy_out = func(*inputs) grad_y = torch.randn_like(dummy_out, device=device, dtype=dtype, requires_grad=True) # Issue #15353: test mkldnn double backward, don't run gradgradcheck due # to imprecision issues if dtype == torch.float: g, = torch.autograd.grad(dummy_out.sum(), x, create_graph=True) return g.requires_grad return gradgradcheck(func, inputs, (grad_y,))
def _assertGradAndGradgradChecks(test_case, apply_fn, inputs): # call assert function rather than returning a bool since it's nicer # if we get whether this failed on the gradcheck or the gradgradcheck. test_case.assertTrue(gradcheck(apply_fn, inputs)) test_case.assertTrue(gradgradcheck(apply_fn, inputs))
def test_hessian_needlemanwunsch_function(self): needle = NeedlemanWunschDecoder(self.operator) inputs = (self.theta, self.A) gradgradcheck(needle, inputs, eps=1e-1, atol=1e-1, rtol=1e-1)
def test_hessian_needlemanwunsch_function_Arand(self): needle = NeedlemanWunschDecoder(self.operator) theta = self.theta.double() A = torch.rand_like(theta) inputs = (theta, A) gradgradcheck(needle, inputs, eps=1e-2)
def forward(self, X): return X**3 torch.manual_seed(0) X = torch.Tensor([3.]) X.requires_grad_() print('x:', X) cube = Cube() Y = cube(X) print('f(x):', Y) S = torch.sum(Y) S.backward() print('<Grad (f)(x), 1>:', X.grad) X.grad.zero_() X.requires_grad_() Y = cube(X) S = torch.sum(Y) G, = torch.autograd.grad(S, (X, ), create_graph=True) S = G.sum() S.backward() print('Grad^2 (f) 1:', X.grad) X.grad.zero_() gradcheck(cube, (X, ), eps=1e-4, atol=1e-2) X.grad.zero_() gradgradcheck(cube, (X, ), eps=1e-4, atol=1e-2)