def _make_tensor(size, device: torch.device, dtype: torch.dtype, *, low, high) -> torch.Tensor: if dtype is torch.bool: return torch.randint(0, 2, size, device=device, dtype=dtype) def _maybe_clamp(t, low_default, low, high_default, high): low = low_default if low is None else max(low_default, low) high = high_default if high is None else min(high_default, high) if low != low_default or high != high_default: return torch.clamp(t, low, high) return t if dtype is torch.uint8: t = torch.randint(0, 10, size, device=device, dtype=dtype) return _maybe_clamp(t, 0, low, 9, high) elif dtype in integral_types(): t = torch.randint(-9, 10, size, device=device, dtype=dtype) return _maybe_clamp(t, -9, low, 9, high) elif dtype in floating_types_and(torch.half, torch.bfloat16): # Windows doesn't support torch.rand(bfloat16) on CUDA if IS_WINDOWS and torch.device( device).type == 'cuda' and dtype is torch.bfloat16: t = (torch.rand(size, device=device, dtype=torch.float32) * 18 - 9).to(torch.bfloat16) else: t = torch.rand(size, device=device, dtype=dtype) * 18 - 9 return _maybe_clamp(t, -9, low, 9, high) else: assert dtype in complex_types() float_dtype = torch.float if dtype is torch.cfloat else torch.double real = torch.rand(size, device=device, dtype=float_dtype) * 18 - 9 real = _maybe_clamp(real, -9, low, 9, high) imag = torch.rand(size, device=device, dtype=float_dtype) * 18 - 9 imag = _maybe_clamp(imag, -9, low, 9, high) return torch.complex(real, imag)
class TestUnaryUfuncs(TestCase): exact_dtype = True # Tests bool tensor negation raises the correct error def test_neg_error_message(self, device): msg = ("Negation, the `\\-` operator, on a bool tensor is not supported." " If you are trying to invert a mask, use the `\\~` or" " `logical_not\\(\\)` operator instead.") t = torch.tensor((False, True), device=device) with self.assertRaisesRegex(RuntimeError, msg): torch.neg(t) @dtypes(*floating_types_and(torch.bfloat16, torch.half)) @ops((_fn for _fn in unary_ufuncs if _fn.domain != (None, None))) def test_float_domains(self, device, dtype, op): if not op.supports_dtype(dtype, torch.device(device).type): raise unittest.SkipTest('unsupported dtype') eps = (1e-5, 1e-3, 1e-1, 1, 2, 10, 20, 50, 100) low, high = op.domain # NOTE: the following two loops are separated for readability if low is not None: low_tensor = torch.tensor(low, device=device, dtype=dtype) for epsilon in eps: lower_tensor = low_tensor - epsilon # Skips the test if the difference is not representable, # which can occur if, for example, the difference is small # and the dtype is imprecise (like bfloat16 is) if lower_tensor.item() == low_tensor.item(): continue result = op(lower_tensor) self.assertEqual(result.item(), float('nan'), msg=("input of {0} outside lower domain boundary" " {1} produced {2}, not nan!").format(lower_tensor.item(), low, result.item())) if high is not None: high_tensor = torch.tensor(high, device=device, dtype=dtype) for epsilon in eps: higher_tensor = high_tensor + epsilon # See above comment if higher_tensor.item() == high_tensor.item(): continue result = op(higher_tensor) self.assertEqual(result.item(), float('nan'), msg=("input of {0} outside upper domain boundary" " {1} produced {2}, not nan!").format(higher_tensor.item(), high, result.item())) # Tests that fn == method == inplace == jit on a simple single tensor input # TODO: should this jitting the method and inplace variants, too? @ops(unary_ufuncs) def test_variant_consistency(self, device, dtype, op): def _fn(t): return op(t) t = make_tensor((5, 5), device, dtype, low=op.domain[0], high=op.domain[1]) expected = op(t) for alt, inplace in ((op.get_method(), False), (op.get_inplace(), True), (torch.jit.script(_fn), False)): if alt is None: with self.assertRaises(RuntimeError): alt(t.clone()) if inplace and op.promotes_integers_to_float and dtype in integral_types() + (torch.bool,): # Assert that RuntimeError is raised # for inplace variant of Operators that # promote integer input to floating dtype. with self.assertRaises(RuntimeError): alt(t.clone()) continue actual = alt(t.clone()) self.assertEqual(actual, expected, rtol=0, atol=0) # Helper for comparing torch tensors and numpy arrays # TODO: should this or assertEqual also validate that strides are equal? def assertEqualHelper(self, actual, expected, msg, *, dtype, exact_dtype=True, **kwargs): assert isinstance(actual, torch.Tensor) # Some NumPy functions return scalars, not arrays if isinstance(expected, Number): self.assertEqual(actual.item(), expected, **kwargs) elif isinstance(expected, np.ndarray): # Handles exact dtype comparisons between arrays and tensors if exact_dtype: # Allows array dtype to be float32 when comparing with bfloat16 tensors # since NumPy doesn't support the bfloat16 dtype if expected.dtype == np.float32: assert actual.dtype in (torch.bfloat16, torch.float32) else: assert expected.dtype == torch_to_numpy_dtype_dict[actual.dtype] self.assertEqual(actual, torch.from_numpy(expected).to(actual.dtype), msg, exact_device=False, **kwargs) else: self.assertEqual(actual, expected, msg, exact_device=False, **kwargs) # Tests that the function and its (array-accepting) reference produce the same # values on a range of tensors, including empty tensors, scalar tensors, # 1D tensors and a large 2D tensor with interesting and extremal values # and discontiguities. @unittest.skipIf(not TEST_NUMPY, "NumPy not found") @suppress_warnings @ops(unary_ufuncs) def test_reference_numerics(self, device, dtype, op): include_extremals = (op.handles_complex_extremals if dtype in (torch.cfloat, torch.cdouble) else op.handles_extremals) tensors = generate_numeric_tensors(device, dtype, domain=op.domain, include_large_values=op.handles_large_floats, include_extremal_values=include_extremals) for t in tensors: if dtype is torch.bfloat16: a = t.cpu().to(torch.float32).numpy() else: a = t.cpu().numpy() actual = op(t) expected = op.ref(a) # Crafts a custom error message for smaller, printable tensors if t.numel() < 10: msg = ("Failed to produce expected results! Input tensor was" " {0}, torch result is {1}, and reference result is" " {2}.").format(t, actual, expected) else: msg = None exact_dtype = True if op.promotes_integers_to_float and dtype in integral_types() + (torch.bool,): exact_dtype = False if dtype in [torch.uint8, torch.int8, torch.bool]: # NOTE: For these dtypes, PyTorch computes in the default scalar type (float) # while NumPy computes in float16 self.assertEqualHelper(actual, expected, msg, dtype=dtype, exact_dtype=exact_dtype, rtol=1e-3, atol=1e-2) continue self.assertEqualHelper(actual, expected, msg, dtype=dtype, exact_dtype=exact_dtype) # Tests for testing (dis)contiguity consistency @ops(unary_ufuncs) def test_contig_vs_every_other(self, device, dtype, op): contig = make_tensor((1026,), device=device, dtype=dtype, low=op.domain[0], high=op.domain[1]) non_contig = contig[::2] self.assertTrue(contig.is_contiguous()) self.assertFalse(non_contig.is_contiguous()) self.assertEqual(op(contig)[::2], op(non_contig)) @ops(unary_ufuncs) def test_contig_vs_transposed(self, device, dtype, op): contig = make_tensor((789, 357), device=device, dtype=dtype, low=op.domain[0], high=op.domain[1]) non_contig = contig.T self.assertTrue(contig.is_contiguous()) self.assertFalse(non_contig.is_contiguous()) self.assertEqual(op(contig).T, op(non_contig)) @ops(unary_ufuncs) def test_non_contig(self, device, dtype, op): shapes = [(5, 7), (1024,)] for shape in shapes: contig = make_tensor(shape, device, dtype, low=op.domain[0], high=op.domain[1]) non_contig = torch.empty(shape + (2,), device=device, dtype=dtype)[..., 0] non_contig.copy_(contig) self.assertTrue(contig.is_contiguous()) self.assertFalse(non_contig.is_contiguous()) self.assertEqual(op(contig), op(non_contig)) @ops(unary_ufuncs) def test_non_contig_index(self, device, dtype, op): contig = make_tensor((2, 2, 1, 2), device, dtype, low=op.domain[0], high=op.domain[1]) non_contig = contig[:, 1, ...] contig = non_contig.contiguous() self.assertTrue(contig.is_contiguous()) self.assertFalse(non_contig.is_contiguous()) self.assertEqual(op(contig), op(non_contig)) @ops(unary_ufuncs) def test_non_contig_expand(self, device, dtype, op): shapes = [(1, 3), (1, 7), (5, 7)] for shape in shapes: contig = make_tensor(shape, device, dtype, low=op.domain[0], high=op.domain[1]) non_contig = contig.clone().expand(3, -1, -1) self.assertTrue(contig.is_contiguous()) self.assertFalse(non_contig.is_contiguous()) contig = op(contig) non_contig = op(non_contig) for i in range(3): self.assertEqual(contig, non_contig[i], msg='non-contiguous expand[' + str(i) + ']') @ops(unary_ufuncs) def test_contig_size1(self, device, dtype, op): contig = make_tensor((5, 100), device, dtype, low=op.domain[0], high=op.domain[1]) contig = contig[:1, :50] contig2 = torch.empty(contig.size(), device=device, dtype=dtype) contig2.copy_(contig) self.assertTrue(contig.is_contiguous()) self.assertTrue(contig2.is_contiguous()) self.assertEqual(op(contig), op(contig2)) @ops(unary_ufuncs) def test_contig_size1_large_dim(self, device, dtype, op): contig = make_tensor((5, 2, 3, 1, 4, 5, 3, 2, 1, 2, 3, 4), device, dtype, low=op.domain[0], high=op.domain[1]) contig = contig[:1, :, :, :, :, :, :, :, :, :, :, :] contig2 = torch.empty(contig.size(), device=device, dtype=dtype) contig2.copy_(contig) self.assertTrue(contig.is_contiguous()) self.assertTrue(contig2.is_contiguous()) self.assertEqual(op(contig), op(contig2)) # Tests that computation on a multiple batches is the same as # per-batch computation. @ops(unary_ufuncs) def test_batch_vs_slicing(self, device, dtype, op): input = make_tensor((1024, 512), dtype=dtype, device=device, low=op.domain[0], high=op.domain[1]) actual = op(input) expected = torch.stack([op(slice) for slice in input]) self.assertEqual(actual, expected) def _test_out_arg(self, op, input, output): dtype = input.dtype out_dtype = output.dtype if dtype is out_dtype: expected = op(input) op(input, out=output) self.assertEqual(output, expected) else: with self.assertRaises(RuntimeError): op(input, out=output) def _test_out_promote_int_to_float_op(self, op, input, output): def compare_out(op, input, out): out_dtype = out.dtype expected = op(input) op(input, out=out) self.assertEqual(out, expected.to(out_dtype)) dtype = input.dtype out_dtype = output.dtype if out_dtype.is_floating_point and not dtype.is_complex: compare_out(op, input, output) elif out_dtype.is_floating_point and dtype.is_complex: # Can't cast complex to float with self.assertRaises(RuntimeError): op(input, out=output) elif out_dtype.is_complex: compare_out(op, input, output) else: # Can't cast to Integral types with self.assertRaises(RuntimeError): op(input, out=output) @ops(unary_ufuncs) def test_out_arg_all_dtypes(self, device, dtype, op): input = make_tensor((64, 64), dtype=dtype, device=device, low=op.domain[0], high=op.domain[1]) for out_dtype in all_types_and_complex_and(torch.bool, torch.half): out = torch.empty_like(input, dtype=out_dtype) if op.promotes_integers_to_float: self._test_out_promote_int_to_float_op(op, input, out) else: self._test_out_arg(op, input, out) @dtypes(*(torch.testing.get_all_int_dtypes() + [torch.bool] + torch.testing.get_all_fp_dtypes(include_bfloat16=False))) def test_nan_to_num(self, device, dtype): for contiguous in [False, True]: x = make_tensor((64, 64), low=0., high=100., dtype=dtype, device=device) if dtype.is_floating_point: # Add extremal values. extremals = [float('nan'), float('inf'), -float('inf')] for idx, extremal in zip(torch.randint(0, 63, (3,)), extremals): x[idx, :] = extremal if not contiguous: x = x.T # With args nan = random.random() posinf = random.random() * 5 neginf = random.random() * 10 self.compare_with_numpy(lambda x: x.nan_to_num(nan=nan, posinf=posinf), lambda x: np.nan_to_num(x, nan=nan, posinf=posinf), x) self.compare_with_numpy(lambda x: x.nan_to_num(posinf=posinf, neginf=neginf), lambda x: np.nan_to_num(x, posinf=posinf, neginf=neginf), x) # Out Variant out = torch.empty_like(x) result = torch.nan_to_num(x) torch.nan_to_num(x, out=out) self.assertEqual(result, out) result = torch.nan_to_num(x, nan=nan, posinf=posinf, neginf=neginf) torch.nan_to_num(x, out=out, nan=nan, posinf=posinf, neginf=neginf) self.assertEqual(result, out)
class TestUnaryUfuncs(TestCase): exact_dtype = True # Helper for comparing torch tensors and numpy arrays # TODO: should this or assertEqual also validate that strides are equal? def assertEqualHelper(self, actual, expected, *, dtype, exact_dtype=True, **kwargs): assert isinstance(actual, torch.Tensor) # Some NumPy functions return scalars, not arrays if isinstance(expected, Number): self.assertEqual(actual.item(), expected) elif isinstance(expected, np.ndarray): # Handles exact dtype comparisons between arrays and tensors if exact_dtype: # Allows array dtype to be float32 when comparing with bfloat16 tensors # since NumPy doesn't support the bfloat16 dtype if expected.dtype == np.float32: assert actual.dtype in (torch.bfloat16, torch.float32) else: assert expected.dtype == torch_to_numpy_dtype_dict[ actual.dtype] self.assertEqual(actual, torch.from_numpy(expected).to(actual.dtype), exact_device=False, **kwargs) else: self.assertEqual(actual, expected, exact_device=False, **kwargs) # Verifies that the unary ufuncs have their supported dtypes # registered correctly by testing that each unlisted dtype # throws a runtime error @skipCUDAIfRocm @onlyOnCPUAndCUDA @ops(unary_ufuncs, unsupported_dtypes_only=True) def test_unsupported_dtypes(self, device, dtype, op): t = torch.empty(1, device=device, dtype=dtype) with self.assertRaises(RuntimeError): op(t) # Tests bool tensor negation raises the correct error def test_neg_error_message(self, device): msg = ( "Negation, the `\\-` operator, on a bool tensor is not supported." " If you are trying to invert a mask, use the `\\~` or" " `logical_not\\(\\)` operator instead.") t = torch.tensor((False, True), device=device) with self.assertRaisesRegex(RuntimeError, msg): torch.neg(t) @dtypes(*floating_types_and(torch.bfloat16, torch.half)) @ops((_fn for _fn in unary_ufuncs if _fn.domain != (None, None))) def test_float_domains(self, device, dtype, op): if not op.supports_dtype(dtype, torch.device(device).type): raise unittest.SkipTest('unsupported dtype') eps = (1e-5, 1e-3, 1e-1, 1, 2, 10, 20, 50, 100) low, high = op.domain # NOTE: the following two loops are separated for readability if low is not None: low_tensor = torch.tensor(low, device=device, dtype=dtype) for epsilon in eps: lower_tensor = low_tensor - epsilon # Skips the test if the difference is not representable, # which can occur if, for example, the difference is small # and the dtype is imprecise (like bfloat16 is) if lower_tensor.item() == low_tensor.item(): continue result = op(lower_tensor) self.assertEqual( result.item(), float('nan'), msg=("input of {0} outside lower domain boundary" " {1} produced {2}, not nan!").format( lower_tensor.item(), low, result.item())) if high is not None: high_tensor = torch.tensor(high, device=device, dtype=dtype) for epsilon in eps: higher_tensor = high_tensor + epsilon # See above comment if higher_tensor.item() == high_tensor.item(): continue result = op(higher_tensor) self.assertEqual( result.item(), float('nan'), msg=("input of {0} outside upper domain boundary" " {1} produced {2}, not nan!").format( higher_tensor.item(), high, result.item())) # Tests that fn == method == inplace == jit on a simple single tensor input # TODO: should this jitting the method and inplace variants, too? @ops(unary_ufuncs) def test_variant_consistency(self, device, dtype, op): def _fn(t): return op(t) t = _make_tensor((5, 5), device, dtype, low=op.domain[0], high=op.domain[1]) expected = op(t) for alt in (op.get_method(), op.get_inplace(), torch.jit.script(_fn)): if alt is None: with self.assertRaises(RuntimeError): alt(t.clone()) actual = alt(t.clone()) self.assertEqual(actual, expected, rtol=0, atol=0) # Tests that the function and its (array-accepting) reference produce the same # values on a range of tensors, including empty tensors, scalar tensors, # 1D tensors and a large 2D tensor with interesting and extremal values # and discontiguities. @unittest.skipIf(not TEST_NUMPY, "NumPy not found") @suppress_warnings @ops(unary_ufuncs) def test_reference_numerics(self, device, dtype, op): include_extremals = (op.handles_complex_extremals if dtype in (torch.cfloat, torch.cdouble) else op.handles_extremals) tensors = generate_numeric_tensors( device, dtype, domain=op.domain, include_large_values=op.handles_large_floats, include_extremal_values=include_extremals) for t in tensors: if dtype is torch.bfloat16: a = t.cpu().to(torch.float32).numpy() else: a = t.cpu().numpy() actual = op(t) expected = op.ref(a) # Crafts a custom error message for smaller, printable tensors if t.numel() < 10: msg = ("Failed to produce expected results! Input tensor was" " {0}, torch result is {1}, and reference result is" " {2}.").format(t, actual, expected) else: msg = None self.assertEqualHelper(actual, expected, dtype=dtype, msg=msg) # Tests for testing (dis)contiguity consistency @ops(unary_ufuncs) def test_contig_vs_every_other(self, device, dtype, op): contig = _make_tensor((1026, ), device=device, dtype=dtype, low=op.domain[0], high=op.domain[1]) non_contig = contig[::2] self.assertTrue(contig.is_contiguous()) self.assertFalse(non_contig.is_contiguous()) self.assertEqual(op(contig)[::2], op(non_contig)) @ops(unary_ufuncs) def test_contig_vs_transposed(self, device, dtype, op): contig = _make_tensor((789, 357), device=device, dtype=dtype, low=op.domain[0], high=op.domain[1]) non_contig = contig.T self.assertTrue(contig.is_contiguous()) self.assertFalse(non_contig.is_contiguous()) self.assertEqual(op(contig).T, op(non_contig)) @ops(unary_ufuncs) def test_non_contig(self, device, dtype, op): shapes = [(5, 7), (1024, )] for shape in shapes: contig = _make_tensor(shape, device, dtype, low=op.domain[0], high=op.domain[1]) non_contig = torch.empty(shape + (2, ), device=device, dtype=dtype)[..., 0] non_contig.copy_(contig) self.assertTrue(contig.is_contiguous()) self.assertFalse(non_contig.is_contiguous()) self.assertEqual(op(contig), op(non_contig)) @ops(unary_ufuncs) def test_non_contig_index(self, device, dtype, op): contig = _make_tensor((2, 2, 1, 2), device, dtype, low=op.domain[0], high=op.domain[1]) non_contig = contig[:, 1, ...] contig = non_contig.contiguous() self.assertTrue(contig.is_contiguous()) self.assertFalse(non_contig.is_contiguous()) self.assertEqual(op(contig), op(non_contig)) @ops(unary_ufuncs) def test_non_contig_expand(self, device, dtype, op): shapes = [(1, 3), (1, 7), (5, 7)] for shape in shapes: contig = _make_tensor(shape, device, dtype, low=op.domain[0], high=op.domain[1]) non_contig = contig.clone().expand(3, -1, -1) self.assertTrue(contig.is_contiguous()) self.assertFalse(non_contig.is_contiguous()) contig = op(contig) non_contig = op(non_contig) for i in range(3): self.assertEqual(contig, non_contig[i], msg='non-contiguous expand[' + str(i) + ']') @ops(unary_ufuncs) def test_contig_size1(self, device, dtype, op): contig = _make_tensor((5, 100), device, dtype, low=op.domain[0], high=op.domain[1]) contig = contig[:1, :50] contig2 = torch.empty(contig.size(), device=device, dtype=dtype) contig2.copy_(contig) self.assertTrue(contig.is_contiguous()) self.assertTrue(contig2.is_contiguous()) self.assertEqual(op(contig), op(contig2)) @ops(unary_ufuncs) def test_contig_size1_large_dim(self, device, dtype, op): contig = _make_tensor((5, 2, 3, 1, 4, 5, 3, 2, 1, 2, 3, 4), device, dtype, low=op.domain[0], high=op.domain[1]) contig = contig[:1, :, :, :, :, :, :, :, :, :, :, :] contig2 = torch.empty(contig.size(), device=device, dtype=dtype) contig2.copy_(contig) self.assertTrue(contig.is_contiguous()) self.assertTrue(contig2.is_contiguous()) self.assertEqual(op(contig), op(contig2)) # Tests that computation on a multiple batches is the same as # per-batch computation. @ops(unary_ufuncs) def test_batch_vs_slicing(self, device, dtype, op): input = _make_tensor((1024, 512), dtype=dtype, device=device, low=op.domain[0], high=op.domain[1]) actual = op(input) expected = torch.stack([op(slice) for slice in input]) self.assertEqual(actual, expected)
dtype=dtype, requires_grad=requires_grad, low=-1, high=1) return [SampleInput(inp, args=((weight, bias) + extra_args))] additional_op_db.extend([ OpInfo('nn.functional.conv2d', aten_name="conv2d", variant_test_name='no_bias', supports_autograd=True, supports_forward_ad=True, sample_inputs_func=partial(sample_inputs_conv2d, False), dtypes=floating_types(), dtypesIfCUDA=floating_types_and(torch.half, torch.bfloat16), supports_out=False), OpInfo('nn.functional.conv2d', aten_name="conv2d", variant_test_name='with_bias', supports_autograd=True, supports_forward_ad=True, sample_inputs_func=partial(sample_inputs_conv2d, True), dtypesIfCUDA=floating_types_and(torch.half, torch.bfloat16), dtypes=floating_types(), supports_out=False), OpInfo('nn.functional.conv2d', aten_name="conv2d", variant_test_name='stride_with_bias', supports_autograd=True, supports_forward_ad=True,