def test_save_different_dtype_unallocated(self): devices = ['cpu'] if torch.cuda.is_available(): devices.append('cuda') def save_load_check(a, b): with io.BytesIO() as f: torch.save([a, b], f) f.seek(0) a_loaded, b_loaded = torch.load(f) self.assertEqual(a, a_loaded) self.assertEqual(b, b_loaded) for device, dtype in product(devices, get_all_dtypes()): a = torch.tensor([], dtype=dtype, device=device) for other_dtype in get_all_dtypes(): s = torch.TypedStorage(wrap_storage=a.storage()._untyped(), dtype=other_dtype) save_load_check(a, s) save_load_check(a.storage(), s) b = torch.tensor([], dtype=other_dtype, device=device) save_load_check(a, b)
class TestSortAndSelect(TestCase): def assertIsOrdered(self, order, x, mxx, ixx, task): SIZE = x.size(1) if order == 'descending': def check_order(a, b): # `a != a` because we put NaNs # at the end of ascending sorted lists, # and the beginning of descending ones. return ((a != a) | (a >= b)).all().item() elif order == 'ascending': def check_order(a, b): # see above return ((b != b) | (a <= b)).all().item() else: error('unknown order "{}", must be "ascending" or "descending"'.format(order)) are_ordered = True for k in range(1, SIZE): self.assertTrue(check_order(mxx[:, k - 1], mxx[:, k]), 'torch.sort ({}) values unordered for {}'.format(order, task)) seen = set() indicesCorrect = True size0 = x.size(0) size = x.size(x.dim() - 1) x = x.tolist() mxx = mxx.tolist() ixx = ixx.tolist() for k in range(size0): seen.clear() for j in range(size): self.assertEqual(x[k][ixx[k][j]], mxx[k][j], msg='torch.sort ({}) indices wrong for {}'.format(order, task)) seen.add(ixx[k][j]) self.assertEqual(len(seen), size) def test_sort(self, device): # on CUDA 2048 vs >2048 have different code path for the dim being sorted for SIZE in (4, 2049): x = torch.rand(4, SIZE, device=device) res1val, res1ind = torch.sort(x) # Test inplace y = x.clone() y_inds = torch.tensor((), dtype=torch.int64, device=device) torch.sort(y, out=(y, y_inds)) x_vals, x_inds = torch.sort(x) self.assertEqual(x_vals, y) self.assertEqual(x_inds, y_inds) # Test use of result tensor res2val = torch.tensor((), device=device) res2ind = torch.tensor((), device=device, dtype=torch.long) torch.sort(x, out=(res2val, res2ind)) self.assertEqual(res1val, res2val, atol=0, rtol=0) self.assertEqual(res1ind, res2ind, atol=0, rtol=0) self.assertEqual(torch.argsort(x), res1ind) self.assertEqual(x.argsort(), res1ind) # Test sorting of random numbers self.assertIsOrdered('ascending', x, res2val, res2ind, 'random') # Test simple sort self.assertEqual( torch.sort(torch.tensor((50, 40, 30, 20, 10), device=device))[0], torch.tensor((10, 20, 30, 40, 50), device=device), atol=0, rtol=0 ) # Test that we still have proper sorting with duplicate keys x = torch.floor(torch.rand(4, SIZE, device=device) * 10) torch.sort(x, out=(res2val, res2ind)) self.assertIsOrdered('ascending', x, res2val, res2ind, 'random with duplicate keys') # DESCENDING SORT x = torch.rand(4, SIZE, device=device) res1val, res1ind = torch.sort(x, x.dim() - 1, True) # Test use of result tensor res2val = torch.tensor((), device=device) res2ind = torch.tensor((), device=device, dtype=torch.long) torch.sort(x, x.dim() - 1, True, out=(res2val, res2ind)) self.assertEqual(res1val, res2val, atol=0, rtol=0) self.assertEqual(res1ind, res2ind, atol=0, rtol=0) self.assertEqual(torch.argsort(x, x.dim() - 1, True), res1ind) self.assertEqual(x.argsort(x.dim() - 1, True), res1ind) # Test sorting of random numbers self.assertIsOrdered('descending', x, res2val, res2ind, 'random') # Test simple sort task self.assertEqual( torch.sort(torch.tensor((10, 20, 30, 40, 50), device=device), 0, True)[0], torch.tensor((50, 40, 30, 20, 10), device=device), atol=0, rtol=0 ) # Test that we still have proper sorting with duplicate keys self.assertIsOrdered('descending', x, res2val, res2ind, 'random with duplicate keys') # Test sorting with NaNs x = torch.rand(4, SIZE, device=device) x[1][2] = float('NaN') x[3][0] = float('NaN') torch.sort(x, out=(res2val, res2ind)) self.assertIsOrdered('ascending', x, res2val, res2ind, 'random with NaNs') torch.sort(x, out=(res2val, res2ind), descending=True) self.assertIsOrdered('descending', x, res2val, res2ind, 'random with NaNs') # FIXME: remove torch.bool from unsupported types once support is added for cub sort @dtypes(*set(get_all_dtypes()) - {torch.bool, torch.complex64, torch.complex128}) def test_stable_sort(self, device, dtype): if TEST_WITH_ROCM and dtype == torch.bfloat16: return sizes = (100, 1000, 10000) for ncopies in sizes: x = torch.tensor([0, 1] * ncopies, dtype=dtype, device=device) _, idx = x.sort(stable=True) self.assertEqual( idx[:ncopies], torch.arange(start=0, end=2 * ncopies, step=2, device=device) ) self.assertEqual( idx[ncopies:], torch.arange(start=1, end=2 * ncopies, step=2, device=device) ) @onlyCUDA @dtypes(torch.uint8) @largeTensorTest('200GB') # Unfortunately 80GB A100 is not large enough def test_sort_large(self, device, dtype): t0 = torch.randperm(8192, device=device).to(dtype) t = t0.view(1, 8192).expand(2 ** 18 + 1, -1).contiguous() v, i = t.sort() del t iv, im = i.var_mean(dim=0) del i vv, vm = v.var_mean(dim=0) del v self.assertEqual(vv, torch.zeros_like(vv)) self.assertEqual(iv, torch.zeros_like(iv)) self.assertEqual(vm, torch.arange(255, dtype=dtype, device=device)) self.assertEqual(im, t0.sort().indices) def _test_sort_discontiguous(self, device, dtype): # on CUDA 2048 vs >2048 have different code path for the dim being sorted sizes = (5, 7, 2049) for shape in permutations(sizes): for perm in permutations((0, 1, 2)): for dim in range(3): t = torch.randn(shape, device=device, dtype=dtype).permute(perm) r1 = t.sort(dim=dim) r2 = t.contiguous().sort(dim=dim) self.assertEqual(r1, r2) n = t.size(dim) # assert ordered self.assertTrue((r1.values.narrow(dim, 1, n - 1) >= r1.values.narrow(dim, 0, n - 1)).all()) # assert that different segments does not mix, which can easily happen # if the stride is not handled correctly self.assertTrue((t.unsqueeze(-1).transpose(dim, -1) == r1.values.unsqueeze(-1)).any(dim=dim).any(dim=-1).all()) # assert stride is preserved if self.device_type == 'cuda': # FIXME: this behavior should be true for all cases, not # just the one specified in if condition self.assertEqual(r1.values.stride(), t.stride()) self.assertEqual(r1.indices.stride(), t.stride()) @onlyCUDA @dtypes(torch.float32) def test_sort_discontiguous(self, device, dtype): self._test_sort_discontiguous(device, dtype) @slowTest # this test is slow on CPU, but not on CUDA @onlyCPU @dtypes(torch.float32) def test_sort_discontiguous_slow(self, device, dtype): self._test_sort_discontiguous(device, dtype) @dtypes(torch.float32) def test_sort_1d_output_discontiguous(self, device, dtype): tensor = torch.randn(12, device=device, dtype=dtype)[:6] values = torch.empty_like(tensor)[::2] indices = torch.empty(18, device=device, dtype=torch.long)[::3] torch.sort(tensor, out=(values, indices)) values_cont, indices_cont = tensor.sort() self.assertEqual(indices, indices_cont) self.assertEqual(values, values_cont) @dtypes(torch.float32) def test_topk_1d_output_discontiguous(self, device, dtype): tensor = torch.randn(12, device=device, dtype=dtype) values = torch.empty_like(tensor)[::2] indices = torch.empty(18, device=device, dtype=torch.long)[::3] for sorted in (True, False): # outputs of `sorted=False` test are not guaranteed to be the same, # but with current implementation they are torch.topk(tensor, 6, sorted=sorted, out=(values, indices)) values_cont, indices_cont = tensor.topk(6, sorted=sorted) self.assertEqual(indices, indices_cont) self.assertEqual(values, values_cont) # FIXME: remove torch.bool from unsupported types once support is added for cub sort @dtypes(*set(get_all_dtypes()) - {torch.bool, torch.complex64, torch.complex128}) def test_stable_sort_against_numpy(self, device, dtype): if TEST_WITH_ROCM and dtype == torch.bfloat16: return if dtype in floating_types_and(torch.float16, torch.bfloat16): inf = float('inf') neg_inf = -float('inf') nan = float('nan') else: if dtype != torch.bool: # no torch.iinfo support for torch.bool inf = torch.iinfo(dtype).max neg_inf = torch.iinfo(dtype).min else: inf = True neg_inf = ~inf # no nan for integral types, we use inf instead for simplicity nan = inf def generate_samples(): from itertools import chain, combinations for sizes in [(1025,), (10000,)]: size = sizes[0] # binary strings yield (torch.tensor([0, 1] * size, dtype=dtype, device=device), 0) if self.device_type == 'cuda': return yield (torch.tensor([0, 1] * 100, dtype=dtype, device=device), 0) def repeated_index_fill(t, dim, idxs, vals): res = t for idx, val in zip(idxs, vals): res = res.index_fill(dim, idx, val) return res for sizes in [(1, 10), (10, 1), (10, 10), (10, 10, 10)]: size = min(*sizes) x = (torch.randn(*sizes, device=device) * size).to(dtype) yield (x, 0) # Generate tensors which are being filled at random locations # with values from the non-empty subsets of the set (inf, neg_inf, nan) # for each dimension. n_fill_vals = 3 # cardinality of (inf, neg_inf, nan) for dim in range(len(sizes)): idxs = (torch.randint(high=size, size=(size // 10,)) for i in range(n_fill_vals)) vals = (inf, neg_inf, nan) subsets = chain.from_iterable(combinations(list(zip(idxs, vals)), r) for r in range(1, n_fill_vals + 1)) for subset in subsets: idxs_subset, vals_subset = zip(*subset) yield (repeated_index_fill(x, dim, idxs_subset, vals_subset), dim) for sample, dim in generate_samples(): _, idx_torch = sample.sort(dim=dim, stable=True) if dtype is torch.bfloat16: sample_numpy = sample.float().cpu().numpy() else: sample_numpy = sample.cpu().numpy() idx_numpy = np.argsort(sample_numpy, axis=dim, kind='stable') self.assertEqual(idx_torch, idx_numpy) @dtypes(*(get_all_int_dtypes() + get_all_fp_dtypes())) def test_msort(self, device, dtype): if TEST_WITH_ROCM and dtype == torch.bfloat16: return def test(shape): tensor = make_tensor(shape, device, dtype, low=-9, high=9) if tensor.size() != torch.Size([]): if dtype is torch.bfloat16: expected = torch.from_numpy(np.msort(tensor.float().cpu().numpy())).bfloat16() else: expected = torch.from_numpy(np.msort(tensor.cpu().numpy())) else: expected = tensor # numpy.msort() does not support empty shapes tensor result = torch.msort(tensor) self.assertEqual(result, expected) out = torch.empty_like(result) torch.msort(tensor, out=out) self.assertEqual(out, expected) shapes = ( [], [0, ], [20, ], [1, 20], [30, 30], [10, 20, 30] ) for shape in shapes: test(shape) def test_topk(self, device): def topKViaSort(t, k, dim, dir): sorted, indices = t.sort(dim, dir) return sorted.narrow(dim, 0, k), indices.narrow(dim, 0, k) def compareTensors(t, res1, ind1, res2, ind2, dim): # Values should be exactly equivalent self.assertEqual(res1, res2, atol=0, rtol=0) # Indices might differ based on the implementation, since there is # no guarantee of the relative order of selection if not ind1.eq(ind2).all(): # To verify that the indices represent equivalent elements, # gather from the input using the topk indices and compare against # the sort indices vals = t.gather(dim, ind2) self.assertEqual(res1, vals, atol=0, rtol=0) def compare(t, k, dim, dir): topKVal, topKInd = t.topk(k, dim, dir, True) sortKVal, sortKInd = topKViaSort(t, k, dim, dir) compareTensors(t, sortKVal, sortKInd, topKVal, topKInd, dim) t = torch.rand(random.randint(1, SIZE), random.randint(1, SIZE), random.randint(1, SIZE), device=device) for _kTries in range(3): for _dimTries in range(3): for transpose in (True, False): for dir in (True, False): testTensor = t if transpose: dim1 = random.randrange(t.ndimension()) dim2 = dim1 while dim1 == dim2: dim2 = random.randrange(t.ndimension()) testTensor = t.transpose(dim1, dim2) dim = random.randrange(testTensor.ndimension()) k = random.randint(1, testTensor.size(dim)) compare(testTensor, k, dim, dir) # This tests the code path where on CUDA, topk is implemented with sort. t = torch.randn((2, 100000), device=device) compare(t, 2000, 1, True) compare(t, 2000, 1, False) def test_topk_arguments(self, device): q = torch.randn(10, 2, 10, device=device) # Make sure True isn't mistakenly taken as the 2nd dimension (interpreted as 1) self.assertRaises(TypeError, lambda: q.topk(4, True)) @skipCUDAIfRocm def test_unique_dim(self, device): self.assertFalse(hasattr(torch, 'unique_dim')) def run_test(device, dtype): x = torch.tensor([[[1., 1.], [0., 1.], [2., 1.], [0., 1.]], [[1., 1.], [0., 1.], [2., 1.], [0., 1.]]], dtype=dtype, device=device) x_empty = torch.empty(5, 0, dtype=dtype, device=device) x_ill_formed_empty = torch.empty(5, 0, 0, dtype=dtype, device=device) x_ill_formed_empty_another = torch.empty(5, 0, 5, dtype=dtype, device=device) if dtype in floating_types_and(torch.float16, torch.bfloat16): x_nan = torch.tensor([float("nan"), 0, 0, float("nan"), float("nan"), 1], dtype=dtype, device=device) expected_unique_dim0 = torch.tensor([[[1., 1.], [0., 1.], [2., 1.], [0., 1.]]], dtype=dtype, device=device) expected_inverse_dim0 = torch.tensor([0, 0]) expected_counts_dim0 = torch.tensor([2]) expected_unique_dim1 = torch.tensor([[[0., 1.], [1., 1.], [2., 1.]], [[0., 1.], [1., 1.], [2., 1.]]], dtype=dtype, device=device) expected_unique_dim1_bool = torch.tensor([[[False, True], [True, True]], [[False, True], [True, True]]], dtype=torch.bool, device=device) expected_inverse_dim1 = torch.tensor([1, 0, 2, 0]) expected_inverse_dim1_bool = torch.tensor([1, 0, 1, 0]) expected_counts_dim1 = torch.tensor([2, 1, 1]) expected_counts_dim1_bool = torch.tensor([2, 2]) expected_unique_dim2 = torch.tensor([[[1., 1.], [0., 1.], [2., 1.], [0., 1.]], [[1., 1.], [0., 1.], [2., 1.], [0., 1.]]], dtype=dtype, device=device) expected_inverse_dim2 = torch.tensor([0, 1]) expected_counts_dim2 = torch.tensor([1, 1]) expected_unique_empty = torch.tensor([], dtype=dtype, device=device) expected_inverse_empty = torch.tensor([], dtype=torch.long, device=device) expected_counts_empty = torch.tensor([], dtype=torch.long, device=device) if dtype in floating_types_and(torch.float16, torch.bfloat16): expected_unique_nan = torch.tensor([float("nan"), 0, float("nan"), float("nan"), 1], dtype=dtype, device=device) expected_inverse_nan = torch.tensor([0, 1, 1, 2, 3, 4], dtype=torch.long, device=device) expected_counts_nan = torch.tensor([1, 2, 1, 1, 1], dtype=torch.long, device=device) # dim0 x_unique = torch.unique(x, dim=0) self.assertEqual(expected_unique_dim0, x_unique) x_unique, x_inverse = torch.unique( x, return_inverse=True, dim=0) self.assertEqual(expected_unique_dim0, x_unique) self.assertEqual(expected_inverse_dim0, x_inverse) x_unique, x_counts = torch.unique( x, return_inverse=False, return_counts=True, dim=0) self.assertEqual(expected_unique_dim0, x_unique) self.assertEqual(expected_counts_dim0, x_counts) x_unique, x_inverse, x_counts = torch.unique( x, return_inverse=True, return_counts=True, dim=0) self.assertEqual(expected_unique_dim0, x_unique) self.assertEqual(expected_inverse_dim0, x_inverse) self.assertEqual(expected_counts_dim0, x_counts) # dim1 x_unique = torch.unique(x, dim=1) if x.dtype == torch.bool: self.assertEqual(expected_unique_dim1_bool, x_unique) else: self.assertEqual(expected_unique_dim1, x_unique) x_unique, x_inverse = torch.unique( x, return_inverse=True, dim=1) if x.dtype == torch.bool: self.assertEqual(expected_unique_dim1_bool, x_unique) self.assertEqual(expected_inverse_dim1_bool, x_inverse) else: self.assertEqual(expected_unique_dim1, x_unique) self.assertEqual(expected_inverse_dim1, x_inverse) x_unique, x_counts = torch.unique( x, return_inverse=False, return_counts=True, dim=1) if x.dtype == torch.bool: self.assertEqual(expected_unique_dim1_bool, x_unique) self.assertEqual(expected_counts_dim1_bool, x_counts) else: self.assertEqual(expected_unique_dim1, x_unique) self.assertEqual(expected_counts_dim1, x_counts) x_unique, x_inverse, x_counts = torch.unique( x, return_inverse=True, return_counts=True, dim=1) if x.dtype == torch.bool: self.assertEqual(expected_unique_dim1_bool, x_unique) self.assertEqual(expected_inverse_dim1_bool, x_inverse) self.assertEqual(expected_counts_dim1_bool, x_counts) else: self.assertEqual(expected_unique_dim1, x_unique) self.assertEqual(expected_inverse_dim1, x_inverse) self.assertEqual(expected_counts_dim1, x_counts) # dim2 x_unique = torch.unique(x, dim=2) self.assertEqual(expected_unique_dim2, x_unique) x_unique, x_inverse = torch.unique( x, return_inverse=True, dim=2) self.assertEqual(expected_unique_dim2, x_unique) self.assertEqual(expected_inverse_dim2, x_inverse) x_unique, x_counts = torch.unique( x, return_inverse=False, return_counts=True, dim=2) self.assertEqual(expected_unique_dim2, x_unique) self.assertEqual(expected_counts_dim2, x_counts) x_unique, x_inverse, x_counts = torch.unique( x, return_inverse=True, return_counts=True, dim=2) self.assertEqual(expected_unique_dim2, x_unique) self.assertEqual(expected_inverse_dim2, x_inverse) self.assertEqual(expected_counts_dim2, x_counts) # test empty tensor x_unique, x_inverse, x_counts = torch.unique( x_empty, return_inverse=True, return_counts=True, dim=1) self.assertEqual(expected_unique_empty, x_unique) self.assertEqual(expected_inverse_empty, x_inverse) self.assertEqual(expected_counts_empty, x_counts) # test tensor with nan if dtype in floating_types_and(torch.float16, torch.bfloat16): x_unique, x_inverse, x_counts = torch.unique( x_nan, return_inverse=True, return_counts=True, dim=0) self.assertEqual(expected_unique_nan, x_unique) self.assertEqual(expected_inverse_nan, x_inverse) self.assertEqual(expected_counts_nan, x_counts) # test not a well formed tensor # Checking for runtime error, as this is the expected behaviour with self.assertRaises(RuntimeError): torch.unique( x_ill_formed_empty, return_inverse=True, return_counts=True, dim=1) # test along dim2 with self.assertRaises(RuntimeError): torch.unique( x_ill_formed_empty_another, return_inverse=True, return_counts=True, dim=2) # test consecutive version y = torch.tensor( [[0, 1], [0, 1], [0, 1], [1, 2], [1, 2], [3, 4], [0, 1], [0, 1], [3, 4], [1, 2]], dtype=dtype, device=device ) # test tensor with nan if dtype in floating_types_and(torch.float16, torch.bfloat16): y_nan = torch.tensor([float("nan"), 0, 0, float("nan"), float("nan"), 1], dtype=dtype, device=device) expected_y_unique = torch.tensor( [[0, 1], [1, 2], [3, 4], [0, 1], [3, 4], [1, 2]], dtype=dtype, device=device ) expected_y_inverse = torch.tensor([0, 0, 0, 1, 1, 2, 3, 3, 4, 5], dtype=torch.int64, device=device) expected_y_counts = torch.tensor([3, 2, 1, 2, 1, 1], dtype=torch.int64, device=device) expected_y_inverse_bool = torch.tensor([0, 0, 0, 1, 1, 1, 2, 2, 3, 3], dtype=torch.int64, device=device) expected_y_counts_bool = torch.tensor([3, 3, 2, 2], dtype=torch.int64, device=device) if dtype in floating_types_and(torch.float16, torch.bfloat16): expected_y_unique_nan = torch.tensor([float("nan"), 0, float("nan"), float("nan"), 1], dtype=dtype, device=device) expected_y_inverse_nan = torch.tensor([0, 1, 1, 2, 3, 4], dtype=torch.long, device=device) expected_y_counts_nan = torch.tensor([1, 2, 1, 1, 1], dtype=torch.long, device=device) y_unique, y_inverse, y_counts = torch.unique_consecutive(y, return_inverse=True, return_counts=True, dim=0) if x.dtype == torch.bool: self.assertEqual(expected_y_inverse_bool, y_inverse) self.assertEqual(expected_y_counts_bool, y_counts) else: self.assertEqual(expected_y_inverse, y_inverse) self.assertEqual(expected_y_counts, y_counts) # test tensor with nan if dtype in floating_types_and(torch.float16, torch.bfloat16): y_unique, y_inverse, y_counts = torch.unique_consecutive( y_nan, return_inverse=True, return_counts=True, dim=0) self.assertEqual(expected_y_unique_nan, y_unique) self.assertEqual(expected_y_inverse_nan, y_inverse) self.assertEqual(expected_y_counts_nan, y_counts) run_test(device, torch.float) run_test(device, torch.double) run_test(device, torch.long) run_test(device, torch.uint8) run_test(device, torch.bool) @onlyCUDA def test_topk_noncontiguous_gpu(self, device): t = torch.randn(20, device=device)[::2] top1, idx1 = t.topk(5) top2, idx2 = t.contiguous().topk(5) self.assertEqual(top1, top2) self.assertEqual(idx1, idx2) def _test_topk_dtype(self, device, dtype, integral, size): if integral: a = torch.randint(torch.iinfo(dtype).min, torch.iinfo(dtype).max, size=(size,), dtype=dtype, device=device) else: a = torch.randn(size=(size,), dtype=dtype, device=device) sort_topk = a.sort()[0][-(size // 2):].flip(0) topk = a.topk(size // 2) self.assertEqual(sort_topk, topk[0]) # check values self.assertEqual(sort_topk, a[topk[1]]) # check indices @dtypes(torch.int8, torch.uint8, torch.int16, torch.int32, torch.int64) def test_topk_integral(self, device, dtype): small = 10 large = 4096 for curr_size in (small, large): self._test_topk_dtype(device, dtype, True, curr_size) @onlyCUDA @dtypes(torch.bfloat16) @skipCUDAIfRocm def test_topk_bfloat16(self, device, dtype): small = 10 large = 8192 for curr_size in (small, large): self._test_topk_dtype(device, dtype, False, curr_size) @dtypesIfCUDA(*get_all_fp_dtypes()) @dtypes(torch.float, torch.double, torch.bfloat16) def test_topk_nonfinite(self, device, dtype): if TEST_WITH_ROCM and dtype == torch.bfloat16: return x = torch.tensor([float('nan'), float('inf'), 1e4, 0, -1e4, -float('inf')], device=device, dtype=dtype) val, idx = x.topk(4) expect = torch.tensor([float('nan'), float('inf'), 1e4, 0], device=device, dtype=dtype) self.assertEqual(val, expect) self.assertEqual(idx, [0, 1, 2, 3]) val, idx = x.topk(4, largest=False) expect = torch.tensor([-float('inf'), -1e4, 0, 1e4], device=device, dtype=dtype) self.assertEqual(val, expect) self.assertEqual(idx, [5, 4, 3, 2]) def test_topk_4d(self, device): x = torch.ones(2, 3072, 2, 2, device=device) x[:, 1, :, :] *= 2. x[:, 10, :, :] *= 1.5 val, ind = torch.topk(x, k=2, dim=1) expected_ind = torch.ones(2, 2, 2, 2, dtype=torch.long, device=device) expected_ind[:, 1, :, :] = 10 expected_val = torch.ones(2, 2, 2, 2, device=device) expected_val[:, 0, :, :] *= 2. expected_val[:, 1, :, :] *= 1.5 self.assertEqual(val, expected_val, atol=0, rtol=0) self.assertEqual(ind, expected_ind, atol=0, rtol=0) @onlyNativeDeviceTypes @dtypesIfCUDA(*(get_all_dtypes(include_complex=False, include_bool=False, include_half=False, include_bfloat16=True))) @dtypes(*(get_all_dtypes(include_complex=False, include_bool=False, include_half=False, include_bfloat16=False))) def test_topk_zero(self, device, dtype): if TEST_WITH_ROCM and dtype == torch.bfloat16: return # https://github.com/pytorch/pytorch/issues/49205 t = torch.rand(2, 2, device=device).to(dtype=dtype) val, idx = torch.topk(t, k=0, largest=False) self.assertEqual(val.size(), torch.Size([2, 0])) self.assertEqual(idx.size(), torch.Size([2, 0])) def _test_unique_scalar_empty(self, dtype, device, f): # test scalar x = torch.tensor(0, dtype=dtype, device=device) unique, inverse, counts = f(x, return_inverse=True, return_counts=True) expected_unique = torch.tensor([0], dtype=dtype, device=device) expected_inverse = torch.tensor(0, device=device) expected_counts = torch.tensor([1], device=device) self.assertEqual(unique, expected_unique) self.assertEqual(inverse, expected_inverse) self.assertEqual(counts, expected_counts) # test zero sized tensor x = torch.zeros((0, 0, 3), dtype=dtype, device=device) unique, inverse, counts = f(x, return_inverse=True, return_counts=True) expected_unique = torch.tensor([], dtype=dtype, device=device) expected_inverse = torch.empty((0, 0, 3), dtype=torch.long, device=device) expected_counts = torch.tensor([], dtype=torch.long, device=device) self.assertEqual(unique, expected_unique) self.assertEqual(inverse, expected_inverse) self.assertEqual(counts, expected_counts) def _test_unique_with_expects(self, device, dtype, f, x, expected_unique, expected_inverse, expected_counts, additional_shape): def ensure_tuple(x): if isinstance(x, torch.Tensor): return (x,) return x for return_inverse in [True, False]: for return_counts in [True, False]: # test with expected ret = ensure_tuple(f(x, return_inverse=return_inverse, return_counts=return_counts)) self.assertEqual(len(ret), 1 + int(return_inverse) + int(return_counts)) self.assertEqual(expected_unique, ret[0]) if return_inverse: self.assertEqual(expected_inverse, ret[1]) if return_counts: count_index = 1 + int(return_inverse) self.assertEqual(expected_counts, ret[count_index]) # tests per-element unique on a higher rank tensor. y = x.view(additional_shape) y_unique, y_inverse, y_counts = f(y, return_inverse=True, return_counts=True) self.assertEqual(expected_unique, y_unique) self.assertEqual(expected_inverse.view(additional_shape), y_inverse) self.assertEqual(expected_counts, y_counts) @dtypesIfCPU(*set(get_all_dtypes()) - {torch.complex64, torch.complex128}) @dtypes(*set(get_all_dtypes()) - {torch.bfloat16, torch.complex64, torch.complex128}) def test_unique(self, device, dtype): if dtype is torch.half and self.device_type == 'cpu': return # CPU does not have half support def ensure_tuple(x): if isinstance(x, torch.Tensor): return (x,) return x if dtype is torch.bool: x = torch.tensor([True, False, False, False, True, False, True, False], dtype=torch.bool, device=device) expected_unique = torch.tensor([False, True], dtype=torch.bool, device=device) expected_inverse = torch.tensor([1, 0, 0, 0, 1, 0, 1, 0], dtype=torch.long, device=device) expected_counts = torch.tensor([5, 3], dtype=torch.long, device=device) else: x = torch.tensor([1, 2, 3, 2, 8, 5, 2, 3], dtype=dtype, device=device) expected_unique = torch.tensor([1, 2, 3, 5, 8], dtype=dtype, device=device) expected_inverse = torch.tensor([0, 1, 2, 1, 4, 3, 1, 2], device=device) expected_counts = torch.tensor([1, 3, 2, 1, 1], device=device) # test sorted unique fs = ( lambda x, **kwargs: torch.unique(x, sorted=True, **kwargs), lambda x, **kwargs: x.unique(sorted=True, **kwargs), ) x_sliced = torch.empty(x.size(0) * 2, dtype=dtype, device=device)[::2].copy_(x) xs = (x, x_sliced) for f, x in product(fs, xs): self._test_unique_with_expects(device, dtype, f, x, expected_unique, expected_inverse, expected_counts, (2, 2, 2)) self._test_unique_scalar_empty(dtype, device, f) # test unsorted unique fs = ( lambda x, **kwargs: torch.unique(x, sorted=False, **kwargs), lambda x, **kwargs: x.unique(sorted=False, **kwargs) ) for f, x in product(fs, xs): self._test_unique_scalar_empty(dtype, device, f) for return_inverse, return_counts in product((True, False), repeat=2): ret = ensure_tuple(f(x, return_inverse=return_inverse, return_counts=return_counts)) self.assertEqual(len(ret), 1 + int(return_inverse) + int(return_counts)) x_list = x.tolist() x_unique_list = ret[0].tolist() self.assertEqual(expected_unique.tolist(), sorted(x_unique_list)) if return_inverse: x_inverse_list = ret[1].tolist() for i, j in enumerate(x_inverse_list): self.assertEqual(x_list[i], x_unique_list[j]) if return_counts: count_index = 1 + int(return_inverse) x_counts_list = ret[count_index].tolist() for i, j in zip(x_unique_list, x_counts_list): count = 0 for k in x_list: if k == i: count += 1 self.assertEqual(j, count) @dtypesIfCPU(*set(get_all_dtypes()) - {torch.complex64, torch.complex128}) @dtypes(*set(get_all_dtypes()) - {torch.bfloat16, torch.complex64, torch.complex128}) def test_unique_consecutive(self, device, dtype): if dtype is torch.half and self.device_type == 'cpu': return # CPU does not have half support if dtype is torch.bool: x = torch.tensor([True, False, False, False, True, True, False, False, False], dtype=torch.bool, device=device) expected_unique = torch.tensor([True, False, True, False], dtype=torch.bool, device=device) expected_inverse = torch.tensor([0, 1, 1, 1, 2, 2, 3, 3, 3], dtype=torch.long, device=device) expected_counts = torch.tensor([1, 3, 2, 3], dtype=torch.long, device=device) else: x = torch.tensor([1, 2, 2, 2, 5, 5, 2, 2, 3], dtype=dtype, device=device) expected_unique = torch.tensor([1, 2, 5, 2, 3], dtype=dtype, device=device) expected_inverse = torch.tensor([0, 1, 1, 1, 2, 2, 3, 3, 4], device=device) expected_counts = torch.tensor([1, 3, 2, 2, 1], device=device) for f in [torch.unique_consecutive, lambda x, **kwargs: x.unique_consecutive(**kwargs)]: self._test_unique_with_expects(device, dtype, f, x, expected_unique, expected_inverse, expected_counts, (3, 3)) self._test_unique_scalar_empty(dtype, device, f) @dtypes(torch.double) def test_kthvalue(self, device, dtype): SIZE = 50 x = torch.rand(SIZE, SIZE, SIZE, dtype=dtype, device=device) x0 = x.clone() k = random.randint(1, SIZE) res1val, res1ind = torch.kthvalue(x, k, keepdim=False) res2val, res2ind = torch.sort(x) self.assertEqual(res1val[:, :], res2val[:, :, k - 1], atol=0, rtol=0) self.assertEqual(res1ind[:, :], res2ind[:, :, k - 1], atol=0, rtol=0) # test use of result tensors k = random.randint(1, SIZE) res1val = torch.tensor([], dtype=dtype, device=device) res1ind = torch.tensor([], dtype=torch.long, device=device) torch.kthvalue(x, k, keepdim=False, out=(res1val, res1ind)) res2val, res2ind = torch.sort(x) self.assertEqual(res1val[:, :], res2val[:, :, k - 1], atol=0, rtol=0) self.assertEqual(res1ind[:, :], res2ind[:, :, k - 1], atol=0, rtol=0) # test non-default dim k = random.randint(1, SIZE) res1val, res1ind = torch.kthvalue(x, k, 0, keepdim=False) res2val, res2ind = torch.sort(x, 0) self.assertEqual(res1val, res2val[k - 1], atol=0, rtol=0) self.assertEqual(res1ind, res2ind[k - 1], atol=0, rtol=0) # non-contiguous y = x.narrow(1, 0, 1) y0 = y.contiguous() k = random.randint(1, SIZE) res1val, res1ind = torch.kthvalue(y, k) res2val, res2ind = torch.kthvalue(y0, k) self.assertEqual(res1val, res2val, atol=0, rtol=0) self.assertEqual(res1ind, res2ind, atol=0, rtol=0) # non-contiguous [Reference: https://github.com/pytorch/pytorch/issues/45721] non_contig_t = torch.tensor([0, -1, 1, -2, 2], dtype=dtype, device=device)[::2] expected_val, expected_ind = non_contig_t.contiguous().kthvalue(2) non_contig_cpu_t = non_contig_t.cpu() expected_val_cpu, expected_ind_cpu = non_contig_cpu_t.kthvalue(2) out_val, out_ind = non_contig_t.kthvalue(2) self.assertEqual(expected_val, out_val, atol=0, rtol=0) self.assertEqual(expected_ind, out_ind, atol=0, rtol=0) self.assertEqual(expected_val_cpu, out_val, atol=0, rtol=0) self.assertEqual(expected_ind_cpu, out_ind, atol=0, rtol=0) # check that the input wasn't modified self.assertEqual(x, x0, atol=0, rtol=0) # simple test case (with repetitions) y = torch.tensor((3., 5, 4, 1, 1, 5), dtype=dtype, device=device) self.assertEqual(torch.kthvalue(y, 3)[0], 3, atol=0, rtol=0) self.assertEqual(torch.kthvalue(y, 2)[0], 1, atol=0, rtol=0) # simple test case (with NaN) SIZE = 50 x = torch.rand(SIZE, SIZE, SIZE, dtype=dtype, device=device) x[torch.arange(SIZE), :, torch.randint(50, (50,))] = nan ks = [random.randint(1, SIZE), 1, SIZE, SIZE - 1] res2val, res2ind = torch.sort(x) for k in ks: res1val, res1ind = torch.kthvalue(x, k, keepdim=False) self.assertEqual(res1val[:, :], res2val[:, :, k - 1], atol=0, rtol=0) self.assertEqual(res1ind[:, :], res2ind[:, :, k - 1], atol=0, rtol=0) @dtypes(torch.float) @onlyNativeDeviceTypes # Fails on XLA def test_kthvalue_scalar(self, device, dtype): # Test scalar input (test case from https://github.com/pytorch/pytorch/issues/30818) # Tests that passing a scalar tensor or 1D tensor with 1 element work either way res = torch.tensor(2, device=device, dtype=dtype).kthvalue(1) ref = torch.tensor([2], device=device, dtype=dtype).kthvalue(1) self.assertEqual(res[0], ref[0].squeeze()) self.assertEqual(res[1], ref[1].squeeze()) @dtypes(*all_types()) @dtypesIfCUDA(*all_types_and(torch.half)) def test_isin(self, device, dtype): def assert_isin_equal(a, b): # Compare to the numpy reference implementation. x = torch.isin(a, b) a = a.cpu().numpy() if torch.is_tensor(a) else np.array(a) b = b.cpu().numpy() if torch.is_tensor(b) else np.array(b) y = np.isin(a, b) self.assertEqual(x, y) # multi-dim tensor, multi-dim tensor a = torch.arange(24, device=device, dtype=dtype).reshape([2, 3, 4]) b = torch.tensor([[10, 20, 30], [0, 1, 3], [11, 22, 33]], device=device, dtype=dtype) assert_isin_equal(a, b) # zero-dim tensor zero_d = torch.tensor(3, device=device, dtype=dtype) assert_isin_equal(zero_d, b) assert_isin_equal(a, zero_d) assert_isin_equal(zero_d, zero_d) # empty tensor empty = torch.tensor([], device=device, dtype=dtype) assert_isin_equal(empty, b) assert_isin_equal(a, empty) assert_isin_equal(empty, empty) # scalar assert_isin_equal(a, 6) assert_isin_equal(5, b) def define_expected(lst, invert=False): expected = torch.tensor(lst, device=device) if invert: expected = expected.logical_not() return expected # Adapted from numpy's in1d tests for mult in [1, 10]: for invert in [False, True]: a = torch.tensor([5, 7, 1, 2], device=device, dtype=dtype) b = torch.tensor([2, 4, 3, 1, 5] * mult, device=device, dtype=dtype) ec = define_expected([True, False, True, True], invert=invert) c = torch.isin(a, b, assume_unique=True, invert=invert) self.assertEqual(c, ec) a[0] = 8 ec = define_expected([False, False, True, True], invert=invert) c = torch.isin(a, b, assume_unique=True, invert=invert) self.assertEqual(c, ec) a[0], a[3] = 4, 8 ec = define_expected([True, False, True, False], invert=invert) c = torch.isin(a, b, assume_unique=True, invert=invert) self.assertEqual(c, ec) a = torch.tensor([5, 4, 5, 3, 4, 4, 3, 4, 3, 5, 2, 1, 5, 5], device=device, dtype=dtype) b = torch.tensor([2, 3, 4] * mult, device=device, dtype=dtype) ec = define_expected([False, True, False, True, True, True, True, True, True, False, True, False, False, False], invert=invert) c = torch.isin(a, b, invert=invert) self.assertEqual(c, ec) b = torch.tensor([2, 3, 4] * mult + [5, 5, 4] * mult, device=device, dtype=dtype) ec = define_expected([True, True, True, True, True, True, True, True, True, True, True, False, True, True], invert=invert) c = torch.isin(a, b, invert=invert) self.assertEqual(c, ec) a = torch.tensor([5, 7, 1, 2], device=device, dtype=dtype) b = torch.tensor([2, 4, 3, 1, 5] * mult, device=device, dtype=dtype) ec = define_expected([True, False, True, True], invert=invert) c = torch.isin(a, b, invert=invert) self.assertEqual(c, ec) a = torch.tensor([5, 7, 1, 1, 2], device=device, dtype=dtype) b = torch.tensor([2, 4, 3, 3, 1, 5] * mult, device=device, dtype=dtype) ec = define_expected([True, False, True, True, True], invert=invert) c = torch.isin(a, b, invert=invert) self.assertEqual(c, ec) a = torch.tensor([5, 5], device=device, dtype=dtype) b = torch.tensor([2, 2] * mult, device=device, dtype=dtype) ec = define_expected([False, False], invert=invert) c = torch.isin(a, b, invert=invert) self.assertEqual(c, ec) # multi-dimensional input case using sort-based algo for assume_unique in [False, True]: a = torch.arange(6, device=device, dtype=dtype).reshape([2, 3]) b = torch.arange(3, 30, device=device, dtype=dtype) ec = define_expected([[False, False, False], [True, True, True]], invert=invert) c = torch.isin(a, b, invert=invert, assume_unique=assume_unique) self.assertEqual(c, ec) def test_isin_different_dtypes(self, device): supported_types = all_types() if device == 'cpu' else all_types_and(torch.half) for mult in [1, 10]: for assume_unique in [False, True]: for dtype1, dtype2 in product(supported_types, supported_types): a = torch.tensor([1, 2, 3], device=device, dtype=dtype1) b = torch.tensor([3, 4, 5] * mult, device=device, dtype=dtype2) ec = torch.tensor([False, False, True], device=device) c = torch.isin(a, b, assume_unique=assume_unique) self.assertEqual(c, ec) @onlyCUDA @dtypes(*all_types()) def test_isin_different_devices(self, device, dtype): a = torch.arange(6, device=device, dtype=dtype).reshape([2, 3]) b = torch.arange(3, 30, device='cpu', dtype=dtype) with self.assertRaises(RuntimeError): torch.isin(a, b) c = torch.arange(6, device='cpu', dtype=dtype).reshape([2, 3]) d = torch.arange(3, 30, device=device, dtype=dtype) with self.assertRaises(RuntimeError): torch.isin(c, d)
class TestNumPyInterop(TestCase): # Note: the warning this tests for only appears once per program, so # other instances of this warning should be addressed to avoid # the tests depending on the order in which they're run. @onlyCPU def test_numpy_non_writeable(self, device): arr = np.zeros(5) arr.flags['WRITEABLE'] = False self.assertWarns(UserWarning, lambda: torch.from_numpy(arr)) @onlyCPU def test_numpy_unresizable(self, device) -> None: x = np.zeros((2, 2)) y = torch.from_numpy(x) with self.assertRaises(ValueError): x.resize((5, 5)) z = torch.randn(5, 5) w = z.numpy() with self.assertRaises(RuntimeError): z.resize_(10, 10) with self.assertRaises(ValueError): w.resize((10, 10)) @onlyCPU def test_to_numpy(self, device) -> None: def get_castable_tensor(shape, dtype): if dtype.is_floating_point: dtype_info = torch.finfo(dtype) # can't directly use min and max, because for double, max - min # is greater than double range and sampling always gives inf. low = max(dtype_info.min, -1e10) high = min(dtype_info.max, 1e10) t = torch.empty(shape, dtype=torch.float64).uniform_(low, high) else: # can't directly use min and max, because for int64_t, max - min # is greater than int64_t range and triggers UB. low = max(torch.iinfo(dtype).min, int(-1e10)) high = min(torch.iinfo(dtype).max, int(1e10)) t = torch.empty(shape, dtype=torch.int64).random_(low, high) return t.to(dtype) dtypes = [ torch.uint8, torch.int8, torch.short, torch.int, torch.half, torch.float, torch.double, torch.long, ] for dtp in dtypes: # 1D sz = 10 x = get_castable_tensor(sz, dtp) y = x.numpy() for i in range(sz): self.assertEqual(x[i], y[i]) # 1D > 0 storage offset xm = get_castable_tensor(sz * 2, dtp) x = xm.narrow(0, sz - 1, sz) self.assertTrue(x.storage_offset() > 0) y = x.numpy() for i in range(sz): self.assertEqual(x[i], y[i]) def check2d(x, y): for i in range(sz1): for j in range(sz2): self.assertEqual(x[i][j], y[i][j]) # empty x = torch.tensor([]).to(dtp) y = x.numpy() self.assertEqual(y.size, 0) # contiguous 2D sz1 = 3 sz2 = 5 x = get_castable_tensor((sz1, sz2), dtp) y = x.numpy() check2d(x, y) self.assertTrue(y.flags['C_CONTIGUOUS']) # with storage offset xm = get_castable_tensor((sz1 * 2, sz2), dtp) x = xm.narrow(0, sz1 - 1, sz1) y = x.numpy() self.assertTrue(x.storage_offset() > 0) check2d(x, y) self.assertTrue(y.flags['C_CONTIGUOUS']) # non-contiguous 2D x = get_castable_tensor((sz2, sz1), dtp).t() y = x.numpy() check2d(x, y) self.assertFalse(y.flags['C_CONTIGUOUS']) # with storage offset xm = get_castable_tensor((sz2 * 2, sz1), dtp) x = xm.narrow(0, sz2 - 1, sz2).t() y = x.numpy() self.assertTrue(x.storage_offset() > 0) check2d(x, y) # non-contiguous 2D with holes xm = get_castable_tensor((sz2 * 2, sz1 * 2), dtp) x = xm.narrow(0, sz2 - 1, sz2).narrow(1, sz1 - 1, sz1).t() y = x.numpy() self.assertTrue(x.storage_offset() > 0) check2d(x, y) if dtp != torch.half: # check writeable x = get_castable_tensor((3, 4), dtp) y = x.numpy() self.assertTrue(y.flags.writeable) y[0][1] = 3 self.assertTrue(x[0][1] == 3) y = x.t().numpy() self.assertTrue(y.flags.writeable) y[0][1] = 3 self.assertTrue(x[0][1] == 3) def test_to_numpy_bool(self, device) -> None: x = torch.tensor([True, False], dtype=torch.bool) self.assertEqual(x.dtype, torch.bool) y = x.numpy() self.assertEqual(y.dtype, np.bool_) for i in range(len(x)): self.assertEqual(x[i], y[i]) x = torch.tensor([True], dtype=torch.bool) self.assertEqual(x.dtype, torch.bool) y = x.numpy() self.assertEqual(y.dtype, np.bool_) self.assertEqual(x[0], y[0]) def test_from_numpy(self, device) -> None: dtypes = [ np.double, np.float64, np.float16, np.complex64, np.complex128, np.int64, np.int32, np.int16, np.int8, np.uint8, np.longlong, np.bool_, ] complex_dtypes = [ np.complex64, np.complex128, ] for dtype in dtypes: array = np.array([1, 2, 3, 4], dtype=dtype) tensor_from_array = torch.from_numpy(array) # TODO: change to tensor equality check once HalfTensor # implements `==` for i in range(len(array)): self.assertEqual(tensor_from_array[i], array[i]) # ufunc 'remainder' not supported for complex dtypes if dtype not in complex_dtypes: # This is a special test case for Windows # https://github.com/pytorch/pytorch/issues/22615 array2 = array % 2 tensor_from_array2 = torch.from_numpy(array2) for i in range(len(array2)): self.assertEqual(tensor_from_array2[i], array2[i]) # Test unsupported type array = np.array([1, 2, 3, 4], dtype=np.uint16) with self.assertRaises(TypeError): tensor_from_array = torch.from_numpy(array) # check storage offset x = np.linspace(1, 125, 125) x.shape = (5, 5, 5) x = x[1] expected = torch.arange(1, 126, dtype=torch.float64).view(5, 5, 5)[1] self.assertEqual(torch.from_numpy(x), expected) # check noncontiguous x = np.linspace(1, 25, 25) x.shape = (5, 5) expected = torch.arange(1, 26, dtype=torch.float64).view(5, 5).t() self.assertEqual(torch.from_numpy(x.T), expected) # check noncontiguous with holes x = np.linspace(1, 125, 125) x.shape = (5, 5, 5) x = x[:, 1] expected = torch.arange(1, 126, dtype=torch.float64).view(5, 5, 5)[:, 1] self.assertEqual(torch.from_numpy(x), expected) # check zero dimensional x = np.zeros((0, 2)) self.assertEqual(torch.from_numpy(x).shape, (0, 2)) x = np.zeros((2, 0)) self.assertEqual(torch.from_numpy(x).shape, (2, 0)) # check ill-sized strides raise exception x = np.array([3., 5., 8.]) x.strides = (3, ) self.assertRaises(ValueError, lambda: torch.from_numpy(x)) def test_from_list_of_ndarray_warning(self, device): warning_msg = r"Creating a tensor from a list of numpy.ndarrays is extremely slow" with self.assertWarnsOnceRegex(UserWarning, warning_msg): torch.tensor([np.array([0]), np.array([1])], device=device) @onlyCPU def test_ctor_with_numpy_scalar_ctor(self, device) -> None: dtypes = [ np.double, np.float64, np.float16, np.int64, np.int32, np.int16, np.uint8, np.bool_, ] for dtype in dtypes: self.assertEqual(dtype(42), torch.tensor(dtype(42)).item()) @onlyCPU def test_numpy_index(self, device): i = np.array([0, 1, 2], dtype=np.int32) x = torch.randn(5, 5) for idx in i: self.assertFalse(isinstance(idx, int)) self.assertEqual(x[idx], x[int(idx)]) @onlyCPU def test_numpy_array_interface(self, device): types = [ torch.DoubleTensor, torch.FloatTensor, torch.HalfTensor, torch.LongTensor, torch.IntTensor, torch.ShortTensor, torch.ByteTensor, ] dtypes = [ np.float64, np.float32, np.float16, np.int64, np.int32, np.int16, np.uint8, ] for tp, dtype in zip(types, dtypes): # Only concrete class can be given where "Type[number[_64Bit]]" is expected if np.dtype(dtype).kind == 'u': # type: ignore[misc] # .type expects a XxxTensor, which have no type hints on # purpose, so ignore during mypy type checking x = torch.tensor([1, 2, 3, 4]).type(tp) # type: ignore[call-overload] array = np.array([1, 2, 3, 4], dtype=dtype) else: x = torch.tensor([1, -2, 3, -4]).type(tp) # type: ignore[call-overload] array = np.array([1, -2, 3, -4], dtype=dtype) # Test __array__ w/o dtype argument asarray = np.asarray(x) self.assertIsInstance(asarray, np.ndarray) self.assertEqual(asarray.dtype, dtype) for i in range(len(x)): self.assertEqual(asarray[i], x[i]) # Test __array_wrap__, same dtype abs_x = np.abs(x) abs_array = np.abs(array) self.assertIsInstance(abs_x, tp) for i in range(len(x)): self.assertEqual(abs_x[i], abs_array[i]) # Test __array__ with dtype argument for dtype in dtypes: x = torch.IntTensor([1, -2, 3, -4]) asarray = np.asarray(x, dtype=dtype) self.assertEqual(asarray.dtype, dtype) # Only concrete class can be given where "Type[number[_64Bit]]" is expected if np.dtype(dtype).kind == 'u': # type: ignore[misc] wrapped_x = np.array([1, -2, 3, -4], dtype=dtype) for i in range(len(x)): self.assertEqual(asarray[i], wrapped_x[i]) else: for i in range(len(x)): self.assertEqual(asarray[i], x[i]) # Test some math functions with float types float_types = [torch.DoubleTensor, torch.FloatTensor] float_dtypes = [np.float64, np.float32] for tp, dtype in zip(float_types, float_dtypes): x = torch.tensor([1, 2, 3, 4]).type(tp) # type: ignore[call-overload] array = np.array([1, 2, 3, 4], dtype=dtype) for func in ['sin', 'sqrt', 'ceil']: ufunc = getattr(np, func) res_x = ufunc(x) res_array = ufunc(array) self.assertIsInstance(res_x, tp) for i in range(len(x)): self.assertEqual(res_x[i], res_array[i]) # Test functions with boolean return value for tp, dtype in zip(types, dtypes): x = torch.tensor([1, 2, 3, 4]).type(tp) # type: ignore[call-overload] array = np.array([1, 2, 3, 4], dtype=dtype) geq2_x = np.greater_equal(x, 2) geq2_array = np.greater_equal(array, 2).astype('uint8') self.assertIsInstance(geq2_x, torch.ByteTensor) for i in range(len(x)): self.assertEqual(geq2_x[i], geq2_array[i]) @onlyCPU def test_multiplication_numpy_scalar(self, device) -> None: for np_dtype in [ np.float32, np.float64, np.int32, np.int64, np.int16, np.uint8 ]: for t_dtype in [torch.float, torch.double]: # mypy raises an error when np.floatXY(2.0) is called # even though this is valid code np_sc = np_dtype(2.0) # type: ignore[abstract, arg-type] t = torch.ones(2, requires_grad=True, dtype=t_dtype) r1 = t * np_sc self.assertIsInstance(r1, torch.Tensor) self.assertTrue(r1.dtype == t_dtype) self.assertTrue(r1.requires_grad) r2 = np_sc * t self.assertIsInstance(r2, torch.Tensor) self.assertTrue(r2.dtype == t_dtype) self.assertTrue(r2.requires_grad) @onlyCPU def test_parse_numpy_int(self, device): # Only concrete class can be given where "Type[number[_64Bit]]" is expected self.assertRaisesRegex(RuntimeError, "Overflow", lambda: torch.mean( torch.randn(1, 1), np.uint64(-1))) # type: ignore[call-overload] # https://github.com/pytorch/pytorch/issues/29252 for nptype in [np.int16, np.int8, np.uint8, np.int32, np.int64]: scalar = 3 np_arr = np.array([scalar], dtype=nptype) np_val = np_arr[0] # np integral type can be treated as a python int in native functions with # int parameters: self.assertEqual( torch.ones(5).diag(scalar), torch.ones(5).diag(np_val)) self.assertEqual( torch.ones([2, 2, 2, 2]).mean(scalar), torch.ones([2, 2, 2, 2]).mean(np_val)) # numpy integral type parses like a python int in custom python bindings: self.assertEqual(torch.Storage(np_val).size(), scalar) # type: ignore[attr-defined] tensor = torch.tensor([2], dtype=torch.int) tensor[0] = np_val self.assertEqual(tensor[0], np_val) # Original reported issue, np integral type parses to the correct # PyTorch integral type when passed for a `Scalar` parameter in # arithmetic operations: t = torch.from_numpy(np_arr) self.assertEqual((t + np_val).dtype, t.dtype) self.assertEqual((np_val + t).dtype, t.dtype) def test_has_storage_numpy(self, device): for dtype in [ np.float32, np.float64, np.int64, np.int32, np.int16, np.uint8 ]: arr = np.array([1], dtype=dtype) self.assertIsNotNone( torch.tensor(arr, device=device, dtype=torch.float32).storage()) self.assertIsNotNone( torch.tensor(arr, device=device, dtype=torch.double).storage()) self.assertIsNotNone( torch.tensor(arr, device=device, dtype=torch.int).storage()) self.assertIsNotNone( torch.tensor(arr, device=device, dtype=torch.long).storage()) self.assertIsNotNone( torch.tensor(arr, device=device, dtype=torch.uint8).storage()) @dtypes(*get_all_dtypes()) def test_numpy_scalar_cmp(self, device, dtype): if dtype.is_complex: tensors = (torch.tensor(complex(1, 3), dtype=dtype, device=device), torch.tensor([complex(1, 3), 0, 2j], dtype=dtype, device=device), torch.tensor([[complex(3, 1), 0], [-1j, 5]], dtype=dtype, device=device)) else: tensors = (torch.tensor(3, dtype=dtype, device=device), torch.tensor([1, 0, -3], dtype=dtype, device=device), torch.tensor([[3, 0, -1], [3, 5, 4]], dtype=dtype, device=device)) for tensor in tensors: if dtype == torch.bfloat16: with self.assertRaises(TypeError): np_array = tensor.cpu().numpy() continue np_array = tensor.cpu().numpy() for t, a in product( (tensor.flatten()[0], tensor.flatten()[0].item()), (np_array.flatten()[0], np_array.flatten()[0].item())): self.assertEqual(t, a) if dtype == torch.complex64 and torch.is_tensor(t) and type( a) == np.complex64: # TODO: Imaginary part is dropped in this case. Need fix. # https://github.com/pytorch/pytorch/issues/43579 self.assertFalse(t == a) else: self.assertTrue(t == a)
class TestShapeOps(TestCase): # TODO: update to work on CUDA, too @onlyCPU def test_unbind(self, device): x = torch.rand(2, 3, 4, 5) for dim in range(4): res = torch.unbind(x, dim) res2 = x.unbind(dim) self.assertEqual(x.size(dim), len(res)) self.assertEqual(x.size(dim), len(res2)) for i in range(dim): self.assertEqual(x.select(dim, i), res[i]) self.assertEqual(x.select(dim, i), res2[i]) # TODO: update to work on CUDA, too? @onlyCPU def test_tolist(self, device): list0D = [] tensor0D = torch.tensor(list0D) self.assertEqual(tensor0D.tolist(), list0D) table1D = [1., 2., 3.] tensor1D = torch.tensor(table1D) storage = torch.Storage(table1D) self.assertEqual(tensor1D.tolist(), table1D) self.assertEqual(storage.tolist(), table1D) self.assertEqual(tensor1D.tolist(), table1D) self.assertEqual(storage.tolist(), table1D) table2D = [[1, 2], [3, 4]] tensor2D = torch.tensor(table2D) self.assertEqual(tensor2D.tolist(), table2D) tensor3D = torch.tensor([[[1, 2], [3, 4]], [[5, 6], [7, 8]]]) tensorNonContig = tensor3D.select(1, 1) self.assertFalse(tensorNonContig.is_contiguous()) self.assertEqual(tensorNonContig.tolist(), [[3, 4], [7, 8]]) @dtypes(torch.int64, torch.float, torch.complex128) def test_movedim_invalid(self, device, dtype): shape = self._rand_shape(4, min_size=5, max_size=10) x = _generate_input(shape, dtype, device, False) for fn in [torch.movedim, torch.moveaxis]: # Invalid `source` and `destination` dimension with self.assertRaisesRegex(IndexError, "Dimension out of range"): fn(x, 5, 0) with self.assertRaisesRegex(IndexError, "Dimension out of range"): fn(x, 0, 5) # Mismatch in size of `source` and `destination` with self.assertRaisesRegex( RuntimeError, "movedim: Invalid source or destination dims:"): fn(x, (1, 0), (0, )) with self.assertRaisesRegex(RuntimeError, "movedim: repeated dim in `source`"): fn(x, (0, 0), (0, 1)) with self.assertRaisesRegex(RuntimeError, "movedim: repeated dim in `source`"): fn(x, (0, 1, 0), (0, 1, 2)) with self.assertRaisesRegex( RuntimeError, "movedim: repeated dim in `destination`"): fn(x, (0, 1), (1, 1)) with self.assertRaisesRegex( RuntimeError, "movedim: repeated dim in `destination`"): fn(x, (0, 1, 2), (1, 0, 1)) @dtypes(torch.int64, torch.float, torch.complex128) def test_movedim(self, device, dtype): for fn in [torch.moveaxis, torch.movedim]: for nd in range(5): shape = self._rand_shape(nd, min_size=5, max_size=10) x = _generate_input(shape, dtype, device, with_extremal=False) for random_negative in [True, False]: for src_dim, dst_dim in permutations(range(nd), r=2): random_prob = random.random() if random_negative and random_prob > 0.66: src_dim = src_dim - nd elif random_negative and random_prob > 0.33: dst_dim = dst_dim - nd elif random_negative: src_dim = src_dim - nd dst_dim = dst_dim - nd # Integer `source` and `destination` torch_fn = partial(fn, source=src_dim, destination=dst_dim) np_fn = partial(np.moveaxis, source=src_dim, destination=dst_dim) self.compare_with_numpy(torch_fn, np_fn, x, device=None, dtype=None) if nd == 0: continue def make_index_negative(sequence, idx): sequence = list(sequence) sequence[random_idx] = sequence[random_idx] - nd return tuple(src_sequence) for src_sequence in permutations(range(nd), r=random.randint(1, nd)): # Sequence `source` and `destination` dst_sequence = tuple( random.sample(range(nd), len(src_sequence))) # Randomly change a dim to a negative dim representation of itself. random_prob = random.random() if random_negative and random_prob > 0.66: random_idx = random.randint( 0, len(src_sequence) - 1) src_sequence = make_index_negative( src_sequence, random_idx) elif random_negative and random_prob > 0.33: random_idx = random.randint( 0, len(src_sequence) - 1) dst_sequence = make_index_negative( dst_sequence, random_idx) elif random_negative: random_idx = random.randint( 0, len(src_sequence) - 1) dst_sequence = make_index_negative( dst_sequence, random_idx) random_idx = random.randint( 0, len(src_sequence) - 1) src_sequence = make_index_negative( src_sequence, random_idx) torch_fn = partial(fn, source=src_sequence, destination=dst_sequence) np_fn = partial(np.moveaxis, source=src_sequence, destination=dst_sequence) self.compare_with_numpy(torch_fn, np_fn, x, device=None, dtype=None) # Move dim to same position x = torch.randn(2, 3, 5, 7, 11) torch_fn = partial(fn, source=(0, 1), destination=(0, 1)) np_fn = partial(np.moveaxis, source=(0, 1), destination=(0, 1)) self.compare_with_numpy(torch_fn, np_fn, x, device=None, dtype=None) torch_fn = partial(fn, source=1, destination=1) np_fn = partial(np.moveaxis, source=1, destination=1) self.compare_with_numpy(torch_fn, np_fn, x, device=None, dtype=None) # Empty Sequence torch_fn = partial(fn, source=(), destination=()) np_fn = partial(np.moveaxis, source=(), destination=()) self.compare_with_numpy(torch_fn, np_fn, x, device=None, dtype=None) @dtypes(torch.float, torch.bool) def test_diag(self, device, dtype): if dtype is torch.bool: x = torch.rand(100, 100, device=device) >= 0.5 else: x = torch.rand(100, 100, dtype=dtype, device=device) res1 = torch.diag(x) res2 = torch.tensor((), dtype=dtype, device=device) torch.diag(x, out=res2) self.assertEqual(res1, res2) def test_diagonal(self, device): x = torch.randn((100, 100), device=device) result = torch.diagonal(x) expected = torch.diag(x) self.assertEqual(result, expected) x = torch.randn((100, 100), device=device) result = torch.diagonal(x, 17) expected = torch.diag(x, 17) self.assertEqual(result, expected) @onlyCPU @dtypes(torch.float) def test_diagonal_multidim(self, device, dtype): x = torch.randn(10, 11, 12, 13, dtype=dtype, device=device) xn = x.numpy() for args in [(2, 2, 3), (2, ), (-2, 1, 2), (0, -2, -1)]: result = torch.diagonal(x, *args) expected = xn.diagonal(*args) self.assertEqual(expected.shape, result.shape) self.assertEqual(expected, result) # test non-continguous xp = x.permute(1, 2, 3, 0) result = torch.diagonal(xp, 0, -2, -1) expected = xp.numpy().diagonal(0, -2, -1) self.assertEqual(expected.shape, result.shape) self.assertEqual(expected, result) @onlyOnCPUAndCUDA @dtypesIfCPU(*get_all_dtypes(include_complex=False, include_bool=False, include_half=False, include_bfloat16=False)) @dtypesIfCUDA(*get_all_dtypes(include_complex=False, include_bool=False, include_bfloat16=False)) def test_trace(self, device, dtype): def test(shape): tensor = make_tensor(shape, device, dtype, low=-9, high=9) expected_dtype = tensor.sum().dtype expected_dtype = torch_to_numpy_dtype_dict[expected_dtype] result = np.trace(tensor.cpu().numpy(), dtype=expected_dtype) expected = torch.tensor(result, device=device) self.assertEqual(tensor.trace(), expected) shapes = ( [10, 1], [1, 10], [100, 100], [20, 100], [100, 20], ) for shape in shapes: test(shape) def generate_clamp_baseline(self, device, dtype, *, min_vals, max_vals, with_nans): """ Creates a random tensor for a given device and dtype, and computes the expected clamped values given the min_vals and/or max_vals. If with_nans is provided, then some values are randomly set to nan. """ X = torch.rand(100, device=device).mul(50).add(-25) # uniform in [-25, 25] X = X.to(dtype) if with_nans: mask = torch.randint(0, 2, X.shape, dtype=torch.bool, device=device) X[mask] = nan if isinstance(min_vals, torch.Tensor): min_vals = min_vals.cpu().numpy() if isinstance(max_vals, torch.Tensor): max_vals = max_vals.cpu().numpy() # Use NumPy implementation as reference X_clamped = torch.tensor(np.clip(X.cpu().numpy(), a_min=min_vals, a_max=max_vals), device=device) return X, X_clamped # Tests clamp and its alias, clip @dtypes(torch.int64, torch.float32) def test_clamp(self, device, dtype): op_list = (torch.clamp, torch.Tensor.clamp, torch.Tensor.clamp_, torch.clip, torch.Tensor.clip, torch.Tensor.clip_) # min/max argument product args = product((-10, None), (10, None)) for op in op_list: for min_val, max_val in args: if min_val is None and max_val is None: continue X, Y_expected = self.generate_clamp_baseline(device, dtype, min_vals=min_val, max_vals=max_val, with_nans=False) # Test op X1 = X.clone() # So that the in-place ops do not change X Y_actual = op(X1, min_val, max_val) self.assertEqual(Y_expected, Y_actual) # Test op-out behavior (out does not exist for method versions) if op in (torch.clamp, torch.clip): Y_out = torch.empty_like(X) op(X, min=min_val, max=max_val, out=Y_out) self.assertEqual(Y_expected, Y_out) def test_clamp_propagates_nans(self, device): op_list = (torch.clamp, torch.Tensor.clamp, torch.Tensor.clamp_, torch.clip, torch.Tensor.clip, torch.Tensor.clip_) # min/max argument product args = product((-10, None), (10, None)) for op in op_list: for min_val, max_val in args: if min_val is None and max_val is None: continue X, Y_expected = self.generate_clamp_baseline(device, torch.float, min_vals=min_val, max_vals=max_val, with_nans=True) Y_expected = torch.isnan(Y_expected) # Test op X1 = X.clone() # So that the in-place ops do not change X Y_actual = op(X1, min_val, max_val) self.assertEqual(Y_expected, torch.isnan(Y_actual)) # Test op-out behavior (out does not exist for method versions) if op in (torch.clamp, torch.clip): Y_out = torch.empty_like(X) op(X, min_val, max_val, out=Y_out) self.assertEqual(Y_expected, torch.isnan(Y_out)) def test_clamp_raises_arg_errors(self, device): X = torch.randn(100, dtype=torch.float, device=device) error_msg = 'At least one of \'min\' or \'max\' must not be None' with self.assertRaisesRegex(RuntimeError, error_msg): X.clamp() with self.assertRaisesRegex(RuntimeError, error_msg): X.clamp_() with self.assertRaisesRegex(RuntimeError, error_msg): torch.clamp(X) @dtypes(*get_all_dtypes()) def test_flip(self, device, dtype): make_from_data = partial(torch.tensor, device=device, dtype=dtype) make_from_size = partial(make_tensor, device=device, dtype=dtype) def test_flip_impl(input_t, dims, output_t): def all_t(): yield input_t, output_t if dtype is torch.float: # We generate quantized versions as well for qdtype in (torch.quint8, torch.qint8, torch.qint32): qinput_t = torch.quantize_per_tensor( input_t, 0.1, 5, qdtype) qoutput_t = torch.quantize_per_tensor( output_t, 0.1, 5, qdtype) yield qinput_t, qoutput_t for in_t, out_t in all_t(): self.assertEqual(in_t.flip(dims), out_t) n = in_t.ndim if not isinstance(dims, tuple): # Wrap dim self.assertEqual(in_t.flip(-n + dims), out_t) else: # Permute dimensions for p_dims in permutations(dims): self.assertEqual(in_t.flip(p_dims), out_t) if len(p_dims) > 0: # Wrap 1st dim self.assertEqual( in_t.flip((-n + p_dims[0], ) + p_dims[1:]), out_t) def gen_data(): # Basic tests data = make_from_data([1, 2, 3, 4, 5, 6, 7, 8]).view(2, 2, 2) nonctg = make_from_size((2, 2, 2), noncontiguous=True).copy_(data) dims_result = ((0, make_from_data([5, 6, 7, 8, 1, 2, 3, 4]).view(2, 2, 2)), (1, make_from_data([3, 4, 1, 2, 7, 8, 5, 6]).view(2, 2, 2)), (2, make_from_data([2, 1, 4, 3, 6, 5, 8, 7]).view(2, 2, 2)), ((0, 1), make_from_data([7, 8, 5, 6, 3, 4, 1, 2]).view(2, 2, 2)), ((0, 1, 2), make_from_data([8, 7, 6, 5, 4, 3, 2, 1]).view(2, 2, 2))) for in_tensor, (dims, out_tensor) in product((data, nonctg), dims_result): yield in_tensor, dims, out_tensor # Expanded in_t = make_from_data([1, 2, 3]).view(3, 1).expand(3, 2) dims = 0 out_t = make_from_data([3, 3, 2, 2, 1, 1]).view(3, 2) yield in_t, dims, out_t # Noop on expanded dimension yield in_t, 1, in_t # Transposed in_t = make_from_data([1, 2, 3, 4, 5, 6, 7, 8]).view(2, 2, 2).transpose(0, 1) dims = (0, 1, 2) out_t = make_from_data([8, 7, 4, 3, 6, 5, 2, 1]).view(2, 2, 2) yield in_t, dims, out_t # Rectangular case in_t = make_from_data([1, 2, 3, 4, 5, 6]).view(2, 3) dims = 0 out_t = make_from_data([[4, 5, 6], [1, 2, 3]]) yield in_t, dims, out_t dims = 1 out_t = make_from_data([[3, 2, 1], [6, 5, 4]]) yield in_t, dims, out_t # Noops (edge cases) # Size 0 in_t = make_from_data(()) yield in_t, 0, in_t yield in_t, (), in_t # dims = () in_t = make_from_size((3, 2, 1)) yield in_t, (), in_t # Zero elements, non-zero size in_t = make_from_size((3, 0, 2)) for i in range(in_t.ndim): yield in_t, i, in_t # Size 1 in_t = make_from_size(()) yield in_t, 0, in_t in_t = make_from_size((1, )) yield in_t, 0, in_t for in_tensor, dims, out_tensor in gen_data(): test_flip_impl(in_tensor, dims, out_tensor) # test for shape size = [2, 3, 4] data = make_from_size(size) possible_dims = range(len(size)) test_dims = chain(combinations(possible_dims, 1), combinations(possible_dims, 2)) for dims in test_dims: self.assertEqual(size, list(data.flip(dims).size())) @dtypes(*get_all_dtypes()) def test_flip_errors(self, device, dtype): make_arg = partial(make_tensor, dtype=dtype, device=device) data = make_arg((2, 2, 2)) # not allow flip on the same dim more than once self.assertRaises(RuntimeError, lambda: data.flip(0, 1, 1)) # not allow empty list as input self.assertRaises(TypeError, lambda: data.flip()) # not allow dim > max dim self.assertRaises(IndexError, lambda: data.flip(0, 1, 2, 3)) self.assertRaises(IndexError, lambda: data.flip(3)) def _rand_shape(self, dim, min_size, max_size): return tuple(torch.randint(min_size, max_size + 1, (dim, ))) @dtypes(*get_all_dtypes()) def test_flip_numpy(self, device, dtype): make_arg = partial(make_tensor, dtype=dtype, device=device) for ndim in [3, 4]: shape = self._rand_shape(ndim, 5, 10) data = make_arg(shape) # Axis to sample for given shape. for i in range(1, ndim + 1): # Check all combinations of `i` axis. for flip_dim in combinations(range(ndim), i): torch_fn = partial(torch.flip, dims=flip_dim) np_fn = partial(np.flip, axis=flip_dim) self.compare_with_numpy(torch_fn, np_fn, data) @onlyCUDA # CPU is too slow @largeTensorTest('17GB' ) # 4 tensors of 4GB (in, out) x (torch, numpy) + 1GB def test_flip_large_tensor(self, device): t_in = torch.empty(2**32 + 1, dtype=torch.uint8).random_() torch_fn = partial(torch.flip, dims=(0, )) np_fn = partial(np.flip, axis=0) self.compare_with_numpy(torch_fn, np_fn, t_in) del t_in def _test_fliplr_flipud(self, torch_fn, np_fn, min_dim, max_dim, device, dtype): for dim in range(min_dim, max_dim + 1): shape = self._rand_shape(dim, 5, 10) # Randomly scale the input if dtype.is_floating_point or dtype.is_complex: data = torch.randn(*shape, device=device, dtype=dtype) else: data = torch.randint(0, 10, shape, device=device, dtype=dtype) self.compare_with_numpy(torch_fn, np_fn, data) @dtypes(torch.int64, torch.double, torch.cdouble) def test_fliplr(self, device, dtype): self._test_fliplr_flipud(torch.fliplr, np.fliplr, 2, 4, device, dtype) @dtypes(torch.int64, torch.double, torch.cdouble) def test_fliplr_invalid(self, device, dtype): x = torch.randn(42).to(dtype) with self.assertRaisesRegex(RuntimeError, "Input must be >= 2-d."): torch.fliplr(x) with self.assertRaisesRegex(RuntimeError, "Input must be >= 2-d."): torch.fliplr(torch.tensor(42, device=device, dtype=dtype)) @dtypes(torch.int64, torch.double, torch.cdouble) def test_flipud(self, device, dtype): self._test_fliplr_flipud(torch.flipud, np.flipud, 1, 4, device, dtype) @dtypes(torch.int64, torch.double, torch.cdouble) def test_flipud_invalid(self, device, dtype): with self.assertRaisesRegex(RuntimeError, "Input must be >= 1-d."): torch.flipud(torch.tensor(42, device=device, dtype=dtype)) def test_rot90(self, device): data = torch.arange(1, 5, device=device).view(2, 2) self.assertEqual( torch.tensor([1, 2, 3, 4]).view(2, 2), data.rot90(0, [0, 1])) self.assertEqual( torch.tensor([2, 4, 1, 3]).view(2, 2), data.rot90(1, [0, 1])) self.assertEqual( torch.tensor([4, 3, 2, 1]).view(2, 2), data.rot90(2, [0, 1])) self.assertEqual( torch.tensor([3, 1, 4, 2]).view(2, 2), data.rot90(3, [0, 1])) # test for default args k=1, dims=[0, 1] self.assertEqual(data.rot90(), data.rot90(1, [0, 1])) # test for reversed order of dims self.assertEqual(data.rot90(3, [0, 1]), data.rot90(1, [1, 0])) # test for modulo of k self.assertEqual(data.rot90(5, [0, 1]), data.rot90(1, [0, 1])) self.assertEqual(data.rot90(3, [0, 1]), data.rot90(-1, [0, 1])) self.assertEqual(data.rot90(-5, [0, 1]), data.rot90(-1, [0, 1])) # test for dims out-of-range error self.assertRaises(RuntimeError, lambda: data.rot90(1, [0, -3])) self.assertRaises(RuntimeError, lambda: data.rot90(1, [0, 2])) # test tensor with more than 2D data = torch.arange(1, 9, device=device).view(2, 2, 2) self.assertEqual( torch.tensor([2, 4, 1, 3, 6, 8, 5, 7]).view(2, 2, 2), data.rot90(1, [1, 2])) self.assertEqual(data.rot90(1, [1, -1]), data.rot90(1, [1, 2])) # test for errors self.assertRaises(RuntimeError, lambda: data.rot90(1, [0, 3])) self.assertRaises(RuntimeError, lambda: data.rot90(1, [1, 1])) self.assertRaises(RuntimeError, lambda: data.rot90(1, [0, 1, 2])) self.assertRaises(RuntimeError, lambda: data.rot90(1, [0])) @dtypes(torch.cfloat, torch.cdouble) def test_complex_rot90(self, device, dtype): shape = self._rand_shape(random.randint(2, 4), 5, 10) for rot_times in range(4): data = torch.randn(*shape, device=device, dtype=dtype) torch_fn = partial(torch.rot90, k=rot_times, dims=[0, 1]) np_fn = partial(np.rot90, k=rot_times, axes=[0, 1]) self.compare_with_numpy(torch_fn, np_fn, data) # TODO: update once warning flag is available to always trigger ONCE warnings # Ensures nonzero does not throw a warning, even when the as_tuple argument # is not provided def test_nonzero_no_warning(self, device): t = torch.randn((2, 2), device=device) with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") torch.nonzero(t) t.nonzero() self.assertEqual(len(w), 0) @dtypes(*get_all_dtypes(include_complex=False)) def test_nonzero(self, device, dtype): shapes = [ torch.Size((12, )), torch.Size((12, 1)), torch.Size((1, 12)), torch.Size((6, 2)), torch.Size((3, 2, 2)), torch.Size((5, 5, 5)), ] def gen_nontrivial_input(shape, dtype, device): if dtype != torch.bfloat16: return torch.randint(2, shape, device=device, dtype=dtype) else: # windows does not work for bfloat16 randing return torch.randint(2, shape, device=device, dtype=torch.float).to(dtype) for shape in shapes: tensor = gen_nontrivial_input(shape, dtype, device) dst1 = torch.nonzero(tensor, as_tuple=False) dst2 = tensor.nonzero(as_tuple=False) dst3 = torch.empty([], dtype=torch.long, device=device) torch.nonzero(tensor, out=dst3) if self.device_type != 'xla': # xla does not raise runtime error self.assertRaisesRegex( RuntimeError, "scalar type Long", lambda: torch.nonzero( tensor, out=torch.empty([], dtype=torch.float, device=device))) if self.device_type == 'cuda': self.assertRaisesRegex( RuntimeError, "on the same device", lambda: torch.nonzero( tensor, out=torch.empty([], dtype=torch.long))) np_array = tensor.cpu().numpy( ) if dtype != torch.bfloat16 else tensor.float().cpu().numpy() np_result = torch.from_numpy(np.stack(np_array.nonzero())).t() self.assertEqual(dst1.cpu(), np_result, atol=0, rtol=0) self.assertEqual(dst2.cpu(), np_result, atol=0, rtol=0) self.assertEqual(dst3.cpu(), np_result, atol=0, rtol=0) tup1 = torch.nonzero(tensor, as_tuple=True) tup2 = tensor.nonzero(as_tuple=True) tup1 = torch.stack(tup1).t().cpu() tup2 = torch.stack(tup2).t().cpu() self.assertEqual(tup1, np_result, atol=0, rtol=0) self.assertEqual(tup2, np_result, atol=0, rtol=0) def test_nonzero_astuple_out(self, device): t = torch.randn((3, 3, 3), device=device) out = torch.empty_like(t, dtype=torch.long) with self.assertRaises(RuntimeError): torch.nonzero(t, as_tuple=True, out=out) self.assertEqual(torch.nonzero(t, as_tuple=False, out=out), torch.nonzero(t, out=out)) # Verifies that JIT script cannot handle the as_tuple kwarg # See Issue https://github.com/pytorch/pytorch/issues/45499. def _foo(t): tuple_result = torch.nonzero(t, as_tuple=True) nontuple_result = torch.nonzero(t, as_tuple=False) out = torch.empty_like(nontuple_result) torch.nonzero(t, as_tuple=False, out=out) return tuple_result, nontuple_result, out with self.assertRaises(RuntimeError): scripted_foo = torch.jit.script(_foo) # Verifies that JIT tracing works fine traced_foo = torch.jit.trace(_foo, t) traced_tuple, traced_nontuple, traced_out = traced_foo(t) expected_tuple = torch.nonzero(t, as_tuple=True) expected_nontuple = torch.nonzero(t) self.assertEqual(traced_tuple, expected_tuple) self.assertEqual(traced_nontuple, expected_nontuple) self.assertEqual(traced_out, expected_nontuple) @onlyOnCPUAndCUDA def test_nonzero_discontiguous(self, device): shape = (4, 4) tensor = torch.randint(2, shape, device=device) tensor_nc = torch.empty(shape[0], shape[1] * 2, device=device)[:, ::2].copy_(tensor) dst1 = tensor.nonzero(as_tuple=False) dst2 = tensor_nc.nonzero(as_tuple=False) self.assertEqual(dst1, dst2, atol=0, rtol=0) dst3 = torch.empty_like(dst1) data_ptr = dst3.data_ptr() # expect dst3 storage to be reused torch.nonzero(tensor, out=dst3) self.assertEqual(data_ptr, dst3.data_ptr()) self.assertEqual(dst1, dst3, atol=0, rtol=0) # discontiguous out dst4 = torch.empty(dst1.size(0), dst1.size(1) * 2, dtype=torch.long, device=device)[:, ::2] data_ptr = dst4.data_ptr() strides = dst4.stride() torch.nonzero(tensor, out=dst4) self.assertEqual(data_ptr, dst4.data_ptr()) self.assertEqual(dst1, dst4, atol=0, rtol=0) self.assertEqual(strides, dst4.stride()) def test_nonzero_non_diff(self, device): x = torch.randn(10, requires_grad=True) nz = x.nonzero() self.assertFalse(nz.requires_grad)
class TestForeach(TestCase): @property def is_cuda(self): return self.device_type == 'cuda' # note(mkozuki): It might be the case that the expected number of `cudaLaunchKernel`s # is greater than 1 once foreach functions internally separate their input `TensorList`s by # devices & dtypes into vectors of tensors. def _get_funcs(self, op, n_expected_cudaLaunchKernels): return ( ForeachFuncWrapper(op.method_variant, n_expected_cudaLaunchKernels), RegularFuncWrapper(op.ref), ForeachFuncWrapper(op.inplace_variant, n_expected_cudaLaunchKernels), RegularFuncWrapper(op.ref_inplace), ) def _binary_test(self, dtype, op, ref, inputs, is_fastpath, is_inplace, *, alpha=None): ref_inputs = [[t.clone().detach() for t in inputs[0]], inputs[1] ] if is_inplace else inputs try: actual = op(inputs, self.is_cuda, is_fastpath) except RuntimeError as e: with self.assertRaisesRegex(type(e), re.escape(str(e))): ref(ref_inputs) else: expected = ref(ref_inputs) self.assertEqual(actual, expected) if alpha is not None: kwargs = {'alpha': alpha} ref_inputs = inputs try: actual = op(inputs, self.is_cuda, is_fastpath, **kwargs) except RuntimeError as e: with self.assertRaisesRegex(type(e), re.escape(str(e))): ref(ref_inputs, **kwargs) else: expected = ref(ref_inputs, **kwargs) if dtype in (torch.float16, torch.bfloat16) and TEST_WITH_ROCM: self.assertEqual(expected, actual, atol=1.e-3, rtol=self.dtype_precisions[dtype][0]) else: self.assertEqual(expected, actual) def _test_binary_op_tensorlists(self, device, dtype, opinfo, N, is_fastpath, disable_fastpath): n_expected_cudaLaunchKernels = N if disable_fastpath else 1 op, ref, inplace_op, inplace_ref = self._get_funcs( opinfo, n_expected_cudaLaunchKernels) inputs = [ opinfo.sample_inputs(device, dtype, N, noncontiguous=not is_fastpath), opinfo.sample_inputs(device, dtype, N, noncontiguous=not is_fastpath), ] self._binary_test(dtype, op, ref, inputs, is_fastpath, is_inplace=False) self._binary_test(dtype, inplace_op, inplace_ref, inputs, is_fastpath, is_inplace=True) if opinfo.supports_alpha_param: alpha = None if dtype in get_all_int_dtypes(): alpha = 3 elif dtype.is_complex: alpha = complex(3, 3) else: alpha = 3.14 self._binary_test(dtype, op, ref, inputs, is_fastpath, is_inplace=False, alpha=alpha) self._binary_test(dtype, inplace_op, inplace_ref, inputs, is_fastpath, is_inplace=True, alpha=alpha) # Tests of implicit broadcasting # When sizes of tensors don't match, foreach functions are supposed to choose slow path # even if this methods's argument `is_fastpath` is True. # `cudaLaunchKernel` will be equal to `N`. For assert in `ForeachFuncWrapper` to pass, # we pass `is_fastpath and disable_fastpath` to `_binary_test`'s argument of is_fastpath. # as n_expected_cudaLaunchKernels is N if disable_fastpath. inputs = [ opinfo.sample_inputs(device, dtype, N, noncontiguous=not is_fastpath), [ make_tensor((N - i, 1), device=device, dtype=dtype, noncontiguous=not is_fastpath) for i in range(N) ], ] self._binary_test(dtype, op, ref, inputs, is_fastpath and disable_fastpath, is_inplace=False) self._binary_test(dtype, inplace_op, inplace_ref, inputs, is_fastpath and disable_fastpath, is_inplace=True) # note(mkozuki): Why ROCm? # ROCm is supposed to compile slow path as in # https://github.com/pytorch/pytorch/blob/7e032f18cf1405804c4f787b05ea2de5e08a091e/aten/src/ATen/native/ForeachUtils.h#L148-L164, # noqa: E501 # Therefore `[torch.add(*args, alpha=alpha) for args in zip(tensors1, tensors2)]` and # `torch._foreach_add(tensors1, tensors2, alpha=alpha)` # are expected to return the same outputs, however, the outputs look unstable for torch.bfloat16 and torch.half. # log: https://ci.pytorch.org/jenkins/job/pytorch-builds/job/pytorch-linux-bionic-rocm4.2-py3.6-test1/2741/console @skipCUDAIfRocm @skipMeta @ops(foreach_binary_op_db) def test_binary_op_tensorlists_fastpath(self, device, dtype, op): for N in N_values: disable_fastpath = op.ref == torch.div and dtype in get_all_int_dtypes( ) + [torch.bool] if op.ref == torch.add and dtype == torch.bool: disable_fastpath = True self._test_binary_op_tensorlists(device, dtype, op, N, True, disable_fastpath) @ops(foreach_binary_op_db) def test_binary_op_tensorlists_slowpath(self, device, dtype, op): for N in N_values: self._test_binary_op_tensorlists(device, dtype, op, N, False, False) def _test_binary_op_scalar(self, device, dtype, opinfo, N, scalar, is_fastpath, disable_fastpath): n_expected_cudaLaunchKernels = N if disable_fastpath else 1 op, ref, inplace_op, inplace_ref = self._get_funcs( opinfo, n_expected_cudaLaunchKernels) inputs = [ opinfo.sample_inputs(device, dtype, N, noncontiguous=not is_fastpath), scalar ] self._binary_test(dtype, op, ref, inputs, is_fastpath, is_inplace=False) self._binary_test(dtype, inplace_op, inplace_ref, inputs, is_fastpath, is_inplace=True) @skipCUDAIfRocm @skipMeta @ops(foreach_binary_op_db) def test_binary_op_scalar_fastpath(self, device, dtype, op): for N, scalar in itertools.product(N_values, Scalars): disable_fastpath = op.ref == torch.div and dtype in get_all_int_dtypes( ) + [torch.bool] if isinstance(scalar, int): disable_fastpath |= dtype == torch.bool if isinstance(scalar, float): disable_fastpath |= dtype in get_all_int_dtypes() + [ torch.bool ] if isinstance(scalar, bool): disable_fastpath |= dtype == torch.bool if op.ref in (torch.add, torch.mul): disable_fastpath = False if isinstance(scalar, complex): disable_fastpath |= dtype not in get_all_complex_dtypes() self._test_binary_op_scalar(device, dtype, op, N, scalar, True, disable_fastpath) @ops(foreach_binary_op_db) def test_binary_op_scalar_slowpath(self, device, dtype, op): for N, scalar in itertools.product(N_values, Scalars): self._test_binary_op_scalar(device, dtype, op, N, scalar, False, False) def _test_binary_op_scalarlist(self, device, dtype, opinfo, N, scalarlist, is_fastpath, disable_fastpath): n_expected_cudaLaunchKernels = N if disable_fastpath else 1 op, ref, inplace_op, inplace_ref = self._get_funcs( opinfo, n_expected_cudaLaunchKernels) inputs = [ opinfo.sample_inputs(device, dtype, N, noncontiguous=not is_fastpath), scalarlist ] self._binary_test(dtype, op, ref, inputs, is_fastpath, is_inplace=False) self._binary_test(dtype, inplace_op, inplace_ref, inputs, is_fastpath, is_inplace=True) # note(mkozuki): Why two functions depending on with/without bool? # `foreach_sub` & `foreach_sub_` do `sub_check(tensors[i], scalars[i])` from i=1...N. # So, if scalarlist has one or more bool values, `foreach_sub` and `foreach_sub_` # raise bool subtraction error before doing any math. # While regular `sub` and `sub_` do some math until they encounter bool. # So, foreach sub's throw bool sub error first. However, regular sub's throw different # errors depending on the order of scalarlist. To keep actual unit test impl simple, # separating mixed scalarlist tests. By setting the first element of scalarlist to bool, # they are expected to throw bool sub error even in inplace test. @skipCUDAIfRocm @skipMeta @ops(foreach_binary_op_db) def test_binary_op_scalarlist_fastpath(self, device, dtype, op): for N in N_values: for type_str, scalarlist in getScalarLists(N): bool_int_div = op.ref == torch.div and dtype in get_all_int_dtypes( ) + [torch.bool] disable_fastpath = bool_int_div if type_str == "int": disable_fastpath |= dtype == torch.bool if type_str == "float": disable_fastpath |= dtype in get_all_int_dtypes() + [ torch.bool ] if type_str == "complex": disable_fastpath |= dtype not in get_all_complex_dtypes() if type_str == "mixed": disable_fastpath |= True and dtype not in get_all_complex_dtypes( ) self._test_binary_op_scalarlist(device, dtype, op, N, scalarlist, True, disable_fastpath) @ops(foreach_binary_op_db) def test_binary_op_scalarlist_slowpath(self, device, dtype, op): for N in N_values: for _, scalarlist in getScalarLists(N): self._test_binary_op_scalarlist(device, dtype, op, N, scalarlist, False, False) def _pointwise_test(self, dtype, op, ref, inputs, is_fastpath, is_inplace, *, values=None): ref_inputs = [[t.clone().detach() for t in inputs[0]], inputs[1], inputs[2]] if is_inplace else inputs try: actual = op(inputs, self.is_cuda, is_fastpath) except RuntimeError as e: with self.assertRaisesRegex(type(e), re.escape(str(e))): ref(ref_inputs) else: expected = ref(ref_inputs) self.assertEqual(expected, actual) if values is not None: try: actual = op(inputs + [values], self.is_cuda, is_fastpath) except RuntimeError as e: with self.assertRaisesRegex(type(e), re.escape(str(e))): ref(ref_inputs, values=values) else: expected = ref(ref_inputs, values=values) self.assertEqual(expected, actual) def _test_pointwise_op(self, device, dtype, opinfo, N, is_fastpath, disable_fastpath, *, values=None): n_expected_cudaLaunchKernels = N if disable_fastpath else 1 op, ref, inplace_op, inplace_ref = self._get_funcs( opinfo, n_expected_cudaLaunchKernels) inputs = [ opinfo.sample_inputs(device, dtype, N, noncontiguous=not is_fastpath), opinfo.sample_inputs(device, dtype, N, noncontiguous=not is_fastpath), opinfo.sample_inputs(device, dtype, N, noncontiguous=not is_fastpath), ] self._pointwise_test(dtype, op, ref, inputs, is_fastpath, is_inplace=False, values=values) self._pointwise_test(dtype, inplace_op, inplace_ref, inputs, is_fastpath, is_inplace=True, values=values) # Tests of implicit broadcasting inputs = [ opinfo.sample_inputs(device, dtype, N, noncontiguous=not is_fastpath, same_size=True), [ make_tensor((N - i, 1), device=device, dtype=dtype, noncontiguous=not is_fastpath) for i in range(N) ], [ make_tensor((1, N - i), device=device, dtype=dtype, noncontiguous=not is_fastpath) for i in range(N) ], ] self._pointwise_test(dtype, op, ref, inputs, is_fastpath and disable_fastpath, is_inplace=False, values=values) self._pointwise_test(dtype, inplace_op, inplace_ref, inputs, is_fastpath and disable_fastpath, is_inplace=True, values=values) @skipMeta @ops(foreach_pointwise_op_db) def test_pointwise_op_fastpath(self, device, dtype, op): disable_fastpath = dtype in get_all_int_dtypes() + [torch.bool] # for N, scalar in itertools.product(N_values, Scalars): for N in N_values: self._test_pointwise_op(device, dtype, op, N, True, disable_fastpath) for scalar in Scalars: self._test_pointwise_op(device, dtype, op, N, True, disable_fastpath, values=scalar) for _, scalarlist in getScalarLists(N): self._test_pointwise_op(device, dtype, op, N, True, disable_fastpath, values=scalarlist) @ops(foreach_pointwise_op_db) def test_pointwise_op_slowpath(self, device, dtype, op): # for N, scalar in itertools.product(N_values, Scalars): for N in N_values: self._test_pointwise_op(device, dtype, op, N, False, False) for scalar in Scalars: self._test_pointwise_op(device, dtype, op, N, False, False, values=scalar) for _, scalarlist in getScalarLists(N): self._test_pointwise_op(device, dtype, op, N, False, False, values=scalarlist) # note(mkozuki): fastpath test uses dtypes which fastpath implementation supports. # To confirm the dtypes of `OpInfo` cover the dtypes that the function support, # this test does not use `try-except` for fastpath. def _regular_unary_test(self, dtype, op, ref, inputs, is_fastpath): if is_fastpath: self.assertEqual(ref(inputs), op(inputs, self.is_cuda, is_fastpath)) return try: actual = op(inputs, self.is_cuda, is_fastpath) except RuntimeError as e: with self.assertRaisesRegex(type(e), re.escape(str(e))): ref(inputs) else: expected = ref(inputs) self.assertEqual(actual, expected) # note(mkozuki): why `try-except` for both fastpath? # - inputs for fastpath can be integer tensors. # - this is becase opinfo dtypes are configured for outpulace implementation # - for integer inputs, trigonometric functions and exponential function returns float outputs, # which causes "result type Float can't be case to the desired type" error. # Thus, `try-except` is used even if `is_fastpath` is `True`. def _inplace_unary_test(self, dtype, inplace, inplace_ref, inputs, is_fastpath): copied_inputs = [[t.clone().detach() for t in tensors] for tensors in inputs] try: inplace(inputs, self.is_cuda, is_fastpath) except RuntimeError as e: with self.assertRaisesRegex(type(e), re.escape(str(e))): inplace_ref(copied_inputs) else: inplace_ref(copied_inputs), self.assertEqual(copied_inputs, inputs) def _test_unary(self, device, dtype, opinfo, N, is_fastpath): op, ref, inplace_op, inplace_ref = self._get_funcs(opinfo, 1) inputs = opinfo.sample_inputs(device, dtype, N, noncontiguous=not is_fastpath), # note(mkozuki): Complex inputs for `_foreach_abs` go through slowpath. if opinfo.name == "_foreach_abs" and dtype in get_all_complex_dtypes(): is_fastpath = False self._regular_unary_test(dtype, op, ref, inputs, is_fastpath) self._inplace_unary_test(dtype, inplace_op, inplace_ref, inputs, is_fastpath) @skipMeta @ops(foreach_unary_op_db) def test_unary_fastpath(self, device, dtype, op): for N in N_values: self._test_unary(device, dtype, op, N, is_fastpath=True) @ops(foreach_unary_op_db, dtypes=get_all_dtypes()) def test_unary_slowpath(self, device, dtype, op): for N in N_values: self._test_unary(device, dtype, op, N, is_fastpath=False) def _minmax_test(self, opinfo, inputs, is_fastpath, n_expected_cudaLaunchKernels): op, ref, _, _ = self._get_funcs(opinfo, n_expected_cudaLaunchKernels) self.assertEqual(ref(inputs), op(inputs, self.is_cuda, is_fastpath)) # note(mkozuki): in-place of foreach_minimum and foreach_maximum aren't implemented. @ops(foreach_minmax_op_db) def test_minmax_fastpath(self, device, dtype, op): for N in N_values: inputs = tuple( op.sample_inputs(device, dtype, N) for _ in range(2)) self._minmax_test(op, inputs, True, N if dtype == torch.bool else 1) @ops(foreach_minmax_op_db, dtypes=get_all_dtypes(include_half=True, include_bfloat16=True, include_complex=False)) def test_minmax_slowpath(self, device, dtype, op): for N in N_values: inputs = tuple( op.sample_inputs(device, dtype, N, noncontiguous=True) for _ in range(2)) self._minmax_test(op, inputs, False, 1) # note(mkozuki): ForeachFuncInfo's of both `_foreach_maximum` and `_foreach_minimum` include integer types. # so, manually limit dtypes to fp types for inf&nan tests. @ops(foreach_minmax_op_db, dtypes=get_all_fp_dtypes(include_bfloat16=True, include_half=True)) def test_minmax_float_inf_nan(self, device, dtype, op): inputs = ( [ torch.tensor([float('inf')], device=device, dtype=dtype), torch.tensor([-float('inf')], device=device, dtype=dtype), torch.tensor([float('nan')], device=device, dtype=dtype), torch.tensor([float('nan')], device=device, dtype=dtype) ], [ torch.tensor([-float('inf')], device=device, dtype=dtype), torch.tensor([float('inf')], device=device, dtype=dtype), torch.tensor([float('inf')], device=device, dtype=dtype), torch.tensor([float('nan')], device=device, dtype=dtype) ], ) self._minmax_test(op, inputs, True, 1) def _reduce_test(self, opinfo, inputs, ord, is_fastpath, n_expected_cudaLaunchKernels): op, ref, _, _ = self._get_funcs(opinfo, n_expected_cudaLaunchKernels) self.assertEqual(ref(inputs, ord=ord), op(inputs, self.is_cuda, is_fastpath, ord=ord)) @ops(foreach_reduce_op_db) def test_reduce_fastpath(self, device, dtype, op): for N, ord in itertools.product(N_values, (0, 1, 2, -1, -2)): if ord in (1, 2) and dtype in torch.testing.get_all_fp_dtypes(): n_expected_cudaLaunchKernels = 3 else: n_expected_cudaLaunchKernels = N inputs = op.sample_inputs(device, dtype, N, noncontiguous=False), self._reduce_test(op, inputs, ord, True, n_expected_cudaLaunchKernels) @ops(foreach_reduce_op_db) def test_reduce_slowpath(self, device, dtype, op): for N, ord in itertools.product(N_values, (0, 1, 2, -1, -2)): inputs = op.sample_inputs(device, dtype, N, noncontiguous=True), self._reduce_test(op, inputs, ord, False, 1) @dtypes(*get_all_dtypes()) def test_add_scalar_with_empty_list_and_empty_tensor(self, device, dtype): # TODO: enable empty list case for tensors in [[torch.randn([0])]]: res = torch._foreach_add(tensors, 1) self.assertEqual(res, tensors) torch._foreach_add_(tensors, 1) self.assertEqual(res, tensors) @ops(foreach_binary_op_db, dtypes=get_all_dtypes()) def test_binary_op_scalar_with_overlapping_tensors(self, device, dtype, op): foreach_op, ref = op.method_variant, op.ref tensors = [ torch.ones(1, 1, device=device, dtype=dtype).expand(2, 1, 3) ] if ref == torch.sub and dtype == torch.bool: with self.assertRaisesRegex(RuntimeError, re.escape(_BOOL_SUB_ERR_MSG)): [ref(t, 1) for t in tensors] with self.assertRaisesRegex(RuntimeError, re.escape(_BOOL_SUB_ERR_MSG)): foreach_op(tensors, 1) return expected = [ref(t, 1) for t in tensors] res = foreach_op(tensors, 1) self.assertEqual(res, expected) # note(mkozuki): this test case fails with Meta at least in my local environment. # The message was # `AssertionError: NotImplementedError("Could not run 'aten::_foreach_add.Scalar' with arguments from the 'Meta' backend.` @skipMeta @ops(foreach_binary_op_db, allowed_dtypes=[torch.float]) def test_binary_op_scalar_with_different_tensor_dtypes( self, device, dtype, op): foreach_op = op.method_variant tensors = [ torch.tensor([1.1], dtype=torch.float, device=device), torch.tensor([1], dtype=torch.long, device=device) ] runtime_error = None try: foreach_op(tensors, 1) except RuntimeError as e: runtime_error = e self.assertIsNone(runtime_error) @ops(foreach_binary_op_db, dtypes=get_all_dtypes()) def test_binary_op_list_error_cases(self, device, dtype, op): foreach_op, foreach_op_, ref, ref_ = op.method_variant, op.inplace_variant, op.ref, op.ref_inplace tensors1 = [] tensors2 = [] # Empty lists with self.assertRaisesRegex( RuntimeError, "There were no tensor arguments to this function"): foreach_op(tensors1, tensors2) with self.assertRaisesRegex( RuntimeError, "There were no tensor arguments to this function"): foreach_op_(tensors1, tensors2) # One empty list tensors1.append(torch.tensor([1], device=device, dtype=dtype)) with self.assertRaisesRegex( RuntimeError, "Tensor list must have same number of elements as scalar list." ): foreach_op(tensors1, tensors2) with self.assertRaisesRegex( RuntimeError, "Tensor list must have same number of elements as scalar list." ): foreach_op_(tensors1, tensors2) # Lists have different amount of tensors tensors2.append(torch.tensor([1], device=device)) tensors2.append(torch.tensor([1], device=device)) with self.assertRaisesRegex( RuntimeError, "Tensor lists must have the same number of tensors, got 1 and 2" ): foreach_op(tensors1, tensors2) with self.assertRaisesRegex( RuntimeError, "Tensor lists must have the same number of tensors, got 1 and 2" ): foreach_op_(tensors1, tensors2) # Corresponding tensors with different sizes that aren't compatible with broadcast # If sizes are different then foreach chooses slow path, thus error messages are expected # to be the same as torch regular function. tensors1 = [ torch.zeros(10, 10, device=device, dtype=dtype) for _ in range(10) ] tensors2 = [ torch.ones(11, 11, device=device, dtype=dtype) for _ in range(10) ] try: foreach_op(tensors1, tensors2) except RuntimeError as e: with self.assertRaisesRegex(type(e), re.escape(str(e))): [ref(t1, t2) for t1, t2 in zip(tensors1, tensors2)] try: foreach_op_(tensors1, tensors2) except RuntimeError as e: with self.assertRaisesRegex(type(e), re.escape(str(e))): [ref_(t1, t2) for t1, t2 in zip(tensors1, tensors2)] # different devices if self.device_type == "cuda" and torch.cuda.device_count() > 1: tensor1 = torch.zeros(10, 10, device="cuda:0", dtype=dtype) tensor2 = torch.ones(10, 10, device="cuda:1", dtype=dtype) if dtype == torch.bool and foreach_op == torch._foreach_sub: with self.assertRaisesRegex(RuntimeError, re.escape(_BOOL_SUB_ERR_MSG)): foreach_op([tensor1], [tensor2]) with self.assertRaisesRegex(RuntimeError, re.escape(_BOOL_SUB_ERR_MSG)): foreach_op_([tensor1], [tensor2]) return with self.assertRaisesRegex( RuntimeError, "Expected all tensors to be on the same device"): foreach_op([tensor1], [tensor2]) if dtype in get_all_int_dtypes() + [ torch.bool ] and foreach_op == torch._foreach_div: with self.assertRaisesRegex(RuntimeError, "result type"): foreach_op_([tensor1], [tensor2]) else: with self.assertRaisesRegex( RuntimeError, "Expected all tensors to be on the same device"): foreach_op_([tensor1], [tensor2]) @skipMeta @unittest.skipIf(not torch.cuda.is_available(), "CUDA not found") @ops(foreach_binary_op_db, dtypes=get_all_dtypes()) def test_binary_op_list_slow_path(self, device, dtype, op): # note(mkozuki): why `n_expected_cudaLaunchKernels=0`? # In this test, foreach functions don't go through fast path, # but as there is only one tensor in each list of tensors, # `cudaLaunchKernel` is 1 so ForeachFuncWrapper internal assert fails. foreach_op, native_op, foreach_op_, native_op_ = self._get_funcs( op, n_expected_cudaLaunchKernels=0) # 0-strides tensor1 = make_tensor((10, 10), dtype=dtype, device=device) tensor2 = make_tensor((1, ), device=device, dtype=dtype).expand_as(tensor1) inputs = ([tensor1], [tensor2]) self._binary_test(dtype, foreach_op, native_op, inputs, is_fastpath=False, is_inplace=False) self._binary_test(dtype, foreach_op_, native_op_, inputs, is_fastpath=False, is_inplace=True) # different strides tensor1 = torch.zeros(10, 10, device=device, dtype=dtype) tensor2 = torch.ones(10, 10, device=device, dtype=dtype) inputs = ([tensor1], [tensor2.t()]) self._binary_test(dtype, foreach_op, native_op, inputs, is_fastpath=False, is_inplace=False) self._binary_test(dtype, foreach_op_, native_op_, inputs, is_fastpath=False, is_inplace=True) # non contiguous tensor1 = make_tensor((5, 2, 1, 3), device=device, dtype=dtype, noncontiguous=True) tensor2 = make_tensor((5, 2, 1, 3), device=device, dtype=dtype, noncontiguous=True) self.assertFalse(tensor1.is_contiguous()) self.assertFalse(tensor2.is_contiguous()) inputs = ([tensor1], [tensor2]) self._binary_test(dtype, foreach_op, native_op, inputs, is_fastpath=False, is_inplace=False) self._binary_test(dtype, foreach_op_, native_op_, inputs, is_fastpath=False, is_inplace=True) # sliced tensor tensor1 = make_tensor((5, 2, 1, 3), device=device, dtype=dtype) tensor2 = make_tensor((5, 2, 1, 3 * 7), device=device, dtype=dtype)[:, :, :, ::7] inputs = ([tensor1], [tensor2]) self._binary_test(dtype, foreach_op, native_op, inputs, is_fastpath=False, is_inplace=False) self._binary_test(dtype, foreach_op_, native_op_, inputs, is_fastpath=False, is_inplace=True) # note: Below three tests (postfixed with `_tensors_on_different_devices`) # checks whether foreach works with lists of tensors on different devices # but tensors of the same index are on the same device, e.g., ['cuda', 'cpu]. @onlyCUDA @ops(foreach_unary_op_db) def test_unary_op_tensors_on_different_devices(self, device, dtype, op): method, ref, inplace_method, ref_inplace = self._get_funcs(op, 1) # tensors: ['cuda', 'cpu] tensors = op.sample_inputs(device, dtype, 2) tensors[1] = tensors[1].to('cpu') try: actual = method((tensors, ), False, False) except RuntimeError as e: with self.assertRaisesRegex(type(e), str(e)): ref((tensors, )) else: expected = ref((tensors, )) self.assertEqual(expected, actual) try: inplace_method((tensors, ), False, False) except RuntimeError as e: with self.assertRaisesRegex(type(e), str(e)): ref_inplace((tensors, )) else: self.assertEqual(expected, tensors) @onlyCUDA @ops(foreach_binary_op_db) def test_binary_op_tensors_on_different_devices(self, device, dtype, op): # `tensors1`: ['cuda', 'cpu'] # `tensors2`: ['cuda', 'cpu'] _cuda_tensors = op.sample_inputs(device, dtype, 2, same_size=True) _cpu_tensors = op.sample_inputs('cpu', dtype, 2, same_size=True) tensors1, tensors2 = list( tensors for tensors in zip(_cuda_tensors, _cpu_tensors)) foreach_op, foreach_op_ = op.method_variant, op.inplace_variant native_op, native_op_ = op.ref, op.ref_inplace try: actual = foreach_op(tensors1, tensors2) except RuntimeError as e: with self.assertRaisesRegex(type(e), re.escape(str(e))): [native_op(t1, t2) for t1, t2 in zip(tensors1, tensors2)] else: expected = [ native_op(t1, t2) for t1, t2 in zip(tensors1, tensors2) ] self.assertEqual(expected, actual) try: foreach_op_(tensors1, tensors2) except RuntimeError as e: with self.assertRaisesRegex(type(e), re.escape(str(e))): [native_op_(t1, t2) for t1, t2 in zip(tensors1, tensors2)] else: self.assertEqual(actual, tensors1) @onlyCUDA @ops(foreach_pointwise_op_db, allowed_dtypes=get_all_fp_dtypes(include_half=False, include_bfloat16=False)) def test_pointwise_op_tensors_on_different_devices(self, device, dtype, op): # tensors1: ['cuda', 'cpu] # tensors2: ['cuda', 'cpu] # tensors3: ['cuda', 'cpu] _cuda_tensors = op.sample_inputs(device, dtype, 3, same_size=True) _cpu_tensors = op.sample_inputs('cpu', dtype, 3, same_size=True) tensors1, tensors2, tensors3 = list( tensors for tensors in zip(_cuda_tensors, _cpu_tensors)) foreach_op, foreach_op_, native_op = op.method_variant, op.inplace_variant, op.ref actual = foreach_op(tensors1, tensors2, tensors3) expected = [native_op(*_cuda_tensors), native_op(*_cpu_tensors)] self.assertEqual(expected, actual) # note(mkozuki): Limiting dtypes to FP32&FP64, we can safely run inplace ops. foreach_op_(tensors1, tensors2, tensors3) self.assertEqual(expected, tensors1)
class TestSparseCSR(TestCase): @onlyCPU def test_csr_layout(self): self.assertEqual(str(torch.sparse_csr), 'torch.sparse_csr') self.assertEqual(type(torch.sparse_csr), torch.layout) @dtypes(*get_all_dtypes()) def test_sparse_csr_constructor_shape_inference(self, device, dtype): crow_indices = [0, 2, 4] col_indices = [0, 1, 0, 1] values = [1, 2, 3, 4] sparse = torch.sparse_csr_tensor(torch.tensor(crow_indices, dtype=torch.int64), torch.tensor(col_indices, dtype=torch.int64), torch.tensor(values), dtype=dtype, device=device) self.assertEqual(torch.tensor(crow_indices, dtype=torch.int64), sparse.crow_indices()) self.assertEqual((len(crow_indices) - 1, max(col_indices) + 1), sparse.shape) self.assertEqual(dtype, sparse.dtype) self.assertEqual(torch.device(device), sparse.device) @dtypes(*get_all_dtypes()) def test_sparse_csr_constructor(self, device, dtype): crow_indices = [0, 2, 4] col_indices = [0, 1, 0, 1] values = [1, 2, 3, 4] for index_dtype in [torch.int32, torch.int64]: sparse = torch.sparse_csr_tensor(torch.tensor(crow_indices, dtype=index_dtype), torch.tensor(col_indices, dtype=index_dtype), torch.tensor(values), size=(2, 10), dtype=dtype, device=device) self.assertEqual((2, 10), sparse.shape) self.assertEqual(torch.tensor(crow_indices, dtype=index_dtype), sparse.crow_indices()) self.assertEqual(torch.tensor(col_indices, dtype=index_dtype), sparse.col_indices()) self.assertEqual(torch.tensor(values, dtype=dtype), sparse.values()) @dtypes(*get_all_dtypes()) def test_sparse_csr_constructor_from_lists(self, device, dtype): # without size sparse = torch.sparse_csr_tensor([0, 2, 4], [0, 1, 0, 1], [1, 2, 3, 4], dtype=dtype, device=device) self.assertEqual((2, 2), sparse.shape) self.assertEqual(4, sparse.numel()) self.assertEqual(torch.tensor([0, 2, 4], dtype=torch.int64, device=device), sparse.crow_indices()) self.assertEqual(torch.tensor([0, 1, 0, 1], dtype=torch.int64, device=device), sparse.col_indices()) self.assertEqual(torch.tensor([1, 2, 3, 4], dtype=dtype, device=device), sparse.values()) # with size for sparse_csr_tensor in [torch.sparse_csr_tensor, torch._sparse_csr_tensor_unsafe]: sparse = sparse_csr_tensor([0, 2, 4], [0, 1, 0, 1], [1, 2, 3, 4], size=(2, 10), dtype=dtype, device=device) self.assertEqual((2, 10), sparse.shape) self.assertEqual(torch.tensor([0, 2, 4], dtype=torch.int64, device=device), sparse.crow_indices()) self.assertEqual(torch.tensor([0, 1, 0, 1], dtype=torch.int64, device=device), sparse.col_indices()) self.assertEqual(torch.tensor([1, 2, 3, 4], dtype=dtype, device=device), sparse.values()) @skipMeta @dtypes(*get_all_dtypes()) def test_empty(self, device, dtype): ns = [5, 2, 0] for shape in itertools.product(ns, ns): result = torch.empty(shape, dtype=dtype, device=device, layout=torch.sparse_csr) self.assertEqual(result.shape, shape) self.assertEqual(result.dtype, dtype) self.assertEqual(result.device, torch.device(device)) self.assertEqual(result.layout, torch.sparse_csr) self.assertEqual(result.crow_indices().shape, (shape[0] + 1,)) self.assertEqual(result.col_indices().shape, (0,)) self.assertEqual(result.values().shape, (0,)) self.assertEqual(result._nnz(), 0) self.assertEqual(result.crow_indices().device, torch.device(device)) self.assertEqual(result.col_indices().device, torch.device(device)) self.assertEqual(result.values().device, torch.device(device)) self.assertEqual(result.crow_indices().dtype, torch.int64) self.assertEqual(result.col_indices().dtype, torch.int64) self.assertEqual(result.values().dtype, dtype) @skipMeta @dtypes(*get_all_dtypes()) def test_empty_errors(self, device, dtype): with self.assertRaisesRegex(RuntimeError, "torch.empty: Only 2D sparse CSR tensors are supported."): torch.empty((5,), dtype=dtype, device=device, layout=torch.sparse_csr) with self.assertRaisesRegex(RuntimeError, "torch.empty: Only 2D sparse CSR tensors are supported."): torch.empty((2, 3, 4), dtype=dtype, device=device, layout=torch.sparse_csr) @skipMeta @dtypes(*get_all_dtypes()) def test_copy(self, device, dtype): def run_test(shape, nnz, index_type): a = self.genSparseCSRTensor(shape, nnz, dtype=dtype, device=device, index_dtype=index_dtype) b = self.genSparseCSRTensor(shape, nnz, dtype=dtype, device=device, index_dtype=index_dtype) a.copy_(b) self.assertEqual(a.crow_indices(), b.crow_indices()) self.assertEqual(a.col_indices(), b.col_indices()) self.assertEqual(a.values(), b.values()) ns = [5, 2, 0] for shape, index_dtype in zip(itertools.product(ns, ns), [torch.int32, torch.int64]): run_test(shape, 0, index_dtype) run_test(shape, shape[0] * shape[1], index_dtype) @skipMeta @dtypes(*get_all_dtypes()) def test_copy_errors(self, device, dtype): for index_dtype in [torch.int32, torch.int64]: shape1 = (2, 3) shape2 = (3, 2) a = self.genSparseCSRTensor(shape1, 0, dtype=dtype, device=device, index_dtype=index_dtype) b = self.genSparseCSRTensor(shape2, 0, dtype=dtype, device=device, index_dtype=index_dtype) with self.assertRaisesRegex(RuntimeError, "only same size tensors are supported."): a.copy_(b) with self.assertRaisesRegex(RuntimeError, "copy between different layouts is not supported."): a.copy_(torch.empty(a.shape, dtype=dtype, device=device)) b = self.genSparseCSRTensor(shape1, 1, dtype=dtype, device=device, index_dtype=index_dtype) with self.assertRaisesRegex(RuntimeError, "only tensors with the same number of specified elements are supported."): a.copy_(b) @skipMeta @dtypes(*get_all_dtypes()) def test_resize(self, device, dtype): for index_dtype in [torch.int32, torch.int64]: shape = (2, 3) nnz = 6 a = self.genSparseCSRTensor(shape, nnz, dtype=dtype, device=device, index_dtype=index_dtype) new_shape = (4, 5) a.resize_(new_shape) self.assertEqual(a.shape, new_shape) # resize to larger shape doesn't add specified elements self.assertEqual(a._nnz(), nnz) new_shape = (1, 5) a.resize_(new_shape) self.assertEqual(a.shape, new_shape) # resize to smaller shape trims specified elements self.assertEqual(a._nnz(), 5) @skipMeta @dtypes(*get_all_dtypes()) def test_resize_errors(self, device, dtype): for index_dtype in [torch.int32, torch.int64]: shape = (2, 3) nnz = 6 a = self.genSparseCSRTensor(shape, nnz, dtype=dtype, device=device, index_dtype=index_dtype) with self.assertRaisesRegex(RuntimeError, "torch.resize_: Only 2D sparse CSR tensors are supported."): new_shape = (4,) a.resize_(new_shape) # resizing of columns to smaller size is not implemented with self.assertRaisesRegex( RuntimeError, "torch.resize_: Resizing columns of sparse CSR tensors to a smaller value is not supported.", ): new_shape = (2, 2) a.resize_(new_shape) def test_factory_type_invariants_check(self, device): with self.assertRaisesRegex(RuntimeError, "both crow_indices and col_indices should have the same type."): torch.sparse_csr_tensor(torch.tensor([0, 2, 4], dtype=torch.int64), torch.tensor([0, 1, 0, 1], dtype=torch.int32), torch.tensor([1, 2, 3, 4]), device=device) with self.assertRaisesRegex(RuntimeError, r"\"csr_construct_check\" not implemented for 'Short'"): torch.sparse_csr_tensor(torch.tensor([0, 2, 4], dtype=torch.int16), torch.tensor([0, 1, 0, 1], dtype=torch.int16), torch.tensor([1, 2, 3, 4]), device=device) def test_factory_layout_invariants_check(self, device): with self.assertRaisesRegex(RuntimeError, "expected values to be a strided and contiguous tensor"): values = torch.tensor([1.], device=device).expand(4,) torch.sparse_csr_tensor(torch.tensor([0, 2, 4], device=device), torch.tensor([0, 1, 0, 1], device=device), values) with self.assertRaisesRegex(RuntimeError, "expected col_indices to be a strided and contiguous tensor"): col_indices = torch.tensor([0], device=device).expand(4,) torch.sparse_csr_tensor(torch.tensor([0, 2, 4]), col_indices, torch.tensor([1, 2, 3, 4])) with self.assertRaisesRegex(RuntimeError, "expected crow_indices to be a strided and contiguous tensor"): crow_indices = torch.arange(6, device=device) torch.sparse_csr_tensor(crow_indices[::2], torch.tensor([0, 1, 0, 1], device=device), torch.tensor([1, 2, 3, 4])) def test_factory_shape_invariants_check(self, device): crow_indices = [0, 2, 4] col_indices = [0, 1, 0, 1] values = [1, 2, 3, 4] size = (2, 10) torch.sparse_csr_tensor(torch.tensor(crow_indices), torch.tensor(col_indices), torch.tensor(values), size, device=device) with self.assertRaisesRegex(RuntimeError, r"size of a CSR tensor must be of length 2, but got: 3"): torch.sparse_csr_tensor(torch.tensor(crow_indices), torch.tensor(col_indices), torch.tensor(values), size=(2, 10, 2), device=device) with self.assertRaisesRegex(RuntimeError, r"crow_indices must have dim\=1 but got crow_indices\.dim\(\)\=2"): torch.sparse_csr_tensor(torch.tensor(crow_indices).repeat(2, 1), torch.tensor(col_indices), torch.tensor(values), size, device=device) with self.assertRaisesRegex(RuntimeError, r"col_indices must have dim\=1 but got col_indices\.dim\(\)\=2"): torch.sparse_csr_tensor(torch.tensor(crow_indices), torch.tensor(col_indices).repeat(2, 1), torch.tensor(values), size, device=device) with self.assertRaisesRegex(RuntimeError, r"values must have dim\=1 but got values\.dim\(\)\=2"): torch.sparse_csr_tensor(torch.tensor(crow_indices), torch.tensor(col_indices), torch.tensor(values).repeat(2, 1), size, device=device) with self.assertRaisesRegex(RuntimeError, r"crow_indices\.numel\(\) must be size\(0\) \+ 1, but got: 3"): torch.sparse_csr_tensor(torch.tensor(crow_indices), torch.tensor(col_indices), torch.tensor(values), (1, 1), device=device) with self.assertRaisesRegex(RuntimeError, r"col_indices and values must have equal sizes, " + r"but got col_indices\.numel\(\): 3, values\.numel\(\): 4"): torch.sparse_csr_tensor(torch.tensor(crow_indices), torch.tensor([0, 1, 0]), torch.tensor(values), size, device=device) def test_factory_indices_invariants_check(self, device): crow_indices = [0, 2, 4] col_indices = [0, 1, 0, 1] values = [1, 2, 3, 4] size = (2, 10) with self.assertRaisesRegex(RuntimeError, "0th value of crow_indices must be 0."): torch.sparse_csr_tensor(torch.tensor([-1, 0, 4]), torch.tensor(col_indices), torch.tensor(values), size, device=device) with self.assertRaisesRegex(RuntimeError, "last value of crow_indices should be equal to the length of col_indices."): torch.sparse_csr_tensor(torch.tensor([0, 2, 5]), torch.tensor(col_indices), torch.tensor(values), size, device=device) with self.assertRaisesRegex(RuntimeError, r"at position i \= 2," + r" this condition crow_indices\[i - 1\] <\= crow_indices\[i\] fails"): torch.sparse_csr_tensor(torch.tensor([0, 5, 4]), torch.tensor(col_indices), torch.tensor(values), size, device=device) with self.assertRaisesRegex(RuntimeError, r"col_indices\.min\(\) should be greater or equal to zero"): torch.sparse_csr_tensor(torch.tensor(crow_indices), torch.tensor([0, -1, 0, 1]), torch.tensor(values), size, device=device) with self.assertRaisesRegex(RuntimeError, r"size\(1\) should be greater than col_indices\.max\(\)"): torch.sparse_csr_tensor(torch.tensor(crow_indices), torch.tensor([0, 11, 0, 1]), torch.tensor(values), size, device=device) @onlyCUDA @dtypes(*get_all_dtypes()) def test_factory_device_type_inference(self, device, dtype): cpu_cuda = ('cpu', 'cuda') cpu_cuda_none = cpu_cuda + (None,) for crow_indices_device, col_indices_device, values_device, device in itertools.product(cpu_cuda, cpu_cuda, cpu_cuda, cpu_cuda_none): for index_dtype in [torch.int32, torch.int64]: crow_indices = torch.tensor([0, 2, 4], dtype=index_dtype, device=crow_indices_device) col_indices = torch.tensor([0, 1, 0, 1], dtype=index_dtype, device=col_indices_device) values = torch.tensor([1, 2, 3, 4], dtype=dtype, device=values_device) if device is None and (crow_indices_device != col_indices_device or crow_indices_device != values_device): with self.assertRaises(RuntimeError): torch.sparse_csr_tensor(crow_indices, col_indices, values, size=(2, 10), device=device) else: t = torch.sparse_csr_tensor(crow_indices, col_indices, values, size=(2, 10), device=device) should_be_cuda = (device == 'cuda' or (device is None and values_device == 'cuda')) self.assertEqual(should_be_cuda, t.is_cuda) t.crow_indices().dtype == index_dtype t.col_indices().dtype == index_dtype t.values().dtype == dtype t.crow_indices().device == t.values().device t.col_indices().device == t.values().device def test_sparse_csr_print(self, device): orig_maxDiff = self.maxDiff self.maxDiff = None shape_nnz = [ ((10, 10), 10), ((100, 10), 10), ((1000, 10), 10) ] printed = [] for shape, nnz in shape_nnz: values_shape = torch.Size((nnz,)) col_indices_shape = torch.Size((nnz,)) crow_indices_shape = torch.Size((shape[0] + 1,)) printed.append("# shape: {}".format(torch.Size(shape))) printed.append("# nnz: {}".format(nnz)) printed.append("# crow_indices shape: {}".format(crow_indices_shape)) printed.append("# col_indices shape: {}".format(col_indices_shape)) printed.append("# values_shape: {}".format(values_shape)) for index_dtype in [torch.int32, torch.int64]: for dtype in floating_types(): printed.append("########## {}/{} ##########".format(dtype, index_dtype)) x = torch.sparse_csr_tensor(torch.tensor([0, 2, 4], dtype=index_dtype), torch.tensor([0, 1, 0, 1], dtype=index_dtype), torch.tensor([1, 2, 3, 4]), dtype=dtype, device=device) printed.append("# sparse tensor") printed.append(str(x)) printed.append("# _crow_indices") printed.append(str(x.crow_indices())) printed.append("# _col_indices") printed.append(str(x.col_indices())) printed.append("# _values") printed.append(str(x.values())) printed.append('') printed.append('') self.assertExpected('\n'.join(printed)) self.maxDiff = orig_maxDiff @dtypes(*get_all_dtypes()) def test_sparse_csr_from_dense(self, device, dtype): dense = torch.tensor([[4, 5, 0], [0, 0, 0], [1, 0, 0]], dtype=dtype, device=device) sparse = dense.to_sparse_csr() self.assertEqual(torch.tensor([0, 2, 2, 3], dtype=torch.int64), sparse.crow_indices()) self.assertEqual(torch.tensor([0, 1, 0], dtype=torch.int64), sparse.col_indices()) self.assertEqual(torch.tensor([4, 5, 1], dtype=dtype), sparse.values()) dense = torch.tensor([[0, 0, 0], [0, 0, 1], [1, 0, 0]], dtype=dtype, device=device) sparse = dense.to_sparse_csr() self.assertEqual(torch.tensor([0, 0, 1, 2], dtype=torch.int64), sparse.crow_indices()) self.assertEqual(torch.tensor([2, 0], dtype=torch.int64), sparse.col_indices()) self.assertEqual(torch.tensor([1, 1], dtype=dtype), sparse.values()) dense = torch.tensor([[2, 2, 2], [2, 2, 2], [2, 2, 2]], dtype=dtype, device=device) sparse = dense.to_sparse_csr() self.assertEqual(torch.tensor([0, 3, 6, 9], dtype=torch.int64), sparse.crow_indices()) self.assertEqual(torch.tensor([0, 1, 2] * 3, dtype=torch.int64), sparse.col_indices()) self.assertEqual(torch.tensor([2] * 9, dtype=dtype), sparse.values()) @dtypes(*get_all_dtypes()) def test_sparse_csr_to_dense(self, device, dtype): mn = [5, 2, 0] for (m, n) in itertools.product(mn, mn): size = (m, n) dense = make_tensor(size, dtype=dtype, device=device) sparse = dense.to_sparse_csr() self.assertEqual(sparse.to_dense(), dense) crow_indices = torch.tensor([0, 3, 5]) col_indices = torch.tensor([0, 1, 2, 0, 1]) values = torch.tensor([1, 2, 1, 3, 4], dtype=dtype) csr = torch.sparse_csr_tensor(crow_indices, col_indices, values, dtype=dtype, device=device) dense = torch.tensor([[1, 2, 1], [3, 4, 0]], dtype=dtype, device=device) self.assertEqual(csr.to_dense(), dense) @skipCPUIfNoMklSparse @coalescedonoff @dtypes(torch.double) def test_coo_to_csr_convert(self, device, dtype, coalesced): with self.assertRaisesRegex(RuntimeError, "Input is supposed to be a vector"): torch._convert_indices_from_coo_to_csr( torch.randint(100, (5, 5), device=device), size=100) size = (5, 5) sparse_dim = 2 nnz = 10 sparse_coo, _, _ = self.genSparseTensor(size, sparse_dim, nnz, coalesced, device, dtype) sparse_csr = sparse_coo.to_sparse_csr() self.assertTrue(sparse_csr.is_sparse_csr) self.assertEqual(sparse_csr.to_dense(), sparse_coo.to_dense()) vec = torch.randn((5, 1), dtype=dtype, device=device) coo_product = sparse_coo.matmul(vec) csr_product = sparse_csr.matmul(vec) self.assertEqual(coo_product, csr_product) vec = torch.randn((100, 1), dtype=dtype, device=device) index = torch.tensor([ [1, 0, 35, 14, 39, 6, 71, 66, 40, 27], [92, 31, 62, 50, 22, 65, 89, 74, 56, 34], ], dtype=torch.int32) values = torch.tensor([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], dtype=dtype, device=device) coo = torch.sparse_coo_tensor(index, values, torch.Size([100, 100]), dtype=dtype, device=device) csr = coo.to_sparse_csr() self.assertEqual(coo.matmul(vec), csr.matmul(vec)) col_indices = torch.tensor([ 31, 92, 65, 50, 34, 62, 22, 56, 74, 89 ], dtype=torch.int64, device=device) self.assertEqual(csr.col_indices(), col_indices) values = torch.tensor([2, 1, 6, 4, 10, 3, 5, 9, 8, 7], dtype=dtype, device=device) self.assertEqual(csr.values(), values) @dtypes(*get_all_dtypes()) def test_sparse_csr_from_dense_convert_error(self, device, dtype): size = (4, 2, 4) dense = make_tensor(size, dtype=dtype, device=device) with self.assertRaisesRegex(RuntimeError, "Only 2D"): sparse = dense.to_sparse_csr() # TODO: Support auto generation of device check for sparse tensors # See: https://github.com/pytorch/pytorch/issues/59058 @onlyCUDA @dtypes(torch.double) def test_matmul_device_mismatch(self, device, dtype): cpu = torch.rand((10, 10)) cuda = cpu.cuda() for s, m1, m2 in itertools.product((cpu, cuda), repeat=3): csr = m1.to_sparse() if s.device == csr.device == m2.device: torch.addmm(s, csr, m2) else: with self.assertRaisesRegex(RuntimeError, "Expected all tensors to be on the same device"): torch.addmm(s, csr, m2) @skipCPUIfNoMklSparse @skipCUDAIfNoCusparseGeneric @dtypes(*floating_and_complex_types()) @dtypesIfCUDA(*get_all_complex_dtypes(), *get_all_fp_dtypes(include_half=SM53OrLater, include_bfloat16=SM80OrLater)) def test_csr_matvec(self, device, dtype): side = 100 for index_dtype in [torch.int32, torch.int64]: csr = self.genSparseCSRTensor((side, side), 1000, device=device, dtype=dtype, index_dtype=index_dtype) vec = torch.randn(side, dtype=dtype, device=device) res = csr.matmul(vec) expected = csr.to_dense().matmul(vec) self.assertEqual(res, expected) bad_vec = torch.randn(side + 10, dtype=dtype, device=device) err_msg = "size mismatch, got" with self.assertRaisesRegex(RuntimeError, err_msg): csr.matmul(bad_vec) @skipCPUIfNoMklSparse @dtypes(torch.double) def test_mm(self, device, dtype): def test_shape(di, dj, dk, nnz): for index_dtype in [torch.int32, torch.int64]: x = self.genSparseCSRTensor((di, dj), nnz, device=device, dtype=dtype, index_dtype=index_dtype) t = torch.randn(di, dk, dtype=dtype, device=device) y = torch.randn(dj, dk, dtype=dtype, device=device) alpha = random.random() beta = random.random() # res = beta * t + alpha * (x @ y) res = torch.addmm(t, x, y, beta=beta, alpha=alpha) expected = torch.addmm(t, x.to_dense(), y, beta=beta, alpha=alpha) self.assertEqual(res, expected) res = torch.addmm(t, x, y) expected = torch.addmm(t, x.to_dense(), y) self.assertEqual(res, expected) res = torch.mm(x, y) expected = torch.mm(x.to_dense(), y) self.assertEqual(res, expected) for i in range(2, 5): for j in range(2, 8): for k in range(2, 8): test_shape(i, j, k, i * j // 2) test_shape(4, 4, 4, 0) @skipCPUIfNoMklSparse @dtypes(*floating_and_complex_types()) @dtypesIfCUDA(*get_all_complex_dtypes(), *get_all_fp_dtypes(include_half=SM53OrLater and TEST_CUSPARSE_GENERIC, include_bfloat16=SM80OrLater and TEST_CUSPARSE_GENERIC)) @precisionOverride({torch.bfloat16: 1e-2, torch.float16: 1e-2}) def test_sparse_mm(self, device, dtype): def test_shape(d1, d2, d3, nnz, transposed, index_dtype): if transposed: D = torch.randn(d3, d2, dtype=dtype, device=device).t_() else: D = torch.randn(d2, d3, dtype=dtype, device=device) S = self.genSparseCSRTensor((d1, d2), nnz, device=device, dtype=dtype, index_dtype=index_dtype) S_dense = S.to_dense() self.assertEqual(torch.sparse.mm(S, D), torch.mm(S_dense, D)) for index_dtype in [torch.int32, torch.int64]: test_shape(7, 8, 9, 20, False, index_dtype) test_shape(7, 8, 9, 20, True, index_dtype) @skipCPUIfNoMklSparse @dtypes(*floating_and_complex_types()) @dtypesIfCUDA(*get_all_complex_dtypes(), *get_all_fp_dtypes(include_half=SM53OrLater and TEST_CUSPARSE_GENERIC, include_bfloat16=SM80OrLater and TEST_CUSPARSE_GENERIC)) @precisionOverride({torch.bfloat16: 1e-2, torch.float16: 1e-2}) def test_sparse_addmm(self, device, dtype): def test_shape(m, n, p, nnz, broadcast, index_dtype, alpha_beta=None): if alpha_beta is None: alpha = random.random() beta = random.random() else: alpha, beta = alpha_beta if broadcast: D1 = make_tensor((), dtype=dtype, device=device) else: D1 = make_tensor([n, p], dtype=dtype, device=device) D2 = make_tensor([m, p], dtype=dtype, device=device) S = self.genSparseCSRTensor([n, m], nnz, dtype=dtype, device=device, index_dtype=index_dtype) S_dense = S.to_dense() Y = torch.sparse.addmm(D1, S, D2, beta=beta, alpha=alpha) Y_dense = torch.addmm(D1, S_dense, D2, beta=beta, alpha=alpha) self.assertEqual(Y, Y_dense) for index_dtype in [torch.int32, torch.int64]: test_shape(7, 8, 9, 20, False, index_dtype, None) test_shape(7, 8, 9, 20, True, index_dtype, None) test_shape(7, 8, 9, 20, False, index_dtype, (1, 0)) test_shape(7, 8, 9, 20, True, index_dtype, (1, 0)) test_shape(7, 8, 9, 20, False, index_dtype, (1, 1)) test_shape(7, 8, 9, 20, True, index_dtype, (1, 1)) @skipCPUIfNoMklSparse @dtypes(*floating_and_complex_types()) @precisionOverride({torch.double: 1e-8, torch.float: 1e-4, torch.bfloat16: 0.6, torch.half: 1e-1, torch.cfloat: 1e-4, torch.cdouble: 1e-8}) @dtypesIfCUDA(torch.complex64, *((torch.complex128,) if CUSPARSE_SPMM_COMPLEX128_SUPPORTED else ()), *torch.testing.get_all_fp_dtypes(include_bfloat16=SM80OrLater, include_half=SM53OrLater)) @skipCUDAIf( not _check_cusparse_spgemm_available(), "cuSparse Generic API SpGEMM is not available" ) def test_addmm_all_sparse_csr(self, device, dtype): M = torch.randn(10, 25, device=device).to(dtype) m1 = torch.randn(10, 50, device=device).to(dtype) m2 = torch.randn(50, 25, device=device).to(dtype) _test_addmm_addmv(self, torch.addmm, M, m1, m2, layout=torch.sparse_csr, all_sparse=True) # Test 0-strided M = torch.randn(10, 1, device=device).to(dtype).expand(10, 25) m1 = torch.randn(10, 1, device=device).to(dtype).expand(10, 50) m2 = torch.randn(50, 25, device=device).to(dtype) _test_addmm_addmv(self, torch.addmm, M, m1, m2, layout=torch.sparse_csr, all_sparse=True) # Test beta=0, M=nan M = torch.full((10, 25), float('nan'), device=device).to(dtype) m1 = torch.randn(10, 50, device=device).to(dtype) m2 = torch.randn(50, 25, device=device).to(dtype) _test_addmm_addmv(self, torch.addmm, M, m1, m2, beta=0, layout=torch.sparse_csr, all_sparse=True) # Test transpose for t1, t2, t3, t4 in itertools.product([True, False], repeat=4): def maybe_transpose(cond, m): if not cond: return m return m.t().clone(memory_format=torch.contiguous_format).t() M = maybe_transpose(t1, torch.randn(10, 25, device=device).to(dtype)) m1 = maybe_transpose(t2, torch.randn(10, 50, device=device).to(dtype)) m2 = maybe_transpose(t3, torch.randn(50, 25, device=device).to(dtype)) _test_addmm_addmv(self, torch.addmm, M, m1, m2, transpose_out=t4, layout=torch.sparse_csr, all_sparse=True) @skipCPUIfNoMklSparse @dtypes(*floating_and_complex_types()) @dtypesIfCUDA(torch.complex64, *((torch.complex128,) if CUSPARSE_SPMM_COMPLEX128_SUPPORTED else ()), *torch.testing.get_all_fp_dtypes(include_bfloat16=SM80OrLater, include_half=SM53OrLater)) @skipCUDAIf( not _check_cusparse_spgemm_available(), "cuSparse Generic API SpGEMM is not available" ) @precisionOverride({torch.double: 1e-8, torch.float: 1e-4, torch.bfloat16: 0.6, torch.half: 1e-1, torch.cfloat: 1e-4, torch.cdouble: 1e-8}) def test_addmm_sizes_all_sparse_csr(self, device, dtype): for m in [0, 1, 25]: for n in [0, 1, 10]: for k in [0, 1, 8]: M = torch.randn(n, m, device=device).to(dtype) m1 = torch.randn(n, k, device=device).to(dtype) m2 = torch.randn(k, m, device=device).to(dtype) _test_addmm_addmv(self, torch.addmm, M, m1, m2, layout=torch.sparse_csr, all_sparse=True) M = torch.randn(n, m, device=device).to(dtype).to_sparse_csr() m1 = torch.randn(n, k + 1, device=device).to(dtype).to_sparse_csr() m2 = torch.randn(k, m, device=device).to(dtype).to_sparse_csr() self.assertRaisesRegex(RuntimeError, f"{n}x{k + 1}.*{k}x{m}", lambda: torch.addmm(M, m1, m2)) self.assertRaisesRegex(RuntimeError, f"{n}x{k + 1}.*{k}x{m}", lambda: torch.mm(m1, m2)) @skipCPUIfNoMklSparse @dtypes(torch.float) def test_addmm_errors(self, device, dtype): # test that the errors are the same for dense and sparse versions import re def test1(*, is_sparse): # shapes must be compatible for matrix multiplication a = make_tensor((2, 3), dtype=dtype, device=device) if is_sparse: a_sparse = a.to_sparse_csr() return torch.addmm(a, a_sparse, a) else: return torch.addmm(a, a, a) def test2(*, is_sparse): # mat2 must be a matrix a = make_tensor((2, 3), dtype=dtype, device=device) if is_sparse: a_sparse = a.to_sparse_csr() return torch.addmm(a, a_sparse, a.unsqueeze(0)) else: return torch.addmm(a, a, a.unsqueeze(0)) def test3(*, is_sparse): # the first input needs to be 1D or 2D a = make_tensor((3, 3), dtype=dtype, device=device) if is_sparse: a_sparse = a.to_sparse_csr() return torch.addmm(a.unsqueeze(0), a_sparse, a) else: return torch.addmm(a.unsqueeze(0), a, a) for test in (test1, test2, test3): try: test(is_sparse=False) except RuntimeError as msg: with self.assertRaisesRegex(RuntimeError, re.escape(str(msg))): test(is_sparse=True) @skipCPUIfNoMklSparse @dtypes(torch.float) def test_mm_errors(self, device, dtype): # test that the errors are the same for dense and sparse versions import re def test1(*, is_sparse): # shapes must be compatible for matrix multiplication a = make_tensor((2, 3), dtype=dtype, device=device) if is_sparse: a_sparse = a.to_sparse_csr() return torch.mm(a_sparse, a) else: return torch.mm(a, a) def test2(*, is_sparse): # mat2 must be a matrix a = make_tensor((2, 3), dtype=dtype, device=device) if is_sparse: a_sparse = a.to_sparse_csr() return torch.mm(a_sparse, a.unsqueeze(0)) else: return torch.mm(a, a.unsqueeze(0)) for test in (test1, test2): try: test(is_sparse=False) except RuntimeError as msg: with self.assertRaisesRegex(RuntimeError, re.escape(str(msg))): test(is_sparse=True) @dtypes(torch.float, torch.double) def test_add(self, device, dtype): def _test_spadd_shape(nnz, shape): x = self.genSparseCSRTensor(shape, nnz, dtype=dtype, device=device, index_dtype=torch.int32) y = torch.randn(*shape, dtype=dtype, device=device) r = random.random() res = torch.add(y, x, alpha=r) expected = y + r * x.to_dense() self.assertEqual(res, expected) # Non contiguous dense tensor s = list(shape) s[0] = shape[-1] s[-1] = shape[0] y = torch.randn(*s, dtype=torch.double, device=device) y.transpose_(0, len(s) - 1) r = random.random() res = torch.add(y, x, alpha=r) expected = y + r * x.to_dense() self.assertEqual(res, expected) _test_spadd_shape(10, [100, 100]) _test_spadd_shape(0, [100, 100]) _test_spadd_shape(10, [100, 1]) _test_spadd_shape(10, [1, 100]) @skipCPUIfNoMklSparse @dtypes(torch.float32, torch.float64, torch.complex64, torch.complex128) def test_sparse_add(self, device, dtype): def run_test(m, n, index_dtype): if TEST_WITH_ROCM and dtype.is_complex: self.skipTest("ROCm doesn't work with complex dtype correctly.") alpha = random.random() nnz1 = random.randint(0, m * n) nnz2 = random.randint(0, m * n) nnz3 = random.randint(0, m * n) if TEST_WITH_ROCM: # ROCm fails when nnz = 0 nnz1, nnz2, nnz3 = max(1, nnz1), max(1, nnz2), max(1, nnz3) S1 = self.genSparseCSRTensor([m, n], nnz1, dtype=dtype, device=device, index_dtype=index_dtype) S2 = self.genSparseCSRTensor([m, n], nnz2, dtype=dtype, device=device, index_dtype=index_dtype) S3 = self.genSparseCSRTensor([m, n], nnz3, dtype=dtype, device=device, index_dtype=index_dtype) expected = torch.add(S1.to_dense(), S2.to_dense(), alpha=alpha) actual = torch.add(S1, S2, alpha=alpha, out=S3) self.assertEqual(actual.to_dense(), expected) self.assertEqual(S3.to_dense(), expected) for index_dtype in [torch.int32, torch.int64]: for m, n in itertools.product([3, 5], [3, 5]): run_test(m, n, index_dtype) @dtypes(torch.float32, torch.float64, torch.complex64, torch.complex128) def test_sparse_add_errors(self, device, dtype): def run_test(index_type): a = self.genSparseCSRTensor((2, 2), 3, dtype=dtype, device=device, index_dtype=index_dtype) b = self.genSparseCSRTensor((2, 1), 2, dtype=dtype, device=device, index_dtype=index_dtype) with self.assertRaisesRegex(RuntimeError, "Expected input tensors to have the same shape"): torch.add(a, b) for index_dtype in [torch.int32, torch.int64]: run_test(index_dtype) @onlyCUDA @skipCUDAIf( not _check_cusparse_triangular_solve_available(), "cuSparse Generic API SpSV is not available" ) @dtypes(torch.float32, torch.float64, torch.complex64, torch.complex128) @precisionOverride({torch.float32: 1e-3, torch.complex64: 1e-3, torch.float64: 1e-8, torch.complex128: 1e-8}) def test_sparse_triangular_solve(self, device, dtype): def run_test(n, k, upper, unitriangular, transpose): triangle_function = torch.triu if upper else torch.tril A = make_tensor((n, n), dtype=dtype, device=device) A = triangle_function(A) A_sparse = A.to_sparse_csr() B = make_tensor((n, k), dtype=dtype, device=device) expected = torch.triangular_solve(B, A, upper=upper, unitriangular=unitriangular, transpose=transpose) expected_X = expected.solution actual = torch.triangular_solve(B, A_sparse, upper=upper, unitriangular=unitriangular, transpose=transpose) actual_X = actual.solution actual_A_clone = actual.cloned_coefficient self.assertTrue(actual_A_clone.numel() == 0) self.assertEqual(actual_X, expected_X) # test out with C contiguous strides out = torch.empty_strided((n, k), (k, 1), dtype=dtype, device=device) torch.triangular_solve( B, A_sparse, upper=upper, unitriangular=unitriangular, transpose=transpose, out=(out, actual_A_clone) ) self.assertEqual(out, expected_X) # test out with F contiguous strides out = torch.empty_strided((n, k), (1, n), dtype=dtype, device=device) torch.triangular_solve( B, A_sparse, upper=upper, unitriangular=unitriangular, transpose=transpose, out=(out, actual_A_clone) ) self.assertEqual(out, expected_X) self.assertEqual(out.stride(), (1, n)) # test out with discontiguous strides out = torch.empty_strided((2 * n, k), (1, 2 * n), dtype=dtype, device=device)[::2] if n > 0 and k > 0: self.assertFalse(out.is_contiguous()) self.assertFalse(out.t().is_contiguous()) before_stride = out.stride() torch.triangular_solve( B, A_sparse, upper=upper, unitriangular=unitriangular, transpose=transpose, out=(out, actual_A_clone) ) self.assertEqual(out, expected_X) self.assertEqual(out.stride(), before_stride) ks = [0, 1, 3] ns = [5, 3, 0] for (k, n), (upper, unitriangular, transpose) in itertools.product(itertools.product(ks, ns), itertools.product([True, False], repeat=3)): run_test(n, k, upper, unitriangular, transpose) @dtypes(*get_all_dtypes()) def test_coo_csr_conversion(self, device, dtype): for m, n in itertools.product([5, 2, 0], [5, 2, 0]): size = (m, n) dense = make_tensor(size, dtype=dtype, device=device) coo_sparse = dense.to_sparse() csr_sparse = coo_sparse.to_sparse_csr() self.assertEqual(csr_sparse.to_dense(), dense)
class TestSparseCSR(TestCase): @onlyCPU def test_csr_layout(self): self.assertEqual(str(torch.sparse_csr), 'torch.sparse_csr') self.assertEqual(type(torch.sparse_csr), torch.layout) @dtypes(*get_all_dtypes()) def test_sparse_csr_constructor_shape_inference(self, device, dtype): crow_indices = [0, 2, 4] col_indices = [0, 1, 0, 1] values = [1, 2, 3, 4] sparse = torch.sparse_csr_tensor(torch.tensor(crow_indices, dtype=torch.int64), torch.tensor(col_indices, dtype=torch.int64), torch.tensor(values), dtype=dtype, device=device) self.assertEqual(torch.tensor(crow_indices, dtype=torch.int64), sparse.crow_indices()) self.assertEqual((len(crow_indices) - 1, max(col_indices) + 1), sparse.shape) self.assertEqual(dtype, sparse.dtype) self.assertEqual(torch.device(device), sparse.device) @dtypes(*get_all_dtypes()) def test_sparse_csr_constructor(self, device, dtype): crow_indices = [0, 2, 4] col_indices = [0, 1, 0, 1] values = [1, 2, 3, 4] for index_dtype in [torch.int32, torch.int64]: sparse = torch.sparse_csr_tensor(torch.tensor(crow_indices, dtype=index_dtype), torch.tensor(col_indices, dtype=index_dtype), torch.tensor(values), size=(2, 10), dtype=dtype, device=device) self.assertEqual((2, 10), sparse.shape) self.assertEqual(torch.tensor(crow_indices, dtype=index_dtype), sparse.crow_indices()) self.assertEqual(torch.tensor(col_indices, dtype=index_dtype), sparse.col_indices()) self.assertEqual(torch.tensor(values, dtype=dtype), sparse.values()) @dtypes(*get_all_dtypes()) def test_sparse_csr_constructor_from_lists(self, device, dtype): # without size sparse = torch.sparse_csr_tensor([0, 2, 4], [0, 1, 0, 1], [1, 2, 3, 4], dtype=dtype, device=device) self.assertEqual((2, 2), sparse.shape) self.assertEqual(4, sparse.numel()) self.assertEqual( torch.tensor([0, 2, 4], dtype=torch.int64, device=device), sparse.crow_indices()) self.assertEqual( torch.tensor([0, 1, 0, 1], dtype=torch.int64, device=device), sparse.col_indices()) self.assertEqual( torch.tensor([1, 2, 3, 4], dtype=dtype, device=device), sparse.values()) # with size for sparse_csr_tensor in [ torch.sparse_csr_tensor, torch._sparse_csr_tensor_unsafe ]: sparse = sparse_csr_tensor([0, 2, 4], [0, 1, 0, 1], [1, 2, 3, 4], size=(2, 10), dtype=dtype, device=device) self.assertEqual((2, 10), sparse.shape) self.assertEqual( torch.tensor([0, 2, 4], dtype=torch.int64, device=device), sparse.crow_indices()) self.assertEqual( torch.tensor([0, 1, 0, 1], dtype=torch.int64, device=device), sparse.col_indices()) self.assertEqual( torch.tensor([1, 2, 3, 4], dtype=dtype, device=device), sparse.values()) def test_factory_type_invariants_check(self, device): with self.assertRaisesRegex( RuntimeError, "both crow_indices and col_indices should have the same type." ): torch.sparse_csr_tensor(torch.tensor([0, 2, 4], dtype=torch.int64), torch.tensor([0, 1, 0, 1], dtype=torch.int32), torch.tensor([1, 2, 3, 4]), device=device) with self.assertRaisesRegex( RuntimeError, r"\"csr_construct_check\" not implemented for 'Short'"): torch.sparse_csr_tensor(torch.tensor([0, 2, 4], dtype=torch.int16), torch.tensor([0, 1, 0, 1], dtype=torch.int16), torch.tensor([1, 2, 3, 4]), device=device) def test_factory_layout_invariants_check(self, device): with self.assertRaisesRegex( RuntimeError, "expected values to be a strided and contiguous tensor"): values = torch.tensor([1.], device=device).expand(4, ) torch.sparse_csr_tensor(torch.tensor([0, 2, 4], device=device), torch.tensor([0, 1, 0, 1], device=device), values) with self.assertRaisesRegex( RuntimeError, "expected col_indices to be a strided and contiguous tensor"): col_indices = torch.tensor([0], device=device).expand(4, ) torch.sparse_csr_tensor(torch.tensor([0, 2, 4]), col_indices, torch.tensor([1, 2, 3, 4])) with self.assertRaisesRegex( RuntimeError, "expected crow_indices to be a strided and contiguous tensor"): crow_indices = torch.arange(6, device=device) torch.sparse_csr_tensor(crow_indices[::2], torch.tensor([0, 1, 0, 1], device=device), torch.tensor([1, 2, 3, 4])) def test_factory_shape_invariants_check(self, device): crow_indices = [0, 2, 4] col_indices = [0, 1, 0, 1] values = [1, 2, 3, 4] size = (2, 10) torch.sparse_csr_tensor(torch.tensor(crow_indices), torch.tensor(col_indices), torch.tensor(values), size, device=device) with self.assertRaisesRegex( RuntimeError, r"size of a CSR tensor must be of length 2, but got: 3"): torch.sparse_csr_tensor(torch.tensor(crow_indices), torch.tensor(col_indices), torch.tensor(values), size=(2, 10, 2), device=device) with self.assertRaisesRegex( RuntimeError, r"crow_indices must have dim\=1 but got crow_indices\.dim\(\)\=2" ): torch.sparse_csr_tensor(torch.tensor(crow_indices).repeat(2, 1), torch.tensor(col_indices), torch.tensor(values), size, device=device) with self.assertRaisesRegex( RuntimeError, r"col_indices must have dim\=1 but got col_indices\.dim\(\)\=2" ): torch.sparse_csr_tensor(torch.tensor(crow_indices), torch.tensor(col_indices).repeat(2, 1), torch.tensor(values), size, device=device) with self.assertRaisesRegex( RuntimeError, r"values must have dim\=1 but got values\.dim\(\)\=2"): torch.sparse_csr_tensor(torch.tensor(crow_indices), torch.tensor(col_indices), torch.tensor(values).repeat(2, 1), size, device=device) with self.assertRaisesRegex( RuntimeError, r"crow_indices\.numel\(\) must be size\(0\) \+ 1, but got: 3"): torch.sparse_csr_tensor(torch.tensor(crow_indices), torch.tensor(col_indices), torch.tensor(values), (1, 1), device=device) with self.assertRaisesRegex( RuntimeError, r"col_indices and values must have equal sizes, " + r"but got col_indices\.numel\(\): 3, values\.numel\(\): 4"): torch.sparse_csr_tensor(torch.tensor(crow_indices), torch.tensor([0, 1, 0]), torch.tensor(values), size, device=device) def test_factory_indices_invariants_check(self, device): crow_indices = [0, 2, 4] col_indices = [0, 1, 0, 1] values = [1, 2, 3, 4] size = (2, 10) with self.assertRaisesRegex(RuntimeError, "0th value of crow_indices must be 0."): torch.sparse_csr_tensor(torch.tensor([-1, 0, 4]), torch.tensor(col_indices), torch.tensor(values), size, device=device) with self.assertRaisesRegex( RuntimeError, "last value of crow_indices should be equal to the length of col_indices." ): torch.sparse_csr_tensor(torch.tensor([0, 2, 5]), torch.tensor(col_indices), torch.tensor(values), size, device=device) with self.assertRaisesRegex( RuntimeError, r"at position i \= 2," + r" this condition crow_indices\[i - 1\] <\= crow_indices\[i\] fails" ): torch.sparse_csr_tensor(torch.tensor([0, 5, 4]), torch.tensor(col_indices), torch.tensor(values), size, device=device) with self.assertRaisesRegex( RuntimeError, r"col_indices\.min\(\) should be greater or equal to zero"): torch.sparse_csr_tensor(torch.tensor(crow_indices), torch.tensor([0, -1, 0, 1]), torch.tensor(values), size, device=device) with self.assertRaisesRegex( RuntimeError, r"size\(1\) should be greater than col_indices\.max\(\)"): torch.sparse_csr_tensor(torch.tensor(crow_indices), torch.tensor([0, 11, 0, 1]), torch.tensor(values), size, device=device) @onlyCUDA @dtypes(*get_all_dtypes()) def test_factory_device_type_inference(self, device, dtype): cpu_cuda = ('cpu', 'cuda') cpu_cuda_none = cpu_cuda + (None, ) for crow_indices_device, col_indices_device, values_device, device in itertools.product( cpu_cuda, cpu_cuda, cpu_cuda, cpu_cuda_none): for index_dtype in [torch.int32, torch.int64]: crow_indices = torch.tensor([0, 2, 4], dtype=index_dtype, device=crow_indices_device) col_indices = torch.tensor([0, 1, 0, 1], dtype=index_dtype, device=col_indices_device) values = torch.tensor([1, 2, 3, 4], dtype=dtype, device=values_device) if device is None and ( crow_indices_device != col_indices_device or crow_indices_device != values_device): with self.assertRaises(RuntimeError): torch.sparse_csr_tensor(crow_indices, col_indices, values, size=(2, 10), device=device) else: t = torch.sparse_csr_tensor(crow_indices, col_indices, values, size=(2, 10), device=device) should_be_cuda = (device == 'cuda' or (device is None and values_device == 'cuda')) self.assertEqual(should_be_cuda, t.is_cuda) t.crow_indices().dtype == index_dtype t.col_indices().dtype == index_dtype t.values().dtype == dtype t.crow_indices().device == t.values().device t.col_indices().device == t.values().device def test_sparse_csr_print(self, device): orig_maxDiff = self.maxDiff self.maxDiff = None shape_nnz = [((10, 10), 10), ((100, 10), 10), ((1000, 10), 10)] printed = [] for shape, nnz in shape_nnz: values_shape = torch.Size((nnz, )) col_indices_shape = torch.Size((nnz, )) crow_indices_shape = torch.Size((shape[0] + 1, )) printed.append("# shape: {}".format(torch.Size(shape))) printed.append("# nnz: {}".format(nnz)) printed.append( "# crow_indices shape: {}".format(crow_indices_shape)) printed.append("# col_indices shape: {}".format(col_indices_shape)) printed.append("# values_shape: {}".format(values_shape)) for index_dtype in [torch.int32, torch.int64]: for dtype in floating_types(): printed.append("########## {}/{} ##########".format( dtype, index_dtype)) x = torch.sparse_csr_tensor( torch.tensor([0, 2, 4], dtype=index_dtype), torch.tensor([0, 1, 0, 1], dtype=index_dtype), torch.tensor([1, 2, 3, 4]), dtype=dtype, device=device) printed.append("# sparse tensor") printed.append(str(x)) printed.append("# _crow_indices") printed.append(str(x.crow_indices())) printed.append("# _col_indices") printed.append(str(x.col_indices())) printed.append("# _values") printed.append(str(x.values())) printed.append('') printed.append('') self.assertExpected('\n'.join(printed)) self.maxDiff = orig_maxDiff @dtypes(*get_all_dtypes()) def test_sparse_csr_from_dense(self, device, dtype): dense = torch.tensor([[4, 5, 0], [0, 0, 0], [1, 0, 0]], dtype=dtype, device=device) sparse = dense.to_sparse_csr() self.assertEqual(torch.tensor([0, 2, 2, 3], dtype=torch.int64), sparse.crow_indices()) self.assertEqual(torch.tensor([0, 1, 0], dtype=torch.int64), sparse.col_indices()) self.assertEqual(torch.tensor([4, 5, 1], dtype=dtype), sparse.values()) dense = torch.tensor([[0, 0, 0], [0, 0, 1], [1, 0, 0]], dtype=dtype, device=device) sparse = dense.to_sparse_csr() self.assertEqual(torch.tensor([0, 0, 1, 2], dtype=torch.int64), sparse.crow_indices()) self.assertEqual(torch.tensor([2, 0], dtype=torch.int64), sparse.col_indices()) self.assertEqual(torch.tensor([1, 1], dtype=dtype), sparse.values()) dense = torch.tensor([[2, 2, 2], [2, 2, 2], [2, 2, 2]], dtype=dtype, device=device) sparse = dense.to_sparse_csr() self.assertEqual(torch.tensor([0, 3, 6, 9], dtype=torch.int64), sparse.crow_indices()) self.assertEqual(torch.tensor([0, 1, 2] * 3, dtype=torch.int64), sparse.col_indices()) self.assertEqual(torch.tensor([2] * 9, dtype=dtype), sparse.values()) @dtypes(*get_all_dtypes()) def test_sparse_csr_to_dense(self, device, dtype): mn = [5, 2, 0] for (m, n) in itertools.product(mn, mn): size = (m, n) dense = make_tensor(size, dtype=dtype, device=device) sparse = dense.to_sparse_csr() self.assertEqual(sparse.to_dense(), dense) crow_indices = torch.tensor([0, 3, 5]) col_indices = torch.tensor([0, 1, 2, 0, 1]) values = torch.tensor([1, 2, 1, 3, 4], dtype=dtype) csr = torch.sparse_csr_tensor(crow_indices, col_indices, values, dtype=dtype, device=device) dense = torch.tensor([[1, 2, 1], [3, 4, 0]], dtype=dtype, device=device) self.assertEqual(csr.to_dense(), dense) @coalescedonoff @dtypes(torch.double) def test_coo_to_csr_convert(self, device, dtype, coalesced): with self.assertRaisesRegex(RuntimeError, "Input is supposed to be a vector"): torch._convert_indices_from_coo_to_csr(torch.randint( 100, (5, 5), device=device), size=100) size = (5, 5) sparse_dim = 2 nnz = 10 sparse_coo, _, _ = self.genSparseTensor(size, sparse_dim, nnz, coalesced, device, dtype) sparse_csr = sparse_coo.to_sparse_csr() self.assertTrue(sparse_csr.is_sparse_csr) self.assertEqual(sparse_csr.to_dense(), sparse_coo.to_dense()) vec = torch.randn((5, 1), dtype=dtype, device=device) coo_product = sparse_coo.matmul(vec) csr_product = sparse_csr.matmul(vec) self.assertEqual(coo_product, csr_product) vec = torch.randn((100, 1), dtype=dtype, device=device) index = torch.tensor([ [1, 0, 35, 14, 39, 6, 71, 66, 40, 27], [92, 31, 62, 50, 22, 65, 89, 74, 56, 34], ], dtype=torch.int32) values = torch.tensor([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], dtype=dtype, device=device) coo = torch.sparse_coo_tensor(index, values, torch.Size([100, 100]), dtype=dtype, device=device) csr = coo.to_sparse_csr() self.assertEqual(coo.matmul(vec), csr.matmul(vec)) col_indices = torch.tensor([31, 92, 65, 50, 34, 62, 22, 56, 74, 89], dtype=torch.int64, device=device) self.assertEqual(csr.col_indices(), col_indices) values = torch.tensor([2, 1, 6, 4, 10, 3, 5, 9, 8, 7], dtype=dtype, device=device) self.assertEqual(csr.values(), values) @onlyCPU @unittest.skipIf(IS_MACOS or IS_WINDOWS, "MKL doesn't work on windows or mac") @dtypes(torch.float, torch.double) def test_mkl_matvec_warnings(self, device, dtype): if torch.has_mkl: for index_dtype in [torch.int32, torch.int64]: sp = torch.sparse_csr_tensor( torch.tensor([0, 2, 4]), torch.tensor([0, 1, 0, 1]), torch.tensor([1, 2, 3, 4], dtype=dtype, device=device)) vec = torch.randn((2, 1), dtype=dtype, device=device) with warnings.catch_warnings(record=True) as w: sp.matmul(vec) self.assertEqual(len(w), 2) self.assertIn( "Pytorch is compiled with MKL LP64 and will convert crow_indices to int32", str(w[0].message)) self.assertIn( "Pytorch is compiled with MKL LP64 and will convert col_indices to int32", str(w[1].message)) @dtypes(*get_all_dtypes()) def test_sparse_csr_from_dense_convert_error(self, device, dtype): size = (4, 2, 4) dense = make_tensor(size, dtype=dtype, device=device) with self.assertRaisesRegex(RuntimeError, "Only 2D"): sparse = dense.to_sparse_csr() # TODO: Support auto generation of device check for sparse tensors # See: https://github.com/pytorch/pytorch/issues/59058 @onlyCUDA @dtypes(torch.double) def test_matmul_device_mismatch(self, device, dtype): cpu = torch.rand((10, 10)) cuda = cpu.cuda() for s, m1, m2 in itertools.product((cpu, cuda), repeat=3): csr = m1.to_sparse() if s.device == csr.device == m2.device: torch.addmm(s, csr, m2) else: with self.assertRaisesRegex( RuntimeError, "Expected all tensors to be on the same device"): torch.addmm(s, csr, m2) @dtypes(torch.float, torch.double) def test_csr_matvec(self, device, dtype): side = 100 for index_dtype in [torch.int32, torch.int64]: csr = self.genSparseCSRTensor((side, side), 1000, device=device, dtype=dtype, index_dtype=index_dtype) vec = torch.randn(side, dtype=dtype, device=device) res = csr.matmul(vec) expected = csr.to_dense().matmul(vec) self.assertEqual(res, expected) bad_vec = torch.randn(side + 10, dtype=dtype, device=device) with self.assertRaisesRegex(RuntimeError, "mv: expected"): csr.matmul(bad_vec) @dtypes(torch.double) def test_mm(self, device, dtype): def test_shape(di, dj, dk, nnz): for index_dtype in [torch.int32, torch.int64]: x = self.genSparseCSRTensor((di, dj), nnz, device=device, dtype=dtype, index_dtype=index_dtype) t = torch.randn(di, dk, dtype=dtype, device=device) y = torch.randn(dj, dk, dtype=dtype, device=device) alpha = random.random() beta = random.random() # res = beta * t + alpha * (x @ y) res = torch.addmm(t, x, y, beta=beta, alpha=alpha) expected = torch.addmm(t, x.to_dense(), y, beta=beta, alpha=alpha) self.assertEqual(res, expected) res = torch.addmm(t, x, y) expected = torch.addmm(t, x.to_dense(), y) self.assertEqual(res, expected) res = torch.mm(x, y) expected = torch.mm(x.to_dense(), y) self.assertEqual(res, expected) for i in range(2, 5): for j in range(2, 8): for k in range(2, 8): test_shape(i, j, k, i * j // 2) test_shape(4, 4, 4, 0) @dtypes(*floating_types()) def test_sparse_mm(self, device, dtype): def test_shape(d1, d2, d3, nnz, transposed): if transposed: D = torch.randn(d3, d2, dtype=dtype, device=device).t_() else: D = torch.randn(d2, d3, dtype=dtype, device=device) S = self.genSparseCSRTensor((d1, d2), nnz, device=device, dtype=dtype, index_dtype=torch.int32) S_dense = S.to_dense() self.assertEqual(torch.sparse.mm(S, D), torch.mm(S_dense, D)) test_shape(7, 8, 9, 20, False) test_shape(7, 8, 9, 20, True) @dtypes(*floating_types()) def test_sparse_addmm(self, device, dtype): def test_shape(m, n, p, nnz, broadcast, alpha_beta=None): if alpha_beta is None: alpha = random.random() beta = random.random() else: alpha, beta = alpha_beta if broadcast: D1 = make_tensor((), dtype=dtype, device=device) else: D1 = make_tensor([n, p], dtype=dtype, device=device) D2 = make_tensor([m, p], dtype=dtype, device=device) S = self.genSparseCSRTensor([n, m], nnz, dtype=dtype, device=device, index_dtype=torch.int32) S_dense = S.to_dense() Y = torch.sparse.addmm(D1, S, D2, beta=beta, alpha=alpha) Y_dense = torch.addmm(D1, S_dense, D2, beta=beta, alpha=alpha) self.assertEqual(Y, Y_dense) test_shape(7, 8, 9, 20, False, None) test_shape(7, 8, 9, 20, True, None) test_shape(7, 8, 9, 20, False, (1, 0)) test_shape(7, 8, 9, 20, True, (1, 0)) test_shape(7, 8, 9, 20, False, (1, 1)) test_shape(7, 8, 9, 20, True, (1, 1)) @dtypes(torch.float, torch.double) def test_add(self, device, dtype): def _test_spadd_shape(nnz, shape): x = self.genSparseCSRTensor(shape, nnz, dtype=dtype, device=device, index_dtype=torch.int32) y = torch.randn(*shape, dtype=dtype, device=device) r = random.random() res = torch.add(y, x, alpha=r) expected = y + r * x.to_dense() self.assertEqual(res, expected) # Non contiguous dense tensor s = list(shape) s[0] = shape[-1] s[-1] = shape[0] y = torch.randn(*s, dtype=torch.double, device=device) y.transpose_(0, len(s) - 1) r = random.random() res = torch.add(y, x, alpha=r) expected = y + r * x.to_dense() self.assertEqual(res, expected) _test_spadd_shape(10, [100, 100]) _test_spadd_shape(0, [100, 100]) _test_spadd_shape(10, [100, 1]) _test_spadd_shape(10, [1, 100]) @dtypes(*get_all_dtypes()) def test_coo_csr_conversion(self, device, dtype): for m, n in itertools.product([5, 2, 0], [5, 2, 0]): size = (m, n) dense = make_tensor(size, dtype=dtype, device=device) coo_sparse = dense.to_sparse() csr_sparse = coo_sparse.to_sparse_csr() self.assertEqual(csr_sparse.to_dense(), dense)
class TestScatterGather(TestCase): # Fills an index tensor with valid indices def _fill_indices(self, idx, dim, dim_size, elems_per_row, m, n, o, unique_indices=True): for i in range(1 if dim == 0 else m): for j in range(1 if dim == 1 else n): for k in range(1 if dim == 2 else o): ii = [i, j, k] ii[dim] = slice(0, idx.size(dim) + 1) if unique_indices: idx[tuple(ii)] = torch.randperm(dim_size)[0:elems_per_row] else: idx[tuple(ii)] = torch.randint(dim_size, (elems_per_row,)) @dtypes(torch.float32, torch.complex64) def test_gather(self, device, dtype): m, n, o = random.randint(10, 20), random.randint(10, 20), random.randint(10, 20) elems_per_row = random.randint(1, 10) dim = random.randrange(3) src = make_tensor((m, n, o), device=device, dtype=dtype) idx_size = [m, n, o] idx_size[dim] = elems_per_row idx = make_tensor(idx_size, device=device, dtype=torch.long) self._fill_indices(idx, dim, src.size(dim), elems_per_row, m, n, o) actual = torch.gather(src, dim, idx) expected = torch.zeros(idx_size, device=device, dtype=dtype) for i in range(idx_size[0]): for j in range(idx_size[1]): for k in range(idx_size[2]): ii = [i, j, k] ii[dim] = idx[i, j, k] expected[i, j, k] = src[tuple(ii)] self.assertEqual(actual, expected, atol=0, rtol=0) # Guarded because torch.max isn't defined for complex types if not dtype.is_complex: src = make_tensor((3, 4, 5), device=device, dtype=dtype) expected, idx = src.max(2, True) actual = torch.gather(src, 2, idx) self.assertEqual(actual, expected, atol=0, rtol=0) @dtypes(torch.bool) def test_gather_bool(self, device, dtype): src = torch.tensor(((False, True), (True, True)), device=device, dtype=dtype) idx = torch.tensor(((0, 0), (1, 0)), device=device, dtype=torch.long) actual = torch.gather(src, 1, idx) expected = torch.tensor(((False, False), (True, True)), device=device, dtype=dtype) self.assertEqual(actual, expected, atol=0, rtol=0) @parametrize("sparse_grad", [False, True]) @dtypes(torch.float32, torch.float64) def test_gather_backward_with_empty_index_tensor(self, device, dtype, sparse_grad): dim = -1 input = torch.rand([10, 5], dtype=dtype, device=device, requires_grad=True) index = torch.randint(0, 2, [3, 0], dtype=torch.int64, device=device) res = torch.gather(input, dim, index, sparse_grad=sparse_grad) res.sum().backward() grad = input.grad.to_dense() if sparse_grad else input.grad expected_grad = torch.zeros_like(input, requires_grad=False) self.assertEqual(grad, expected_grad, atol=0, rtol=0) def _test_scatter_base(self, fn, *, device, dtype, is_scalar, reduction, unique_indices=True, include_self=True): m, n, o = random.randint(10, 20), random.randint(10, 20), random.randint(10, 20) elems_per_row = random.randint(1, 10) dim = random.randrange(3) idx_size = [m, n, o] idx_size[dim] = elems_per_row idx = torch.empty(tuple(idx_size), device=device, dtype=torch.long) self._fill_indices(idx, dim, ([m, n, o])[dim], elems_per_row, m, n, o, unique_indices) if is_scalar: src = random.random() else: src_size = [random.randint(1, 5) + s for s in idx_size] src = make_tensor(tuple(src_size), device=device, dtype=dtype) base = make_tensor((m, n, o), device=device, dtype=dtype) if reduction is not None: if fn is torch.Tensor.scatter_reduce_: actual = fn(base.clone(), dim, idx, src, reduce=reduction, include_self=include_self) else: actual = fn(base.clone(), dim, idx, src, reduce=reduction) else: actual = fn(base.clone(), dim, idx, src) expected = base.clone() counts = torch.zeros(base.shape, dtype=torch.long, device=device) + include_self for i in range(idx_size[0]): for j in range(idx_size[1]): for k in range(idx_size[2]): ii = [i, j, k] ii[dim] = idx[i, j, k] if fn is torch.Tensor.scatter_add_: expected[tuple(ii)] += src[i, j, k] else: # method may be 'scatter_', 'scatter', 'scatter_reduce' # or 'scatter_reduce_', the former two might have a reduction argument # while the latter two always do value = src if is_scalar else src[i, j, k] if ((not include_self) and counts[tuple(ii)] == 0): expected[tuple(ii)] = value else: if reduction == "add" or reduction == "sum": expected[tuple(ii)] += value elif reduction == "multiply" or reduction == "prod": expected[tuple(ii)] *= value elif reduction == "amax": expected[tuple(ii)] = max(expected[tuple(ii)], value) elif reduction == "amin": expected[tuple(ii)] = min(expected[tuple(ii)], value) elif reduction == "mean": expected[tuple(ii)] += value else: expected[tuple(ii)] = value counts[tuple(ii)] += 1 if (reduction == "mean"): counts.masked_fill_(counts == 0, 1) if (dtype.is_floating_point or dtype.is_complex): expected /= counts else: expected.div_(counts, rounding_mode="floor") self.assertEqual(actual, expected, atol=0, rtol=0) # Tests empty index dst = make_tensor((2, 2), device=device, dtype=dtype) idx = torch.tensor((), device=device, dtype=torch.long) src = make_tensor((2, 2), device=device, dtype=dtype) if reduction is not None: actual = fn(dst, 0, idx, src, reduce=reduction) else: actual = fn(dst, 0, idx, src) self.assertEqual(actual, dst, atol=0, rtol=0) @dtypes(torch.float16, torch.float32, torch.complex64) def test_scatter_(self, device, dtype): self._test_scatter_base(torch.Tensor.scatter_, device=device, dtype=dtype, is_scalar=False, reduction=None) @dtypes(torch.float16, torch.float32, torch.complex64) def test_scatter__scalar(self, device, dtype): self._test_scatter_base(torch.Tensor.scatter_, device=device, dtype=dtype, is_scalar=True, reduction=None) # FIXME: RuntimeError: "cuda_scatter_gather_base_kernel_reduce_multiply" not implemented for 'ComplexFloat' @toleranceOverride({torch.float16: tol(atol=1e-2, rtol=0)}) @dtypesIfCUDA(torch.float16, torch.float32) @dtypes(torch.float16, torch.float32, torch.complex64) def test_scatter__reductions(self, device, dtype): for reduction in ("add", "multiply"): self._test_scatter_base(torch.Tensor.scatter_, device=device, dtype=dtype, is_scalar=False, reduction=reduction) self._test_scatter_base(torch.Tensor.scatter_, device=device, dtype=dtype, is_scalar=True, reduction=reduction) @dtypes(torch.float16, torch.float32, torch.complex64) def test_scatter_add_(self, device, dtype): self._test_scatter_base(torch.Tensor.scatter_add_, device=device, dtype=dtype, is_scalar=False, reduction=None) @dtypes(torch.float32) def test_scatter_add_mult_index_base(self, device, dtype): m, n = 30, 40 idx = torch.zeros(m, n, device=device, dtype=torch.long) src = torch.ones(m, n, device=device, dtype=dtype) res0 = torch.zeros(m, n, device=device, dtype=dtype).scatter_add_(0, idx, src) res1 = torch.zeros(m, n, device=device, dtype=dtype).scatter_add_(1, idx, src) self.assertEqual(res0[0, :], m * torch.ones(n, device=device, dtype=dtype), atol=0, rtol=0) self.assertEqual(res1[:, 0], n * torch.ones(m, device=device, dtype=dtype), atol=0, rtol=0) # FIXME: discrepancy between bool ReduceAdd on CUDA and CPU (a + b on CPU and buggy a && b on CUDA) @dtypes(*get_all_dtypes(include_half=True, include_bfloat16=True, include_bool=False)) def test_scatter_reduce_sum(self, device, dtype): for include_self in (True, False): self._test_scatter_base(torch.Tensor.scatter_reduce_, device=device, dtype=dtype, is_scalar=False, reduction='sum', unique_indices=False, include_self=include_self) @dtypes(*get_all_dtypes(include_half=True, include_bfloat16=True)) @dtypesIfCUDA(*get_all_dtypes(include_half=True, include_bfloat16=True, include_complex=False, include_bool=False)) def test_scatter_reduce_prod(self, device, dtype): for include_self in (True, False): self._test_scatter_base(torch.Tensor.scatter_reduce_, device=device, dtype=dtype, is_scalar=False, reduction='prod', unique_indices=False, include_self=include_self) @dtypes(*get_all_dtypes(include_half=True, include_bfloat16=True, include_bool=False)) @dtypesIfCUDA(*get_all_dtypes(include_half=True, include_bfloat16=True, include_complex=False, include_bool=False)) def test_scatter_reduce_mean(self, device, dtype): for include_self in (True, False): self._test_scatter_base(torch.Tensor.scatter_reduce_, device=device, dtype=dtype, is_scalar=False, reduction='mean', unique_indices=False, include_self=include_self) @dtypes(*get_all_dtypes(include_half=True, include_bfloat16=True, include_complex=False)) @dtypesIfCUDA(*get_all_dtypes(include_half=True, include_bfloat16=True, include_complex=False, include_bool=False)) def test_scatter_reduce_amax(self, device, dtype): for include_self in (True, False): self._test_scatter_base(torch.Tensor.scatter_reduce_, device=device, dtype=dtype, is_scalar=False, reduction='amax', unique_indices=False, include_self=include_self) # simple test for nan/inf propagation if (dtype.is_floating_point): input = torch.zeros(3, device=device, dtype=dtype) src = torch.tensor([1, float('nan'), -float('inf'), -float('inf'), 2, float('inf')], device=device, dtype=dtype) idx = torch.tensor([0, 0, 1, 1, 2, 2], device=device) input.scatter_reduce_(0, idx, src, 'amax', include_self=include_self) expected_result = torch.tensor([float('nan'), -float('inf'), float('inf')], device=device, dtype=dtype) if (include_self): expected_result[1] = 0 self.assertEqual(input, expected_result) @dtypes(*get_all_dtypes(include_half=True, include_bfloat16=True, include_complex=False)) @dtypesIfCUDA(*get_all_dtypes(include_half=True, include_bfloat16=True, include_complex=False, include_bool=False)) def test_scatter_reduce_amin(self, device, dtype): for include_self in (True, False): self._test_scatter_base(torch.Tensor.scatter_reduce_, device=device, dtype=dtype, is_scalar=False, reduction='amin', unique_indices=False, include_self=include_self) # simple test for nan/inf propagation if (dtype.is_floating_point): input = torch.zeros(3, device=device, dtype=dtype) src = torch.tensor([1, float('nan'), -2, -float('inf'), float('inf'), float('inf')], device=device, dtype=dtype) idx = torch.tensor([0, 0, 1, 1, 2, 2], device=device) input.scatter_reduce_(0, idx, src, 'amin', include_self=include_self) expected_result = torch.tensor([float('nan'), -float('inf'), float('inf')], device=device, dtype=dtype) if (include_self): expected_result[2] = 0 self.assertEqual(input, expected_result)
def test_dtypes(self, device, op): # dtypes to try to backward in allowed_backward_dtypes = floating_and_complex_types_and( torch.bfloat16, torch.float16) # lists for (un)supported dtypes supported_dtypes = [] unsupported_dtypes = [] supported_backward_dtypes = [] unsupported_backward_dtypes = [] def unsupported(dtype): unsupported_dtypes.append(dtype) if dtype in allowed_backward_dtypes: unsupported_backward_dtypes.append(dtype) for dtype in get_all_dtypes(): # tries to acquire samples - failure indicates lack of support requires_grad = (dtype in allowed_backward_dtypes and op.supports_autograd) try: samples = list( op.sample_inputs(device, dtype, requires_grad=requires_grad)) except Exception as e: unsupported(dtype) continue # Counts number of successful backward attempts # NOTE: This exists as a kludge because this only understands how to # request a gradient if the output is a tensor or a sequence with # a tensor as its first element. num_backward_successes = 0 for sample in samples: # tries to call operator with the sample - failure indicates # lack of support try: result = op(sample.input, *sample.args, **sample.kwargs) except Exception as e: # NOTE: some ops will fail in forward if their inputs # require grad but they don't support computing the gradient # in that type! This is a bug in the op! unsupported(dtype) # Short-circuits testing this dtype -- it doesn't work if dtype in unsupported_dtypes: break # Short-circuits if the dtype isn't a backward dtype or # it's already identified as not supported if dtype not in allowed_backward_dtypes or dtype in unsupported_backward_dtypes: continue # Checks for backward support in the same dtype try: result = sample.output_process_fn_grad(result) if isinstance(result, torch.Tensor): backward_tensor = result elif isinstance(result, Sequence) and isinstance( result[0], torch.Tensor): backward_tensor = result[0] else: continue # Note: this grad may not have the same dtype as dtype # For functions like complex (float -> complex) or abs # (complex -> float) the grad tensor will have a # different dtype than the input. # For simplicity, this is still modeled as these ops # supporting grad in the input dtype. grad = torch.randn_like(backward_tensor) backward_tensor.backward(grad) num_backward_successes += 1 except Exception as e: unsupported_backward_dtypes.append(dtype) if dtype not in unsupported_dtypes: supported_dtypes.append(dtype) if num_backward_successes > 0 and dtype not in unsupported_backward_dtypes: supported_backward_dtypes.append(dtype) # Checks that dtypes are listed correctly and generates an informative # error message device_type = torch.device(device).type claimed_supported = set(op.supported_dtypes(device_type)) supported_dtypes = set(supported_dtypes) supported_but_unclaimed = supported_dtypes - claimed_supported claimed_but_unsupported = claimed_supported - supported_dtypes msg = """The supported dtypes for {0} on {1} according to its OpInfo are {2}, but the detected supported dtypes are {3}. """.format(op.name, device_type, claimed_supported, supported_dtypes) if len(supported_but_unclaimed) > 0: msg += "The following dtypes should be added to the OpInfo: {0}. ".format( supported_but_unclaimed) if len(claimed_but_unsupported) > 0: msg += "The following dtypes should be removed from the OpInfo: {0}.".format( claimed_but_unsupported) self.assertEqual(supported_dtypes, claimed_supported, msg=msg) # Checks that backward dtypes are listed correctly and generates an # informative error message # NOTE: this code is nearly identical to the check + msg generation claimed_backward_supported = set( op.supported_backward_dtypes(device_type)) supported_backward_dtypes = set(supported_backward_dtypes) supported_but_unclaimed = supported_backward_dtypes - claimed_backward_supported claimed_but_unsupported = claimed_backward_supported - supported_backward_dtypes msg = """The supported backward dtypes for {0} on {1} according to its OpInfo are {2}, but the detected supported backward dtypes are {3}. """.format(op.name, device_type, claimed_backward_supported, supported_backward_dtypes) if len(supported_but_unclaimed) > 0: msg += "The following backward dtypes should be added to the OpInfo: {0}. ".format( supported_but_unclaimed) if len(claimed_but_unsupported) > 0: msg += "The following backward dtypes should be removed from the OpInfo: {0}.".format( claimed_but_unsupported) self.assertEqual(supported_backward_dtypes, claimed_backward_supported, msg=msg)
class TestSparseCSR(TestCase): @onlyCPU def test_csr_layout(self): self.assertEqual(str(torch.sparse_csr), 'torch.sparse_csr') self.assertEqual(type(torch.sparse_csr), torch.layout) @dtypes(*get_all_dtypes()) def test_sparse_csr_constructor_shape_inference(self, device, dtype): crow_indices = [0, 2, 4] col_indices = [0, 1, 0, 1] values = [1, 2, 3, 4] sparse = torch.sparse_csr_tensor(torch.tensor(crow_indices, dtype=torch.int64), torch.tensor(col_indices, dtype=torch.int64), torch.tensor(values), dtype=dtype, device=device) self.assertEqual(torch.tensor(crow_indices, dtype=torch.int64), sparse.crow_indices()) self.assertEqual((len(crow_indices) - 1, max(col_indices) + 1), sparse.shape) self.assertEqual(dtype, sparse.dtype) self.assertEqual(torch.device(device), sparse.device) @dtypes(*get_all_dtypes()) def test_sparse_csr_constructor(self, device, dtype): crow_indices = [0, 2, 4] col_indices = [0, 1, 0, 1] values = [1, 2, 3, 4] for index_dtype in [torch.int32, torch.int64]: sparse = torch.sparse_csr_tensor(torch.tensor(crow_indices, dtype=index_dtype), torch.tensor(col_indices, dtype=index_dtype), torch.tensor(values), size=(2, 10), dtype=dtype, device=device) self.assertEqual((2, 10), sparse.shape) self.assertEqual(torch.tensor(crow_indices, dtype=index_dtype), sparse.crow_indices()) self.assertEqual(torch.tensor(col_indices, dtype=index_dtype), sparse.col_indices()) self.assertEqual(torch.tensor(values, dtype=dtype), sparse.values()) @dtypes(*get_all_dtypes()) def test_sparse_csr_constructor_from_lists(self, device, dtype): # without size sparse = torch.sparse_csr_tensor([0, 2, 4], [0, 1, 0, 1], [1, 2, 3, 4], dtype=dtype, device=device) self.assertEqual((2, 2), sparse.shape) self.assertEqual(4, sparse.numel()) self.assertEqual(torch.tensor([0, 2, 4], dtype=torch.int64, device=device), sparse.crow_indices()) self.assertEqual(torch.tensor([0, 1, 0, 1], dtype=torch.int64, device=device), sparse.col_indices()) self.assertEqual(torch.tensor([1, 2, 3, 4], dtype=dtype, device=device), sparse.values()) # with size for sparse_csr_tensor in [torch.sparse_csr_tensor, torch._sparse_csr_tensor_unsafe]: sparse = sparse_csr_tensor([0, 2, 4], [0, 1, 0, 1], [1, 2, 3, 4], size=(2, 10), dtype=dtype, device=device) self.assertEqual((2, 10), sparse.shape) self.assertEqual(torch.tensor([0, 2, 4], dtype=torch.int64, device=device), sparse.crow_indices()) self.assertEqual(torch.tensor([0, 1, 0, 1], dtype=torch.int64, device=device), sparse.col_indices()) self.assertEqual(torch.tensor([1, 2, 3, 4], dtype=dtype, device=device), sparse.values()) @skipMeta @dtypes(*get_all_dtypes()) def test_empty(self, device, dtype): ns = [5, 2, 0] for shape in itertools.product(ns, ns): result = torch.empty(shape, dtype=dtype, device=device, layout=torch.sparse_csr) self.assertEqual(result.shape, shape) self.assertEqual(result.dtype, dtype) self.assertEqual(result.device, torch.device(device)) self.assertEqual(result.layout, torch.sparse_csr) self.assertEqual(result.crow_indices().shape, (shape[0] + 1,)) self.assertEqual(result.col_indices().shape, (0,)) self.assertEqual(result.values().shape, (0,)) self.assertEqual(result._nnz(), 0) self.assertEqual(result.crow_indices().device, torch.device(device)) self.assertEqual(result.col_indices().device, torch.device(device)) self.assertEqual(result.values().device, torch.device(device)) self.assertEqual(result.crow_indices().dtype, torch.int64) self.assertEqual(result.col_indices().dtype, torch.int64) self.assertEqual(result.values().dtype, dtype) @skipMeta @dtypes(*get_all_dtypes()) def test_empty_errors(self, device, dtype): with self.assertRaisesRegex(RuntimeError, "torch.empty: Only 2D sparse CSR tensors are supported."): torch.empty((5,), dtype=dtype, device=device, layout=torch.sparse_csr) with self.assertRaisesRegex(RuntimeError, "torch.empty: Only 2D sparse CSR tensors are supported."): torch.empty((2, 3, 4), dtype=dtype, device=device, layout=torch.sparse_csr) @skipMeta @dtypes(*get_all_dtypes()) def test_copy(self, device, dtype): def run_test(shape, nnz, index_type): a = self.genSparseCSRTensor(shape, nnz, dtype=dtype, device=device, index_dtype=index_dtype) b = self.genSparseCSRTensor(shape, nnz, dtype=dtype, device=device, index_dtype=index_dtype) a.copy_(b) self.assertEqual(a.crow_indices(), b.crow_indices()) self.assertEqual(a.col_indices(), b.col_indices()) self.assertEqual(a.values(), b.values()) ns = [5, 2, 0] for shape, index_dtype in zip(itertools.product(ns, ns), [torch.int32, torch.int64]): run_test(shape, 0, index_dtype) run_test(shape, shape[0] * shape[1], index_dtype) @skipMeta @dtypes(*get_all_dtypes()) def test_copy_errors(self, device, dtype): for index_dtype in [torch.int32, torch.int64]: shape1 = (2, 3) shape2 = (3, 2) a = self.genSparseCSRTensor(shape1, 0, dtype=dtype, device=device, index_dtype=index_dtype) b = self.genSparseCSRTensor(shape2, 0, dtype=dtype, device=device, index_dtype=index_dtype) with self.assertRaisesRegex(RuntimeError, "only same size tensors are supported."): a.copy_(b) with self.assertRaisesRegex(RuntimeError, "copy between different layouts is not supported."): a.copy_(torch.empty(a.shape, dtype=dtype, device=device)) b = self.genSparseCSRTensor(shape1, 1, dtype=dtype, device=device, index_dtype=index_dtype) with self.assertRaisesRegex(RuntimeError, "only tensors with the same number of specified elements are supported."): a.copy_(b) @skipMeta @dtypes(*get_all_dtypes()) def test_resize(self, device, dtype): for index_dtype in [torch.int32, torch.int64]: shape = (2, 3) nnz = 6 a = self.genSparseCSRTensor(shape, nnz, dtype=dtype, device=device, index_dtype=index_dtype) new_shape = (4, 5) a.resize_(new_shape) self.assertEqual(a.shape, new_shape) # resize to larger shape doesn't add specified elements self.assertEqual(a._nnz(), nnz) new_shape = (1, 5) a.resize_(new_shape) self.assertEqual(a.shape, new_shape) # resize to smaller shape trims specified elements self.assertEqual(a._nnz(), 5) @skipMeta @dtypes(*get_all_dtypes()) def test_resize_errors(self, device, dtype): for index_dtype in [torch.int32, torch.int64]: shape = (2, 3) nnz = 6 a = self.genSparseCSRTensor(shape, nnz, dtype=dtype, device=device, index_dtype=index_dtype) with self.assertRaisesRegex(RuntimeError, "torch.resize_: Only 2D sparse CSR tensors are supported."): new_shape = (4,) a.resize_(new_shape) # resizing of columns to smaller size is not implemented with self.assertRaisesRegex( RuntimeError, "torch.resize_: Resizing columns of sparse CSR tensors to a smaller value is not supported.", ): new_shape = (2, 2) a.resize_(new_shape) def test_factory_type_invariants_check(self, device): with self.assertRaisesRegex(RuntimeError, "both crow_indices and col_indices should have the same type."): torch.sparse_csr_tensor(torch.tensor([0, 2, 4], dtype=torch.int64), torch.tensor([0, 1, 0, 1], dtype=torch.int32), torch.tensor([1, 2, 3, 4]), device=device) with self.assertRaisesRegex(RuntimeError, r"\"csr_construct_check\" not implemented for 'Short'"): torch.sparse_csr_tensor(torch.tensor([0, 2, 4], dtype=torch.int16), torch.tensor([0, 1, 0, 1], dtype=torch.int16), torch.tensor([1, 2, 3, 4]), device=device) def test_factory_layout_invariants_check(self, device): with self.assertRaisesRegex(RuntimeError, "expected values to be a strided and contiguous tensor"): values = torch.tensor([1.], device=device).expand(4,) torch.sparse_csr_tensor(torch.tensor([0, 2, 4], device=device), torch.tensor([0, 1, 0, 1], device=device), values) with self.assertRaisesRegex(RuntimeError, "expected col_indices to be a strided and contiguous tensor"): col_indices = torch.tensor([0], device=device).expand(4,) torch.sparse_csr_tensor(torch.tensor([0, 2, 4]), col_indices, torch.tensor([1, 2, 3, 4])) with self.assertRaisesRegex(RuntimeError, "expected crow_indices to be a strided and contiguous tensor"): crow_indices = torch.arange(6, device=device) torch.sparse_csr_tensor(crow_indices[::2], torch.tensor([0, 1, 0, 1], device=device), torch.tensor([1, 2, 3, 4])) def test_factory_shape_invariants_check(self, device): crow_indices = [0, 2, 4] col_indices = [0, 1, 0, 1] values = [1, 2, 3, 4] size = (2, 10) torch.sparse_csr_tensor(torch.tensor(crow_indices), torch.tensor(col_indices), torch.tensor(values), size, device=device) with self.assertRaisesRegex(RuntimeError, r"size of a CSR tensor must be of length 2, but got: 3"): torch.sparse_csr_tensor(torch.tensor(crow_indices), torch.tensor(col_indices), torch.tensor(values), size=(2, 10, 2), device=device) with self.assertRaisesRegex(RuntimeError, r"crow_indices must have dim\=1 but got crow_indices\.dim\(\)\=2"): torch.sparse_csr_tensor(torch.tensor(crow_indices).repeat(2, 1), torch.tensor(col_indices), torch.tensor(values), size, device=device) with self.assertRaisesRegex(RuntimeError, r"col_indices must have dim\=1 but got col_indices\.dim\(\)\=2"): torch.sparse_csr_tensor(torch.tensor(crow_indices), torch.tensor(col_indices).repeat(2, 1), torch.tensor(values), size, device=device) with self.assertRaisesRegex(RuntimeError, r"values must have dim\=1 but got values\.dim\(\)\=2"): torch.sparse_csr_tensor(torch.tensor(crow_indices), torch.tensor(col_indices), torch.tensor(values).repeat(2, 1), size, device=device) with self.assertRaisesRegex(RuntimeError, r"crow_indices\.numel\(\) must be size\(0\) \+ 1, but got: 3"): torch.sparse_csr_tensor(torch.tensor(crow_indices), torch.tensor(col_indices), torch.tensor(values), (1, 1), device=device) with self.assertRaisesRegex(RuntimeError, r"col_indices and values must have equal sizes, " + r"but got col_indices\.numel\(\): 3, values\.numel\(\): 4"): torch.sparse_csr_tensor(torch.tensor(crow_indices), torch.tensor([0, 1, 0]), torch.tensor(values), size, device=device) def test_factory_indices_invariants_check(self, device): crow_indices = [0, 2, 4] col_indices = [0, 1, 0, 1] values = [1, 2, 3, 4] size = (2, 10) with self.assertRaisesRegex(RuntimeError, "0th value of crow_indices must be 0."): torch.sparse_csr_tensor(torch.tensor([-1, 0, 4]), torch.tensor(col_indices), torch.tensor(values), size, device=device) with self.assertRaisesRegex(RuntimeError, "last value of crow_indices should be equal to the length of col_indices."): torch.sparse_csr_tensor(torch.tensor([0, 2, 5]), torch.tensor(col_indices), torch.tensor(values), size, device=device) with self.assertRaisesRegex(RuntimeError, r"at position i \= 2," + r" this condition crow_indices\[i - 1\] <\= crow_indices\[i\] fails"): torch.sparse_csr_tensor(torch.tensor([0, 5, 4]), torch.tensor(col_indices), torch.tensor(values), size, device=device) with self.assertRaisesRegex(RuntimeError, r"col_indices\.min\(\) should be greater or equal to zero"): torch.sparse_csr_tensor(torch.tensor(crow_indices), torch.tensor([0, -1, 0, 1]), torch.tensor(values), size, device=device) with self.assertRaisesRegex(RuntimeError, r"size\(1\) should be greater than col_indices\.max\(\)"): torch.sparse_csr_tensor(torch.tensor(crow_indices), torch.tensor([0, 11, 0, 1]), torch.tensor(values), size, device=device) @onlyCUDA @dtypes(*get_all_dtypes()) def test_factory_device_type_inference(self, device, dtype): cpu_cuda = ('cpu', 'cuda') cpu_cuda_none = cpu_cuda + (None,) for crow_indices_device, col_indices_device, values_device, device in itertools.product(cpu_cuda, cpu_cuda, cpu_cuda, cpu_cuda_none): for index_dtype in [torch.int32, torch.int64]: crow_indices = torch.tensor([0, 2, 4], dtype=index_dtype, device=crow_indices_device) col_indices = torch.tensor([0, 1, 0, 1], dtype=index_dtype, device=col_indices_device) values = torch.tensor([1, 2, 3, 4], dtype=dtype, device=values_device) if device is None and (crow_indices_device != col_indices_device or crow_indices_device != values_device): with self.assertRaises(RuntimeError): torch.sparse_csr_tensor(crow_indices, col_indices, values, size=(2, 10), device=device) else: t = torch.sparse_csr_tensor(crow_indices, col_indices, values, size=(2, 10), device=device) should_be_cuda = (device == 'cuda' or (device is None and values_device == 'cuda')) self.assertEqual(should_be_cuda, t.is_cuda) t.crow_indices().dtype == index_dtype t.col_indices().dtype == index_dtype t.values().dtype == dtype t.crow_indices().device == t.values().device t.col_indices().device == t.values().device def test_sparse_csr_print(self, device): orig_maxDiff = self.maxDiff self.maxDiff = None shape_nnz = [ ((10, 10), 10), ((100, 10), 10), ((1000, 10), 10) ] printed = [] for shape, nnz in shape_nnz: values_shape = torch.Size((nnz,)) col_indices_shape = torch.Size((nnz,)) crow_indices_shape = torch.Size((shape[0] + 1,)) printed.append("# shape: {}".format(torch.Size(shape))) printed.append("# nnz: {}".format(nnz)) printed.append("# crow_indices shape: {}".format(crow_indices_shape)) printed.append("# col_indices shape: {}".format(col_indices_shape)) printed.append("# values_shape: {}".format(values_shape)) for index_dtype in [torch.int32, torch.int64]: for dtype in floating_types(): printed.append("########## {}/{} ##########".format(dtype, index_dtype)) x = torch.sparse_csr_tensor(torch.tensor([0, 2, 4], dtype=index_dtype), torch.tensor([0, 1, 0, 1], dtype=index_dtype), torch.tensor([1, 2, 3, 4]), dtype=dtype, device=device) printed.append("# sparse tensor") printed.append(str(x)) printed.append("# _crow_indices") printed.append(str(x.crow_indices())) printed.append("# _col_indices") printed.append(str(x.col_indices())) printed.append("# _values") printed.append(str(x.values())) printed.append('') printed.append('') self.assertExpected('\n'.join(printed)) self.maxDiff = orig_maxDiff @dtypes(*get_all_dtypes()) def test_sparse_csr_from_dense(self, device, dtype): dense = torch.tensor([[4, 5, 0], [0, 0, 0], [1, 0, 0]], dtype=dtype, device=device) sparse = dense.to_sparse_csr() self.assertEqual(torch.tensor([0, 2, 2, 3], dtype=torch.int64), sparse.crow_indices()) self.assertEqual(torch.tensor([0, 1, 0], dtype=torch.int64), sparse.col_indices()) self.assertEqual(torch.tensor([4, 5, 1], dtype=dtype), sparse.values()) dense = torch.tensor([[0, 0, 0], [0, 0, 1], [1, 0, 0]], dtype=dtype, device=device) sparse = dense.to_sparse_csr() self.assertEqual(torch.tensor([0, 0, 1, 2], dtype=torch.int64), sparse.crow_indices()) self.assertEqual(torch.tensor([2, 0], dtype=torch.int64), sparse.col_indices()) self.assertEqual(torch.tensor([1, 1], dtype=dtype), sparse.values()) dense = torch.tensor([[2, 2, 2], [2, 2, 2], [2, 2, 2]], dtype=dtype, device=device) sparse = dense.to_sparse_csr() self.assertEqual(torch.tensor([0, 3, 6, 9], dtype=torch.int64), sparse.crow_indices()) self.assertEqual(torch.tensor([0, 1, 2] * 3, dtype=torch.int64), sparse.col_indices()) self.assertEqual(torch.tensor([2] * 9, dtype=dtype), sparse.values()) @dtypes(*get_all_dtypes()) def test_sparse_csr_to_dense(self, device, dtype): mn = [5, 2, 0] for (m, n) in itertools.product(mn, mn): size = (m, n) dense = make_tensor(size, dtype=dtype, device=device) sparse = dense.to_sparse_csr() self.assertEqual(sparse.to_dense(), dense) crow_indices = torch.tensor([0, 3, 5]) col_indices = torch.tensor([0, 1, 2, 0, 1]) values = torch.tensor([1, 2, 1, 3, 4], dtype=dtype) csr = torch.sparse_csr_tensor(crow_indices, col_indices, values, dtype=dtype, device=device) dense = torch.tensor([[1, 2, 1], [3, 4, 0]], dtype=dtype, device=device) self.assertEqual(csr.to_dense(), dense) @coalescedonoff @dtypes(torch.double) def test_coo_to_csr_convert(self, device, dtype, coalesced): with self.assertRaisesRegex(RuntimeError, "Input is supposed to be a vector"): torch._convert_indices_from_coo_to_csr( torch.randint(100, (5, 5), device=device), size=100) size = (5, 5) sparse_dim = 2 nnz = 10 sparse_coo, _, _ = self.genSparseTensor(size, sparse_dim, nnz, coalesced, device, dtype) sparse_csr = sparse_coo.to_sparse_csr() self.assertTrue(sparse_csr.is_sparse_csr) self.assertEqual(sparse_csr.to_dense(), sparse_coo.to_dense()) vec = torch.randn((5, 1), dtype=dtype, device=device) coo_product = sparse_coo.matmul(vec) csr_product = sparse_csr.matmul(vec) self.assertEqual(coo_product, csr_product) vec = torch.randn((100, 1), dtype=dtype, device=device) index = torch.tensor([ [1, 0, 35, 14, 39, 6, 71, 66, 40, 27], [92, 31, 62, 50, 22, 65, 89, 74, 56, 34], ], dtype=torch.int32) values = torch.tensor([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], dtype=dtype, device=device) coo = torch.sparse_coo_tensor(index, values, torch.Size([100, 100]), dtype=dtype, device=device) csr = coo.to_sparse_csr() self.assertEqual(coo.matmul(vec), csr.matmul(vec)) col_indices = torch.tensor([ 31, 92, 65, 50, 34, 62, 22, 56, 74, 89 ], dtype=torch.int64, device=device) self.assertEqual(csr.col_indices(), col_indices) values = torch.tensor([2, 1, 6, 4, 10, 3, 5, 9, 8, 7], dtype=dtype, device=device) self.assertEqual(csr.values(), values) @onlyCPU @unittest.skipIf(IS_MACOS or IS_WINDOWS, "MKL doesn't work on windows or mac") @dtypes(torch.float, torch.double) def test_mkl_matvec_warnings(self, device, dtype): if torch.has_mkl: for index_dtype in [torch.int32, torch.int64]: sp = torch.sparse_csr_tensor(torch.tensor([0, 2, 4]), torch.tensor([0, 1, 0, 1]), torch.tensor([1, 2, 3, 4], dtype=dtype, device=device)) vec = torch.randn((2, 1), dtype=dtype, device=device) with warnings.catch_warnings(record=True) as w: sp.matmul(vec) self.assertEqual(len(w), 2) self.assertIn("Pytorch is compiled with MKL LP64 and will convert crow_indices to int32", str(w[0].message)) self.assertIn("Pytorch is compiled with MKL LP64 and will convert col_indices to int32", str(w[1].message)) @dtypes(*get_all_dtypes()) def test_sparse_csr_from_dense_convert_error(self, device, dtype): size = (4, 2, 4) dense = make_tensor(size, dtype=dtype, device=device) with self.assertRaisesRegex(RuntimeError, "Only 2D"): sparse = dense.to_sparse_csr() # TODO: Support auto generation of device check for sparse tensors # See: https://github.com/pytorch/pytorch/issues/59058 @onlyCUDA @dtypes(torch.double) def test_matmul_device_mismatch(self, device, dtype): cpu = torch.rand((10, 10)) cuda = cpu.cuda() for s, m1, m2 in itertools.product((cpu, cuda), repeat=3): csr = m1.to_sparse() if s.device == csr.device == m2.device: torch.addmm(s, csr, m2) else: with self.assertRaisesRegex(RuntimeError, "Expected all tensors to be on the same device"): torch.addmm(s, csr, m2) @skipCUDAIfNoCusparseGeneric @dtypes(*torch.testing.floating_types()) @dtypesIfCUDA(*get_all_complex_dtypes(), *get_all_fp_dtypes(include_half=SM53OrLater, include_bfloat16=SM80OrLater)) def test_csr_matvec(self, device, dtype): side = 100 for index_dtype in [torch.int32, torch.int64]: csr = self.genSparseCSRTensor((side, side), 1000, device=device, dtype=dtype, index_dtype=index_dtype) vec = torch.randn(side, dtype=dtype, device=device) res = csr.matmul(vec) expected = csr.to_dense().matmul(vec) self.assertEqual(res, expected) bad_vec = torch.randn(side + 10, dtype=dtype, device=device) err_msg = "mv: expected" # CUDA path now uses generic meta/structured implementation # TODO: move CPU path to not use `mv_sparse` function if self.device_type == 'cuda': err_msg = "size mismatch, got" with self.assertRaisesRegex(RuntimeError, err_msg): csr.matmul(bad_vec) @dtypes(torch.double) def test_mm(self, device, dtype): def test_shape(di, dj, dk, nnz): for index_dtype in [torch.int32, torch.int64]: x = self.genSparseCSRTensor((di, dj), nnz, device=device, dtype=dtype, index_dtype=index_dtype) t = torch.randn(di, dk, dtype=dtype, device=device) y = torch.randn(dj, dk, dtype=dtype, device=device) alpha = random.random() beta = random.random() # res = beta * t + alpha * (x @ y) res = torch.addmm(t, x, y, beta=beta, alpha=alpha) expected = torch.addmm(t, x.to_dense(), y, beta=beta, alpha=alpha) self.assertEqual(res, expected) res = torch.addmm(t, x, y) expected = torch.addmm(t, x.to_dense(), y) self.assertEqual(res, expected) res = torch.mm(x, y) expected = torch.mm(x.to_dense(), y) self.assertEqual(res, expected) for i in range(2, 5): for j in range(2, 8): for k in range(2, 8): test_shape(i, j, k, i * j // 2) test_shape(4, 4, 4, 0) @dtypes(*floating_types()) @dtypesIfCUDA(*get_all_complex_dtypes(), *get_all_fp_dtypes(include_half=SM53OrLater and TEST_CUSPARSE_GENERIC, include_bfloat16=SM80OrLater and TEST_CUSPARSE_GENERIC)) @precisionOverride({torch.bfloat16: 1e-2, torch.float16: 1e-2}) def test_sparse_mm(self, device, dtype): def test_shape(d1, d2, d3, nnz, transposed, index_dtype): if transposed: D = torch.randn(d3, d2, dtype=dtype, device=device).t_() else: D = torch.randn(d2, d3, dtype=dtype, device=device) S = self.genSparseCSRTensor((d1, d2), nnz, device=device, dtype=dtype, index_dtype=index_dtype) S_dense = S.to_dense() self.assertEqual(torch.sparse.mm(S, D), torch.mm(S_dense, D)) for index_dtype in [torch.int32, torch.int64]: test_shape(7, 8, 9, 20, False, index_dtype) test_shape(7, 8, 9, 20, True, index_dtype) @dtypes(*floating_types()) @dtypesIfCUDA(*get_all_complex_dtypes(), *get_all_fp_dtypes(include_half=SM53OrLater and TEST_CUSPARSE_GENERIC, include_bfloat16=SM80OrLater and TEST_CUSPARSE_GENERIC)) @precisionOverride({torch.bfloat16: 1e-2, torch.float16: 1e-2}) def test_sparse_addmm(self, device, dtype): def test_shape(m, n, p, nnz, broadcast, index_dtype, alpha_beta=None): if alpha_beta is None: alpha = random.random() beta = random.random() else: alpha, beta = alpha_beta if broadcast: D1 = make_tensor((), dtype=dtype, device=device) else: D1 = make_tensor([n, p], dtype=dtype, device=device) D2 = make_tensor([m, p], dtype=dtype, device=device) S = self.genSparseCSRTensor([n, m], nnz, dtype=dtype, device=device, index_dtype=index_dtype) S_dense = S.to_dense() Y = torch.sparse.addmm(D1, S, D2, beta=beta, alpha=alpha) Y_dense = torch.addmm(D1, S_dense, D2, beta=beta, alpha=alpha) self.assertEqual(Y, Y_dense) for index_dtype in [torch.int32, torch.int64]: test_shape(7, 8, 9, 20, False, index_dtype, None) test_shape(7, 8, 9, 20, True, index_dtype, None) test_shape(7, 8, 9, 20, False, index_dtype, (1, 0)) test_shape(7, 8, 9, 20, True, index_dtype, (1, 0)) test_shape(7, 8, 9, 20, False, index_dtype, (1, 1)) test_shape(7, 8, 9, 20, True, index_dtype, (1, 1)) @onlyCUDA @dtypes(torch.float) def test_addmm_errors(self, device, dtype): # test that the errors are the same for dense and sparse versions import re def test1(*, is_sparse): # shapes must be compatible for matrix multiplication a = make_tensor((2, 3), dtype=dtype, device=device) if is_sparse: a_sparse = a.to_sparse_csr() return torch.addmm(a, a_sparse, a) else: return torch.addmm(a, a, a) def test2(*, is_sparse): # mat2 must be a matrix a = make_tensor((2, 3), dtype=dtype, device=device) if is_sparse: a_sparse = a.to_sparse_csr() return torch.addmm(a, a_sparse, a.unsqueeze(0)) else: return torch.addmm(a, a, a.unsqueeze(0)) def test3(*, is_sparse): # the first input needs to be 1D or 2D a = make_tensor((3, 3), dtype=dtype, device=device) if is_sparse: a_sparse = a.to_sparse_csr() return torch.addmm(a.unsqueeze(0), a_sparse, a) else: return torch.addmm(a.unsqueeze(0), a, a) for test in (test1, test2, test3): try: test(is_sparse=False) except RuntimeError as msg: with self.assertRaisesRegex(RuntimeError, re.escape(str(msg))): test(is_sparse=True) @onlyCUDA @dtypes(torch.float) def test_mm_errors(self, device, dtype): # test that the errors are the same for dense and sparse versions import re def test1(*, is_sparse): # shapes must be compatible for matrix multiplication a = make_tensor((2, 3), dtype=dtype, device=device) if is_sparse: a_sparse = a.to_sparse_csr() return torch.mm(a_sparse, a) else: return torch.mm(a, a) def test2(*, is_sparse): # mat2 must be a matrix a = make_tensor((2, 3), dtype=dtype, device=device) if is_sparse: a_sparse = a.to_sparse_csr() return torch.mm(a_sparse, a.unsqueeze(0)) else: return torch.mm(a, a.unsqueeze(0)) for test in (test1, test2): try: test(is_sparse=False) except RuntimeError as msg: with self.assertRaisesRegex(RuntimeError, re.escape(str(msg))): test(is_sparse=True) @dtypes(torch.float, torch.double) def test_add(self, device, dtype): def _test_spadd_shape(nnz, shape): x = self.genSparseCSRTensor(shape, nnz, dtype=dtype, device=device, index_dtype=torch.int32) y = torch.randn(*shape, dtype=dtype, device=device) r = random.random() res = torch.add(y, x, alpha=r) expected = y + r * x.to_dense() self.assertEqual(res, expected) # Non contiguous dense tensor s = list(shape) s[0] = shape[-1] s[-1] = shape[0] y = torch.randn(*s, dtype=torch.double, device=device) y.transpose_(0, len(s) - 1) r = random.random() res = torch.add(y, x, alpha=r) expected = y + r * x.to_dense() self.assertEqual(res, expected) _test_spadd_shape(10, [100, 100]) _test_spadd_shape(0, [100, 100]) _test_spadd_shape(10, [100, 1]) _test_spadd_shape(10, [1, 100]) @dtypes(*get_all_dtypes()) def test_coo_csr_conversion(self, device, dtype): for m, n in itertools.product([5, 2, 0], [5, 2, 0]): size = (m, n) dense = make_tensor(size, dtype=dtype, device=device) coo_sparse = dense.to_sparse() csr_sparse = coo_sparse.to_sparse_csr() self.assertEqual(csr_sparse.to_dense(), dense)