def test_save_different_dtype_unallocated(self):
devices = ['cpu']
if torch.cuda.is_available():
devices.append('cuda')
def save_load_check(a, b):
with io.BytesIO() as f:
torch.save([a, b], f)
f.seek(0)
a_loaded, b_loaded = torch.load(f)
self.assertEqual(a, a_loaded)
self.assertEqual(b, b_loaded)
for device, dtype in product(devices, get_all_dtypes()):
a = torch.tensor([], dtype=dtype, device=device)
for other_dtype in get_all_dtypes():
s = torch.TypedStorage(wrap_storage=a.storage()._untyped(),
dtype=other_dtype)
save_load_check(a, s)
save_load_check(a.storage(), s)
b = torch.tensor([], dtype=other_dtype, device=device)
save_load_check(a, b)
class TestSortAndSelect(TestCase):
def assertIsOrdered(self, order, x, mxx, ixx, task):
SIZE = x.size(1)
if order == 'descending':
def check_order(a, b):
# `a != a` because we put NaNs
# at the end of ascending sorted lists,
# and the beginning of descending ones.
return ((a != a) | (a >= b)).all().item()
elif order == 'ascending':
def check_order(a, b):
# see above
return ((b != b) | (a <= b)).all().item()
else:
error('unknown order "{}", must be "ascending" or "descending"'.format(order))
are_ordered = True
for k in range(1, SIZE):
self.assertTrue(check_order(mxx[:, k - 1], mxx[:, k]),
'torch.sort ({}) values unordered for {}'.format(order, task))
seen = set()
indicesCorrect = True
size0 = x.size(0)
size = x.size(x.dim() - 1)
x = x.tolist()
mxx = mxx.tolist()
ixx = ixx.tolist()
for k in range(size0):
seen.clear()
for j in range(size):
self.assertEqual(x[k][ixx[k][j]], mxx[k][j],
msg='torch.sort ({}) indices wrong for {}'.format(order, task))
seen.add(ixx[k][j])
self.assertEqual(len(seen), size)
def test_sort(self, device):
# on CUDA 2048 vs >2048 have different code path for the dim being sorted
for SIZE in (4, 2049):
x = torch.rand(4, SIZE, device=device)
res1val, res1ind = torch.sort(x)
# Test inplace
y = x.clone()
y_inds = torch.tensor((), dtype=torch.int64, device=device)
torch.sort(y, out=(y, y_inds))
x_vals, x_inds = torch.sort(x)
self.assertEqual(x_vals, y)
self.assertEqual(x_inds, y_inds)
# Test use of result tensor
res2val = torch.tensor((), device=device)
res2ind = torch.tensor((), device=device, dtype=torch.long)
torch.sort(x, out=(res2val, res2ind))
self.assertEqual(res1val, res2val, atol=0, rtol=0)
self.assertEqual(res1ind, res2ind, atol=0, rtol=0)
self.assertEqual(torch.argsort(x), res1ind)
self.assertEqual(x.argsort(), res1ind)
# Test sorting of random numbers
self.assertIsOrdered('ascending', x, res2val, res2ind, 'random')
# Test simple sort
self.assertEqual(
torch.sort(torch.tensor((50, 40, 30, 20, 10), device=device))[0],
torch.tensor((10, 20, 30, 40, 50), device=device),
atol=0, rtol=0
)
# Test that we still have proper sorting with duplicate keys
x = torch.floor(torch.rand(4, SIZE, device=device) * 10)
torch.sort(x, out=(res2val, res2ind))
self.assertIsOrdered('ascending', x, res2val, res2ind, 'random with duplicate keys')
# DESCENDING SORT
x = torch.rand(4, SIZE, device=device)
res1val, res1ind = torch.sort(x, x.dim() - 1, True)
# Test use of result tensor
res2val = torch.tensor((), device=device)
res2ind = torch.tensor((), device=device, dtype=torch.long)
torch.sort(x, x.dim() - 1, True, out=(res2val, res2ind))
self.assertEqual(res1val, res2val, atol=0, rtol=0)
self.assertEqual(res1ind, res2ind, atol=0, rtol=0)
self.assertEqual(torch.argsort(x, x.dim() - 1, True), res1ind)
self.assertEqual(x.argsort(x.dim() - 1, True), res1ind)
# Test sorting of random numbers
self.assertIsOrdered('descending', x, res2val, res2ind, 'random')
# Test simple sort task
self.assertEqual(
torch.sort(torch.tensor((10, 20, 30, 40, 50), device=device), 0, True)[0],
torch.tensor((50, 40, 30, 20, 10), device=device),
atol=0, rtol=0
)
# Test that we still have proper sorting with duplicate keys
self.assertIsOrdered('descending', x, res2val, res2ind, 'random with duplicate keys')
# Test sorting with NaNs
x = torch.rand(4, SIZE, device=device)
x[1][2] = float('NaN')
x[3][0] = float('NaN')
torch.sort(x, out=(res2val, res2ind))
self.assertIsOrdered('ascending', x, res2val, res2ind,
'random with NaNs')
torch.sort(x, out=(res2val, res2ind), descending=True)
self.assertIsOrdered('descending', x, res2val, res2ind,
'random with NaNs')
# FIXME: remove torch.bool from unsupported types once support is added for cub sort
@dtypes(*set(get_all_dtypes()) - {torch.bool, torch.complex64, torch.complex128})
def test_stable_sort(self, device, dtype):
if TEST_WITH_ROCM and dtype == torch.bfloat16:
return
sizes = (100, 1000, 10000)
for ncopies in sizes:
x = torch.tensor([0, 1] * ncopies, dtype=dtype, device=device)
_, idx = x.sort(stable=True)
self.assertEqual(
idx[:ncopies],
torch.arange(start=0, end=2 * ncopies, step=2, device=device)
)
self.assertEqual(
idx[ncopies:],
torch.arange(start=1, end=2 * ncopies, step=2, device=device)
)
@onlyCUDA
@dtypes(torch.uint8)
@largeTensorTest('200GB') # Unfortunately 80GB A100 is not large enough
def test_sort_large(self, device, dtype):
t0 = torch.randperm(8192, device=device).to(dtype)
t = t0.view(1, 8192).expand(2 ** 18 + 1, -1).contiguous()
v, i = t.sort()
del t
iv, im = i.var_mean(dim=0)
del i
vv, vm = v.var_mean(dim=0)
del v
self.assertEqual(vv, torch.zeros_like(vv))
self.assertEqual(iv, torch.zeros_like(iv))
self.assertEqual(vm, torch.arange(255, dtype=dtype, device=device))
self.assertEqual(im, t0.sort().indices)
def _test_sort_discontiguous(self, device, dtype):
# on CUDA 2048 vs >2048 have different code path for the dim being sorted
sizes = (5, 7, 2049)
for shape in permutations(sizes):
for perm in permutations((0, 1, 2)):
for dim in range(3):
t = torch.randn(shape, device=device, dtype=dtype).permute(perm)
r1 = t.sort(dim=dim)
r2 = t.contiguous().sort(dim=dim)
self.assertEqual(r1, r2)
n = t.size(dim)
# assert ordered
self.assertTrue((r1.values.narrow(dim, 1, n - 1) >= r1.values.narrow(dim, 0, n - 1)).all())
# assert that different segments does not mix, which can easily happen
# if the stride is not handled correctly
self.assertTrue((t.unsqueeze(-1).transpose(dim, -1) == r1.values.unsqueeze(-1)).any(dim=dim).any(dim=-1).all())
# assert stride is preserved
if self.device_type == 'cuda':
# FIXME: this behavior should be true for all cases, not
# just the one specified in if condition
self.assertEqual(r1.values.stride(), t.stride())
self.assertEqual(r1.indices.stride(), t.stride())
@onlyCUDA
@dtypes(torch.float32)
def test_sort_discontiguous(self, device, dtype):
self._test_sort_discontiguous(device, dtype)
@slowTest # this test is slow on CPU, but not on CUDA
@onlyCPU
@dtypes(torch.float32)
def test_sort_discontiguous_slow(self, device, dtype):
self._test_sort_discontiguous(device, dtype)
@dtypes(torch.float32)
def test_sort_1d_output_discontiguous(self, device, dtype):
tensor = torch.randn(12, device=device, dtype=dtype)[:6]
values = torch.empty_like(tensor)[::2]
indices = torch.empty(18, device=device, dtype=torch.long)[::3]
torch.sort(tensor, out=(values, indices))
values_cont, indices_cont = tensor.sort()
self.assertEqual(indices, indices_cont)
self.assertEqual(values, values_cont)
@dtypes(torch.float32)
def test_topk_1d_output_discontiguous(self, device, dtype):
tensor = torch.randn(12, device=device, dtype=dtype)
values = torch.empty_like(tensor)[::2]
indices = torch.empty(18, device=device, dtype=torch.long)[::3]
for sorted in (True, False):
# outputs of `sorted=False` test are not guaranteed to be the same,
# but with current implementation they are
torch.topk(tensor, 6, sorted=sorted, out=(values, indices))
values_cont, indices_cont = tensor.topk(6, sorted=sorted)
self.assertEqual(indices, indices_cont)
self.assertEqual(values, values_cont)
# FIXME: remove torch.bool from unsupported types once support is added for cub sort
@dtypes(*set(get_all_dtypes()) - {torch.bool, torch.complex64, torch.complex128})
def test_stable_sort_against_numpy(self, device, dtype):
if TEST_WITH_ROCM and dtype == torch.bfloat16:
return
if dtype in floating_types_and(torch.float16, torch.bfloat16):
inf = float('inf')
neg_inf = -float('inf')
nan = float('nan')
else:
if dtype != torch.bool:
# no torch.iinfo support for torch.bool
inf = torch.iinfo(dtype).max
neg_inf = torch.iinfo(dtype).min
else:
inf = True
neg_inf = ~inf
# no nan for integral types, we use inf instead for simplicity
nan = inf
def generate_samples():
from itertools import chain, combinations
for sizes in [(1025,), (10000,)]:
size = sizes[0]
# binary strings
yield (torch.tensor([0, 1] * size, dtype=dtype, device=device), 0)
if self.device_type == 'cuda':
return
yield (torch.tensor([0, 1] * 100, dtype=dtype, device=device), 0)
def repeated_index_fill(t, dim, idxs, vals):
res = t
for idx, val in zip(idxs, vals):
res = res.index_fill(dim, idx, val)
return res
for sizes in [(1, 10), (10, 1), (10, 10), (10, 10, 10)]:
size = min(*sizes)
x = (torch.randn(*sizes, device=device) * size).to(dtype)
yield (x, 0)
# Generate tensors which are being filled at random locations
# with values from the non-empty subsets of the set (inf, neg_inf, nan)
# for each dimension.
n_fill_vals = 3 # cardinality of (inf, neg_inf, nan)
for dim in range(len(sizes)):
idxs = (torch.randint(high=size, size=(size // 10,)) for i in range(n_fill_vals))
vals = (inf, neg_inf, nan)
subsets = chain.from_iterable(combinations(list(zip(idxs, vals)), r)
for r in range(1, n_fill_vals + 1))
for subset in subsets:
idxs_subset, vals_subset = zip(*subset)
yield (repeated_index_fill(x, dim, idxs_subset, vals_subset), dim)
for sample, dim in generate_samples():
_, idx_torch = sample.sort(dim=dim, stable=True)
if dtype is torch.bfloat16:
sample_numpy = sample.float().cpu().numpy()
else:
sample_numpy = sample.cpu().numpy()
idx_numpy = np.argsort(sample_numpy, axis=dim, kind='stable')
self.assertEqual(idx_torch, idx_numpy)
@dtypes(*(get_all_int_dtypes() + get_all_fp_dtypes()))
def test_msort(self, device, dtype):
if TEST_WITH_ROCM and dtype == torch.bfloat16:
return
def test(shape):
tensor = make_tensor(shape, device, dtype, low=-9, high=9)
if tensor.size() != torch.Size([]):
if dtype is torch.bfloat16:
expected = torch.from_numpy(np.msort(tensor.float().cpu().numpy())).bfloat16()
else:
expected = torch.from_numpy(np.msort(tensor.cpu().numpy()))
else:
expected = tensor # numpy.msort() does not support empty shapes tensor
result = torch.msort(tensor)
self.assertEqual(result, expected)
out = torch.empty_like(result)
torch.msort(tensor, out=out)
self.assertEqual(out, expected)
shapes = (
[],
[0, ],
[20, ],
[1, 20],
[30, 30],
[10, 20, 30]
)
for shape in shapes:
test(shape)
def test_topk(self, device):
def topKViaSort(t, k, dim, dir):
sorted, indices = t.sort(dim, dir)
return sorted.narrow(dim, 0, k), indices.narrow(dim, 0, k)
def compareTensors(t, res1, ind1, res2, ind2, dim):
# Values should be exactly equivalent
self.assertEqual(res1, res2, atol=0, rtol=0)
# Indices might differ based on the implementation, since there is
# no guarantee of the relative order of selection
if not ind1.eq(ind2).all():
# To verify that the indices represent equivalent elements,
# gather from the input using the topk indices and compare against
# the sort indices
vals = t.gather(dim, ind2)
self.assertEqual(res1, vals, atol=0, rtol=0)
def compare(t, k, dim, dir):
topKVal, topKInd = t.topk(k, dim, dir, True)
sortKVal, sortKInd = topKViaSort(t, k, dim, dir)
compareTensors(t, sortKVal, sortKInd, topKVal, topKInd, dim)
t = torch.rand(random.randint(1, SIZE),
random.randint(1, SIZE),
random.randint(1, SIZE), device=device)
for _kTries in range(3):
for _dimTries in range(3):
for transpose in (True, False):
for dir in (True, False):
testTensor = t
if transpose:
dim1 = random.randrange(t.ndimension())
dim2 = dim1
while dim1 == dim2:
dim2 = random.randrange(t.ndimension())
testTensor = t.transpose(dim1, dim2)
dim = random.randrange(testTensor.ndimension())
k = random.randint(1, testTensor.size(dim))
compare(testTensor, k, dim, dir)
# This tests the code path where on CUDA, topk is implemented with sort.
t = torch.randn((2, 100000), device=device)
compare(t, 2000, 1, True)
compare(t, 2000, 1, False)
def test_topk_arguments(self, device):
q = torch.randn(10, 2, 10, device=device)
# Make sure True isn't mistakenly taken as the 2nd dimension (interpreted as 1)
self.assertRaises(TypeError, lambda: q.topk(4, True))
@skipCUDAIfRocm
def test_unique_dim(self, device):
self.assertFalse(hasattr(torch, 'unique_dim'))
def run_test(device, dtype):
x = torch.tensor([[[1., 1.],
[0., 1.],
[2., 1.],
[0., 1.]],
[[1., 1.],
[0., 1.],
[2., 1.],
[0., 1.]]],
dtype=dtype,
device=device)
x_empty = torch.empty(5, 0, dtype=dtype, device=device)
x_ill_formed_empty = torch.empty(5, 0, 0, dtype=dtype, device=device)
x_ill_formed_empty_another = torch.empty(5, 0, 5, dtype=dtype, device=device)
if dtype in floating_types_and(torch.float16, torch.bfloat16):
x_nan = torch.tensor([float("nan"), 0, 0, float("nan"), float("nan"), 1], dtype=dtype, device=device)
expected_unique_dim0 = torch.tensor([[[1., 1.],
[0., 1.],
[2., 1.],
[0., 1.]]],
dtype=dtype,
device=device)
expected_inverse_dim0 = torch.tensor([0, 0])
expected_counts_dim0 = torch.tensor([2])
expected_unique_dim1 = torch.tensor([[[0., 1.],
[1., 1.],
[2., 1.]],
[[0., 1.],
[1., 1.],
[2., 1.]]],
dtype=dtype,
device=device)
expected_unique_dim1_bool = torch.tensor([[[False, True], [True, True]],
[[False, True], [True, True]]],
dtype=torch.bool,
device=device)
expected_inverse_dim1 = torch.tensor([1, 0, 2, 0])
expected_inverse_dim1_bool = torch.tensor([1, 0, 1, 0])
expected_counts_dim1 = torch.tensor([2, 1, 1])
expected_counts_dim1_bool = torch.tensor([2, 2])
expected_unique_dim2 = torch.tensor([[[1., 1.],
[0., 1.],
[2., 1.],
[0., 1.]],
[[1., 1.],
[0., 1.],
[2., 1.],
[0., 1.]]],
dtype=dtype,
device=device)
expected_inverse_dim2 = torch.tensor([0, 1])
expected_counts_dim2 = torch.tensor([1, 1])
expected_unique_empty = torch.tensor([], dtype=dtype, device=device)
expected_inverse_empty = torch.tensor([], dtype=torch.long, device=device)
expected_counts_empty = torch.tensor([], dtype=torch.long, device=device)
if dtype in floating_types_and(torch.float16, torch.bfloat16):
expected_unique_nan = torch.tensor([float("nan"), 0, float("nan"), float("nan"), 1], dtype=dtype, device=device)
expected_inverse_nan = torch.tensor([0, 1, 1, 2, 3, 4], dtype=torch.long, device=device)
expected_counts_nan = torch.tensor([1, 2, 1, 1, 1], dtype=torch.long, device=device)
# dim0
x_unique = torch.unique(x, dim=0)
self.assertEqual(expected_unique_dim0, x_unique)
x_unique, x_inverse = torch.unique(
x,
return_inverse=True,
dim=0)
self.assertEqual(expected_unique_dim0, x_unique)
self.assertEqual(expected_inverse_dim0, x_inverse)
x_unique, x_counts = torch.unique(
x,
return_inverse=False,
return_counts=True,
dim=0)
self.assertEqual(expected_unique_dim0, x_unique)
self.assertEqual(expected_counts_dim0, x_counts)
x_unique, x_inverse, x_counts = torch.unique(
x,
return_inverse=True,
return_counts=True,
dim=0)
self.assertEqual(expected_unique_dim0, x_unique)
self.assertEqual(expected_inverse_dim0, x_inverse)
self.assertEqual(expected_counts_dim0, x_counts)
# dim1
x_unique = torch.unique(x, dim=1)
if x.dtype == torch.bool:
self.assertEqual(expected_unique_dim1_bool, x_unique)
else:
self.assertEqual(expected_unique_dim1, x_unique)
x_unique, x_inverse = torch.unique(
x,
return_inverse=True,
dim=1)
if x.dtype == torch.bool:
self.assertEqual(expected_unique_dim1_bool, x_unique)
self.assertEqual(expected_inverse_dim1_bool, x_inverse)
else:
self.assertEqual(expected_unique_dim1, x_unique)
self.assertEqual(expected_inverse_dim1, x_inverse)
x_unique, x_counts = torch.unique(
x,
return_inverse=False,
return_counts=True,
dim=1)
if x.dtype == torch.bool:
self.assertEqual(expected_unique_dim1_bool, x_unique)
self.assertEqual(expected_counts_dim1_bool, x_counts)
else:
self.assertEqual(expected_unique_dim1, x_unique)
self.assertEqual(expected_counts_dim1, x_counts)
x_unique, x_inverse, x_counts = torch.unique(
x,
return_inverse=True,
return_counts=True,
dim=1)
if x.dtype == torch.bool:
self.assertEqual(expected_unique_dim1_bool, x_unique)
self.assertEqual(expected_inverse_dim1_bool, x_inverse)
self.assertEqual(expected_counts_dim1_bool, x_counts)
else:
self.assertEqual(expected_unique_dim1, x_unique)
self.assertEqual(expected_inverse_dim1, x_inverse)
self.assertEqual(expected_counts_dim1, x_counts)
# dim2
x_unique = torch.unique(x, dim=2)
self.assertEqual(expected_unique_dim2, x_unique)
x_unique, x_inverse = torch.unique(
x,
return_inverse=True,
dim=2)
self.assertEqual(expected_unique_dim2, x_unique)
self.assertEqual(expected_inverse_dim2, x_inverse)
x_unique, x_counts = torch.unique(
x,
return_inverse=False,
return_counts=True,
dim=2)
self.assertEqual(expected_unique_dim2, x_unique)
self.assertEqual(expected_counts_dim2, x_counts)
x_unique, x_inverse, x_counts = torch.unique(
x,
return_inverse=True,
return_counts=True,
dim=2)
self.assertEqual(expected_unique_dim2, x_unique)
self.assertEqual(expected_inverse_dim2, x_inverse)
self.assertEqual(expected_counts_dim2, x_counts)
# test empty tensor
x_unique, x_inverse, x_counts = torch.unique(
x_empty,
return_inverse=True,
return_counts=True,
dim=1)
self.assertEqual(expected_unique_empty, x_unique)
self.assertEqual(expected_inverse_empty, x_inverse)
self.assertEqual(expected_counts_empty, x_counts)
# test tensor with nan
if dtype in floating_types_and(torch.float16, torch.bfloat16):
x_unique, x_inverse, x_counts = torch.unique(
x_nan,
return_inverse=True,
return_counts=True,
dim=0)
self.assertEqual(expected_unique_nan, x_unique)
self.assertEqual(expected_inverse_nan, x_inverse)
self.assertEqual(expected_counts_nan, x_counts)
# test not a well formed tensor
# Checking for runtime error, as this is the expected behaviour
with self.assertRaises(RuntimeError):
torch.unique(
x_ill_formed_empty,
return_inverse=True,
return_counts=True,
dim=1)
# test along dim2
with self.assertRaises(RuntimeError):
torch.unique(
x_ill_formed_empty_another,
return_inverse=True,
return_counts=True,
dim=2)
# test consecutive version
y = torch.tensor(
[[0, 1],
[0, 1],
[0, 1],
[1, 2],
[1, 2],
[3, 4],
[0, 1],
[0, 1],
[3, 4],
[1, 2]],
dtype=dtype,
device=device
)
# test tensor with nan
if dtype in floating_types_and(torch.float16, torch.bfloat16):
y_nan = torch.tensor([float("nan"), 0, 0, float("nan"), float("nan"), 1], dtype=dtype, device=device)
expected_y_unique = torch.tensor(
[[0, 1],
[1, 2],
[3, 4],
[0, 1],
[3, 4],
[1, 2]],
dtype=dtype,
device=device
)
expected_y_inverse = torch.tensor([0, 0, 0, 1, 1, 2, 3, 3, 4, 5], dtype=torch.int64, device=device)
expected_y_counts = torch.tensor([3, 2, 1, 2, 1, 1], dtype=torch.int64, device=device)
expected_y_inverse_bool = torch.tensor([0, 0, 0, 1, 1, 1, 2, 2, 3, 3], dtype=torch.int64, device=device)
expected_y_counts_bool = torch.tensor([3, 3, 2, 2], dtype=torch.int64, device=device)
if dtype in floating_types_and(torch.float16, torch.bfloat16):
expected_y_unique_nan = torch.tensor([float("nan"), 0, float("nan"), float("nan"), 1], dtype=dtype, device=device)
expected_y_inverse_nan = torch.tensor([0, 1, 1, 2, 3, 4], dtype=torch.long, device=device)
expected_y_counts_nan = torch.tensor([1, 2, 1, 1, 1], dtype=torch.long, device=device)
y_unique, y_inverse, y_counts = torch.unique_consecutive(y, return_inverse=True, return_counts=True, dim=0)
if x.dtype == torch.bool:
self.assertEqual(expected_y_inverse_bool, y_inverse)
self.assertEqual(expected_y_counts_bool, y_counts)
else:
self.assertEqual(expected_y_inverse, y_inverse)
self.assertEqual(expected_y_counts, y_counts)
# test tensor with nan
if dtype in floating_types_and(torch.float16, torch.bfloat16):
y_unique, y_inverse, y_counts = torch.unique_consecutive(
y_nan,
return_inverse=True,
return_counts=True,
dim=0)
self.assertEqual(expected_y_unique_nan, y_unique)
self.assertEqual(expected_y_inverse_nan, y_inverse)
self.assertEqual(expected_y_counts_nan, y_counts)
run_test(device, torch.float)
run_test(device, torch.double)
run_test(device, torch.long)
run_test(device, torch.uint8)
run_test(device, torch.bool)
@onlyCUDA
def test_topk_noncontiguous_gpu(self, device):
t = torch.randn(20, device=device)[::2]
top1, idx1 = t.topk(5)
top2, idx2 = t.contiguous().topk(5)
self.assertEqual(top1, top2)
self.assertEqual(idx1, idx2)
def _test_topk_dtype(self, device, dtype, integral, size):
if integral:
a = torch.randint(torch.iinfo(dtype).min, torch.iinfo(dtype).max,
size=(size,), dtype=dtype, device=device)
else:
a = torch.randn(size=(size,), dtype=dtype, device=device)
sort_topk = a.sort()[0][-(size // 2):].flip(0)
topk = a.topk(size // 2)
self.assertEqual(sort_topk, topk[0]) # check values
self.assertEqual(sort_topk, a[topk[1]]) # check indices
@dtypes(torch.int8, torch.uint8, torch.int16, torch.int32, torch.int64)
def test_topk_integral(self, device, dtype):
small = 10
large = 4096
for curr_size in (small, large):
self._test_topk_dtype(device, dtype, True, curr_size)
@onlyCUDA
@dtypes(torch.bfloat16)
@skipCUDAIfRocm
def test_topk_bfloat16(self, device, dtype):
small = 10
large = 8192
for curr_size in (small, large):
self._test_topk_dtype(device, dtype, False, curr_size)
@dtypesIfCUDA(*get_all_fp_dtypes())
@dtypes(torch.float, torch.double, torch.bfloat16)
def test_topk_nonfinite(self, device, dtype):
if TEST_WITH_ROCM and dtype == torch.bfloat16:
return
x = torch.tensor([float('nan'), float('inf'), 1e4, 0, -1e4, -float('inf')], device=device, dtype=dtype)
val, idx = x.topk(4)
expect = torch.tensor([float('nan'), float('inf'), 1e4, 0], device=device, dtype=dtype)
self.assertEqual(val, expect)
self.assertEqual(idx, [0, 1, 2, 3])
val, idx = x.topk(4, largest=False)
expect = torch.tensor([-float('inf'), -1e4, 0, 1e4], device=device, dtype=dtype)
self.assertEqual(val, expect)
self.assertEqual(idx, [5, 4, 3, 2])
def test_topk_4d(self, device):
x = torch.ones(2, 3072, 2, 2, device=device)
x[:, 1, :, :] *= 2.
x[:, 10, :, :] *= 1.5
val, ind = torch.topk(x, k=2, dim=1)
expected_ind = torch.ones(2, 2, 2, 2, dtype=torch.long, device=device)
expected_ind[:, 1, :, :] = 10
expected_val = torch.ones(2, 2, 2, 2, device=device)
expected_val[:, 0, :, :] *= 2.
expected_val[:, 1, :, :] *= 1.5
self.assertEqual(val, expected_val, atol=0, rtol=0)
self.assertEqual(ind, expected_ind, atol=0, rtol=0)
@onlyNativeDeviceTypes
@dtypesIfCUDA(*(get_all_dtypes(include_complex=False,
include_bool=False,
include_half=False,
include_bfloat16=True)))
@dtypes(*(get_all_dtypes(include_complex=False, include_bool=False, include_half=False, include_bfloat16=False)))
def test_topk_zero(self, device, dtype):
if TEST_WITH_ROCM and dtype == torch.bfloat16:
return
# https://github.com/pytorch/pytorch/issues/49205
t = torch.rand(2, 2, device=device).to(dtype=dtype)
val, idx = torch.topk(t, k=0, largest=False)
self.assertEqual(val.size(), torch.Size([2, 0]))
self.assertEqual(idx.size(), torch.Size([2, 0]))
def _test_unique_scalar_empty(self, dtype, device, f):
# test scalar
x = torch.tensor(0, dtype=dtype, device=device)
unique, inverse, counts = f(x, return_inverse=True, return_counts=True)
expected_unique = torch.tensor([0], dtype=dtype, device=device)
expected_inverse = torch.tensor(0, device=device)
expected_counts = torch.tensor([1], device=device)
self.assertEqual(unique, expected_unique)
self.assertEqual(inverse, expected_inverse)
self.assertEqual(counts, expected_counts)
# test zero sized tensor
x = torch.zeros((0, 0, 3), dtype=dtype, device=device)
unique, inverse, counts = f(x, return_inverse=True, return_counts=True)
expected_unique = torch.tensor([], dtype=dtype, device=device)
expected_inverse = torch.empty((0, 0, 3), dtype=torch.long, device=device)
expected_counts = torch.tensor([], dtype=torch.long, device=device)
self.assertEqual(unique, expected_unique)
self.assertEqual(inverse, expected_inverse)
self.assertEqual(counts, expected_counts)
def _test_unique_with_expects(self, device, dtype, f, x, expected_unique, expected_inverse, expected_counts, additional_shape):
def ensure_tuple(x):
if isinstance(x, torch.Tensor):
return (x,)
return x
for return_inverse in [True, False]:
for return_counts in [True, False]:
# test with expected
ret = ensure_tuple(f(x, return_inverse=return_inverse, return_counts=return_counts))
self.assertEqual(len(ret), 1 + int(return_inverse) + int(return_counts))
self.assertEqual(expected_unique, ret[0])
if return_inverse:
self.assertEqual(expected_inverse, ret[1])
if return_counts:
count_index = 1 + int(return_inverse)
self.assertEqual(expected_counts, ret[count_index])
# tests per-element unique on a higher rank tensor.
y = x.view(additional_shape)
y_unique, y_inverse, y_counts = f(y, return_inverse=True, return_counts=True)
self.assertEqual(expected_unique, y_unique)
self.assertEqual(expected_inverse.view(additional_shape), y_inverse)
self.assertEqual(expected_counts, y_counts)
@dtypesIfCPU(*set(get_all_dtypes()) - {torch.complex64, torch.complex128})
@dtypes(*set(get_all_dtypes()) - {torch.bfloat16, torch.complex64, torch.complex128})
def test_unique(self, device, dtype):
if dtype is torch.half and self.device_type == 'cpu':
return # CPU does not have half support
def ensure_tuple(x):
if isinstance(x, torch.Tensor):
return (x,)
return x
if dtype is torch.bool:
x = torch.tensor([True, False, False, False, True, False, True, False], dtype=torch.bool, device=device)
expected_unique = torch.tensor([False, True], dtype=torch.bool, device=device)
expected_inverse = torch.tensor([1, 0, 0, 0, 1, 0, 1, 0], dtype=torch.long, device=device)
expected_counts = torch.tensor([5, 3], dtype=torch.long, device=device)
else:
x = torch.tensor([1, 2, 3, 2, 8, 5, 2, 3], dtype=dtype, device=device)
expected_unique = torch.tensor([1, 2, 3, 5, 8], dtype=dtype, device=device)
expected_inverse = torch.tensor([0, 1, 2, 1, 4, 3, 1, 2], device=device)
expected_counts = torch.tensor([1, 3, 2, 1, 1], device=device)
# test sorted unique
fs = (
lambda x, **kwargs: torch.unique(x, sorted=True, **kwargs),
lambda x, **kwargs: x.unique(sorted=True, **kwargs),
)
x_sliced = torch.empty(x.size(0) * 2, dtype=dtype, device=device)[::2].copy_(x)
xs = (x, x_sliced)
for f, x in product(fs, xs):
self._test_unique_with_expects(device, dtype, f, x, expected_unique, expected_inverse, expected_counts, (2, 2, 2))
self._test_unique_scalar_empty(dtype, device, f)
# test unsorted unique
fs = (
lambda x, **kwargs: torch.unique(x, sorted=False, **kwargs),
lambda x, **kwargs: x.unique(sorted=False, **kwargs)
)
for f, x in product(fs, xs):
self._test_unique_scalar_empty(dtype, device, f)
for return_inverse, return_counts in product((True, False), repeat=2):
ret = ensure_tuple(f(x, return_inverse=return_inverse, return_counts=return_counts))
self.assertEqual(len(ret), 1 + int(return_inverse) + int(return_counts))
x_list = x.tolist()
x_unique_list = ret[0].tolist()
self.assertEqual(expected_unique.tolist(), sorted(x_unique_list))
if return_inverse:
x_inverse_list = ret[1].tolist()
for i, j in enumerate(x_inverse_list):
self.assertEqual(x_list[i], x_unique_list[j])
if return_counts:
count_index = 1 + int(return_inverse)
x_counts_list = ret[count_index].tolist()
for i, j in zip(x_unique_list, x_counts_list):
count = 0
for k in x_list:
if k == i:
count += 1
self.assertEqual(j, count)
@dtypesIfCPU(*set(get_all_dtypes()) - {torch.complex64, torch.complex128})
@dtypes(*set(get_all_dtypes()) - {torch.bfloat16, torch.complex64, torch.complex128})
def test_unique_consecutive(self, device, dtype):
if dtype is torch.half and self.device_type == 'cpu':
return # CPU does not have half support
if dtype is torch.bool:
x = torch.tensor([True, False, False, False, True, True, False, False, False], dtype=torch.bool, device=device)
expected_unique = torch.tensor([True, False, True, False], dtype=torch.bool, device=device)
expected_inverse = torch.tensor([0, 1, 1, 1, 2, 2, 3, 3, 3], dtype=torch.long, device=device)
expected_counts = torch.tensor([1, 3, 2, 3], dtype=torch.long, device=device)
else:
x = torch.tensor([1, 2, 2, 2, 5, 5, 2, 2, 3], dtype=dtype, device=device)
expected_unique = torch.tensor([1, 2, 5, 2, 3], dtype=dtype, device=device)
expected_inverse = torch.tensor([0, 1, 1, 1, 2, 2, 3, 3, 4], device=device)
expected_counts = torch.tensor([1, 3, 2, 2, 1], device=device)
for f in [torch.unique_consecutive, lambda x, **kwargs: x.unique_consecutive(**kwargs)]:
self._test_unique_with_expects(device, dtype, f, x, expected_unique, expected_inverse, expected_counts, (3, 3))
self._test_unique_scalar_empty(dtype, device, f)
@dtypes(torch.double)
def test_kthvalue(self, device, dtype):
SIZE = 50
x = torch.rand(SIZE, SIZE, SIZE, dtype=dtype, device=device)
x0 = x.clone()
k = random.randint(1, SIZE)
res1val, res1ind = torch.kthvalue(x, k, keepdim=False)
res2val, res2ind = torch.sort(x)
self.assertEqual(res1val[:, :], res2val[:, :, k - 1], atol=0, rtol=0)
self.assertEqual(res1ind[:, :], res2ind[:, :, k - 1], atol=0, rtol=0)
# test use of result tensors
k = random.randint(1, SIZE)
res1val = torch.tensor([], dtype=dtype, device=device)
res1ind = torch.tensor([], dtype=torch.long, device=device)
torch.kthvalue(x, k, keepdim=False, out=(res1val, res1ind))
res2val, res2ind = torch.sort(x)
self.assertEqual(res1val[:, :], res2val[:, :, k - 1], atol=0, rtol=0)
self.assertEqual(res1ind[:, :], res2ind[:, :, k - 1], atol=0, rtol=0)
# test non-default dim
k = random.randint(1, SIZE)
res1val, res1ind = torch.kthvalue(x, k, 0, keepdim=False)
res2val, res2ind = torch.sort(x, 0)
self.assertEqual(res1val, res2val[k - 1], atol=0, rtol=0)
self.assertEqual(res1ind, res2ind[k - 1], atol=0, rtol=0)
# non-contiguous
y = x.narrow(1, 0, 1)
y0 = y.contiguous()
k = random.randint(1, SIZE)
res1val, res1ind = torch.kthvalue(y, k)
res2val, res2ind = torch.kthvalue(y0, k)
self.assertEqual(res1val, res2val, atol=0, rtol=0)
self.assertEqual(res1ind, res2ind, atol=0, rtol=0)
# non-contiguous [Reference: https://github.com/pytorch/pytorch/issues/45721]
non_contig_t = torch.tensor([0, -1, 1, -2, 2], dtype=dtype, device=device)[::2]
expected_val, expected_ind = non_contig_t.contiguous().kthvalue(2)
non_contig_cpu_t = non_contig_t.cpu()
expected_val_cpu, expected_ind_cpu = non_contig_cpu_t.kthvalue(2)
out_val, out_ind = non_contig_t.kthvalue(2)
self.assertEqual(expected_val, out_val, atol=0, rtol=0)
self.assertEqual(expected_ind, out_ind, atol=0, rtol=0)
self.assertEqual(expected_val_cpu, out_val, atol=0, rtol=0)
self.assertEqual(expected_ind_cpu, out_ind, atol=0, rtol=0)
# check that the input wasn't modified
self.assertEqual(x, x0, atol=0, rtol=0)
# simple test case (with repetitions)
y = torch.tensor((3., 5, 4, 1, 1, 5), dtype=dtype, device=device)
self.assertEqual(torch.kthvalue(y, 3)[0], 3, atol=0, rtol=0)
self.assertEqual(torch.kthvalue(y, 2)[0], 1, atol=0, rtol=0)
# simple test case (with NaN)
SIZE = 50
x = torch.rand(SIZE, SIZE, SIZE, dtype=dtype, device=device)
x[torch.arange(SIZE), :, torch.randint(50, (50,))] = nan
ks = [random.randint(1, SIZE), 1, SIZE, SIZE - 1]
res2val, res2ind = torch.sort(x)
for k in ks:
res1val, res1ind = torch.kthvalue(x, k, keepdim=False)
self.assertEqual(res1val[:, :], res2val[:, :, k - 1], atol=0, rtol=0)
self.assertEqual(res1ind[:, :], res2ind[:, :, k - 1], atol=0, rtol=0)
@dtypes(torch.float)
@onlyNativeDeviceTypes # Fails on XLA
def test_kthvalue_scalar(self, device, dtype):
# Test scalar input (test case from https://github.com/pytorch/pytorch/issues/30818)
# Tests that passing a scalar tensor or 1D tensor with 1 element work either way
res = torch.tensor(2, device=device, dtype=dtype).kthvalue(1)
ref = torch.tensor([2], device=device, dtype=dtype).kthvalue(1)
self.assertEqual(res[0], ref[0].squeeze())
self.assertEqual(res[1], ref[1].squeeze())
@dtypes(*all_types())
@dtypesIfCUDA(*all_types_and(torch.half))
def test_isin(self, device, dtype):
def assert_isin_equal(a, b):
# Compare to the numpy reference implementation.
x = torch.isin(a, b)
a = a.cpu().numpy() if torch.is_tensor(a) else np.array(a)
b = b.cpu().numpy() if torch.is_tensor(b) else np.array(b)
y = np.isin(a, b)
self.assertEqual(x, y)
# multi-dim tensor, multi-dim tensor
a = torch.arange(24, device=device, dtype=dtype).reshape([2, 3, 4])
b = torch.tensor([[10, 20, 30], [0, 1, 3], [11, 22, 33]], device=device, dtype=dtype)
assert_isin_equal(a, b)
# zero-dim tensor
zero_d = torch.tensor(3, device=device, dtype=dtype)
assert_isin_equal(zero_d, b)
assert_isin_equal(a, zero_d)
assert_isin_equal(zero_d, zero_d)
# empty tensor
empty = torch.tensor([], device=device, dtype=dtype)
assert_isin_equal(empty, b)
assert_isin_equal(a, empty)
assert_isin_equal(empty, empty)
# scalar
assert_isin_equal(a, 6)
assert_isin_equal(5, b)
def define_expected(lst, invert=False):
expected = torch.tensor(lst, device=device)
if invert:
expected = expected.logical_not()
return expected
# Adapted from numpy's in1d tests
for mult in [1, 10]:
for invert in [False, True]:
a = torch.tensor([5, 7, 1, 2], device=device, dtype=dtype)
b = torch.tensor([2, 4, 3, 1, 5] * mult, device=device, dtype=dtype)
ec = define_expected([True, False, True, True], invert=invert)
c = torch.isin(a, b, assume_unique=True, invert=invert)
self.assertEqual(c, ec)
a[0] = 8
ec = define_expected([False, False, True, True], invert=invert)
c = torch.isin(a, b, assume_unique=True, invert=invert)
self.assertEqual(c, ec)
a[0], a[3] = 4, 8
ec = define_expected([True, False, True, False], invert=invert)
c = torch.isin(a, b, assume_unique=True, invert=invert)
self.assertEqual(c, ec)
a = torch.tensor([5, 4, 5, 3, 4, 4, 3, 4, 3, 5, 2, 1, 5, 5], device=device, dtype=dtype)
b = torch.tensor([2, 3, 4] * mult, device=device, dtype=dtype)
ec = define_expected([False, True, False, True, True, True, True, True, True,
False, True, False, False, False], invert=invert)
c = torch.isin(a, b, invert=invert)
self.assertEqual(c, ec)
b = torch.tensor([2, 3, 4] * mult + [5, 5, 4] * mult, device=device, dtype=dtype)
ec = define_expected([True, True, True, True, True, True, True, True, True, True,
True, False, True, True], invert=invert)
c = torch.isin(a, b, invert=invert)
self.assertEqual(c, ec)
a = torch.tensor([5, 7, 1, 2], device=device, dtype=dtype)
b = torch.tensor([2, 4, 3, 1, 5] * mult, device=device, dtype=dtype)
ec = define_expected([True, False, True, True], invert=invert)
c = torch.isin(a, b, invert=invert)
self.assertEqual(c, ec)
a = torch.tensor([5, 7, 1, 1, 2], device=device, dtype=dtype)
b = torch.tensor([2, 4, 3, 3, 1, 5] * mult, device=device, dtype=dtype)
ec = define_expected([True, False, True, True, True], invert=invert)
c = torch.isin(a, b, invert=invert)
self.assertEqual(c, ec)
a = torch.tensor([5, 5], device=device, dtype=dtype)
b = torch.tensor([2, 2] * mult, device=device, dtype=dtype)
ec = define_expected([False, False], invert=invert)
c = torch.isin(a, b, invert=invert)
self.assertEqual(c, ec)
# multi-dimensional input case using sort-based algo
for assume_unique in [False, True]:
a = torch.arange(6, device=device, dtype=dtype).reshape([2, 3])
b = torch.arange(3, 30, device=device, dtype=dtype)
ec = define_expected([[False, False, False], [True, True, True]], invert=invert)
c = torch.isin(a, b, invert=invert, assume_unique=assume_unique)
self.assertEqual(c, ec)
def test_isin_different_dtypes(self, device):
supported_types = all_types() if device == 'cpu' else all_types_and(torch.half)
for mult in [1, 10]:
for assume_unique in [False, True]:
for dtype1, dtype2 in product(supported_types, supported_types):
a = torch.tensor([1, 2, 3], device=device, dtype=dtype1)
b = torch.tensor([3, 4, 5] * mult, device=device, dtype=dtype2)
ec = torch.tensor([False, False, True], device=device)
c = torch.isin(a, b, assume_unique=assume_unique)
self.assertEqual(c, ec)
@onlyCUDA
@dtypes(*all_types())
def test_isin_different_devices(self, device, dtype):
a = torch.arange(6, device=device, dtype=dtype).reshape([2, 3])
b = torch.arange(3, 30, device='cpu', dtype=dtype)
with self.assertRaises(RuntimeError):
torch.isin(a, b)
c = torch.arange(6, device='cpu', dtype=dtype).reshape([2, 3])
d = torch.arange(3, 30, device=device, dtype=dtype)
with self.assertRaises(RuntimeError):
torch.isin(c, d)
class TestNumPyInterop(TestCase):
# Note: the warning this tests for only appears once per program, so
# other instances of this warning should be addressed to avoid
# the tests depending on the order in which they're run.
@onlyCPU
def test_numpy_non_writeable(self, device):
arr = np.zeros(5)
arr.flags['WRITEABLE'] = False
self.assertWarns(UserWarning, lambda: torch.from_numpy(arr))
@onlyCPU
def test_numpy_unresizable(self, device) -> None:
x = np.zeros((2, 2))
y = torch.from_numpy(x)
with self.assertRaises(ValueError):
x.resize((5, 5))
z = torch.randn(5, 5)
w = z.numpy()
with self.assertRaises(RuntimeError):
z.resize_(10, 10)
with self.assertRaises(ValueError):
w.resize((10, 10))
@onlyCPU
def test_to_numpy(self, device) -> None:
def get_castable_tensor(shape, dtype):
if dtype.is_floating_point:
dtype_info = torch.finfo(dtype)
# can't directly use min and max, because for double, max - min
# is greater than double range and sampling always gives inf.
low = max(dtype_info.min, -1e10)
high = min(dtype_info.max, 1e10)
t = torch.empty(shape, dtype=torch.float64).uniform_(low, high)
else:
# can't directly use min and max, because for int64_t, max - min
# is greater than int64_t range and triggers UB.
low = max(torch.iinfo(dtype).min, int(-1e10))
high = min(torch.iinfo(dtype).max, int(1e10))
t = torch.empty(shape, dtype=torch.int64).random_(low, high)
return t.to(dtype)
dtypes = [
torch.uint8,
torch.int8,
torch.short,
torch.int,
torch.half,
torch.float,
torch.double,
torch.long,
]
for dtp in dtypes:
# 1D
sz = 10
x = get_castable_tensor(sz, dtp)
y = x.numpy()
for i in range(sz):
self.assertEqual(x[i], y[i])
# 1D > 0 storage offset
xm = get_castable_tensor(sz * 2, dtp)
x = xm.narrow(0, sz - 1, sz)
self.assertTrue(x.storage_offset() > 0)
y = x.numpy()
for i in range(sz):
self.assertEqual(x[i], y[i])
def check2d(x, y):
for i in range(sz1):
for j in range(sz2):
self.assertEqual(x[i][j], y[i][j])
# empty
x = torch.tensor([]).to(dtp)
y = x.numpy()
self.assertEqual(y.size, 0)
# contiguous 2D
sz1 = 3
sz2 = 5
x = get_castable_tensor((sz1, sz2), dtp)
y = x.numpy()
check2d(x, y)
self.assertTrue(y.flags['C_CONTIGUOUS'])
# with storage offset
xm = get_castable_tensor((sz1 * 2, sz2), dtp)
x = xm.narrow(0, sz1 - 1, sz1)
y = x.numpy()
self.assertTrue(x.storage_offset() > 0)
check2d(x, y)
self.assertTrue(y.flags['C_CONTIGUOUS'])
# non-contiguous 2D
x = get_castable_tensor((sz2, sz1), dtp).t()
y = x.numpy()
check2d(x, y)
self.assertFalse(y.flags['C_CONTIGUOUS'])
# with storage offset
xm = get_castable_tensor((sz2 * 2, sz1), dtp)
x = xm.narrow(0, sz2 - 1, sz2).t()
y = x.numpy()
self.assertTrue(x.storage_offset() > 0)
check2d(x, y)
# non-contiguous 2D with holes
xm = get_castable_tensor((sz2 * 2, sz1 * 2), dtp)
x = xm.narrow(0, sz2 - 1, sz2).narrow(1, sz1 - 1, sz1).t()
y = x.numpy()
self.assertTrue(x.storage_offset() > 0)
check2d(x, y)
if dtp != torch.half:
# check writeable
x = get_castable_tensor((3, 4), dtp)
y = x.numpy()
self.assertTrue(y.flags.writeable)
y[0][1] = 3
self.assertTrue(x[0][1] == 3)
y = x.t().numpy()
self.assertTrue(y.flags.writeable)
y[0][1] = 3
self.assertTrue(x[0][1] == 3)
def test_to_numpy_bool(self, device) -> None:
x = torch.tensor([True, False], dtype=torch.bool)
self.assertEqual(x.dtype, torch.bool)
y = x.numpy()
self.assertEqual(y.dtype, np.bool_)
for i in range(len(x)):
self.assertEqual(x[i], y[i])
x = torch.tensor([True], dtype=torch.bool)
self.assertEqual(x.dtype, torch.bool)
y = x.numpy()
self.assertEqual(y.dtype, np.bool_)
self.assertEqual(x[0], y[0])
def test_from_numpy(self, device) -> None:
dtypes = [
np.double,
np.float64,
np.float16,
np.complex64,
np.complex128,
np.int64,
np.int32,
np.int16,
np.int8,
np.uint8,
np.longlong,
np.bool_,
]
complex_dtypes = [
np.complex64,
np.complex128,
]
for dtype in dtypes:
array = np.array([1, 2, 3, 4], dtype=dtype)
tensor_from_array = torch.from_numpy(array)
# TODO: change to tensor equality check once HalfTensor
# implements `==`
for i in range(len(array)):
self.assertEqual(tensor_from_array[i], array[i])
# ufunc 'remainder' not supported for complex dtypes
if dtype not in complex_dtypes:
# This is a special test case for Windows
# https://github.com/pytorch/pytorch/issues/22615
array2 = array % 2
tensor_from_array2 = torch.from_numpy(array2)
for i in range(len(array2)):
self.assertEqual(tensor_from_array2[i], array2[i])
# Test unsupported type
array = np.array([1, 2, 3, 4], dtype=np.uint16)
with self.assertRaises(TypeError):
tensor_from_array = torch.from_numpy(array)
# check storage offset
x = np.linspace(1, 125, 125)
x.shape = (5, 5, 5)
x = x[1]
expected = torch.arange(1, 126, dtype=torch.float64).view(5, 5, 5)[1]
self.assertEqual(torch.from_numpy(x), expected)
# check noncontiguous
x = np.linspace(1, 25, 25)
x.shape = (5, 5)
expected = torch.arange(1, 26, dtype=torch.float64).view(5, 5).t()
self.assertEqual(torch.from_numpy(x.T), expected)
# check noncontiguous with holes
x = np.linspace(1, 125, 125)
x.shape = (5, 5, 5)
x = x[:, 1]
expected = torch.arange(1, 126, dtype=torch.float64).view(5, 5, 5)[:,
1]
self.assertEqual(torch.from_numpy(x), expected)
# check zero dimensional
x = np.zeros((0, 2))
self.assertEqual(torch.from_numpy(x).shape, (0, 2))
x = np.zeros((2, 0))
self.assertEqual(torch.from_numpy(x).shape, (2, 0))
# check ill-sized strides raise exception
x = np.array([3., 5., 8.])
x.strides = (3, )
self.assertRaises(ValueError, lambda: torch.from_numpy(x))
def test_from_list_of_ndarray_warning(self, device):
warning_msg = r"Creating a tensor from a list of numpy.ndarrays is extremely slow"
with self.assertWarnsOnceRegex(UserWarning, warning_msg):
torch.tensor([np.array([0]), np.array([1])], device=device)
@onlyCPU
def test_ctor_with_numpy_scalar_ctor(self, device) -> None:
dtypes = [
np.double,
np.float64,
np.float16,
np.int64,
np.int32,
np.int16,
np.uint8,
np.bool_,
]
for dtype in dtypes:
self.assertEqual(dtype(42), torch.tensor(dtype(42)).item())
@onlyCPU
def test_numpy_index(self, device):
i = np.array([0, 1, 2], dtype=np.int32)
x = torch.randn(5, 5)
for idx in i:
self.assertFalse(isinstance(idx, int))
self.assertEqual(x[idx], x[int(idx)])
@onlyCPU
def test_numpy_array_interface(self, device):
types = [
torch.DoubleTensor,
torch.FloatTensor,
torch.HalfTensor,
torch.LongTensor,
torch.IntTensor,
torch.ShortTensor,
torch.ByteTensor,
]
dtypes = [
np.float64,
np.float32,
np.float16,
np.int64,
np.int32,
np.int16,
np.uint8,
]
for tp, dtype in zip(types, dtypes):
# Only concrete class can be given where "Type[number[_64Bit]]" is expected
if np.dtype(dtype).kind == 'u': # type: ignore[misc]
# .type expects a XxxTensor, which have no type hints on
# purpose, so ignore during mypy type checking
x = torch.tensor([1, 2, 3,
4]).type(tp) # type: ignore[call-overload]
array = np.array([1, 2, 3, 4], dtype=dtype)
else:
x = torch.tensor([1, -2, 3,
-4]).type(tp) # type: ignore[call-overload]
array = np.array([1, -2, 3, -4], dtype=dtype)
# Test __array__ w/o dtype argument
asarray = np.asarray(x)
self.assertIsInstance(asarray, np.ndarray)
self.assertEqual(asarray.dtype, dtype)
for i in range(len(x)):
self.assertEqual(asarray[i], x[i])
# Test __array_wrap__, same dtype
abs_x = np.abs(x)
abs_array = np.abs(array)
self.assertIsInstance(abs_x, tp)
for i in range(len(x)):
self.assertEqual(abs_x[i], abs_array[i])
# Test __array__ with dtype argument
for dtype in dtypes:
x = torch.IntTensor([1, -2, 3, -4])
asarray = np.asarray(x, dtype=dtype)
self.assertEqual(asarray.dtype, dtype)
# Only concrete class can be given where "Type[number[_64Bit]]" is expected
if np.dtype(dtype).kind == 'u': # type: ignore[misc]
wrapped_x = np.array([1, -2, 3, -4], dtype=dtype)
for i in range(len(x)):
self.assertEqual(asarray[i], wrapped_x[i])
else:
for i in range(len(x)):
self.assertEqual(asarray[i], x[i])
# Test some math functions with float types
float_types = [torch.DoubleTensor, torch.FloatTensor]
float_dtypes = [np.float64, np.float32]
for tp, dtype in zip(float_types, float_dtypes):
x = torch.tensor([1, 2, 3,
4]).type(tp) # type: ignore[call-overload]
array = np.array([1, 2, 3, 4], dtype=dtype)
for func in ['sin', 'sqrt', 'ceil']:
ufunc = getattr(np, func)
res_x = ufunc(x)
res_array = ufunc(array)
self.assertIsInstance(res_x, tp)
for i in range(len(x)):
self.assertEqual(res_x[i], res_array[i])
# Test functions with boolean return value
for tp, dtype in zip(types, dtypes):
x = torch.tensor([1, 2, 3,
4]).type(tp) # type: ignore[call-overload]
array = np.array([1, 2, 3, 4], dtype=dtype)
geq2_x = np.greater_equal(x, 2)
geq2_array = np.greater_equal(array, 2).astype('uint8')
self.assertIsInstance(geq2_x, torch.ByteTensor)
for i in range(len(x)):
self.assertEqual(geq2_x[i], geq2_array[i])
@onlyCPU
def test_multiplication_numpy_scalar(self, device) -> None:
for np_dtype in [
np.float32, np.float64, np.int32, np.int64, np.int16, np.uint8
]:
for t_dtype in [torch.float, torch.double]:
# mypy raises an error when np.floatXY(2.0) is called
# even though this is valid code
np_sc = np_dtype(2.0) # type: ignore[abstract, arg-type]
t = torch.ones(2, requires_grad=True, dtype=t_dtype)
r1 = t * np_sc
self.assertIsInstance(r1, torch.Tensor)
self.assertTrue(r1.dtype == t_dtype)
self.assertTrue(r1.requires_grad)
r2 = np_sc * t
self.assertIsInstance(r2, torch.Tensor)
self.assertTrue(r2.dtype == t_dtype)
self.assertTrue(r2.requires_grad)
@onlyCPU
def test_parse_numpy_int(self, device):
# Only concrete class can be given where "Type[number[_64Bit]]" is expected
self.assertRaisesRegex(RuntimeError, "Overflow", lambda: torch.mean(
torch.randn(1, 1), np.uint64(-1))) # type: ignore[call-overload]
# https://github.com/pytorch/pytorch/issues/29252
for nptype in [np.int16, np.int8, np.uint8, np.int32, np.int64]:
scalar = 3
np_arr = np.array([scalar], dtype=nptype)
np_val = np_arr[0]
# np integral type can be treated as a python int in native functions with
# int parameters:
self.assertEqual(
torch.ones(5).diag(scalar),
torch.ones(5).diag(np_val))
self.assertEqual(
torch.ones([2, 2, 2, 2]).mean(scalar),
torch.ones([2, 2, 2, 2]).mean(np_val))
# numpy integral type parses like a python int in custom python bindings:
self.assertEqual(torch.Storage(np_val).size(),
scalar) # type: ignore[attr-defined]
tensor = torch.tensor([2], dtype=torch.int)
tensor[0] = np_val
self.assertEqual(tensor[0], np_val)
# Original reported issue, np integral type parses to the correct
# PyTorch integral type when passed for a `Scalar` parameter in
# arithmetic operations:
t = torch.from_numpy(np_arr)
self.assertEqual((t + np_val).dtype, t.dtype)
self.assertEqual((np_val + t).dtype, t.dtype)
def test_has_storage_numpy(self, device):
for dtype in [
np.float32, np.float64, np.int64, np.int32, np.int16, np.uint8
]:
arr = np.array([1], dtype=dtype)
self.assertIsNotNone(
torch.tensor(arr, device=device,
dtype=torch.float32).storage())
self.assertIsNotNone(
torch.tensor(arr, device=device, dtype=torch.double).storage())
self.assertIsNotNone(
torch.tensor(arr, device=device, dtype=torch.int).storage())
self.assertIsNotNone(
torch.tensor(arr, device=device, dtype=torch.long).storage())
self.assertIsNotNone(
torch.tensor(arr, device=device, dtype=torch.uint8).storage())
@dtypes(*get_all_dtypes())
def test_numpy_scalar_cmp(self, device, dtype):
if dtype.is_complex:
tensors = (torch.tensor(complex(1, 3), dtype=dtype, device=device),
torch.tensor([complex(1, 3), 0, 2j],
dtype=dtype,
device=device),
torch.tensor([[complex(3, 1), 0], [-1j, 5]],
dtype=dtype,
device=device))
else:
tensors = (torch.tensor(3, dtype=dtype, device=device),
torch.tensor([1, 0, -3], dtype=dtype, device=device),
torch.tensor([[3, 0, -1], [3, 5, 4]],
dtype=dtype,
device=device))
for tensor in tensors:
if dtype == torch.bfloat16:
with self.assertRaises(TypeError):
np_array = tensor.cpu().numpy()
continue
np_array = tensor.cpu().numpy()
for t, a in product(
(tensor.flatten()[0], tensor.flatten()[0].item()),
(np_array.flatten()[0], np_array.flatten()[0].item())):
self.assertEqual(t, a)
if dtype == torch.complex64 and torch.is_tensor(t) and type(
a) == np.complex64:
# TODO: Imaginary part is dropped in this case. Need fix.
# https://github.com/pytorch/pytorch/issues/43579
self.assertFalse(t == a)
else:
self.assertTrue(t == a)
class TestShapeOps(TestCase):
# TODO: update to work on CUDA, too
@onlyCPU
def test_unbind(self, device):
x = torch.rand(2, 3, 4, 5)
for dim in range(4):
res = torch.unbind(x, dim)
res2 = x.unbind(dim)
self.assertEqual(x.size(dim), len(res))
self.assertEqual(x.size(dim), len(res2))
for i in range(dim):
self.assertEqual(x.select(dim, i), res[i])
self.assertEqual(x.select(dim, i), res2[i])
# TODO: update to work on CUDA, too?
@onlyCPU
def test_tolist(self, device):
list0D = []
tensor0D = torch.tensor(list0D)
self.assertEqual(tensor0D.tolist(), list0D)
table1D = [1., 2., 3.]
tensor1D = torch.tensor(table1D)
storage = torch.Storage(table1D)
self.assertEqual(tensor1D.tolist(), table1D)
self.assertEqual(storage.tolist(), table1D)
self.assertEqual(tensor1D.tolist(), table1D)
self.assertEqual(storage.tolist(), table1D)
table2D = [[1, 2], [3, 4]]
tensor2D = torch.tensor(table2D)
self.assertEqual(tensor2D.tolist(), table2D)
tensor3D = torch.tensor([[[1, 2], [3, 4]], [[5, 6], [7, 8]]])
tensorNonContig = tensor3D.select(1, 1)
self.assertFalse(tensorNonContig.is_contiguous())
self.assertEqual(tensorNonContig.tolist(), [[3, 4], [7, 8]])
@dtypes(torch.int64, torch.float, torch.complex128)
def test_movedim_invalid(self, device, dtype):
shape = self._rand_shape(4, min_size=5, max_size=10)
x = _generate_input(shape, dtype, device, False)
for fn in [torch.movedim, torch.moveaxis]:
# Invalid `source` and `destination` dimension
with self.assertRaisesRegex(IndexError, "Dimension out of range"):
fn(x, 5, 0)
with self.assertRaisesRegex(IndexError, "Dimension out of range"):
fn(x, 0, 5)
# Mismatch in size of `source` and `destination`
with self.assertRaisesRegex(
RuntimeError,
"movedim: Invalid source or destination dims:"):
fn(x, (1, 0), (0, ))
with self.assertRaisesRegex(RuntimeError,
"movedim: repeated dim in `source`"):
fn(x, (0, 0), (0, 1))
with self.assertRaisesRegex(RuntimeError,
"movedim: repeated dim in `source`"):
fn(x, (0, 1, 0), (0, 1, 2))
with self.assertRaisesRegex(
RuntimeError, "movedim: repeated dim in `destination`"):
fn(x, (0, 1), (1, 1))
with self.assertRaisesRegex(
RuntimeError, "movedim: repeated dim in `destination`"):
fn(x, (0, 1, 2), (1, 0, 1))
@dtypes(torch.int64, torch.float, torch.complex128)
def test_movedim(self, device, dtype):
for fn in [torch.moveaxis, torch.movedim]:
for nd in range(5):
shape = self._rand_shape(nd, min_size=5, max_size=10)
x = _generate_input(shape, dtype, device, with_extremal=False)
for random_negative in [True, False]:
for src_dim, dst_dim in permutations(range(nd), r=2):
random_prob = random.random()
if random_negative and random_prob > 0.66:
src_dim = src_dim - nd
elif random_negative and random_prob > 0.33:
dst_dim = dst_dim - nd
elif random_negative:
src_dim = src_dim - nd
dst_dim = dst_dim - nd
# Integer `source` and `destination`
torch_fn = partial(fn,
source=src_dim,
destination=dst_dim)
np_fn = partial(np.moveaxis,
source=src_dim,
destination=dst_dim)
self.compare_with_numpy(torch_fn,
np_fn,
x,
device=None,
dtype=None)
if nd == 0:
continue
def make_index_negative(sequence, idx):
sequence = list(sequence)
sequence[random_idx] = sequence[random_idx] - nd
return tuple(src_sequence)
for src_sequence in permutations(range(nd),
r=random.randint(1, nd)):
# Sequence `source` and `destination`
dst_sequence = tuple(
random.sample(range(nd), len(src_sequence)))
# Randomly change a dim to a negative dim representation of itself.
random_prob = random.random()
if random_negative and random_prob > 0.66:
random_idx = random.randint(
0,
len(src_sequence) - 1)
src_sequence = make_index_negative(
src_sequence, random_idx)
elif random_negative and random_prob > 0.33:
random_idx = random.randint(
0,
len(src_sequence) - 1)
dst_sequence = make_index_negative(
dst_sequence, random_idx)
elif random_negative:
random_idx = random.randint(
0,
len(src_sequence) - 1)
dst_sequence = make_index_negative(
dst_sequence, random_idx)
random_idx = random.randint(
0,
len(src_sequence) - 1)
src_sequence = make_index_negative(
src_sequence, random_idx)
torch_fn = partial(fn,
source=src_sequence,
destination=dst_sequence)
np_fn = partial(np.moveaxis,
source=src_sequence,
destination=dst_sequence)
self.compare_with_numpy(torch_fn,
np_fn,
x,
device=None,
dtype=None)
# Move dim to same position
x = torch.randn(2, 3, 5, 7, 11)
torch_fn = partial(fn, source=(0, 1), destination=(0, 1))
np_fn = partial(np.moveaxis, source=(0, 1), destination=(0, 1))
self.compare_with_numpy(torch_fn,
np_fn,
x,
device=None,
dtype=None)
torch_fn = partial(fn, source=1, destination=1)
np_fn = partial(np.moveaxis, source=1, destination=1)
self.compare_with_numpy(torch_fn,
np_fn,
x,
device=None,
dtype=None)
# Empty Sequence
torch_fn = partial(fn, source=(), destination=())
np_fn = partial(np.moveaxis, source=(), destination=())
self.compare_with_numpy(torch_fn,
np_fn,
x,
device=None,
dtype=None)
@dtypes(torch.float, torch.bool)
def test_diag(self, device, dtype):
if dtype is torch.bool:
x = torch.rand(100, 100, device=device) >= 0.5
else:
x = torch.rand(100, 100, dtype=dtype, device=device)
res1 = torch.diag(x)
res2 = torch.tensor((), dtype=dtype, device=device)
torch.diag(x, out=res2)
self.assertEqual(res1, res2)
def test_diagonal(self, device):
x = torch.randn((100, 100), device=device)
result = torch.diagonal(x)
expected = torch.diag(x)
self.assertEqual(result, expected)
x = torch.randn((100, 100), device=device)
result = torch.diagonal(x, 17)
expected = torch.diag(x, 17)
self.assertEqual(result, expected)
@onlyCPU
@dtypes(torch.float)
def test_diagonal_multidim(self, device, dtype):
x = torch.randn(10, 11, 12, 13, dtype=dtype, device=device)
xn = x.numpy()
for args in [(2, 2, 3), (2, ), (-2, 1, 2), (0, -2, -1)]:
result = torch.diagonal(x, *args)
expected = xn.diagonal(*args)
self.assertEqual(expected.shape, result.shape)
self.assertEqual(expected, result)
# test non-continguous
xp = x.permute(1, 2, 3, 0)
result = torch.diagonal(xp, 0, -2, -1)
expected = xp.numpy().diagonal(0, -2, -1)
self.assertEqual(expected.shape, result.shape)
self.assertEqual(expected, result)
@onlyOnCPUAndCUDA
@dtypesIfCPU(*get_all_dtypes(include_complex=False,
include_bool=False,
include_half=False,
include_bfloat16=False))
@dtypesIfCUDA(*get_all_dtypes(include_complex=False,
include_bool=False,
include_bfloat16=False))
def test_trace(self, device, dtype):
def test(shape):
tensor = make_tensor(shape, device, dtype, low=-9, high=9)
expected_dtype = tensor.sum().dtype
expected_dtype = torch_to_numpy_dtype_dict[expected_dtype]
result = np.trace(tensor.cpu().numpy(), dtype=expected_dtype)
expected = torch.tensor(result, device=device)
self.assertEqual(tensor.trace(), expected)
shapes = (
[10, 1],
[1, 10],
[100, 100],
[20, 100],
[100, 20],
)
for shape in shapes:
test(shape)
def generate_clamp_baseline(self, device, dtype, *, min_vals, max_vals,
with_nans):
"""
Creates a random tensor for a given device and dtype, and computes the expected clamped
values given the min_vals and/or max_vals.
If with_nans is provided, then some values are randomly set to nan.
"""
X = torch.rand(100,
device=device).mul(50).add(-25) # uniform in [-25, 25]
X = X.to(dtype)
if with_nans:
mask = torch.randint(0,
2,
X.shape,
dtype=torch.bool,
device=device)
X[mask] = nan
if isinstance(min_vals, torch.Tensor):
min_vals = min_vals.cpu().numpy()
if isinstance(max_vals, torch.Tensor):
max_vals = max_vals.cpu().numpy()
# Use NumPy implementation as reference
X_clamped = torch.tensor(np.clip(X.cpu().numpy(),
a_min=min_vals,
a_max=max_vals),
device=device)
return X, X_clamped
# Tests clamp and its alias, clip
@dtypes(torch.int64, torch.float32)
def test_clamp(self, device, dtype):
op_list = (torch.clamp, torch.Tensor.clamp, torch.Tensor.clamp_,
torch.clip, torch.Tensor.clip, torch.Tensor.clip_)
# min/max argument product
args = product((-10, None), (10, None))
for op in op_list:
for min_val, max_val in args:
if min_val is None and max_val is None:
continue
X, Y_expected = self.generate_clamp_baseline(device,
dtype,
min_vals=min_val,
max_vals=max_val,
with_nans=False)
# Test op
X1 = X.clone() # So that the in-place ops do not change X
Y_actual = op(X1, min_val, max_val)
self.assertEqual(Y_expected, Y_actual)
# Test op-out behavior (out does not exist for method versions)
if op in (torch.clamp, torch.clip):
Y_out = torch.empty_like(X)
op(X, min=min_val, max=max_val, out=Y_out)
self.assertEqual(Y_expected, Y_out)
def test_clamp_propagates_nans(self, device):
op_list = (torch.clamp, torch.Tensor.clamp, torch.Tensor.clamp_,
torch.clip, torch.Tensor.clip, torch.Tensor.clip_)
# min/max argument product
args = product((-10, None), (10, None))
for op in op_list:
for min_val, max_val in args:
if min_val is None and max_val is None:
continue
X, Y_expected = self.generate_clamp_baseline(device,
torch.float,
min_vals=min_val,
max_vals=max_val,
with_nans=True)
Y_expected = torch.isnan(Y_expected)
# Test op
X1 = X.clone() # So that the in-place ops do not change X
Y_actual = op(X1, min_val, max_val)
self.assertEqual(Y_expected, torch.isnan(Y_actual))
# Test op-out behavior (out does not exist for method versions)
if op in (torch.clamp, torch.clip):
Y_out = torch.empty_like(X)
op(X, min_val, max_val, out=Y_out)
self.assertEqual(Y_expected, torch.isnan(Y_out))
def test_clamp_raises_arg_errors(self, device):
X = torch.randn(100, dtype=torch.float, device=device)
error_msg = 'At least one of \'min\' or \'max\' must not be None'
with self.assertRaisesRegex(RuntimeError, error_msg):
X.clamp()
with self.assertRaisesRegex(RuntimeError, error_msg):
X.clamp_()
with self.assertRaisesRegex(RuntimeError, error_msg):
torch.clamp(X)
@dtypes(*get_all_dtypes())
def test_flip(self, device, dtype):
make_from_data = partial(torch.tensor, device=device, dtype=dtype)
make_from_size = partial(make_tensor, device=device, dtype=dtype)
def test_flip_impl(input_t, dims, output_t):
def all_t():
yield input_t, output_t
if dtype is torch.float:
# We generate quantized versions as well
for qdtype in (torch.quint8, torch.qint8, torch.qint32):
qinput_t = torch.quantize_per_tensor(
input_t, 0.1, 5, qdtype)
qoutput_t = torch.quantize_per_tensor(
output_t, 0.1, 5, qdtype)
yield qinput_t, qoutput_t
for in_t, out_t in all_t():
self.assertEqual(in_t.flip(dims), out_t)
n = in_t.ndim
if not isinstance(dims, tuple):
# Wrap dim
self.assertEqual(in_t.flip(-n + dims), out_t)
else:
# Permute dimensions
for p_dims in permutations(dims):
self.assertEqual(in_t.flip(p_dims), out_t)
if len(p_dims) > 0:
# Wrap 1st dim
self.assertEqual(
in_t.flip((-n + p_dims[0], ) + p_dims[1:]),
out_t)
def gen_data():
# Basic tests
data = make_from_data([1, 2, 3, 4, 5, 6, 7, 8]).view(2, 2, 2)
nonctg = make_from_size((2, 2, 2), noncontiguous=True).copy_(data)
dims_result = ((0, make_from_data([5, 6, 7, 8, 1, 2, 3,
4]).view(2, 2, 2)),
(1, make_from_data([3, 4, 1, 2, 7, 8, 5,
6]).view(2, 2, 2)),
(2, make_from_data([2, 1, 4, 3, 6, 5, 8,
7]).view(2, 2, 2)),
((0, 1), make_from_data([7, 8, 5, 6, 3, 4, 1,
2]).view(2, 2, 2)),
((0, 1, 2), make_from_data([8, 7, 6, 5, 4, 3, 2,
1]).view(2, 2, 2)))
for in_tensor, (dims, out_tensor) in product((data, nonctg),
dims_result):
yield in_tensor, dims, out_tensor
# Expanded
in_t = make_from_data([1, 2, 3]).view(3, 1).expand(3, 2)
dims = 0
out_t = make_from_data([3, 3, 2, 2, 1, 1]).view(3, 2)
yield in_t, dims, out_t
# Noop on expanded dimension
yield in_t, 1, in_t
# Transposed
in_t = make_from_data([1, 2, 3, 4, 5, 6, 7,
8]).view(2, 2, 2).transpose(0, 1)
dims = (0, 1, 2)
out_t = make_from_data([8, 7, 4, 3, 6, 5, 2, 1]).view(2, 2, 2)
yield in_t, dims, out_t
# Rectangular case
in_t = make_from_data([1, 2, 3, 4, 5, 6]).view(2, 3)
dims = 0
out_t = make_from_data([[4, 5, 6], [1, 2, 3]])
yield in_t, dims, out_t
dims = 1
out_t = make_from_data([[3, 2, 1], [6, 5, 4]])
yield in_t, dims, out_t
# Noops (edge cases)
# Size 0
in_t = make_from_data(())
yield in_t, 0, in_t
yield in_t, (), in_t
# dims = ()
in_t = make_from_size((3, 2, 1))
yield in_t, (), in_t
# Zero elements, non-zero size
in_t = make_from_size((3, 0, 2))
for i in range(in_t.ndim):
yield in_t, i, in_t
# Size 1
in_t = make_from_size(())
yield in_t, 0, in_t
in_t = make_from_size((1, ))
yield in_t, 0, in_t
for in_tensor, dims, out_tensor in gen_data():
test_flip_impl(in_tensor, dims, out_tensor)
# test for shape
size = [2, 3, 4]
data = make_from_size(size)
possible_dims = range(len(size))
test_dims = chain(combinations(possible_dims, 1),
combinations(possible_dims, 2))
for dims in test_dims:
self.assertEqual(size, list(data.flip(dims).size()))
@dtypes(*get_all_dtypes())
def test_flip_errors(self, device, dtype):
make_arg = partial(make_tensor, dtype=dtype, device=device)
data = make_arg((2, 2, 2))
# not allow flip on the same dim more than once
self.assertRaises(RuntimeError, lambda: data.flip(0, 1, 1))
# not allow empty list as input
self.assertRaises(TypeError, lambda: data.flip())
# not allow dim > max dim
self.assertRaises(IndexError, lambda: data.flip(0, 1, 2, 3))
self.assertRaises(IndexError, lambda: data.flip(3))
def _rand_shape(self, dim, min_size, max_size):
return tuple(torch.randint(min_size, max_size + 1, (dim, )))
@dtypes(*get_all_dtypes())
def test_flip_numpy(self, device, dtype):
make_arg = partial(make_tensor, dtype=dtype, device=device)
for ndim in [3, 4]:
shape = self._rand_shape(ndim, 5, 10)
data = make_arg(shape)
# Axis to sample for given shape.
for i in range(1, ndim + 1):
# Check all combinations of `i` axis.
for flip_dim in combinations(range(ndim), i):
torch_fn = partial(torch.flip, dims=flip_dim)
np_fn = partial(np.flip, axis=flip_dim)
self.compare_with_numpy(torch_fn, np_fn, data)
@onlyCUDA # CPU is too slow
@largeTensorTest('17GB'
) # 4 tensors of 4GB (in, out) x (torch, numpy) + 1GB
def test_flip_large_tensor(self, device):
t_in = torch.empty(2**32 + 1, dtype=torch.uint8).random_()
torch_fn = partial(torch.flip, dims=(0, ))
np_fn = partial(np.flip, axis=0)
self.compare_with_numpy(torch_fn, np_fn, t_in)
del t_in
def _test_fliplr_flipud(self, torch_fn, np_fn, min_dim, max_dim, device,
dtype):
for dim in range(min_dim, max_dim + 1):
shape = self._rand_shape(dim, 5, 10)
# Randomly scale the input
if dtype.is_floating_point or dtype.is_complex:
data = torch.randn(*shape, device=device, dtype=dtype)
else:
data = torch.randint(0, 10, shape, device=device, dtype=dtype)
self.compare_with_numpy(torch_fn, np_fn, data)
@dtypes(torch.int64, torch.double, torch.cdouble)
def test_fliplr(self, device, dtype):
self._test_fliplr_flipud(torch.fliplr, np.fliplr, 2, 4, device, dtype)
@dtypes(torch.int64, torch.double, torch.cdouble)
def test_fliplr_invalid(self, device, dtype):
x = torch.randn(42).to(dtype)
with self.assertRaisesRegex(RuntimeError, "Input must be >= 2-d."):
torch.fliplr(x)
with self.assertRaisesRegex(RuntimeError, "Input must be >= 2-d."):
torch.fliplr(torch.tensor(42, device=device, dtype=dtype))
@dtypes(torch.int64, torch.double, torch.cdouble)
def test_flipud(self, device, dtype):
self._test_fliplr_flipud(torch.flipud, np.flipud, 1, 4, device, dtype)
@dtypes(torch.int64, torch.double, torch.cdouble)
def test_flipud_invalid(self, device, dtype):
with self.assertRaisesRegex(RuntimeError, "Input must be >= 1-d."):
torch.flipud(torch.tensor(42, device=device, dtype=dtype))
def test_rot90(self, device):
data = torch.arange(1, 5, device=device).view(2, 2)
self.assertEqual(
torch.tensor([1, 2, 3, 4]).view(2, 2), data.rot90(0, [0, 1]))
self.assertEqual(
torch.tensor([2, 4, 1, 3]).view(2, 2), data.rot90(1, [0, 1]))
self.assertEqual(
torch.tensor([4, 3, 2, 1]).view(2, 2), data.rot90(2, [0, 1]))
self.assertEqual(
torch.tensor([3, 1, 4, 2]).view(2, 2), data.rot90(3, [0, 1]))
# test for default args k=1, dims=[0, 1]
self.assertEqual(data.rot90(), data.rot90(1, [0, 1]))
# test for reversed order of dims
self.assertEqual(data.rot90(3, [0, 1]), data.rot90(1, [1, 0]))
# test for modulo of k
self.assertEqual(data.rot90(5, [0, 1]), data.rot90(1, [0, 1]))
self.assertEqual(data.rot90(3, [0, 1]), data.rot90(-1, [0, 1]))
self.assertEqual(data.rot90(-5, [0, 1]), data.rot90(-1, [0, 1]))
# test for dims out-of-range error
self.assertRaises(RuntimeError, lambda: data.rot90(1, [0, -3]))
self.assertRaises(RuntimeError, lambda: data.rot90(1, [0, 2]))
# test tensor with more than 2D
data = torch.arange(1, 9, device=device).view(2, 2, 2)
self.assertEqual(
torch.tensor([2, 4, 1, 3, 6, 8, 5, 7]).view(2, 2, 2),
data.rot90(1, [1, 2]))
self.assertEqual(data.rot90(1, [1, -1]), data.rot90(1, [1, 2]))
# test for errors
self.assertRaises(RuntimeError, lambda: data.rot90(1, [0, 3]))
self.assertRaises(RuntimeError, lambda: data.rot90(1, [1, 1]))
self.assertRaises(RuntimeError, lambda: data.rot90(1, [0, 1, 2]))
self.assertRaises(RuntimeError, lambda: data.rot90(1, [0]))
@dtypes(torch.cfloat, torch.cdouble)
def test_complex_rot90(self, device, dtype):
shape = self._rand_shape(random.randint(2, 4), 5, 10)
for rot_times in range(4):
data = torch.randn(*shape, device=device, dtype=dtype)
torch_fn = partial(torch.rot90, k=rot_times, dims=[0, 1])
np_fn = partial(np.rot90, k=rot_times, axes=[0, 1])
self.compare_with_numpy(torch_fn, np_fn, data)
# TODO: update once warning flag is available to always trigger ONCE warnings
# Ensures nonzero does not throw a warning, even when the as_tuple argument
# is not provided
def test_nonzero_no_warning(self, device):
t = torch.randn((2, 2), device=device)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
torch.nonzero(t)
t.nonzero()
self.assertEqual(len(w), 0)
@dtypes(*get_all_dtypes(include_complex=False))
def test_nonzero(self, device, dtype):
shapes = [
torch.Size((12, )),
torch.Size((12, 1)),
torch.Size((1, 12)),
torch.Size((6, 2)),
torch.Size((3, 2, 2)),
torch.Size((5, 5, 5)),
]
def gen_nontrivial_input(shape, dtype, device):
if dtype != torch.bfloat16:
return torch.randint(2, shape, device=device, dtype=dtype)
else:
# windows does not work for bfloat16 randing
return torch.randint(2,
shape,
device=device,
dtype=torch.float).to(dtype)
for shape in shapes:
tensor = gen_nontrivial_input(shape, dtype, device)
dst1 = torch.nonzero(tensor, as_tuple=False)
dst2 = tensor.nonzero(as_tuple=False)
dst3 = torch.empty([], dtype=torch.long, device=device)
torch.nonzero(tensor, out=dst3)
if self.device_type != 'xla':
# xla does not raise runtime error
self.assertRaisesRegex(
RuntimeError, "scalar type Long", lambda: torch.nonzero(
tensor,
out=torch.empty([], dtype=torch.float, device=device)))
if self.device_type == 'cuda':
self.assertRaisesRegex(
RuntimeError, "on the same device", lambda: torch.nonzero(
tensor, out=torch.empty([], dtype=torch.long)))
np_array = tensor.cpu().numpy(
) if dtype != torch.bfloat16 else tensor.float().cpu().numpy()
np_result = torch.from_numpy(np.stack(np_array.nonzero())).t()
self.assertEqual(dst1.cpu(), np_result, atol=0, rtol=0)
self.assertEqual(dst2.cpu(), np_result, atol=0, rtol=0)
self.assertEqual(dst3.cpu(), np_result, atol=0, rtol=0)
tup1 = torch.nonzero(tensor, as_tuple=True)
tup2 = tensor.nonzero(as_tuple=True)
tup1 = torch.stack(tup1).t().cpu()
tup2 = torch.stack(tup2).t().cpu()
self.assertEqual(tup1, np_result, atol=0, rtol=0)
self.assertEqual(tup2, np_result, atol=0, rtol=0)
def test_nonzero_astuple_out(self, device):
t = torch.randn((3, 3, 3), device=device)
out = torch.empty_like(t, dtype=torch.long)
with self.assertRaises(RuntimeError):
torch.nonzero(t, as_tuple=True, out=out)
self.assertEqual(torch.nonzero(t, as_tuple=False, out=out),
torch.nonzero(t, out=out))
# Verifies that JIT script cannot handle the as_tuple kwarg
# See Issue https://github.com/pytorch/pytorch/issues/45499.
def _foo(t):
tuple_result = torch.nonzero(t, as_tuple=True)
nontuple_result = torch.nonzero(t, as_tuple=False)
out = torch.empty_like(nontuple_result)
torch.nonzero(t, as_tuple=False, out=out)
return tuple_result, nontuple_result, out
with self.assertRaises(RuntimeError):
scripted_foo = torch.jit.script(_foo)
# Verifies that JIT tracing works fine
traced_foo = torch.jit.trace(_foo, t)
traced_tuple, traced_nontuple, traced_out = traced_foo(t)
expected_tuple = torch.nonzero(t, as_tuple=True)
expected_nontuple = torch.nonzero(t)
self.assertEqual(traced_tuple, expected_tuple)
self.assertEqual(traced_nontuple, expected_nontuple)
self.assertEqual(traced_out, expected_nontuple)
@onlyOnCPUAndCUDA
def test_nonzero_discontiguous(self, device):
shape = (4, 4)
tensor = torch.randint(2, shape, device=device)
tensor_nc = torch.empty(shape[0], shape[1] * 2,
device=device)[:, ::2].copy_(tensor)
dst1 = tensor.nonzero(as_tuple=False)
dst2 = tensor_nc.nonzero(as_tuple=False)
self.assertEqual(dst1, dst2, atol=0, rtol=0)
dst3 = torch.empty_like(dst1)
data_ptr = dst3.data_ptr()
# expect dst3 storage to be reused
torch.nonzero(tensor, out=dst3)
self.assertEqual(data_ptr, dst3.data_ptr())
self.assertEqual(dst1, dst3, atol=0, rtol=0)
# discontiguous out
dst4 = torch.empty(dst1.size(0),
dst1.size(1) * 2,
dtype=torch.long,
device=device)[:, ::2]
data_ptr = dst4.data_ptr()
strides = dst4.stride()
torch.nonzero(tensor, out=dst4)
self.assertEqual(data_ptr, dst4.data_ptr())
self.assertEqual(dst1, dst4, atol=0, rtol=0)
self.assertEqual(strides, dst4.stride())
def test_nonzero_non_diff(self, device):
x = torch.randn(10, requires_grad=True)
nz = x.nonzero()
self.assertFalse(nz.requires_grad)
class TestForeach(TestCase):
@property
def is_cuda(self):
return self.device_type == 'cuda'
# note(mkozuki): It might be the case that the expected number of `cudaLaunchKernel`s
# is greater than 1 once foreach functions internally separate their input `TensorList`s by
# devices & dtypes into vectors of tensors.
def _get_funcs(self, op, n_expected_cudaLaunchKernels):
return (
ForeachFuncWrapper(op.method_variant,
n_expected_cudaLaunchKernels),
RegularFuncWrapper(op.ref),
ForeachFuncWrapper(op.inplace_variant,
n_expected_cudaLaunchKernels),
RegularFuncWrapper(op.ref_inplace),
)
def _binary_test(self,
dtype,
op,
ref,
inputs,
is_fastpath,
is_inplace,
*,
alpha=None):
ref_inputs = [[t.clone().detach() for t in inputs[0]], inputs[1]
] if is_inplace else inputs
try:
actual = op(inputs, self.is_cuda, is_fastpath)
except RuntimeError as e:
with self.assertRaisesRegex(type(e), re.escape(str(e))):
ref(ref_inputs)
else:
expected = ref(ref_inputs)
self.assertEqual(actual, expected)
if alpha is not None:
kwargs = {'alpha': alpha}
ref_inputs = inputs
try:
actual = op(inputs, self.is_cuda, is_fastpath, **kwargs)
except RuntimeError as e:
with self.assertRaisesRegex(type(e), re.escape(str(e))):
ref(ref_inputs, **kwargs)
else:
expected = ref(ref_inputs, **kwargs)
if dtype in (torch.float16, torch.bfloat16) and TEST_WITH_ROCM:
self.assertEqual(expected,
actual,
atol=1.e-3,
rtol=self.dtype_precisions[dtype][0])
else:
self.assertEqual(expected, actual)
def _test_binary_op_tensorlists(self, device, dtype, opinfo, N,
is_fastpath, disable_fastpath):
n_expected_cudaLaunchKernels = N if disable_fastpath else 1
op, ref, inplace_op, inplace_ref = self._get_funcs(
opinfo, n_expected_cudaLaunchKernels)
inputs = [
opinfo.sample_inputs(device,
dtype,
N,
noncontiguous=not is_fastpath),
opinfo.sample_inputs(device,
dtype,
N,
noncontiguous=not is_fastpath),
]
self._binary_test(dtype,
op,
ref,
inputs,
is_fastpath,
is_inplace=False)
self._binary_test(dtype,
inplace_op,
inplace_ref,
inputs,
is_fastpath,
is_inplace=True)
if opinfo.supports_alpha_param:
alpha = None
if dtype in get_all_int_dtypes():
alpha = 3
elif dtype.is_complex:
alpha = complex(3, 3)
else:
alpha = 3.14
self._binary_test(dtype,
op,
ref,
inputs,
is_fastpath,
is_inplace=False,
alpha=alpha)
self._binary_test(dtype,
inplace_op,
inplace_ref,
inputs,
is_fastpath,
is_inplace=True,
alpha=alpha)
# Tests of implicit broadcasting
# When sizes of tensors don't match, foreach functions are supposed to choose slow path
# even if this methods's argument `is_fastpath` is True.
# `cudaLaunchKernel` will be equal to `N`. For assert in `ForeachFuncWrapper` to pass,
# we pass `is_fastpath and disable_fastpath` to `_binary_test`'s argument of is_fastpath.
# as n_expected_cudaLaunchKernels is N if disable_fastpath.
inputs = [
opinfo.sample_inputs(device,
dtype,
N,
noncontiguous=not is_fastpath),
[
make_tensor((N - i, 1),
device=device,
dtype=dtype,
noncontiguous=not is_fastpath) for i in range(N)
],
]
self._binary_test(dtype,
op,
ref,
inputs,
is_fastpath and disable_fastpath,
is_inplace=False)
self._binary_test(dtype,
inplace_op,
inplace_ref,
inputs,
is_fastpath and disable_fastpath,
is_inplace=True)
# note(mkozuki): Why ROCm?
# ROCm is supposed to compile slow path as in
# https://github.com/pytorch/pytorch/blob/7e032f18cf1405804c4f787b05ea2de5e08a091e/aten/src/ATen/native/ForeachUtils.h#L148-L164, # noqa: E501
# Therefore `[torch.add(*args, alpha=alpha) for args in zip(tensors1, tensors2)]` and
# `torch._foreach_add(tensors1, tensors2, alpha=alpha)`
# are expected to return the same outputs, however, the outputs look unstable for torch.bfloat16 and torch.half.
# log: https://ci.pytorch.org/jenkins/job/pytorch-builds/job/pytorch-linux-bionic-rocm4.2-py3.6-test1/2741/console
@skipCUDAIfRocm
@skipMeta
@ops(foreach_binary_op_db)
def test_binary_op_tensorlists_fastpath(self, device, dtype, op):
for N in N_values:
disable_fastpath = op.ref == torch.div and dtype in get_all_int_dtypes(
) + [torch.bool]
if op.ref == torch.add and dtype == torch.bool:
disable_fastpath = True
self._test_binary_op_tensorlists(device, dtype, op, N, True,
disable_fastpath)
@ops(foreach_binary_op_db)
def test_binary_op_tensorlists_slowpath(self, device, dtype, op):
for N in N_values:
self._test_binary_op_tensorlists(device, dtype, op, N, False,
False)
def _test_binary_op_scalar(self, device, dtype, opinfo, N, scalar,
is_fastpath, disable_fastpath):
n_expected_cudaLaunchKernels = N if disable_fastpath else 1
op, ref, inplace_op, inplace_ref = self._get_funcs(
opinfo, n_expected_cudaLaunchKernels)
inputs = [
opinfo.sample_inputs(device,
dtype,
N,
noncontiguous=not is_fastpath), scalar
]
self._binary_test(dtype,
op,
ref,
inputs,
is_fastpath,
is_inplace=False)
self._binary_test(dtype,
inplace_op,
inplace_ref,
inputs,
is_fastpath,
is_inplace=True)
@skipCUDAIfRocm
@skipMeta
@ops(foreach_binary_op_db)
def test_binary_op_scalar_fastpath(self, device, dtype, op):
for N, scalar in itertools.product(N_values, Scalars):
disable_fastpath = op.ref == torch.div and dtype in get_all_int_dtypes(
) + [torch.bool]
if isinstance(scalar, int):
disable_fastpath |= dtype == torch.bool
if isinstance(scalar, float):
disable_fastpath |= dtype in get_all_int_dtypes() + [
torch.bool
]
if isinstance(scalar, bool):
disable_fastpath |= dtype == torch.bool
if op.ref in (torch.add, torch.mul):
disable_fastpath = False
if isinstance(scalar, complex):
disable_fastpath |= dtype not in get_all_complex_dtypes()
self._test_binary_op_scalar(device, dtype, op, N, scalar, True,
disable_fastpath)
@ops(foreach_binary_op_db)
def test_binary_op_scalar_slowpath(self, device, dtype, op):
for N, scalar in itertools.product(N_values, Scalars):
self._test_binary_op_scalar(device, dtype, op, N, scalar, False,
False)
def _test_binary_op_scalarlist(self, device, dtype, opinfo, N, scalarlist,
is_fastpath, disable_fastpath):
n_expected_cudaLaunchKernels = N if disable_fastpath else 1
op, ref, inplace_op, inplace_ref = self._get_funcs(
opinfo, n_expected_cudaLaunchKernels)
inputs = [
opinfo.sample_inputs(device,
dtype,
N,
noncontiguous=not is_fastpath), scalarlist
]
self._binary_test(dtype,
op,
ref,
inputs,
is_fastpath,
is_inplace=False)
self._binary_test(dtype,
inplace_op,
inplace_ref,
inputs,
is_fastpath,
is_inplace=True)
# note(mkozuki): Why two functions depending on with/without bool?
# `foreach_sub` & `foreach_sub_` do `sub_check(tensors[i], scalars[i])` from i=1...N.
# So, if scalarlist has one or more bool values, `foreach_sub` and `foreach_sub_`
# raise bool subtraction error before doing any math.
# While regular `sub` and `sub_` do some math until they encounter bool.
# So, foreach sub's throw bool sub error first. However, regular sub's throw different
# errors depending on the order of scalarlist. To keep actual unit test impl simple,
# separating mixed scalarlist tests. By setting the first element of scalarlist to bool,
# they are expected to throw bool sub error even in inplace test.
@skipCUDAIfRocm
@skipMeta
@ops(foreach_binary_op_db)
def test_binary_op_scalarlist_fastpath(self, device, dtype, op):
for N in N_values:
for type_str, scalarlist in getScalarLists(N):
bool_int_div = op.ref == torch.div and dtype in get_all_int_dtypes(
) + [torch.bool]
disable_fastpath = bool_int_div
if type_str == "int":
disable_fastpath |= dtype == torch.bool
if type_str == "float":
disable_fastpath |= dtype in get_all_int_dtypes() + [
torch.bool
]
if type_str == "complex":
disable_fastpath |= dtype not in get_all_complex_dtypes()
if type_str == "mixed":
disable_fastpath |= True and dtype not in get_all_complex_dtypes(
)
self._test_binary_op_scalarlist(device, dtype, op, N,
scalarlist, True,
disable_fastpath)
@ops(foreach_binary_op_db)
def test_binary_op_scalarlist_slowpath(self, device, dtype, op):
for N in N_values:
for _, scalarlist in getScalarLists(N):
self._test_binary_op_scalarlist(device, dtype, op, N,
scalarlist, False, False)
def _pointwise_test(self,
dtype,
op,
ref,
inputs,
is_fastpath,
is_inplace,
*,
values=None):
ref_inputs = [[t.clone().detach() for t in inputs[0]], inputs[1],
inputs[2]] if is_inplace else inputs
try:
actual = op(inputs, self.is_cuda, is_fastpath)
except RuntimeError as e:
with self.assertRaisesRegex(type(e), re.escape(str(e))):
ref(ref_inputs)
else:
expected = ref(ref_inputs)
self.assertEqual(expected, actual)
if values is not None:
try:
actual = op(inputs + [values], self.is_cuda, is_fastpath)
except RuntimeError as e:
with self.assertRaisesRegex(type(e), re.escape(str(e))):
ref(ref_inputs, values=values)
else:
expected = ref(ref_inputs, values=values)
self.assertEqual(expected, actual)
def _test_pointwise_op(self,
device,
dtype,
opinfo,
N,
is_fastpath,
disable_fastpath,
*,
values=None):
n_expected_cudaLaunchKernels = N if disable_fastpath else 1
op, ref, inplace_op, inplace_ref = self._get_funcs(
opinfo, n_expected_cudaLaunchKernels)
inputs = [
opinfo.sample_inputs(device,
dtype,
N,
noncontiguous=not is_fastpath),
opinfo.sample_inputs(device,
dtype,
N,
noncontiguous=not is_fastpath),
opinfo.sample_inputs(device,
dtype,
N,
noncontiguous=not is_fastpath),
]
self._pointwise_test(dtype,
op,
ref,
inputs,
is_fastpath,
is_inplace=False,
values=values)
self._pointwise_test(dtype,
inplace_op,
inplace_ref,
inputs,
is_fastpath,
is_inplace=True,
values=values)
# Tests of implicit broadcasting
inputs = [
opinfo.sample_inputs(device,
dtype,
N,
noncontiguous=not is_fastpath,
same_size=True),
[
make_tensor((N - i, 1),
device=device,
dtype=dtype,
noncontiguous=not is_fastpath) for i in range(N)
],
[
make_tensor((1, N - i),
device=device,
dtype=dtype,
noncontiguous=not is_fastpath) for i in range(N)
],
]
self._pointwise_test(dtype,
op,
ref,
inputs,
is_fastpath and disable_fastpath,
is_inplace=False,
values=values)
self._pointwise_test(dtype,
inplace_op,
inplace_ref,
inputs,
is_fastpath and disable_fastpath,
is_inplace=True,
values=values)
@skipMeta
@ops(foreach_pointwise_op_db)
def test_pointwise_op_fastpath(self, device, dtype, op):
disable_fastpath = dtype in get_all_int_dtypes() + [torch.bool]
# for N, scalar in itertools.product(N_values, Scalars):
for N in N_values:
self._test_pointwise_op(device, dtype, op, N, True,
disable_fastpath)
for scalar in Scalars:
self._test_pointwise_op(device,
dtype,
op,
N,
True,
disable_fastpath,
values=scalar)
for _, scalarlist in getScalarLists(N):
self._test_pointwise_op(device,
dtype,
op,
N,
True,
disable_fastpath,
values=scalarlist)
@ops(foreach_pointwise_op_db)
def test_pointwise_op_slowpath(self, device, dtype, op):
# for N, scalar in itertools.product(N_values, Scalars):
for N in N_values:
self._test_pointwise_op(device, dtype, op, N, False, False)
for scalar in Scalars:
self._test_pointwise_op(device,
dtype,
op,
N,
False,
False,
values=scalar)
for _, scalarlist in getScalarLists(N):
self._test_pointwise_op(device,
dtype,
op,
N,
False,
False,
values=scalarlist)
# note(mkozuki): fastpath test uses dtypes which fastpath implementation supports.
# To confirm the dtypes of `OpInfo` cover the dtypes that the function support,
# this test does not use `try-except` for fastpath.
def _regular_unary_test(self, dtype, op, ref, inputs, is_fastpath):
if is_fastpath:
self.assertEqual(ref(inputs), op(inputs, self.is_cuda,
is_fastpath))
return
try:
actual = op(inputs, self.is_cuda, is_fastpath)
except RuntimeError as e:
with self.assertRaisesRegex(type(e), re.escape(str(e))):
ref(inputs)
else:
expected = ref(inputs)
self.assertEqual(actual, expected)
# note(mkozuki): why `try-except` for both fastpath?
# - inputs for fastpath can be integer tensors.
# - this is becase opinfo dtypes are configured for outpulace implementation
# - for integer inputs, trigonometric functions and exponential function returns float outputs,
# which causes "result type Float can't be case to the desired type" error.
# Thus, `try-except` is used even if `is_fastpath` is `True`.
def _inplace_unary_test(self, dtype, inplace, inplace_ref, inputs,
is_fastpath):
copied_inputs = [[t.clone().detach() for t in tensors]
for tensors in inputs]
try:
inplace(inputs, self.is_cuda, is_fastpath)
except RuntimeError as e:
with self.assertRaisesRegex(type(e), re.escape(str(e))):
inplace_ref(copied_inputs)
else:
inplace_ref(copied_inputs),
self.assertEqual(copied_inputs, inputs)
def _test_unary(self, device, dtype, opinfo, N, is_fastpath):
op, ref, inplace_op, inplace_ref = self._get_funcs(opinfo, 1)
inputs = opinfo.sample_inputs(device,
dtype,
N,
noncontiguous=not is_fastpath),
# note(mkozuki): Complex inputs for `_foreach_abs` go through slowpath.
if opinfo.name == "_foreach_abs" and dtype in get_all_complex_dtypes():
is_fastpath = False
self._regular_unary_test(dtype, op, ref, inputs, is_fastpath)
self._inplace_unary_test(dtype, inplace_op, inplace_ref, inputs,
is_fastpath)
@skipMeta
@ops(foreach_unary_op_db)
def test_unary_fastpath(self, device, dtype, op):
for N in N_values:
self._test_unary(device, dtype, op, N, is_fastpath=True)
@ops(foreach_unary_op_db, dtypes=get_all_dtypes())
def test_unary_slowpath(self, device, dtype, op):
for N in N_values:
self._test_unary(device, dtype, op, N, is_fastpath=False)
def _minmax_test(self, opinfo, inputs, is_fastpath,
n_expected_cudaLaunchKernels):
op, ref, _, _ = self._get_funcs(opinfo, n_expected_cudaLaunchKernels)
self.assertEqual(ref(inputs), op(inputs, self.is_cuda, is_fastpath))
# note(mkozuki): in-place of foreach_minimum and foreach_maximum aren't implemented.
@ops(foreach_minmax_op_db)
def test_minmax_fastpath(self, device, dtype, op):
for N in N_values:
inputs = tuple(
op.sample_inputs(device, dtype, N) for _ in range(2))
self._minmax_test(op, inputs, True,
N if dtype == torch.bool else 1)
@ops(foreach_minmax_op_db,
dtypes=get_all_dtypes(include_half=True,
include_bfloat16=True,
include_complex=False))
def test_minmax_slowpath(self, device, dtype, op):
for N in N_values:
inputs = tuple(
op.sample_inputs(device, dtype, N, noncontiguous=True)
for _ in range(2))
self._minmax_test(op, inputs, False, 1)
# note(mkozuki): ForeachFuncInfo's of both `_foreach_maximum` and `_foreach_minimum` include integer types.
# so, manually limit dtypes to fp types for inf&nan tests.
@ops(foreach_minmax_op_db,
dtypes=get_all_fp_dtypes(include_bfloat16=True, include_half=True))
def test_minmax_float_inf_nan(self, device, dtype, op):
inputs = (
[
torch.tensor([float('inf')], device=device, dtype=dtype),
torch.tensor([-float('inf')], device=device, dtype=dtype),
torch.tensor([float('nan')], device=device, dtype=dtype),
torch.tensor([float('nan')], device=device, dtype=dtype)
],
[
torch.tensor([-float('inf')], device=device, dtype=dtype),
torch.tensor([float('inf')], device=device, dtype=dtype),
torch.tensor([float('inf')], device=device, dtype=dtype),
torch.tensor([float('nan')], device=device, dtype=dtype)
],
)
self._minmax_test(op, inputs, True, 1)
def _reduce_test(self, opinfo, inputs, ord, is_fastpath,
n_expected_cudaLaunchKernels):
op, ref, _, _ = self._get_funcs(opinfo, n_expected_cudaLaunchKernels)
self.assertEqual(ref(inputs, ord=ord),
op(inputs, self.is_cuda, is_fastpath, ord=ord))
@ops(foreach_reduce_op_db)
def test_reduce_fastpath(self, device, dtype, op):
for N, ord in itertools.product(N_values, (0, 1, 2, -1, -2)):
if ord in (1, 2) and dtype in torch.testing.get_all_fp_dtypes():
n_expected_cudaLaunchKernels = 3
else:
n_expected_cudaLaunchKernels = N
inputs = op.sample_inputs(device, dtype, N, noncontiguous=False),
self._reduce_test(op, inputs, ord, True,
n_expected_cudaLaunchKernels)
@ops(foreach_reduce_op_db)
def test_reduce_slowpath(self, device, dtype, op):
for N, ord in itertools.product(N_values, (0, 1, 2, -1, -2)):
inputs = op.sample_inputs(device, dtype, N, noncontiguous=True),
self._reduce_test(op, inputs, ord, False, 1)
@dtypes(*get_all_dtypes())
def test_add_scalar_with_empty_list_and_empty_tensor(self, device, dtype):
# TODO: enable empty list case
for tensors in [[torch.randn([0])]]:
res = torch._foreach_add(tensors, 1)
self.assertEqual(res, tensors)
torch._foreach_add_(tensors, 1)
self.assertEqual(res, tensors)
@ops(foreach_binary_op_db, dtypes=get_all_dtypes())
def test_binary_op_scalar_with_overlapping_tensors(self, device, dtype,
op):
foreach_op, ref = op.method_variant, op.ref
tensors = [
torch.ones(1, 1, device=device, dtype=dtype).expand(2, 1, 3)
]
if ref == torch.sub and dtype == torch.bool:
with self.assertRaisesRegex(RuntimeError,
re.escape(_BOOL_SUB_ERR_MSG)):
[ref(t, 1) for t in tensors]
with self.assertRaisesRegex(RuntimeError,
re.escape(_BOOL_SUB_ERR_MSG)):
foreach_op(tensors, 1)
return
expected = [ref(t, 1) for t in tensors]
res = foreach_op(tensors, 1)
self.assertEqual(res, expected)
# note(mkozuki): this test case fails with Meta at least in my local environment.
# The message was
# `AssertionError: NotImplementedError("Could not run 'aten::_foreach_add.Scalar' with arguments from the 'Meta' backend.`
@skipMeta
@ops(foreach_binary_op_db, allowed_dtypes=[torch.float])
def test_binary_op_scalar_with_different_tensor_dtypes(
self, device, dtype, op):
foreach_op = op.method_variant
tensors = [
torch.tensor([1.1], dtype=torch.float, device=device),
torch.tensor([1], dtype=torch.long, device=device)
]
runtime_error = None
try:
foreach_op(tensors, 1)
except RuntimeError as e:
runtime_error = e
self.assertIsNone(runtime_error)
@ops(foreach_binary_op_db, dtypes=get_all_dtypes())
def test_binary_op_list_error_cases(self, device, dtype, op):
foreach_op, foreach_op_, ref, ref_ = op.method_variant, op.inplace_variant, op.ref, op.ref_inplace
tensors1 = []
tensors2 = []
# Empty lists
with self.assertRaisesRegex(
RuntimeError,
"There were no tensor arguments to this function"):
foreach_op(tensors1, tensors2)
with self.assertRaisesRegex(
RuntimeError,
"There were no tensor arguments to this function"):
foreach_op_(tensors1, tensors2)
# One empty list
tensors1.append(torch.tensor([1], device=device, dtype=dtype))
with self.assertRaisesRegex(
RuntimeError,
"Tensor list must have same number of elements as scalar list."
):
foreach_op(tensors1, tensors2)
with self.assertRaisesRegex(
RuntimeError,
"Tensor list must have same number of elements as scalar list."
):
foreach_op_(tensors1, tensors2)
# Lists have different amount of tensors
tensors2.append(torch.tensor([1], device=device))
tensors2.append(torch.tensor([1], device=device))
with self.assertRaisesRegex(
RuntimeError,
"Tensor lists must have the same number of tensors, got 1 and 2"
):
foreach_op(tensors1, tensors2)
with self.assertRaisesRegex(
RuntimeError,
"Tensor lists must have the same number of tensors, got 1 and 2"
):
foreach_op_(tensors1, tensors2)
# Corresponding tensors with different sizes that aren't compatible with broadcast
# If sizes are different then foreach chooses slow path, thus error messages are expected
# to be the same as torch regular function.
tensors1 = [
torch.zeros(10, 10, device=device, dtype=dtype) for _ in range(10)
]
tensors2 = [
torch.ones(11, 11, device=device, dtype=dtype) for _ in range(10)
]
try:
foreach_op(tensors1, tensors2)
except RuntimeError as e:
with self.assertRaisesRegex(type(e), re.escape(str(e))):
[ref(t1, t2) for t1, t2 in zip(tensors1, tensors2)]
try:
foreach_op_(tensors1, tensors2)
except RuntimeError as e:
with self.assertRaisesRegex(type(e), re.escape(str(e))):
[ref_(t1, t2) for t1, t2 in zip(tensors1, tensors2)]
# different devices
if self.device_type == "cuda" and torch.cuda.device_count() > 1:
tensor1 = torch.zeros(10, 10, device="cuda:0", dtype=dtype)
tensor2 = torch.ones(10, 10, device="cuda:1", dtype=dtype)
if dtype == torch.bool and foreach_op == torch._foreach_sub:
with self.assertRaisesRegex(RuntimeError,
re.escape(_BOOL_SUB_ERR_MSG)):
foreach_op([tensor1], [tensor2])
with self.assertRaisesRegex(RuntimeError,
re.escape(_BOOL_SUB_ERR_MSG)):
foreach_op_([tensor1], [tensor2])
return
with self.assertRaisesRegex(
RuntimeError,
"Expected all tensors to be on the same device"):
foreach_op([tensor1], [tensor2])
if dtype in get_all_int_dtypes() + [
torch.bool
] and foreach_op == torch._foreach_div:
with self.assertRaisesRegex(RuntimeError, "result type"):
foreach_op_([tensor1], [tensor2])
else:
with self.assertRaisesRegex(
RuntimeError,
"Expected all tensors to be on the same device"):
foreach_op_([tensor1], [tensor2])
@skipMeta
@unittest.skipIf(not torch.cuda.is_available(), "CUDA not found")
@ops(foreach_binary_op_db, dtypes=get_all_dtypes())
def test_binary_op_list_slow_path(self, device, dtype, op):
# note(mkozuki): why `n_expected_cudaLaunchKernels=0`?
# In this test, foreach functions don't go through fast path,
# but as there is only one tensor in each list of tensors,
# `cudaLaunchKernel` is 1 so ForeachFuncWrapper internal assert fails.
foreach_op, native_op, foreach_op_, native_op_ = self._get_funcs(
op, n_expected_cudaLaunchKernels=0)
# 0-strides
tensor1 = make_tensor((10, 10), dtype=dtype, device=device)
tensor2 = make_tensor((1, ), device=device,
dtype=dtype).expand_as(tensor1)
inputs = ([tensor1], [tensor2])
self._binary_test(dtype,
foreach_op,
native_op,
inputs,
is_fastpath=False,
is_inplace=False)
self._binary_test(dtype,
foreach_op_,
native_op_,
inputs,
is_fastpath=False,
is_inplace=True)
# different strides
tensor1 = torch.zeros(10, 10, device=device, dtype=dtype)
tensor2 = torch.ones(10, 10, device=device, dtype=dtype)
inputs = ([tensor1], [tensor2.t()])
self._binary_test(dtype,
foreach_op,
native_op,
inputs,
is_fastpath=False,
is_inplace=False)
self._binary_test(dtype,
foreach_op_,
native_op_,
inputs,
is_fastpath=False,
is_inplace=True)
# non contiguous
tensor1 = make_tensor((5, 2, 1, 3),
device=device,
dtype=dtype,
noncontiguous=True)
tensor2 = make_tensor((5, 2, 1, 3),
device=device,
dtype=dtype,
noncontiguous=True)
self.assertFalse(tensor1.is_contiguous())
self.assertFalse(tensor2.is_contiguous())
inputs = ([tensor1], [tensor2])
self._binary_test(dtype,
foreach_op,
native_op,
inputs,
is_fastpath=False,
is_inplace=False)
self._binary_test(dtype,
foreach_op_,
native_op_,
inputs,
is_fastpath=False,
is_inplace=True)
# sliced tensor
tensor1 = make_tensor((5, 2, 1, 3), device=device, dtype=dtype)
tensor2 = make_tensor((5, 2, 1, 3 * 7), device=device,
dtype=dtype)[:, :, :, ::7]
inputs = ([tensor1], [tensor2])
self._binary_test(dtype,
foreach_op,
native_op,
inputs,
is_fastpath=False,
is_inplace=False)
self._binary_test(dtype,
foreach_op_,
native_op_,
inputs,
is_fastpath=False,
is_inplace=True)
# note: Below three tests (postfixed with `_tensors_on_different_devices`)
# checks whether foreach works with lists of tensors on different devices
# but tensors of the same index are on the same device, e.g., ['cuda', 'cpu].
@onlyCUDA
@ops(foreach_unary_op_db)
def test_unary_op_tensors_on_different_devices(self, device, dtype, op):
method, ref, inplace_method, ref_inplace = self._get_funcs(op, 1)
# tensors: ['cuda', 'cpu]
tensors = op.sample_inputs(device, dtype, 2)
tensors[1] = tensors[1].to('cpu')
try:
actual = method((tensors, ), False, False)
except RuntimeError as e:
with self.assertRaisesRegex(type(e), str(e)):
ref((tensors, ))
else:
expected = ref((tensors, ))
self.assertEqual(expected, actual)
try:
inplace_method((tensors, ), False, False)
except RuntimeError as e:
with self.assertRaisesRegex(type(e), str(e)):
ref_inplace((tensors, ))
else:
self.assertEqual(expected, tensors)
@onlyCUDA
@ops(foreach_binary_op_db)
def test_binary_op_tensors_on_different_devices(self, device, dtype, op):
# `tensors1`: ['cuda', 'cpu']
# `tensors2`: ['cuda', 'cpu']
_cuda_tensors = op.sample_inputs(device, dtype, 2, same_size=True)
_cpu_tensors = op.sample_inputs('cpu', dtype, 2, same_size=True)
tensors1, tensors2 = list(
tensors for tensors in zip(_cuda_tensors, _cpu_tensors))
foreach_op, foreach_op_ = op.method_variant, op.inplace_variant
native_op, native_op_ = op.ref, op.ref_inplace
try:
actual = foreach_op(tensors1, tensors2)
except RuntimeError as e:
with self.assertRaisesRegex(type(e), re.escape(str(e))):
[native_op(t1, t2) for t1, t2 in zip(tensors1, tensors2)]
else:
expected = [
native_op(t1, t2) for t1, t2 in zip(tensors1, tensors2)
]
self.assertEqual(expected, actual)
try:
foreach_op_(tensors1, tensors2)
except RuntimeError as e:
with self.assertRaisesRegex(type(e), re.escape(str(e))):
[native_op_(t1, t2) for t1, t2 in zip(tensors1, tensors2)]
else:
self.assertEqual(actual, tensors1)
@onlyCUDA
@ops(foreach_pointwise_op_db,
allowed_dtypes=get_all_fp_dtypes(include_half=False,
include_bfloat16=False))
def test_pointwise_op_tensors_on_different_devices(self, device, dtype,
op):
# tensors1: ['cuda', 'cpu]
# tensors2: ['cuda', 'cpu]
# tensors3: ['cuda', 'cpu]
_cuda_tensors = op.sample_inputs(device, dtype, 3, same_size=True)
_cpu_tensors = op.sample_inputs('cpu', dtype, 3, same_size=True)
tensors1, tensors2, tensors3 = list(
tensors for tensors in zip(_cuda_tensors, _cpu_tensors))
foreach_op, foreach_op_, native_op = op.method_variant, op.inplace_variant, op.ref
actual = foreach_op(tensors1, tensors2, tensors3)
expected = [native_op(*_cuda_tensors), native_op(*_cpu_tensors)]
self.assertEqual(expected, actual)
# note(mkozuki): Limiting dtypes to FP32&FP64, we can safely run inplace ops.
foreach_op_(tensors1, tensors2, tensors3)
self.assertEqual(expected, tensors1)
class TestSparseCSR(TestCase):
@onlyCPU
def test_csr_layout(self):
self.assertEqual(str(torch.sparse_csr), 'torch.sparse_csr')
self.assertEqual(type(torch.sparse_csr), torch.layout)
@dtypes(*get_all_dtypes())
def test_sparse_csr_constructor_shape_inference(self, device, dtype):
crow_indices = [0, 2, 4]
col_indices = [0, 1, 0, 1]
values = [1, 2, 3, 4]
sparse = torch.sparse_csr_tensor(torch.tensor(crow_indices, dtype=torch.int64),
torch.tensor(col_indices, dtype=torch.int64),
torch.tensor(values), dtype=dtype, device=device)
self.assertEqual(torch.tensor(crow_indices, dtype=torch.int64), sparse.crow_indices())
self.assertEqual((len(crow_indices) - 1, max(col_indices) + 1), sparse.shape)
self.assertEqual(dtype, sparse.dtype)
self.assertEqual(torch.device(device), sparse.device)
@dtypes(*get_all_dtypes())
def test_sparse_csr_constructor(self, device, dtype):
crow_indices = [0, 2, 4]
col_indices = [0, 1, 0, 1]
values = [1, 2, 3, 4]
for index_dtype in [torch.int32, torch.int64]:
sparse = torch.sparse_csr_tensor(torch.tensor(crow_indices, dtype=index_dtype),
torch.tensor(col_indices, dtype=index_dtype),
torch.tensor(values),
size=(2, 10),
dtype=dtype,
device=device)
self.assertEqual((2, 10), sparse.shape)
self.assertEqual(torch.tensor(crow_indices, dtype=index_dtype), sparse.crow_indices())
self.assertEqual(torch.tensor(col_indices, dtype=index_dtype), sparse.col_indices())
self.assertEqual(torch.tensor(values, dtype=dtype), sparse.values())
@dtypes(*get_all_dtypes())
def test_sparse_csr_constructor_from_lists(self, device, dtype):
# without size
sparse = torch.sparse_csr_tensor([0, 2, 4],
[0, 1, 0, 1],
[1, 2, 3, 4],
dtype=dtype,
device=device)
self.assertEqual((2, 2), sparse.shape)
self.assertEqual(4, sparse.numel())
self.assertEqual(torch.tensor([0, 2, 4], dtype=torch.int64, device=device), sparse.crow_indices())
self.assertEqual(torch.tensor([0, 1, 0, 1], dtype=torch.int64, device=device), sparse.col_indices())
self.assertEqual(torch.tensor([1, 2, 3, 4], dtype=dtype, device=device), sparse.values())
# with size
for sparse_csr_tensor in [torch.sparse_csr_tensor, torch._sparse_csr_tensor_unsafe]:
sparse = sparse_csr_tensor([0, 2, 4],
[0, 1, 0, 1],
[1, 2, 3, 4],
size=(2, 10),
dtype=dtype,
device=device)
self.assertEqual((2, 10), sparse.shape)
self.assertEqual(torch.tensor([0, 2, 4], dtype=torch.int64, device=device), sparse.crow_indices())
self.assertEqual(torch.tensor([0, 1, 0, 1], dtype=torch.int64, device=device), sparse.col_indices())
self.assertEqual(torch.tensor([1, 2, 3, 4], dtype=dtype, device=device), sparse.values())
@skipMeta
@dtypes(*get_all_dtypes())
def test_empty(self, device, dtype):
ns = [5, 2, 0]
for shape in itertools.product(ns, ns):
result = torch.empty(shape, dtype=dtype, device=device, layout=torch.sparse_csr)
self.assertEqual(result.shape, shape)
self.assertEqual(result.dtype, dtype)
self.assertEqual(result.device, torch.device(device))
self.assertEqual(result.layout, torch.sparse_csr)
self.assertEqual(result.crow_indices().shape, (shape[0] + 1,))
self.assertEqual(result.col_indices().shape, (0,))
self.assertEqual(result.values().shape, (0,))
self.assertEqual(result._nnz(), 0)
self.assertEqual(result.crow_indices().device, torch.device(device))
self.assertEqual(result.col_indices().device, torch.device(device))
self.assertEqual(result.values().device, torch.device(device))
self.assertEqual(result.crow_indices().dtype, torch.int64)
self.assertEqual(result.col_indices().dtype, torch.int64)
self.assertEqual(result.values().dtype, dtype)
@skipMeta
@dtypes(*get_all_dtypes())
def test_empty_errors(self, device, dtype):
with self.assertRaisesRegex(RuntimeError, "torch.empty: Only 2D sparse CSR tensors are supported."):
torch.empty((5,), dtype=dtype, device=device, layout=torch.sparse_csr)
with self.assertRaisesRegex(RuntimeError, "torch.empty: Only 2D sparse CSR tensors are supported."):
torch.empty((2, 3, 4), dtype=dtype, device=device, layout=torch.sparse_csr)
@skipMeta
@dtypes(*get_all_dtypes())
def test_copy(self, device, dtype):
def run_test(shape, nnz, index_type):
a = self.genSparseCSRTensor(shape, nnz, dtype=dtype, device=device, index_dtype=index_dtype)
b = self.genSparseCSRTensor(shape, nnz, dtype=dtype, device=device, index_dtype=index_dtype)
a.copy_(b)
self.assertEqual(a.crow_indices(), b.crow_indices())
self.assertEqual(a.col_indices(), b.col_indices())
self.assertEqual(a.values(), b.values())
ns = [5, 2, 0]
for shape, index_dtype in zip(itertools.product(ns, ns), [torch.int32, torch.int64]):
run_test(shape, 0, index_dtype)
run_test(shape, shape[0] * shape[1], index_dtype)
@skipMeta
@dtypes(*get_all_dtypes())
def test_copy_errors(self, device, dtype):
for index_dtype in [torch.int32, torch.int64]:
shape1 = (2, 3)
shape2 = (3, 2)
a = self.genSparseCSRTensor(shape1, 0, dtype=dtype, device=device, index_dtype=index_dtype)
b = self.genSparseCSRTensor(shape2, 0, dtype=dtype, device=device, index_dtype=index_dtype)
with self.assertRaisesRegex(RuntimeError, "only same size tensors are supported."):
a.copy_(b)
with self.assertRaisesRegex(RuntimeError, "copy between different layouts is not supported."):
a.copy_(torch.empty(a.shape, dtype=dtype, device=device))
b = self.genSparseCSRTensor(shape1, 1, dtype=dtype, device=device, index_dtype=index_dtype)
with self.assertRaisesRegex(RuntimeError, "only tensors with the same number of specified elements are supported."):
a.copy_(b)
@skipMeta
@dtypes(*get_all_dtypes())
def test_resize(self, device, dtype):
for index_dtype in [torch.int32, torch.int64]:
shape = (2, 3)
nnz = 6
a = self.genSparseCSRTensor(shape, nnz, dtype=dtype, device=device, index_dtype=index_dtype)
new_shape = (4, 5)
a.resize_(new_shape)
self.assertEqual(a.shape, new_shape)
# resize to larger shape doesn't add specified elements
self.assertEqual(a._nnz(), nnz)
new_shape = (1, 5)
a.resize_(new_shape)
self.assertEqual(a.shape, new_shape)
# resize to smaller shape trims specified elements
self.assertEqual(a._nnz(), 5)
@skipMeta
@dtypes(*get_all_dtypes())
def test_resize_errors(self, device, dtype):
for index_dtype in [torch.int32, torch.int64]:
shape = (2, 3)
nnz = 6
a = self.genSparseCSRTensor(shape, nnz, dtype=dtype, device=device, index_dtype=index_dtype)
with self.assertRaisesRegex(RuntimeError, "torch.resize_: Only 2D sparse CSR tensors are supported."):
new_shape = (4,)
a.resize_(new_shape)
# resizing of columns to smaller size is not implemented
with self.assertRaisesRegex(
RuntimeError,
"torch.resize_: Resizing columns of sparse CSR tensors to a smaller value is not supported.",
):
new_shape = (2, 2)
a.resize_(new_shape)
def test_factory_type_invariants_check(self, device):
with self.assertRaisesRegex(RuntimeError, "both crow_indices and col_indices should have the same type."):
torch.sparse_csr_tensor(torch.tensor([0, 2, 4], dtype=torch.int64),
torch.tensor([0, 1, 0, 1], dtype=torch.int32),
torch.tensor([1, 2, 3, 4]),
device=device)
with self.assertRaisesRegex(RuntimeError, r"\"csr_construct_check\" not implemented for 'Short'"):
torch.sparse_csr_tensor(torch.tensor([0, 2, 4], dtype=torch.int16),
torch.tensor([0, 1, 0, 1], dtype=torch.int16),
torch.tensor([1, 2, 3, 4]),
device=device)
def test_factory_layout_invariants_check(self, device):
with self.assertRaisesRegex(RuntimeError, "expected values to be a strided and contiguous tensor"):
values = torch.tensor([1.], device=device).expand(4,)
torch.sparse_csr_tensor(torch.tensor([0, 2, 4], device=device),
torch.tensor([0, 1, 0, 1], device=device),
values)
with self.assertRaisesRegex(RuntimeError, "expected col_indices to be a strided and contiguous tensor"):
col_indices = torch.tensor([0], device=device).expand(4,)
torch.sparse_csr_tensor(torch.tensor([0, 2, 4]),
col_indices,
torch.tensor([1, 2, 3, 4]))
with self.assertRaisesRegex(RuntimeError, "expected crow_indices to be a strided and contiguous tensor"):
crow_indices = torch.arange(6, device=device)
torch.sparse_csr_tensor(crow_indices[::2],
torch.tensor([0, 1, 0, 1], device=device),
torch.tensor([1, 2, 3, 4]))
def test_factory_shape_invariants_check(self, device):
crow_indices = [0, 2, 4]
col_indices = [0, 1, 0, 1]
values = [1, 2, 3, 4]
size = (2, 10)
torch.sparse_csr_tensor(torch.tensor(crow_indices), torch.tensor(col_indices), torch.tensor(values), size,
device=device)
with self.assertRaisesRegex(RuntimeError, r"size of a CSR tensor must be of length 2, but got: 3"):
torch.sparse_csr_tensor(torch.tensor(crow_indices), torch.tensor(col_indices), torch.tensor(values),
size=(2, 10, 2),
device=device)
with self.assertRaisesRegex(RuntimeError, r"crow_indices must have dim\=1 but got crow_indices\.dim\(\)\=2"):
torch.sparse_csr_tensor(torch.tensor(crow_indices).repeat(2, 1),
torch.tensor(col_indices),
torch.tensor(values),
size,
device=device)
with self.assertRaisesRegex(RuntimeError, r"col_indices must have dim\=1 but got col_indices\.dim\(\)\=2"):
torch.sparse_csr_tensor(torch.tensor(crow_indices),
torch.tensor(col_indices).repeat(2, 1),
torch.tensor(values),
size,
device=device)
with self.assertRaisesRegex(RuntimeError, r"values must have dim\=1 but got values\.dim\(\)\=2"):
torch.sparse_csr_tensor(torch.tensor(crow_indices),
torch.tensor(col_indices),
torch.tensor(values).repeat(2, 1),
size,
device=device)
with self.assertRaisesRegex(RuntimeError,
r"crow_indices\.numel\(\) must be size\(0\) \+ 1, but got: 3"):
torch.sparse_csr_tensor(torch.tensor(crow_indices), torch.tensor(col_indices), torch.tensor(values), (1, 1),
device=device)
with self.assertRaisesRegex(RuntimeError,
r"col_indices and values must have equal sizes, " +
r"but got col_indices\.numel\(\): 3, values\.numel\(\): 4"):
torch.sparse_csr_tensor(torch.tensor(crow_indices), torch.tensor([0, 1, 0]), torch.tensor(values), size,
device=device)
def test_factory_indices_invariants_check(self, device):
crow_indices = [0, 2, 4]
col_indices = [0, 1, 0, 1]
values = [1, 2, 3, 4]
size = (2, 10)
with self.assertRaisesRegex(RuntimeError, "0th value of crow_indices must be 0."):
torch.sparse_csr_tensor(torch.tensor([-1, 0, 4]), torch.tensor(col_indices), torch.tensor(values), size,
device=device)
with self.assertRaisesRegex(RuntimeError,
"last value of crow_indices should be equal to the length of col_indices."):
torch.sparse_csr_tensor(torch.tensor([0, 2, 5]), torch.tensor(col_indices), torch.tensor(values), size,
device=device)
with self.assertRaisesRegex(RuntimeError,
r"at position i \= 2," +
r" this condition crow_indices\[i - 1\] <\= crow_indices\[i\] fails"):
torch.sparse_csr_tensor(torch.tensor([0, 5, 4]), torch.tensor(col_indices), torch.tensor(values), size,
device=device)
with self.assertRaisesRegex(RuntimeError, r"col_indices\.min\(\) should be greater or equal to zero"):
torch.sparse_csr_tensor(torch.tensor(crow_indices), torch.tensor([0, -1, 0, 1]), torch.tensor(values), size,
device=device)
with self.assertRaisesRegex(RuntimeError, r"size\(1\) should be greater than col_indices\.max\(\)"):
torch.sparse_csr_tensor(torch.tensor(crow_indices), torch.tensor([0, 11, 0, 1]), torch.tensor(values), size,
device=device)
@onlyCUDA
@dtypes(*get_all_dtypes())
def test_factory_device_type_inference(self, device, dtype):
cpu_cuda = ('cpu', 'cuda')
cpu_cuda_none = cpu_cuda + (None,)
for crow_indices_device, col_indices_device, values_device, device in itertools.product(cpu_cuda,
cpu_cuda,
cpu_cuda,
cpu_cuda_none):
for index_dtype in [torch.int32, torch.int64]:
crow_indices = torch.tensor([0, 2, 4], dtype=index_dtype, device=crow_indices_device)
col_indices = torch.tensor([0, 1, 0, 1], dtype=index_dtype, device=col_indices_device)
values = torch.tensor([1, 2, 3, 4], dtype=dtype, device=values_device)
if device is None and (crow_indices_device != col_indices_device or
crow_indices_device != values_device):
with self.assertRaises(RuntimeError):
torch.sparse_csr_tensor(crow_indices,
col_indices,
values,
size=(2, 10),
device=device)
else:
t = torch.sparse_csr_tensor(crow_indices,
col_indices,
values,
size=(2, 10),
device=device)
should_be_cuda = (device == 'cuda' or (device is None and values_device == 'cuda'))
self.assertEqual(should_be_cuda, t.is_cuda)
t.crow_indices().dtype == index_dtype
t.col_indices().dtype == index_dtype
t.values().dtype == dtype
t.crow_indices().device == t.values().device
t.col_indices().device == t.values().device
def test_sparse_csr_print(self, device):
orig_maxDiff = self.maxDiff
self.maxDiff = None
shape_nnz = [
((10, 10), 10),
((100, 10), 10),
((1000, 10), 10)
]
printed = []
for shape, nnz in shape_nnz:
values_shape = torch.Size((nnz,))
col_indices_shape = torch.Size((nnz,))
crow_indices_shape = torch.Size((shape[0] + 1,))
printed.append("# shape: {}".format(torch.Size(shape)))
printed.append("# nnz: {}".format(nnz))
printed.append("# crow_indices shape: {}".format(crow_indices_shape))
printed.append("# col_indices shape: {}".format(col_indices_shape))
printed.append("# values_shape: {}".format(values_shape))
for index_dtype in [torch.int32, torch.int64]:
for dtype in floating_types():
printed.append("########## {}/{} ##########".format(dtype, index_dtype))
x = torch.sparse_csr_tensor(torch.tensor([0, 2, 4], dtype=index_dtype),
torch.tensor([0, 1, 0, 1], dtype=index_dtype),
torch.tensor([1, 2, 3, 4]), dtype=dtype, device=device)
printed.append("# sparse tensor")
printed.append(str(x))
printed.append("# _crow_indices")
printed.append(str(x.crow_indices()))
printed.append("# _col_indices")
printed.append(str(x.col_indices()))
printed.append("# _values")
printed.append(str(x.values()))
printed.append('')
printed.append('')
self.assertExpected('\n'.join(printed))
self.maxDiff = orig_maxDiff
@dtypes(*get_all_dtypes())
def test_sparse_csr_from_dense(self, device, dtype):
dense = torch.tensor([[4, 5, 0], [0, 0, 0], [1, 0, 0]], dtype=dtype, device=device)
sparse = dense.to_sparse_csr()
self.assertEqual(torch.tensor([0, 2, 2, 3], dtype=torch.int64), sparse.crow_indices())
self.assertEqual(torch.tensor([0, 1, 0], dtype=torch.int64), sparse.col_indices())
self.assertEqual(torch.tensor([4, 5, 1], dtype=dtype), sparse.values())
dense = torch.tensor([[0, 0, 0], [0, 0, 1], [1, 0, 0]], dtype=dtype, device=device)
sparse = dense.to_sparse_csr()
self.assertEqual(torch.tensor([0, 0, 1, 2], dtype=torch.int64), sparse.crow_indices())
self.assertEqual(torch.tensor([2, 0], dtype=torch.int64), sparse.col_indices())
self.assertEqual(torch.tensor([1, 1], dtype=dtype), sparse.values())
dense = torch.tensor([[2, 2, 2], [2, 2, 2], [2, 2, 2]], dtype=dtype, device=device)
sparse = dense.to_sparse_csr()
self.assertEqual(torch.tensor([0, 3, 6, 9], dtype=torch.int64), sparse.crow_indices())
self.assertEqual(torch.tensor([0, 1, 2] * 3, dtype=torch.int64), sparse.col_indices())
self.assertEqual(torch.tensor([2] * 9, dtype=dtype), sparse.values())
@dtypes(*get_all_dtypes())
def test_sparse_csr_to_dense(self, device, dtype):
mn = [5, 2, 0]
for (m, n) in itertools.product(mn, mn):
size = (m, n)
dense = make_tensor(size, dtype=dtype, device=device)
sparse = dense.to_sparse_csr()
self.assertEqual(sparse.to_dense(), dense)
crow_indices = torch.tensor([0, 3, 5])
col_indices = torch.tensor([0, 1, 2, 0, 1])
values = torch.tensor([1, 2, 1, 3, 4], dtype=dtype)
csr = torch.sparse_csr_tensor(crow_indices, col_indices,
values, dtype=dtype, device=device)
dense = torch.tensor([[1, 2, 1], [3, 4, 0]], dtype=dtype, device=device)
self.assertEqual(csr.to_dense(), dense)
@skipCPUIfNoMklSparse
@coalescedonoff
@dtypes(torch.double)
def test_coo_to_csr_convert(self, device, dtype, coalesced):
with self.assertRaisesRegex(RuntimeError, "Input is supposed to be a vector"):
torch._convert_indices_from_coo_to_csr(
torch.randint(100, (5, 5), device=device),
size=100)
size = (5, 5)
sparse_dim = 2
nnz = 10
sparse_coo, _, _ = self.genSparseTensor(size, sparse_dim, nnz, coalesced, device, dtype)
sparse_csr = sparse_coo.to_sparse_csr()
self.assertTrue(sparse_csr.is_sparse_csr)
self.assertEqual(sparse_csr.to_dense(), sparse_coo.to_dense())
vec = torch.randn((5, 1), dtype=dtype, device=device)
coo_product = sparse_coo.matmul(vec)
csr_product = sparse_csr.matmul(vec)
self.assertEqual(coo_product, csr_product)
vec = torch.randn((100, 1), dtype=dtype, device=device)
index = torch.tensor([
[1, 0, 35, 14, 39, 6, 71, 66, 40, 27],
[92, 31, 62, 50, 22, 65, 89, 74, 56, 34],
], dtype=torch.int32)
values = torch.tensor([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], dtype=dtype, device=device)
coo = torch.sparse_coo_tensor(index, values, torch.Size([100, 100]), dtype=dtype, device=device)
csr = coo.to_sparse_csr()
self.assertEqual(coo.matmul(vec), csr.matmul(vec))
col_indices = torch.tensor([
31, 92, 65, 50, 34, 62, 22, 56, 74, 89
], dtype=torch.int64, device=device)
self.assertEqual(csr.col_indices(), col_indices)
values = torch.tensor([2, 1, 6, 4, 10, 3, 5, 9, 8, 7], dtype=dtype, device=device)
self.assertEqual(csr.values(), values)
@dtypes(*get_all_dtypes())
def test_sparse_csr_from_dense_convert_error(self, device, dtype):
size = (4, 2, 4)
dense = make_tensor(size, dtype=dtype, device=device)
with self.assertRaisesRegex(RuntimeError, "Only 2D"):
sparse = dense.to_sparse_csr()
# TODO: Support auto generation of device check for sparse tensors
# See: https://github.com/pytorch/pytorch/issues/59058
@onlyCUDA
@dtypes(torch.double)
def test_matmul_device_mismatch(self, device, dtype):
cpu = torch.rand((10, 10))
cuda = cpu.cuda()
for s, m1, m2 in itertools.product((cpu, cuda), repeat=3):
csr = m1.to_sparse()
if s.device == csr.device == m2.device:
torch.addmm(s, csr, m2)
else:
with self.assertRaisesRegex(RuntimeError, "Expected all tensors to be on the same device"):
torch.addmm(s, csr, m2)
@skipCPUIfNoMklSparse
@skipCUDAIfNoCusparseGeneric
@dtypes(*floating_and_complex_types())
@dtypesIfCUDA(*get_all_complex_dtypes(),
*get_all_fp_dtypes(include_half=SM53OrLater, include_bfloat16=SM80OrLater))
def test_csr_matvec(self, device, dtype):
side = 100
for index_dtype in [torch.int32, torch.int64]:
csr = self.genSparseCSRTensor((side, side), 1000, device=device, dtype=dtype, index_dtype=index_dtype)
vec = torch.randn(side, dtype=dtype, device=device)
res = csr.matmul(vec)
expected = csr.to_dense().matmul(vec)
self.assertEqual(res, expected)
bad_vec = torch.randn(side + 10, dtype=dtype, device=device)
err_msg = "size mismatch, got"
with self.assertRaisesRegex(RuntimeError, err_msg):
csr.matmul(bad_vec)
@skipCPUIfNoMklSparse
@dtypes(torch.double)
def test_mm(self, device, dtype):
def test_shape(di, dj, dk, nnz):
for index_dtype in [torch.int32, torch.int64]:
x = self.genSparseCSRTensor((di, dj), nnz, device=device, dtype=dtype, index_dtype=index_dtype)
t = torch.randn(di, dk, dtype=dtype, device=device)
y = torch.randn(dj, dk, dtype=dtype, device=device)
alpha = random.random()
beta = random.random()
# res = beta * t + alpha * (x @ y)
res = torch.addmm(t, x, y, beta=beta, alpha=alpha)
expected = torch.addmm(t, x.to_dense(), y, beta=beta, alpha=alpha)
self.assertEqual(res, expected)
res = torch.addmm(t, x, y)
expected = torch.addmm(t, x.to_dense(), y)
self.assertEqual(res, expected)
res = torch.mm(x, y)
expected = torch.mm(x.to_dense(), y)
self.assertEqual(res, expected)
for i in range(2, 5):
for j in range(2, 8):
for k in range(2, 8):
test_shape(i, j, k, i * j // 2)
test_shape(4, 4, 4, 0)
@skipCPUIfNoMklSparse
@dtypes(*floating_and_complex_types())
@dtypesIfCUDA(*get_all_complex_dtypes(),
*get_all_fp_dtypes(include_half=SM53OrLater and TEST_CUSPARSE_GENERIC,
include_bfloat16=SM80OrLater and TEST_CUSPARSE_GENERIC))
@precisionOverride({torch.bfloat16: 1e-2, torch.float16: 1e-2})
def test_sparse_mm(self, device, dtype):
def test_shape(d1, d2, d3, nnz, transposed, index_dtype):
if transposed:
D = torch.randn(d3, d2, dtype=dtype, device=device).t_()
else:
D = torch.randn(d2, d3, dtype=dtype, device=device)
S = self.genSparseCSRTensor((d1, d2), nnz, device=device, dtype=dtype, index_dtype=index_dtype)
S_dense = S.to_dense()
self.assertEqual(torch.sparse.mm(S, D), torch.mm(S_dense, D))
for index_dtype in [torch.int32, torch.int64]:
test_shape(7, 8, 9, 20, False, index_dtype)
test_shape(7, 8, 9, 20, True, index_dtype)
@skipCPUIfNoMklSparse
@dtypes(*floating_and_complex_types())
@dtypesIfCUDA(*get_all_complex_dtypes(),
*get_all_fp_dtypes(include_half=SM53OrLater and TEST_CUSPARSE_GENERIC,
include_bfloat16=SM80OrLater and TEST_CUSPARSE_GENERIC))
@precisionOverride({torch.bfloat16: 1e-2, torch.float16: 1e-2})
def test_sparse_addmm(self, device, dtype):
def test_shape(m, n, p, nnz, broadcast, index_dtype, alpha_beta=None):
if alpha_beta is None:
alpha = random.random()
beta = random.random()
else:
alpha, beta = alpha_beta
if broadcast:
D1 = make_tensor((), dtype=dtype, device=device)
else:
D1 = make_tensor([n, p], dtype=dtype, device=device)
D2 = make_tensor([m, p], dtype=dtype, device=device)
S = self.genSparseCSRTensor([n, m], nnz, dtype=dtype, device=device, index_dtype=index_dtype)
S_dense = S.to_dense()
Y = torch.sparse.addmm(D1, S, D2, beta=beta, alpha=alpha)
Y_dense = torch.addmm(D1, S_dense, D2, beta=beta, alpha=alpha)
self.assertEqual(Y, Y_dense)
for index_dtype in [torch.int32, torch.int64]:
test_shape(7, 8, 9, 20, False, index_dtype, None)
test_shape(7, 8, 9, 20, True, index_dtype, None)
test_shape(7, 8, 9, 20, False, index_dtype, (1, 0))
test_shape(7, 8, 9, 20, True, index_dtype, (1, 0))
test_shape(7, 8, 9, 20, False, index_dtype, (1, 1))
test_shape(7, 8, 9, 20, True, index_dtype, (1, 1))
@skipCPUIfNoMklSparse
@dtypes(*floating_and_complex_types())
@precisionOverride({torch.double: 1e-8, torch.float: 1e-4, torch.bfloat16: 0.6,
torch.half: 1e-1, torch.cfloat: 1e-4, torch.cdouble: 1e-8})
@dtypesIfCUDA(torch.complex64,
*((torch.complex128,) if CUSPARSE_SPMM_COMPLEX128_SUPPORTED else ()),
*torch.testing.get_all_fp_dtypes(include_bfloat16=SM80OrLater,
include_half=SM53OrLater))
@skipCUDAIf(
not _check_cusparse_spgemm_available(),
"cuSparse Generic API SpGEMM is not available"
)
def test_addmm_all_sparse_csr(self, device, dtype):
M = torch.randn(10, 25, device=device).to(dtype)
m1 = torch.randn(10, 50, device=device).to(dtype)
m2 = torch.randn(50, 25, device=device).to(dtype)
_test_addmm_addmv(self, torch.addmm, M, m1, m2, layout=torch.sparse_csr, all_sparse=True)
# Test 0-strided
M = torch.randn(10, 1, device=device).to(dtype).expand(10, 25)
m1 = torch.randn(10, 1, device=device).to(dtype).expand(10, 50)
m2 = torch.randn(50, 25, device=device).to(dtype)
_test_addmm_addmv(self, torch.addmm, M, m1, m2, layout=torch.sparse_csr, all_sparse=True)
# Test beta=0, M=nan
M = torch.full((10, 25), float('nan'), device=device).to(dtype)
m1 = torch.randn(10, 50, device=device).to(dtype)
m2 = torch.randn(50, 25, device=device).to(dtype)
_test_addmm_addmv(self, torch.addmm, M, m1, m2, beta=0, layout=torch.sparse_csr, all_sparse=True)
# Test transpose
for t1, t2, t3, t4 in itertools.product([True, False], repeat=4):
def maybe_transpose(cond, m):
if not cond:
return m
return m.t().clone(memory_format=torch.contiguous_format).t()
M = maybe_transpose(t1, torch.randn(10, 25, device=device).to(dtype))
m1 = maybe_transpose(t2, torch.randn(10, 50, device=device).to(dtype))
m2 = maybe_transpose(t3, torch.randn(50, 25, device=device).to(dtype))
_test_addmm_addmv(self, torch.addmm, M, m1, m2, transpose_out=t4, layout=torch.sparse_csr, all_sparse=True)
@skipCPUIfNoMklSparse
@dtypes(*floating_and_complex_types())
@dtypesIfCUDA(torch.complex64,
*((torch.complex128,) if CUSPARSE_SPMM_COMPLEX128_SUPPORTED else ()),
*torch.testing.get_all_fp_dtypes(include_bfloat16=SM80OrLater,
include_half=SM53OrLater))
@skipCUDAIf(
not _check_cusparse_spgemm_available(),
"cuSparse Generic API SpGEMM is not available"
)
@precisionOverride({torch.double: 1e-8, torch.float: 1e-4, torch.bfloat16: 0.6,
torch.half: 1e-1, torch.cfloat: 1e-4, torch.cdouble: 1e-8})
def test_addmm_sizes_all_sparse_csr(self, device, dtype):
for m in [0, 1, 25]:
for n in [0, 1, 10]:
for k in [0, 1, 8]:
M = torch.randn(n, m, device=device).to(dtype)
m1 = torch.randn(n, k, device=device).to(dtype)
m2 = torch.randn(k, m, device=device).to(dtype)
_test_addmm_addmv(self, torch.addmm, M, m1, m2, layout=torch.sparse_csr, all_sparse=True)
M = torch.randn(n, m, device=device).to(dtype).to_sparse_csr()
m1 = torch.randn(n, k + 1, device=device).to(dtype).to_sparse_csr()
m2 = torch.randn(k, m, device=device).to(dtype).to_sparse_csr()
self.assertRaisesRegex(RuntimeError, f"{n}x{k + 1}.*{k}x{m}", lambda: torch.addmm(M, m1, m2))
self.assertRaisesRegex(RuntimeError, f"{n}x{k + 1}.*{k}x{m}", lambda: torch.mm(m1, m2))
@skipCPUIfNoMklSparse
@dtypes(torch.float)
def test_addmm_errors(self, device, dtype):
# test that the errors are the same for dense and sparse versions
import re
def test1(*, is_sparse):
# shapes must be compatible for matrix multiplication
a = make_tensor((2, 3), dtype=dtype, device=device)
if is_sparse:
a_sparse = a.to_sparse_csr()
return torch.addmm(a, a_sparse, a)
else:
return torch.addmm(a, a, a)
def test2(*, is_sparse):
# mat2 must be a matrix
a = make_tensor((2, 3), dtype=dtype, device=device)
if is_sparse:
a_sparse = a.to_sparse_csr()
return torch.addmm(a, a_sparse, a.unsqueeze(0))
else:
return torch.addmm(a, a, a.unsqueeze(0))
def test3(*, is_sparse):
# the first input needs to be 1D or 2D
a = make_tensor((3, 3), dtype=dtype, device=device)
if is_sparse:
a_sparse = a.to_sparse_csr()
return torch.addmm(a.unsqueeze(0), a_sparse, a)
else:
return torch.addmm(a.unsqueeze(0), a, a)
for test in (test1, test2, test3):
try:
test(is_sparse=False)
except RuntimeError as msg:
with self.assertRaisesRegex(RuntimeError, re.escape(str(msg))):
test(is_sparse=True)
@skipCPUIfNoMklSparse
@dtypes(torch.float)
def test_mm_errors(self, device, dtype):
# test that the errors are the same for dense and sparse versions
import re
def test1(*, is_sparse):
# shapes must be compatible for matrix multiplication
a = make_tensor((2, 3), dtype=dtype, device=device)
if is_sparse:
a_sparse = a.to_sparse_csr()
return torch.mm(a_sparse, a)
else:
return torch.mm(a, a)
def test2(*, is_sparse):
# mat2 must be a matrix
a = make_tensor((2, 3), dtype=dtype, device=device)
if is_sparse:
a_sparse = a.to_sparse_csr()
return torch.mm(a_sparse, a.unsqueeze(0))
else:
return torch.mm(a, a.unsqueeze(0))
for test in (test1, test2):
try:
test(is_sparse=False)
except RuntimeError as msg:
with self.assertRaisesRegex(RuntimeError, re.escape(str(msg))):
test(is_sparse=True)
@dtypes(torch.float, torch.double)
def test_add(self, device, dtype):
def _test_spadd_shape(nnz, shape):
x = self.genSparseCSRTensor(shape, nnz, dtype=dtype, device=device, index_dtype=torch.int32)
y = torch.randn(*shape, dtype=dtype, device=device)
r = random.random()
res = torch.add(y, x, alpha=r)
expected = y + r * x.to_dense()
self.assertEqual(res, expected)
# Non contiguous dense tensor
s = list(shape)
s[0] = shape[-1]
s[-1] = shape[0]
y = torch.randn(*s, dtype=torch.double, device=device)
y.transpose_(0, len(s) - 1)
r = random.random()
res = torch.add(y, x, alpha=r)
expected = y + r * x.to_dense()
self.assertEqual(res, expected)
_test_spadd_shape(10, [100, 100])
_test_spadd_shape(0, [100, 100])
_test_spadd_shape(10, [100, 1])
_test_spadd_shape(10, [1, 100])
@skipCPUIfNoMklSparse
@dtypes(torch.float32, torch.float64, torch.complex64, torch.complex128)
def test_sparse_add(self, device, dtype):
def run_test(m, n, index_dtype):
if TEST_WITH_ROCM and dtype.is_complex:
self.skipTest("ROCm doesn't work with complex dtype correctly.")
alpha = random.random()
nnz1 = random.randint(0, m * n)
nnz2 = random.randint(0, m * n)
nnz3 = random.randint(0, m * n)
if TEST_WITH_ROCM:
# ROCm fails when nnz = 0
nnz1, nnz2, nnz3 = max(1, nnz1), max(1, nnz2), max(1, nnz3)
S1 = self.genSparseCSRTensor([m, n], nnz1, dtype=dtype, device=device, index_dtype=index_dtype)
S2 = self.genSparseCSRTensor([m, n], nnz2, dtype=dtype, device=device, index_dtype=index_dtype)
S3 = self.genSparseCSRTensor([m, n], nnz3, dtype=dtype, device=device, index_dtype=index_dtype)
expected = torch.add(S1.to_dense(), S2.to_dense(), alpha=alpha)
actual = torch.add(S1, S2, alpha=alpha, out=S3)
self.assertEqual(actual.to_dense(), expected)
self.assertEqual(S3.to_dense(), expected)
for index_dtype in [torch.int32, torch.int64]:
for m, n in itertools.product([3, 5], [3, 5]):
run_test(m, n, index_dtype)
@dtypes(torch.float32, torch.float64, torch.complex64, torch.complex128)
def test_sparse_add_errors(self, device, dtype):
def run_test(index_type):
a = self.genSparseCSRTensor((2, 2), 3, dtype=dtype, device=device, index_dtype=index_dtype)
b = self.genSparseCSRTensor((2, 1), 2, dtype=dtype, device=device, index_dtype=index_dtype)
with self.assertRaisesRegex(RuntimeError, "Expected input tensors to have the same shape"):
torch.add(a, b)
for index_dtype in [torch.int32, torch.int64]:
run_test(index_dtype)
@onlyCUDA
@skipCUDAIf(
not _check_cusparse_triangular_solve_available(),
"cuSparse Generic API SpSV is not available"
)
@dtypes(torch.float32, torch.float64, torch.complex64, torch.complex128)
@precisionOverride({torch.float32: 1e-3, torch.complex64: 1e-3,
torch.float64: 1e-8, torch.complex128: 1e-8})
def test_sparse_triangular_solve(self, device, dtype):
def run_test(n, k, upper, unitriangular, transpose):
triangle_function = torch.triu if upper else torch.tril
A = make_tensor((n, n), dtype=dtype, device=device)
A = triangle_function(A)
A_sparse = A.to_sparse_csr()
B = make_tensor((n, k), dtype=dtype, device=device)
expected = torch.triangular_solve(B, A, upper=upper, unitriangular=unitriangular, transpose=transpose)
expected_X = expected.solution
actual = torch.triangular_solve(B, A_sparse, upper=upper, unitriangular=unitriangular, transpose=transpose)
actual_X = actual.solution
actual_A_clone = actual.cloned_coefficient
self.assertTrue(actual_A_clone.numel() == 0)
self.assertEqual(actual_X, expected_X)
# test out with C contiguous strides
out = torch.empty_strided((n, k), (k, 1), dtype=dtype, device=device)
torch.triangular_solve(
B, A_sparse,
upper=upper, unitriangular=unitriangular, transpose=transpose, out=(out, actual_A_clone)
)
self.assertEqual(out, expected_X)
# test out with F contiguous strides
out = torch.empty_strided((n, k), (1, n), dtype=dtype, device=device)
torch.triangular_solve(
B, A_sparse,
upper=upper, unitriangular=unitriangular, transpose=transpose, out=(out, actual_A_clone)
)
self.assertEqual(out, expected_X)
self.assertEqual(out.stride(), (1, n))
# test out with discontiguous strides
out = torch.empty_strided((2 * n, k), (1, 2 * n), dtype=dtype, device=device)[::2]
if n > 0 and k > 0:
self.assertFalse(out.is_contiguous())
self.assertFalse(out.t().is_contiguous())
before_stride = out.stride()
torch.triangular_solve(
B, A_sparse,
upper=upper, unitriangular=unitriangular, transpose=transpose, out=(out, actual_A_clone)
)
self.assertEqual(out, expected_X)
self.assertEqual(out.stride(), before_stride)
ks = [0, 1, 3]
ns = [5, 3, 0]
for (k, n), (upper, unitriangular, transpose) in itertools.product(itertools.product(ks, ns),
itertools.product([True, False], repeat=3)):
run_test(n, k, upper, unitriangular, transpose)
@dtypes(*get_all_dtypes())
def test_coo_csr_conversion(self, device, dtype):
for m, n in itertools.product([5, 2, 0], [5, 2, 0]):
size = (m, n)
dense = make_tensor(size, dtype=dtype, device=device)
coo_sparse = dense.to_sparse()
csr_sparse = coo_sparse.to_sparse_csr()
self.assertEqual(csr_sparse.to_dense(), dense)
class TestSparseCSR(TestCase):
@onlyCPU
def test_csr_layout(self):
self.assertEqual(str(torch.sparse_csr), 'torch.sparse_csr')
self.assertEqual(type(torch.sparse_csr), torch.layout)
@dtypes(*get_all_dtypes())
def test_sparse_csr_constructor_shape_inference(self, device, dtype):
crow_indices = [0, 2, 4]
col_indices = [0, 1, 0, 1]
values = [1, 2, 3, 4]
sparse = torch.sparse_csr_tensor(torch.tensor(crow_indices,
dtype=torch.int64),
torch.tensor(col_indices,
dtype=torch.int64),
torch.tensor(values),
dtype=dtype,
device=device)
self.assertEqual(torch.tensor(crow_indices, dtype=torch.int64),
sparse.crow_indices())
self.assertEqual((len(crow_indices) - 1, max(col_indices) + 1),
sparse.shape)
self.assertEqual(dtype, sparse.dtype)
self.assertEqual(torch.device(device), sparse.device)
@dtypes(*get_all_dtypes())
def test_sparse_csr_constructor(self, device, dtype):
crow_indices = [0, 2, 4]
col_indices = [0, 1, 0, 1]
values = [1, 2, 3, 4]
for index_dtype in [torch.int32, torch.int64]:
sparse = torch.sparse_csr_tensor(torch.tensor(crow_indices,
dtype=index_dtype),
torch.tensor(col_indices,
dtype=index_dtype),
torch.tensor(values),
size=(2, 10),
dtype=dtype,
device=device)
self.assertEqual((2, 10), sparse.shape)
self.assertEqual(torch.tensor(crow_indices, dtype=index_dtype),
sparse.crow_indices())
self.assertEqual(torch.tensor(col_indices, dtype=index_dtype),
sparse.col_indices())
self.assertEqual(torch.tensor(values, dtype=dtype),
sparse.values())
@dtypes(*get_all_dtypes())
def test_sparse_csr_constructor_from_lists(self, device, dtype):
# without size
sparse = torch.sparse_csr_tensor([0, 2, 4], [0, 1, 0, 1], [1, 2, 3, 4],
dtype=dtype,
device=device)
self.assertEqual((2, 2), sparse.shape)
self.assertEqual(4, sparse.numel())
self.assertEqual(
torch.tensor([0, 2, 4], dtype=torch.int64, device=device),
sparse.crow_indices())
self.assertEqual(
torch.tensor([0, 1, 0, 1], dtype=torch.int64, device=device),
sparse.col_indices())
self.assertEqual(
torch.tensor([1, 2, 3, 4], dtype=dtype, device=device),
sparse.values())
# with size
for sparse_csr_tensor in [
torch.sparse_csr_tensor, torch._sparse_csr_tensor_unsafe
]:
sparse = sparse_csr_tensor([0, 2, 4], [0, 1, 0, 1], [1, 2, 3, 4],
size=(2, 10),
dtype=dtype,
device=device)
self.assertEqual((2, 10), sparse.shape)
self.assertEqual(
torch.tensor([0, 2, 4], dtype=torch.int64, device=device),
sparse.crow_indices())
self.assertEqual(
torch.tensor([0, 1, 0, 1], dtype=torch.int64, device=device),
sparse.col_indices())
self.assertEqual(
torch.tensor([1, 2, 3, 4], dtype=dtype, device=device),
sparse.values())
def test_factory_type_invariants_check(self, device):
with self.assertRaisesRegex(
RuntimeError,
"both crow_indices and col_indices should have the same type."
):
torch.sparse_csr_tensor(torch.tensor([0, 2, 4], dtype=torch.int64),
torch.tensor([0, 1, 0, 1],
dtype=torch.int32),
torch.tensor([1, 2, 3, 4]),
device=device)
with self.assertRaisesRegex(
RuntimeError,
r"\"csr_construct_check\" not implemented for 'Short'"):
torch.sparse_csr_tensor(torch.tensor([0, 2, 4], dtype=torch.int16),
torch.tensor([0, 1, 0, 1],
dtype=torch.int16),
torch.tensor([1, 2, 3, 4]),
device=device)
def test_factory_layout_invariants_check(self, device):
with self.assertRaisesRegex(
RuntimeError,
"expected values to be a strided and contiguous tensor"):
values = torch.tensor([1.], device=device).expand(4, )
torch.sparse_csr_tensor(torch.tensor([0, 2, 4], device=device),
torch.tensor([0, 1, 0, 1], device=device),
values)
with self.assertRaisesRegex(
RuntimeError,
"expected col_indices to be a strided and contiguous tensor"):
col_indices = torch.tensor([0], device=device).expand(4, )
torch.sparse_csr_tensor(torch.tensor([0, 2, 4]), col_indices,
torch.tensor([1, 2, 3, 4]))
with self.assertRaisesRegex(
RuntimeError,
"expected crow_indices to be a strided and contiguous tensor"):
crow_indices = torch.arange(6, device=device)
torch.sparse_csr_tensor(crow_indices[::2],
torch.tensor([0, 1, 0, 1], device=device),
torch.tensor([1, 2, 3, 4]))
def test_factory_shape_invariants_check(self, device):
crow_indices = [0, 2, 4]
col_indices = [0, 1, 0, 1]
values = [1, 2, 3, 4]
size = (2, 10)
torch.sparse_csr_tensor(torch.tensor(crow_indices),
torch.tensor(col_indices),
torch.tensor(values),
size,
device=device)
with self.assertRaisesRegex(
RuntimeError,
r"size of a CSR tensor must be of length 2, but got: 3"):
torch.sparse_csr_tensor(torch.tensor(crow_indices),
torch.tensor(col_indices),
torch.tensor(values),
size=(2, 10, 2),
device=device)
with self.assertRaisesRegex(
RuntimeError,
r"crow_indices must have dim\=1 but got crow_indices\.dim\(\)\=2"
):
torch.sparse_csr_tensor(torch.tensor(crow_indices).repeat(2, 1),
torch.tensor(col_indices),
torch.tensor(values),
size,
device=device)
with self.assertRaisesRegex(
RuntimeError,
r"col_indices must have dim\=1 but got col_indices\.dim\(\)\=2"
):
torch.sparse_csr_tensor(torch.tensor(crow_indices),
torch.tensor(col_indices).repeat(2, 1),
torch.tensor(values),
size,
device=device)
with self.assertRaisesRegex(
RuntimeError,
r"values must have dim\=1 but got values\.dim\(\)\=2"):
torch.sparse_csr_tensor(torch.tensor(crow_indices),
torch.tensor(col_indices),
torch.tensor(values).repeat(2, 1),
size,
device=device)
with self.assertRaisesRegex(
RuntimeError,
r"crow_indices\.numel\(\) must be size\(0\) \+ 1, but got: 3"):
torch.sparse_csr_tensor(torch.tensor(crow_indices),
torch.tensor(col_indices),
torch.tensor(values), (1, 1),
device=device)
with self.assertRaisesRegex(
RuntimeError,
r"col_indices and values must have equal sizes, " +
r"but got col_indices\.numel\(\): 3, values\.numel\(\): 4"):
torch.sparse_csr_tensor(torch.tensor(crow_indices),
torch.tensor([0, 1, 0]),
torch.tensor(values),
size,
device=device)
def test_factory_indices_invariants_check(self, device):
crow_indices = [0, 2, 4]
col_indices = [0, 1, 0, 1]
values = [1, 2, 3, 4]
size = (2, 10)
with self.assertRaisesRegex(RuntimeError,
"0th value of crow_indices must be 0."):
torch.sparse_csr_tensor(torch.tensor([-1, 0, 4]),
torch.tensor(col_indices),
torch.tensor(values),
size,
device=device)
with self.assertRaisesRegex(
RuntimeError,
"last value of crow_indices should be equal to the length of col_indices."
):
torch.sparse_csr_tensor(torch.tensor([0, 2, 5]),
torch.tensor(col_indices),
torch.tensor(values),
size,
device=device)
with self.assertRaisesRegex(
RuntimeError, r"at position i \= 2," +
r" this condition crow_indices\[i - 1\] <\= crow_indices\[i\] fails"
):
torch.sparse_csr_tensor(torch.tensor([0, 5, 4]),
torch.tensor(col_indices),
torch.tensor(values),
size,
device=device)
with self.assertRaisesRegex(
RuntimeError,
r"col_indices\.min\(\) should be greater or equal to zero"):
torch.sparse_csr_tensor(torch.tensor(crow_indices),
torch.tensor([0, -1, 0, 1]),
torch.tensor(values),
size,
device=device)
with self.assertRaisesRegex(
RuntimeError,
r"size\(1\) should be greater than col_indices\.max\(\)"):
torch.sparse_csr_tensor(torch.tensor(crow_indices),
torch.tensor([0, 11, 0, 1]),
torch.tensor(values),
size,
device=device)
@onlyCUDA
@dtypes(*get_all_dtypes())
def test_factory_device_type_inference(self, device, dtype):
cpu_cuda = ('cpu', 'cuda')
cpu_cuda_none = cpu_cuda + (None, )
for crow_indices_device, col_indices_device, values_device, device in itertools.product(
cpu_cuda, cpu_cuda, cpu_cuda, cpu_cuda_none):
for index_dtype in [torch.int32, torch.int64]:
crow_indices = torch.tensor([0, 2, 4],
dtype=index_dtype,
device=crow_indices_device)
col_indices = torch.tensor([0, 1, 0, 1],
dtype=index_dtype,
device=col_indices_device)
values = torch.tensor([1, 2, 3, 4],
dtype=dtype,
device=values_device)
if device is None and (
crow_indices_device != col_indices_device
or crow_indices_device != values_device):
with self.assertRaises(RuntimeError):
torch.sparse_csr_tensor(crow_indices,
col_indices,
values,
size=(2, 10),
device=device)
else:
t = torch.sparse_csr_tensor(crow_indices,
col_indices,
values,
size=(2, 10),
device=device)
should_be_cuda = (device == 'cuda'
or (device is None
and values_device == 'cuda'))
self.assertEqual(should_be_cuda, t.is_cuda)
t.crow_indices().dtype == index_dtype
t.col_indices().dtype == index_dtype
t.values().dtype == dtype
t.crow_indices().device == t.values().device
t.col_indices().device == t.values().device
def test_sparse_csr_print(self, device):
orig_maxDiff = self.maxDiff
self.maxDiff = None
shape_nnz = [((10, 10), 10), ((100, 10), 10), ((1000, 10), 10)]
printed = []
for shape, nnz in shape_nnz:
values_shape = torch.Size((nnz, ))
col_indices_shape = torch.Size((nnz, ))
crow_indices_shape = torch.Size((shape[0] + 1, ))
printed.append("# shape: {}".format(torch.Size(shape)))
printed.append("# nnz: {}".format(nnz))
printed.append(
"# crow_indices shape: {}".format(crow_indices_shape))
printed.append("# col_indices shape: {}".format(col_indices_shape))
printed.append("# values_shape: {}".format(values_shape))
for index_dtype in [torch.int32, torch.int64]:
for dtype in floating_types():
printed.append("########## {}/{} ##########".format(
dtype, index_dtype))
x = torch.sparse_csr_tensor(
torch.tensor([0, 2, 4], dtype=index_dtype),
torch.tensor([0, 1, 0, 1], dtype=index_dtype),
torch.tensor([1, 2, 3, 4]),
dtype=dtype,
device=device)
printed.append("# sparse tensor")
printed.append(str(x))
printed.append("# _crow_indices")
printed.append(str(x.crow_indices()))
printed.append("# _col_indices")
printed.append(str(x.col_indices()))
printed.append("# _values")
printed.append(str(x.values()))
printed.append('')
printed.append('')
self.assertExpected('\n'.join(printed))
self.maxDiff = orig_maxDiff
@dtypes(*get_all_dtypes())
def test_sparse_csr_from_dense(self, device, dtype):
dense = torch.tensor([[4, 5, 0], [0, 0, 0], [1, 0, 0]],
dtype=dtype,
device=device)
sparse = dense.to_sparse_csr()
self.assertEqual(torch.tensor([0, 2, 2, 3], dtype=torch.int64),
sparse.crow_indices())
self.assertEqual(torch.tensor([0, 1, 0], dtype=torch.int64),
sparse.col_indices())
self.assertEqual(torch.tensor([4, 5, 1], dtype=dtype), sparse.values())
dense = torch.tensor([[0, 0, 0], [0, 0, 1], [1, 0, 0]],
dtype=dtype,
device=device)
sparse = dense.to_sparse_csr()
self.assertEqual(torch.tensor([0, 0, 1, 2], dtype=torch.int64),
sparse.crow_indices())
self.assertEqual(torch.tensor([2, 0], dtype=torch.int64),
sparse.col_indices())
self.assertEqual(torch.tensor([1, 1], dtype=dtype), sparse.values())
dense = torch.tensor([[2, 2, 2], [2, 2, 2], [2, 2, 2]],
dtype=dtype,
device=device)
sparse = dense.to_sparse_csr()
self.assertEqual(torch.tensor([0, 3, 6, 9], dtype=torch.int64),
sparse.crow_indices())
self.assertEqual(torch.tensor([0, 1, 2] * 3, dtype=torch.int64),
sparse.col_indices())
self.assertEqual(torch.tensor([2] * 9, dtype=dtype), sparse.values())
@dtypes(*get_all_dtypes())
def test_sparse_csr_to_dense(self, device, dtype):
mn = [5, 2, 0]
for (m, n) in itertools.product(mn, mn):
size = (m, n)
dense = make_tensor(size, dtype=dtype, device=device)
sparse = dense.to_sparse_csr()
self.assertEqual(sparse.to_dense(), dense)
crow_indices = torch.tensor([0, 3, 5])
col_indices = torch.tensor([0, 1, 2, 0, 1])
values = torch.tensor([1, 2, 1, 3, 4], dtype=dtype)
csr = torch.sparse_csr_tensor(crow_indices,
col_indices,
values,
dtype=dtype,
device=device)
dense = torch.tensor([[1, 2, 1], [3, 4, 0]],
dtype=dtype,
device=device)
self.assertEqual(csr.to_dense(), dense)
@coalescedonoff
@dtypes(torch.double)
def test_coo_to_csr_convert(self, device, dtype, coalesced):
with self.assertRaisesRegex(RuntimeError,
"Input is supposed to be a vector"):
torch._convert_indices_from_coo_to_csr(torch.randint(
100, (5, 5), device=device),
size=100)
size = (5, 5)
sparse_dim = 2
nnz = 10
sparse_coo, _, _ = self.genSparseTensor(size, sparse_dim, nnz,
coalesced, device, dtype)
sparse_csr = sparse_coo.to_sparse_csr()
self.assertTrue(sparse_csr.is_sparse_csr)
self.assertEqual(sparse_csr.to_dense(), sparse_coo.to_dense())
vec = torch.randn((5, 1), dtype=dtype, device=device)
coo_product = sparse_coo.matmul(vec)
csr_product = sparse_csr.matmul(vec)
self.assertEqual(coo_product, csr_product)
vec = torch.randn((100, 1), dtype=dtype, device=device)
index = torch.tensor([
[1, 0, 35, 14, 39, 6, 71, 66, 40, 27],
[92, 31, 62, 50, 22, 65, 89, 74, 56, 34],
],
dtype=torch.int32)
values = torch.tensor([1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
dtype=dtype,
device=device)
coo = torch.sparse_coo_tensor(index,
values,
torch.Size([100, 100]),
dtype=dtype,
device=device)
csr = coo.to_sparse_csr()
self.assertEqual(coo.matmul(vec), csr.matmul(vec))
col_indices = torch.tensor([31, 92, 65, 50, 34, 62, 22, 56, 74, 89],
dtype=torch.int64,
device=device)
self.assertEqual(csr.col_indices(), col_indices)
values = torch.tensor([2, 1, 6, 4, 10, 3, 5, 9, 8, 7],
dtype=dtype,
device=device)
self.assertEqual(csr.values(), values)
@onlyCPU
@unittest.skipIf(IS_MACOS or IS_WINDOWS,
"MKL doesn't work on windows or mac")
@dtypes(torch.float, torch.double)
def test_mkl_matvec_warnings(self, device, dtype):
if torch.has_mkl:
for index_dtype in [torch.int32, torch.int64]:
sp = torch.sparse_csr_tensor(
torch.tensor([0, 2, 4]), torch.tensor([0, 1, 0, 1]),
torch.tensor([1, 2, 3, 4], dtype=dtype, device=device))
vec = torch.randn((2, 1), dtype=dtype, device=device)
with warnings.catch_warnings(record=True) as w:
sp.matmul(vec)
self.assertEqual(len(w), 2)
self.assertIn(
"Pytorch is compiled with MKL LP64 and will convert crow_indices to int32",
str(w[0].message))
self.assertIn(
"Pytorch is compiled with MKL LP64 and will convert col_indices to int32",
str(w[1].message))
@dtypes(*get_all_dtypes())
def test_sparse_csr_from_dense_convert_error(self, device, dtype):
size = (4, 2, 4)
dense = make_tensor(size, dtype=dtype, device=device)
with self.assertRaisesRegex(RuntimeError, "Only 2D"):
sparse = dense.to_sparse_csr()
# TODO: Support auto generation of device check for sparse tensors
# See: https://github.com/pytorch/pytorch/issues/59058
@onlyCUDA
@dtypes(torch.double)
def test_matmul_device_mismatch(self, device, dtype):
cpu = torch.rand((10, 10))
cuda = cpu.cuda()
for s, m1, m2 in itertools.product((cpu, cuda), repeat=3):
csr = m1.to_sparse()
if s.device == csr.device == m2.device:
torch.addmm(s, csr, m2)
else:
with self.assertRaisesRegex(
RuntimeError,
"Expected all tensors to be on the same device"):
torch.addmm(s, csr, m2)
@dtypes(torch.float, torch.double)
def test_csr_matvec(self, device, dtype):
side = 100
for index_dtype in [torch.int32, torch.int64]:
csr = self.genSparseCSRTensor((side, side),
1000,
device=device,
dtype=dtype,
index_dtype=index_dtype)
vec = torch.randn(side, dtype=dtype, device=device)
res = csr.matmul(vec)
expected = csr.to_dense().matmul(vec)
self.assertEqual(res, expected)
bad_vec = torch.randn(side + 10, dtype=dtype, device=device)
with self.assertRaisesRegex(RuntimeError, "mv: expected"):
csr.matmul(bad_vec)
@dtypes(torch.double)
def test_mm(self, device, dtype):
def test_shape(di, dj, dk, nnz):
for index_dtype in [torch.int32, torch.int64]:
x = self.genSparseCSRTensor((di, dj),
nnz,
device=device,
dtype=dtype,
index_dtype=index_dtype)
t = torch.randn(di, dk, dtype=dtype, device=device)
y = torch.randn(dj, dk, dtype=dtype, device=device)
alpha = random.random()
beta = random.random()
# res = beta * t + alpha * (x @ y)
res = torch.addmm(t, x, y, beta=beta, alpha=alpha)
expected = torch.addmm(t,
x.to_dense(),
y,
beta=beta,
alpha=alpha)
self.assertEqual(res, expected)
res = torch.addmm(t, x, y)
expected = torch.addmm(t, x.to_dense(), y)
self.assertEqual(res, expected)
res = torch.mm(x, y)
expected = torch.mm(x.to_dense(), y)
self.assertEqual(res, expected)
for i in range(2, 5):
for j in range(2, 8):
for k in range(2, 8):
test_shape(i, j, k, i * j // 2)
test_shape(4, 4, 4, 0)
@dtypes(*floating_types())
def test_sparse_mm(self, device, dtype):
def test_shape(d1, d2, d3, nnz, transposed):
if transposed:
D = torch.randn(d3, d2, dtype=dtype, device=device).t_()
else:
D = torch.randn(d2, d3, dtype=dtype, device=device)
S = self.genSparseCSRTensor((d1, d2),
nnz,
device=device,
dtype=dtype,
index_dtype=torch.int32)
S_dense = S.to_dense()
self.assertEqual(torch.sparse.mm(S, D), torch.mm(S_dense, D))
test_shape(7, 8, 9, 20, False)
test_shape(7, 8, 9, 20, True)
@dtypes(*floating_types())
def test_sparse_addmm(self, device, dtype):
def test_shape(m, n, p, nnz, broadcast, alpha_beta=None):
if alpha_beta is None:
alpha = random.random()
beta = random.random()
else:
alpha, beta = alpha_beta
if broadcast:
D1 = make_tensor((), dtype=dtype, device=device)
else:
D1 = make_tensor([n, p], dtype=dtype, device=device)
D2 = make_tensor([m, p], dtype=dtype, device=device)
S = self.genSparseCSRTensor([n, m],
nnz,
dtype=dtype,
device=device,
index_dtype=torch.int32)
S_dense = S.to_dense()
Y = torch.sparse.addmm(D1, S, D2, beta=beta, alpha=alpha)
Y_dense = torch.addmm(D1, S_dense, D2, beta=beta, alpha=alpha)
self.assertEqual(Y, Y_dense)
test_shape(7, 8, 9, 20, False, None)
test_shape(7, 8, 9, 20, True, None)
test_shape(7, 8, 9, 20, False, (1, 0))
test_shape(7, 8, 9, 20, True, (1, 0))
test_shape(7, 8, 9, 20, False, (1, 1))
test_shape(7, 8, 9, 20, True, (1, 1))
@dtypes(torch.float, torch.double)
def test_add(self, device, dtype):
def _test_spadd_shape(nnz, shape):
x = self.genSparseCSRTensor(shape,
nnz,
dtype=dtype,
device=device,
index_dtype=torch.int32)
y = torch.randn(*shape, dtype=dtype, device=device)
r = random.random()
res = torch.add(y, x, alpha=r)
expected = y + r * x.to_dense()
self.assertEqual(res, expected)
# Non contiguous dense tensor
s = list(shape)
s[0] = shape[-1]
s[-1] = shape[0]
y = torch.randn(*s, dtype=torch.double, device=device)
y.transpose_(0, len(s) - 1)
r = random.random()
res = torch.add(y, x, alpha=r)
expected = y + r * x.to_dense()
self.assertEqual(res, expected)
_test_spadd_shape(10, [100, 100])
_test_spadd_shape(0, [100, 100])
_test_spadd_shape(10, [100, 1])
_test_spadd_shape(10, [1, 100])
@dtypes(*get_all_dtypes())
def test_coo_csr_conversion(self, device, dtype):
for m, n in itertools.product([5, 2, 0], [5, 2, 0]):
size = (m, n)
dense = make_tensor(size, dtype=dtype, device=device)
coo_sparse = dense.to_sparse()
csr_sparse = coo_sparse.to_sparse_csr()
self.assertEqual(csr_sparse.to_dense(), dense)
class TestScatterGather(TestCase):
# Fills an index tensor with valid indices
def _fill_indices(self, idx, dim, dim_size, elems_per_row, m, n, o, unique_indices=True):
for i in range(1 if dim == 0 else m):
for j in range(1 if dim == 1 else n):
for k in range(1 if dim == 2 else o):
ii = [i, j, k]
ii[dim] = slice(0, idx.size(dim) + 1)
if unique_indices:
idx[tuple(ii)] = torch.randperm(dim_size)[0:elems_per_row]
else:
idx[tuple(ii)] = torch.randint(dim_size, (elems_per_row,))
@dtypes(torch.float32, torch.complex64)
def test_gather(self, device, dtype):
m, n, o = random.randint(10, 20), random.randint(10, 20), random.randint(10, 20)
elems_per_row = random.randint(1, 10)
dim = random.randrange(3)
src = make_tensor((m, n, o), device=device, dtype=dtype)
idx_size = [m, n, o]
idx_size[dim] = elems_per_row
idx = make_tensor(idx_size, device=device, dtype=torch.long)
self._fill_indices(idx, dim, src.size(dim), elems_per_row, m, n, o)
actual = torch.gather(src, dim, idx)
expected = torch.zeros(idx_size, device=device, dtype=dtype)
for i in range(idx_size[0]):
for j in range(idx_size[1]):
for k in range(idx_size[2]):
ii = [i, j, k]
ii[dim] = idx[i, j, k]
expected[i, j, k] = src[tuple(ii)]
self.assertEqual(actual, expected, atol=0, rtol=0)
# Guarded because torch.max isn't defined for complex types
if not dtype.is_complex:
src = make_tensor((3, 4, 5), device=device, dtype=dtype)
expected, idx = src.max(2, True)
actual = torch.gather(src, 2, idx)
self.assertEqual(actual, expected, atol=0, rtol=0)
@dtypes(torch.bool)
def test_gather_bool(self, device, dtype):
src = torch.tensor(((False, True), (True, True)), device=device, dtype=dtype)
idx = torch.tensor(((0, 0), (1, 0)), device=device, dtype=torch.long)
actual = torch.gather(src, 1, idx)
expected = torch.tensor(((False, False), (True, True)), device=device, dtype=dtype)
self.assertEqual(actual, expected, atol=0, rtol=0)
@parametrize("sparse_grad", [False, True])
@dtypes(torch.float32, torch.float64)
def test_gather_backward_with_empty_index_tensor(self, device, dtype, sparse_grad):
dim = -1
input = torch.rand([10, 5], dtype=dtype, device=device, requires_grad=True)
index = torch.randint(0, 2, [3, 0], dtype=torch.int64, device=device)
res = torch.gather(input, dim, index, sparse_grad=sparse_grad)
res.sum().backward()
grad = input.grad.to_dense() if sparse_grad else input.grad
expected_grad = torch.zeros_like(input, requires_grad=False)
self.assertEqual(grad, expected_grad, atol=0, rtol=0)
def _test_scatter_base(self, fn, *, device, dtype, is_scalar, reduction,
unique_indices=True, include_self=True):
m, n, o = random.randint(10, 20), random.randint(10, 20), random.randint(10, 20)
elems_per_row = random.randint(1, 10)
dim = random.randrange(3)
idx_size = [m, n, o]
idx_size[dim] = elems_per_row
idx = torch.empty(tuple(idx_size), device=device, dtype=torch.long)
self._fill_indices(idx, dim, ([m, n, o])[dim], elems_per_row, m, n, o, unique_indices)
if is_scalar:
src = random.random()
else:
src_size = [random.randint(1, 5) + s for s in idx_size]
src = make_tensor(tuple(src_size), device=device, dtype=dtype)
base = make_tensor((m, n, o), device=device, dtype=dtype)
if reduction is not None:
if fn is torch.Tensor.scatter_reduce_:
actual = fn(base.clone(), dim, idx, src, reduce=reduction, include_self=include_self)
else:
actual = fn(base.clone(), dim, idx, src, reduce=reduction)
else:
actual = fn(base.clone(), dim, idx, src)
expected = base.clone()
counts = torch.zeros(base.shape, dtype=torch.long, device=device) + include_self
for i in range(idx_size[0]):
for j in range(idx_size[1]):
for k in range(idx_size[2]):
ii = [i, j, k]
ii[dim] = idx[i, j, k]
if fn is torch.Tensor.scatter_add_:
expected[tuple(ii)] += src[i, j, k]
else:
# method may be 'scatter_', 'scatter', 'scatter_reduce'
# or 'scatter_reduce_', the former two might have a reduction argument
# while the latter two always do
value = src if is_scalar else src[i, j, k]
if ((not include_self) and counts[tuple(ii)] == 0):
expected[tuple(ii)] = value
else:
if reduction == "add" or reduction == "sum":
expected[tuple(ii)] += value
elif reduction == "multiply" or reduction == "prod":
expected[tuple(ii)] *= value
elif reduction == "amax":
expected[tuple(ii)] = max(expected[tuple(ii)], value)
elif reduction == "amin":
expected[tuple(ii)] = min(expected[tuple(ii)], value)
elif reduction == "mean":
expected[tuple(ii)] += value
else:
expected[tuple(ii)] = value
counts[tuple(ii)] += 1
if (reduction == "mean"):
counts.masked_fill_(counts == 0, 1)
if (dtype.is_floating_point or dtype.is_complex):
expected /= counts
else:
expected.div_(counts, rounding_mode="floor")
self.assertEqual(actual, expected, atol=0, rtol=0)
# Tests empty index
dst = make_tensor((2, 2), device=device, dtype=dtype)
idx = torch.tensor((), device=device, dtype=torch.long)
src = make_tensor((2, 2), device=device, dtype=dtype)
if reduction is not None:
actual = fn(dst, 0, idx, src, reduce=reduction)
else:
actual = fn(dst, 0, idx, src)
self.assertEqual(actual, dst, atol=0, rtol=0)
@dtypes(torch.float16, torch.float32, torch.complex64)
def test_scatter_(self, device, dtype):
self._test_scatter_base(torch.Tensor.scatter_, device=device, dtype=dtype,
is_scalar=False, reduction=None)
@dtypes(torch.float16, torch.float32, torch.complex64)
def test_scatter__scalar(self, device, dtype):
self._test_scatter_base(torch.Tensor.scatter_, device=device, dtype=dtype,
is_scalar=True, reduction=None)
# FIXME: RuntimeError: "cuda_scatter_gather_base_kernel_reduce_multiply" not implemented for 'ComplexFloat'
@toleranceOverride({torch.float16: tol(atol=1e-2, rtol=0)})
@dtypesIfCUDA(torch.float16, torch.float32)
@dtypes(torch.float16, torch.float32, torch.complex64)
def test_scatter__reductions(self, device, dtype):
for reduction in ("add", "multiply"):
self._test_scatter_base(torch.Tensor.scatter_, device=device, dtype=dtype,
is_scalar=False, reduction=reduction)
self._test_scatter_base(torch.Tensor.scatter_, device=device, dtype=dtype,
is_scalar=True, reduction=reduction)
@dtypes(torch.float16, torch.float32, torch.complex64)
def test_scatter_add_(self, device, dtype):
self._test_scatter_base(torch.Tensor.scatter_add_, device=device, dtype=dtype,
is_scalar=False, reduction=None)
@dtypes(torch.float32)
def test_scatter_add_mult_index_base(self, device, dtype):
m, n = 30, 40
idx = torch.zeros(m, n, device=device, dtype=torch.long)
src = torch.ones(m, n, device=device, dtype=dtype)
res0 = torch.zeros(m, n, device=device, dtype=dtype).scatter_add_(0, idx, src)
res1 = torch.zeros(m, n, device=device, dtype=dtype).scatter_add_(1, idx, src)
self.assertEqual(res0[0, :], m * torch.ones(n, device=device, dtype=dtype), atol=0, rtol=0)
self.assertEqual(res1[:, 0], n * torch.ones(m, device=device, dtype=dtype), atol=0, rtol=0)
# FIXME: discrepancy between bool ReduceAdd on CUDA and CPU (a + b on CPU and buggy a && b on CUDA)
@dtypes(*get_all_dtypes(include_half=True, include_bfloat16=True, include_bool=False))
def test_scatter_reduce_sum(self, device, dtype):
for include_self in (True, False):
self._test_scatter_base(torch.Tensor.scatter_reduce_, device=device, dtype=dtype,
is_scalar=False, reduction='sum', unique_indices=False,
include_self=include_self)
@dtypes(*get_all_dtypes(include_half=True, include_bfloat16=True))
@dtypesIfCUDA(*get_all_dtypes(include_half=True, include_bfloat16=True, include_complex=False, include_bool=False))
def test_scatter_reduce_prod(self, device, dtype):
for include_self in (True, False):
self._test_scatter_base(torch.Tensor.scatter_reduce_, device=device, dtype=dtype,
is_scalar=False, reduction='prod', unique_indices=False,
include_self=include_self)
@dtypes(*get_all_dtypes(include_half=True, include_bfloat16=True, include_bool=False))
@dtypesIfCUDA(*get_all_dtypes(include_half=True, include_bfloat16=True, include_complex=False, include_bool=False))
def test_scatter_reduce_mean(self, device, dtype):
for include_self in (True, False):
self._test_scatter_base(torch.Tensor.scatter_reduce_, device=device, dtype=dtype,
is_scalar=False, reduction='mean', unique_indices=False,
include_self=include_self)
@dtypes(*get_all_dtypes(include_half=True, include_bfloat16=True, include_complex=False))
@dtypesIfCUDA(*get_all_dtypes(include_half=True, include_bfloat16=True, include_complex=False, include_bool=False))
def test_scatter_reduce_amax(self, device, dtype):
for include_self in (True, False):
self._test_scatter_base(torch.Tensor.scatter_reduce_, device=device, dtype=dtype,
is_scalar=False, reduction='amax', unique_indices=False,
include_self=include_self)
# simple test for nan/inf propagation
if (dtype.is_floating_point):
input = torch.zeros(3, device=device, dtype=dtype)
src = torch.tensor([1, float('nan'), -float('inf'), -float('inf'), 2, float('inf')], device=device, dtype=dtype)
idx = torch.tensor([0, 0, 1, 1, 2, 2], device=device)
input.scatter_reduce_(0, idx, src, 'amax', include_self=include_self)
expected_result = torch.tensor([float('nan'), -float('inf'), float('inf')], device=device, dtype=dtype)
if (include_self):
expected_result[1] = 0
self.assertEqual(input, expected_result)
@dtypes(*get_all_dtypes(include_half=True, include_bfloat16=True, include_complex=False))
@dtypesIfCUDA(*get_all_dtypes(include_half=True, include_bfloat16=True, include_complex=False, include_bool=False))
def test_scatter_reduce_amin(self, device, dtype):
for include_self in (True, False):
self._test_scatter_base(torch.Tensor.scatter_reduce_, device=device, dtype=dtype,
is_scalar=False, reduction='amin', unique_indices=False,
include_self=include_self)
# simple test for nan/inf propagation
if (dtype.is_floating_point):
input = torch.zeros(3, device=device, dtype=dtype)
src = torch.tensor([1, float('nan'), -2, -float('inf'), float('inf'), float('inf')], device=device, dtype=dtype)
idx = torch.tensor([0, 0, 1, 1, 2, 2], device=device)
input.scatter_reduce_(0, idx, src, 'amin', include_self=include_self)
expected_result = torch.tensor([float('nan'), -float('inf'), float('inf')], device=device, dtype=dtype)
if (include_self):
expected_result[2] = 0
self.assertEqual(input, expected_result)
def test_dtypes(self, device, op):
# dtypes to try to backward in
allowed_backward_dtypes = floating_and_complex_types_and(
torch.bfloat16, torch.float16)
# lists for (un)supported dtypes
supported_dtypes = []
unsupported_dtypes = []
supported_backward_dtypes = []
unsupported_backward_dtypes = []
def unsupported(dtype):
unsupported_dtypes.append(dtype)
if dtype in allowed_backward_dtypes:
unsupported_backward_dtypes.append(dtype)
for dtype in get_all_dtypes():
# tries to acquire samples - failure indicates lack of support
requires_grad = (dtype in allowed_backward_dtypes
and op.supports_autograd)
try:
samples = list(
op.sample_inputs(device,
dtype,
requires_grad=requires_grad))
except Exception as e:
unsupported(dtype)
continue
# Counts number of successful backward attempts
# NOTE: This exists as a kludge because this only understands how to
# request a gradient if the output is a tensor or a sequence with
# a tensor as its first element.
num_backward_successes = 0
for sample in samples:
# tries to call operator with the sample - failure indicates
# lack of support
try:
result = op(sample.input, *sample.args, **sample.kwargs)
except Exception as e:
# NOTE: some ops will fail in forward if their inputs
# require grad but they don't support computing the gradient
# in that type! This is a bug in the op!
unsupported(dtype)
# Short-circuits testing this dtype -- it doesn't work
if dtype in unsupported_dtypes:
break
# Short-circuits if the dtype isn't a backward dtype or
# it's already identified as not supported
if dtype not in allowed_backward_dtypes or dtype in unsupported_backward_dtypes:
continue
# Checks for backward support in the same dtype
try:
result = sample.output_process_fn_grad(result)
if isinstance(result, torch.Tensor):
backward_tensor = result
elif isinstance(result, Sequence) and isinstance(
result[0], torch.Tensor):
backward_tensor = result[0]
else:
continue
# Note: this grad may not have the same dtype as dtype
# For functions like complex (float -> complex) or abs
# (complex -> float) the grad tensor will have a
# different dtype than the input.
# For simplicity, this is still modeled as these ops
# supporting grad in the input dtype.
grad = torch.randn_like(backward_tensor)
backward_tensor.backward(grad)
num_backward_successes += 1
except Exception as e:
unsupported_backward_dtypes.append(dtype)
if dtype not in unsupported_dtypes:
supported_dtypes.append(dtype)
if num_backward_successes > 0 and dtype not in unsupported_backward_dtypes:
supported_backward_dtypes.append(dtype)
# Checks that dtypes are listed correctly and generates an informative
# error message
device_type = torch.device(device).type
claimed_supported = set(op.supported_dtypes(device_type))
supported_dtypes = set(supported_dtypes)
supported_but_unclaimed = supported_dtypes - claimed_supported
claimed_but_unsupported = claimed_supported - supported_dtypes
msg = """The supported dtypes for {0} on {1} according to its OpInfo are
{2}, but the detected supported dtypes are {3}.
""".format(op.name, device_type, claimed_supported, supported_dtypes)
if len(supported_but_unclaimed) > 0:
msg += "The following dtypes should be added to the OpInfo: {0}. ".format(
supported_but_unclaimed)
if len(claimed_but_unsupported) > 0:
msg += "The following dtypes should be removed from the OpInfo: {0}.".format(
claimed_but_unsupported)
self.assertEqual(supported_dtypes, claimed_supported, msg=msg)
# Checks that backward dtypes are listed correctly and generates an
# informative error message
# NOTE: this code is nearly identical to the check + msg generation
claimed_backward_supported = set(
op.supported_backward_dtypes(device_type))
supported_backward_dtypes = set(supported_backward_dtypes)
supported_but_unclaimed = supported_backward_dtypes - claimed_backward_supported
claimed_but_unsupported = claimed_backward_supported - supported_backward_dtypes
msg = """The supported backward dtypes for {0} on {1} according to its OpInfo are
{2}, but the detected supported backward dtypes are {3}.
""".format(op.name, device_type, claimed_backward_supported,
supported_backward_dtypes)
if len(supported_but_unclaimed) > 0:
msg += "The following backward dtypes should be added to the OpInfo: {0}. ".format(
supported_but_unclaimed)
if len(claimed_but_unsupported) > 0:
msg += "The following backward dtypes should be removed from the OpInfo: {0}.".format(
claimed_but_unsupported)
self.assertEqual(supported_backward_dtypes,
claimed_backward_supported,
msg=msg)
class TestSparseCSR(TestCase):
@onlyCPU
def test_csr_layout(self):
self.assertEqual(str(torch.sparse_csr), 'torch.sparse_csr')
self.assertEqual(type(torch.sparse_csr), torch.layout)
@dtypes(*get_all_dtypes())
def test_sparse_csr_constructor_shape_inference(self, device, dtype):
crow_indices = [0, 2, 4]
col_indices = [0, 1, 0, 1]
values = [1, 2, 3, 4]
sparse = torch.sparse_csr_tensor(torch.tensor(crow_indices, dtype=torch.int64),
torch.tensor(col_indices, dtype=torch.int64),
torch.tensor(values), dtype=dtype, device=device)
self.assertEqual(torch.tensor(crow_indices, dtype=torch.int64), sparse.crow_indices())
self.assertEqual((len(crow_indices) - 1, max(col_indices) + 1), sparse.shape)
self.assertEqual(dtype, sparse.dtype)
self.assertEqual(torch.device(device), sparse.device)
@dtypes(*get_all_dtypes())
def test_sparse_csr_constructor(self, device, dtype):
crow_indices = [0, 2, 4]
col_indices = [0, 1, 0, 1]
values = [1, 2, 3, 4]
for index_dtype in [torch.int32, torch.int64]:
sparse = torch.sparse_csr_tensor(torch.tensor(crow_indices, dtype=index_dtype),
torch.tensor(col_indices, dtype=index_dtype),
torch.tensor(values),
size=(2, 10),
dtype=dtype,
device=device)
self.assertEqual((2, 10), sparse.shape)
self.assertEqual(torch.tensor(crow_indices, dtype=index_dtype), sparse.crow_indices())
self.assertEqual(torch.tensor(col_indices, dtype=index_dtype), sparse.col_indices())
self.assertEqual(torch.tensor(values, dtype=dtype), sparse.values())
@dtypes(*get_all_dtypes())
def test_sparse_csr_constructor_from_lists(self, device, dtype):
# without size
sparse = torch.sparse_csr_tensor([0, 2, 4],
[0, 1, 0, 1],
[1, 2, 3, 4],
dtype=dtype,
device=device)
self.assertEqual((2, 2), sparse.shape)
self.assertEqual(4, sparse.numel())
self.assertEqual(torch.tensor([0, 2, 4], dtype=torch.int64, device=device), sparse.crow_indices())
self.assertEqual(torch.tensor([0, 1, 0, 1], dtype=torch.int64, device=device), sparse.col_indices())
self.assertEqual(torch.tensor([1, 2, 3, 4], dtype=dtype, device=device), sparse.values())
# with size
for sparse_csr_tensor in [torch.sparse_csr_tensor, torch._sparse_csr_tensor_unsafe]:
sparse = sparse_csr_tensor([0, 2, 4],
[0, 1, 0, 1],
[1, 2, 3, 4],
size=(2, 10),
dtype=dtype,
device=device)
self.assertEqual((2, 10), sparse.shape)
self.assertEqual(torch.tensor([0, 2, 4], dtype=torch.int64, device=device), sparse.crow_indices())
self.assertEqual(torch.tensor([0, 1, 0, 1], dtype=torch.int64, device=device), sparse.col_indices())
self.assertEqual(torch.tensor([1, 2, 3, 4], dtype=dtype, device=device), sparse.values())
@skipMeta
@dtypes(*get_all_dtypes())
def test_empty(self, device, dtype):
ns = [5, 2, 0]
for shape in itertools.product(ns, ns):
result = torch.empty(shape, dtype=dtype, device=device, layout=torch.sparse_csr)
self.assertEqual(result.shape, shape)
self.assertEqual(result.dtype, dtype)
self.assertEqual(result.device, torch.device(device))
self.assertEqual(result.layout, torch.sparse_csr)
self.assertEqual(result.crow_indices().shape, (shape[0] + 1,))
self.assertEqual(result.col_indices().shape, (0,))
self.assertEqual(result.values().shape, (0,))
self.assertEqual(result._nnz(), 0)
self.assertEqual(result.crow_indices().device, torch.device(device))
self.assertEqual(result.col_indices().device, torch.device(device))
self.assertEqual(result.values().device, torch.device(device))
self.assertEqual(result.crow_indices().dtype, torch.int64)
self.assertEqual(result.col_indices().dtype, torch.int64)
self.assertEqual(result.values().dtype, dtype)
@skipMeta
@dtypes(*get_all_dtypes())
def test_empty_errors(self, device, dtype):
with self.assertRaisesRegex(RuntimeError, "torch.empty: Only 2D sparse CSR tensors are supported."):
torch.empty((5,), dtype=dtype, device=device, layout=torch.sparse_csr)
with self.assertRaisesRegex(RuntimeError, "torch.empty: Only 2D sparse CSR tensors are supported."):
torch.empty((2, 3, 4), dtype=dtype, device=device, layout=torch.sparse_csr)
@skipMeta
@dtypes(*get_all_dtypes())
def test_copy(self, device, dtype):
def run_test(shape, nnz, index_type):
a = self.genSparseCSRTensor(shape, nnz, dtype=dtype, device=device, index_dtype=index_dtype)
b = self.genSparseCSRTensor(shape, nnz, dtype=dtype, device=device, index_dtype=index_dtype)
a.copy_(b)
self.assertEqual(a.crow_indices(), b.crow_indices())
self.assertEqual(a.col_indices(), b.col_indices())
self.assertEqual(a.values(), b.values())
ns = [5, 2, 0]
for shape, index_dtype in zip(itertools.product(ns, ns), [torch.int32, torch.int64]):
run_test(shape, 0, index_dtype)
run_test(shape, shape[0] * shape[1], index_dtype)
@skipMeta
@dtypes(*get_all_dtypes())
def test_copy_errors(self, device, dtype):
for index_dtype in [torch.int32, torch.int64]:
shape1 = (2, 3)
shape2 = (3, 2)
a = self.genSparseCSRTensor(shape1, 0, dtype=dtype, device=device, index_dtype=index_dtype)
b = self.genSparseCSRTensor(shape2, 0, dtype=dtype, device=device, index_dtype=index_dtype)
with self.assertRaisesRegex(RuntimeError, "only same size tensors are supported."):
a.copy_(b)
with self.assertRaisesRegex(RuntimeError, "copy between different layouts is not supported."):
a.copy_(torch.empty(a.shape, dtype=dtype, device=device))
b = self.genSparseCSRTensor(shape1, 1, dtype=dtype, device=device, index_dtype=index_dtype)
with self.assertRaisesRegex(RuntimeError, "only tensors with the same number of specified elements are supported."):
a.copy_(b)
@skipMeta
@dtypes(*get_all_dtypes())
def test_resize(self, device, dtype):
for index_dtype in [torch.int32, torch.int64]:
shape = (2, 3)
nnz = 6
a = self.genSparseCSRTensor(shape, nnz, dtype=dtype, device=device, index_dtype=index_dtype)
new_shape = (4, 5)
a.resize_(new_shape)
self.assertEqual(a.shape, new_shape)
# resize to larger shape doesn't add specified elements
self.assertEqual(a._nnz(), nnz)
new_shape = (1, 5)
a.resize_(new_shape)
self.assertEqual(a.shape, new_shape)
# resize to smaller shape trims specified elements
self.assertEqual(a._nnz(), 5)
@skipMeta
@dtypes(*get_all_dtypes())
def test_resize_errors(self, device, dtype):
for index_dtype in [torch.int32, torch.int64]:
shape = (2, 3)
nnz = 6
a = self.genSparseCSRTensor(shape, nnz, dtype=dtype, device=device, index_dtype=index_dtype)
with self.assertRaisesRegex(RuntimeError, "torch.resize_: Only 2D sparse CSR tensors are supported."):
new_shape = (4,)
a.resize_(new_shape)
# resizing of columns to smaller size is not implemented
with self.assertRaisesRegex(
RuntimeError,
"torch.resize_: Resizing columns of sparse CSR tensors to a smaller value is not supported.",
):
new_shape = (2, 2)
a.resize_(new_shape)
def test_factory_type_invariants_check(self, device):
with self.assertRaisesRegex(RuntimeError, "both crow_indices and col_indices should have the same type."):
torch.sparse_csr_tensor(torch.tensor([0, 2, 4], dtype=torch.int64),
torch.tensor([0, 1, 0, 1], dtype=torch.int32),
torch.tensor([1, 2, 3, 4]),
device=device)
with self.assertRaisesRegex(RuntimeError, r"\"csr_construct_check\" not implemented for 'Short'"):
torch.sparse_csr_tensor(torch.tensor([0, 2, 4], dtype=torch.int16),
torch.tensor([0, 1, 0, 1], dtype=torch.int16),
torch.tensor([1, 2, 3, 4]),
device=device)
def test_factory_layout_invariants_check(self, device):
with self.assertRaisesRegex(RuntimeError, "expected values to be a strided and contiguous tensor"):
values = torch.tensor([1.], device=device).expand(4,)
torch.sparse_csr_tensor(torch.tensor([0, 2, 4], device=device),
torch.tensor([0, 1, 0, 1], device=device),
values)
with self.assertRaisesRegex(RuntimeError, "expected col_indices to be a strided and contiguous tensor"):
col_indices = torch.tensor([0], device=device).expand(4,)
torch.sparse_csr_tensor(torch.tensor([0, 2, 4]),
col_indices,
torch.tensor([1, 2, 3, 4]))
with self.assertRaisesRegex(RuntimeError, "expected crow_indices to be a strided and contiguous tensor"):
crow_indices = torch.arange(6, device=device)
torch.sparse_csr_tensor(crow_indices[::2],
torch.tensor([0, 1, 0, 1], device=device),
torch.tensor([1, 2, 3, 4]))
def test_factory_shape_invariants_check(self, device):
crow_indices = [0, 2, 4]
col_indices = [0, 1, 0, 1]
values = [1, 2, 3, 4]
size = (2, 10)
torch.sparse_csr_tensor(torch.tensor(crow_indices), torch.tensor(col_indices), torch.tensor(values), size,
device=device)
with self.assertRaisesRegex(RuntimeError, r"size of a CSR tensor must be of length 2, but got: 3"):
torch.sparse_csr_tensor(torch.tensor(crow_indices), torch.tensor(col_indices), torch.tensor(values),
size=(2, 10, 2),
device=device)
with self.assertRaisesRegex(RuntimeError, r"crow_indices must have dim\=1 but got crow_indices\.dim\(\)\=2"):
torch.sparse_csr_tensor(torch.tensor(crow_indices).repeat(2, 1),
torch.tensor(col_indices),
torch.tensor(values),
size,
device=device)
with self.assertRaisesRegex(RuntimeError, r"col_indices must have dim\=1 but got col_indices\.dim\(\)\=2"):
torch.sparse_csr_tensor(torch.tensor(crow_indices),
torch.tensor(col_indices).repeat(2, 1),
torch.tensor(values),
size,
device=device)
with self.assertRaisesRegex(RuntimeError, r"values must have dim\=1 but got values\.dim\(\)\=2"):
torch.sparse_csr_tensor(torch.tensor(crow_indices),
torch.tensor(col_indices),
torch.tensor(values).repeat(2, 1),
size,
device=device)
with self.assertRaisesRegex(RuntimeError,
r"crow_indices\.numel\(\) must be size\(0\) \+ 1, but got: 3"):
torch.sparse_csr_tensor(torch.tensor(crow_indices), torch.tensor(col_indices), torch.tensor(values), (1, 1),
device=device)
with self.assertRaisesRegex(RuntimeError,
r"col_indices and values must have equal sizes, " +
r"but got col_indices\.numel\(\): 3, values\.numel\(\): 4"):
torch.sparse_csr_tensor(torch.tensor(crow_indices), torch.tensor([0, 1, 0]), torch.tensor(values), size,
device=device)
def test_factory_indices_invariants_check(self, device):
crow_indices = [0, 2, 4]
col_indices = [0, 1, 0, 1]
values = [1, 2, 3, 4]
size = (2, 10)
with self.assertRaisesRegex(RuntimeError, "0th value of crow_indices must be 0."):
torch.sparse_csr_tensor(torch.tensor([-1, 0, 4]), torch.tensor(col_indices), torch.tensor(values), size,
device=device)
with self.assertRaisesRegex(RuntimeError,
"last value of crow_indices should be equal to the length of col_indices."):
torch.sparse_csr_tensor(torch.tensor([0, 2, 5]), torch.tensor(col_indices), torch.tensor(values), size,
device=device)
with self.assertRaisesRegex(RuntimeError,
r"at position i \= 2," +
r" this condition crow_indices\[i - 1\] <\= crow_indices\[i\] fails"):
torch.sparse_csr_tensor(torch.tensor([0, 5, 4]), torch.tensor(col_indices), torch.tensor(values), size,
device=device)
with self.assertRaisesRegex(RuntimeError, r"col_indices\.min\(\) should be greater or equal to zero"):
torch.sparse_csr_tensor(torch.tensor(crow_indices), torch.tensor([0, -1, 0, 1]), torch.tensor(values), size,
device=device)
with self.assertRaisesRegex(RuntimeError, r"size\(1\) should be greater than col_indices\.max\(\)"):
torch.sparse_csr_tensor(torch.tensor(crow_indices), torch.tensor([0, 11, 0, 1]), torch.tensor(values), size,
device=device)
@onlyCUDA
@dtypes(*get_all_dtypes())
def test_factory_device_type_inference(self, device, dtype):
cpu_cuda = ('cpu', 'cuda')
cpu_cuda_none = cpu_cuda + (None,)
for crow_indices_device, col_indices_device, values_device, device in itertools.product(cpu_cuda,
cpu_cuda,
cpu_cuda,
cpu_cuda_none):
for index_dtype in [torch.int32, torch.int64]:
crow_indices = torch.tensor([0, 2, 4], dtype=index_dtype, device=crow_indices_device)
col_indices = torch.tensor([0, 1, 0, 1], dtype=index_dtype, device=col_indices_device)
values = torch.tensor([1, 2, 3, 4], dtype=dtype, device=values_device)
if device is None and (crow_indices_device != col_indices_device or
crow_indices_device != values_device):
with self.assertRaises(RuntimeError):
torch.sparse_csr_tensor(crow_indices,
col_indices,
values,
size=(2, 10),
device=device)
else:
t = torch.sparse_csr_tensor(crow_indices,
col_indices,
values,
size=(2, 10),
device=device)
should_be_cuda = (device == 'cuda' or (device is None and values_device == 'cuda'))
self.assertEqual(should_be_cuda, t.is_cuda)
t.crow_indices().dtype == index_dtype
t.col_indices().dtype == index_dtype
t.values().dtype == dtype
t.crow_indices().device == t.values().device
t.col_indices().device == t.values().device
def test_sparse_csr_print(self, device):
orig_maxDiff = self.maxDiff
self.maxDiff = None
shape_nnz = [
((10, 10), 10),
((100, 10), 10),
((1000, 10), 10)
]
printed = []
for shape, nnz in shape_nnz:
values_shape = torch.Size((nnz,))
col_indices_shape = torch.Size((nnz,))
crow_indices_shape = torch.Size((shape[0] + 1,))
printed.append("# shape: {}".format(torch.Size(shape)))
printed.append("# nnz: {}".format(nnz))
printed.append("# crow_indices shape: {}".format(crow_indices_shape))
printed.append("# col_indices shape: {}".format(col_indices_shape))
printed.append("# values_shape: {}".format(values_shape))
for index_dtype in [torch.int32, torch.int64]:
for dtype in floating_types():
printed.append("########## {}/{} ##########".format(dtype, index_dtype))
x = torch.sparse_csr_tensor(torch.tensor([0, 2, 4], dtype=index_dtype),
torch.tensor([0, 1, 0, 1], dtype=index_dtype),
torch.tensor([1, 2, 3, 4]), dtype=dtype, device=device)
printed.append("# sparse tensor")
printed.append(str(x))
printed.append("# _crow_indices")
printed.append(str(x.crow_indices()))
printed.append("# _col_indices")
printed.append(str(x.col_indices()))
printed.append("# _values")
printed.append(str(x.values()))
printed.append('')
printed.append('')
self.assertExpected('\n'.join(printed))
self.maxDiff = orig_maxDiff
@dtypes(*get_all_dtypes())
def test_sparse_csr_from_dense(self, device, dtype):
dense = torch.tensor([[4, 5, 0], [0, 0, 0], [1, 0, 0]], dtype=dtype, device=device)
sparse = dense.to_sparse_csr()
self.assertEqual(torch.tensor([0, 2, 2, 3], dtype=torch.int64), sparse.crow_indices())
self.assertEqual(torch.tensor([0, 1, 0], dtype=torch.int64), sparse.col_indices())
self.assertEqual(torch.tensor([4, 5, 1], dtype=dtype), sparse.values())
dense = torch.tensor([[0, 0, 0], [0, 0, 1], [1, 0, 0]], dtype=dtype, device=device)
sparse = dense.to_sparse_csr()
self.assertEqual(torch.tensor([0, 0, 1, 2], dtype=torch.int64), sparse.crow_indices())
self.assertEqual(torch.tensor([2, 0], dtype=torch.int64), sparse.col_indices())
self.assertEqual(torch.tensor([1, 1], dtype=dtype), sparse.values())
dense = torch.tensor([[2, 2, 2], [2, 2, 2], [2, 2, 2]], dtype=dtype, device=device)
sparse = dense.to_sparse_csr()
self.assertEqual(torch.tensor([0, 3, 6, 9], dtype=torch.int64), sparse.crow_indices())
self.assertEqual(torch.tensor([0, 1, 2] * 3, dtype=torch.int64), sparse.col_indices())
self.assertEqual(torch.tensor([2] * 9, dtype=dtype), sparse.values())
@dtypes(*get_all_dtypes())
def test_sparse_csr_to_dense(self, device, dtype):
mn = [5, 2, 0]
for (m, n) in itertools.product(mn, mn):
size = (m, n)
dense = make_tensor(size, dtype=dtype, device=device)
sparse = dense.to_sparse_csr()
self.assertEqual(sparse.to_dense(), dense)
crow_indices = torch.tensor([0, 3, 5])
col_indices = torch.tensor([0, 1, 2, 0, 1])
values = torch.tensor([1, 2, 1, 3, 4], dtype=dtype)
csr = torch.sparse_csr_tensor(crow_indices, col_indices,
values, dtype=dtype, device=device)
dense = torch.tensor([[1, 2, 1], [3, 4, 0]], dtype=dtype, device=device)
self.assertEqual(csr.to_dense(), dense)
@coalescedonoff
@dtypes(torch.double)
def test_coo_to_csr_convert(self, device, dtype, coalesced):
with self.assertRaisesRegex(RuntimeError, "Input is supposed to be a vector"):
torch._convert_indices_from_coo_to_csr(
torch.randint(100, (5, 5), device=device),
size=100)
size = (5, 5)
sparse_dim = 2
nnz = 10
sparse_coo, _, _ = self.genSparseTensor(size, sparse_dim, nnz, coalesced, device, dtype)
sparse_csr = sparse_coo.to_sparse_csr()
self.assertTrue(sparse_csr.is_sparse_csr)
self.assertEqual(sparse_csr.to_dense(), sparse_coo.to_dense())
vec = torch.randn((5, 1), dtype=dtype, device=device)
coo_product = sparse_coo.matmul(vec)
csr_product = sparse_csr.matmul(vec)
self.assertEqual(coo_product, csr_product)
vec = torch.randn((100, 1), dtype=dtype, device=device)
index = torch.tensor([
[1, 0, 35, 14, 39, 6, 71, 66, 40, 27],
[92, 31, 62, 50, 22, 65, 89, 74, 56, 34],
], dtype=torch.int32)
values = torch.tensor([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], dtype=dtype, device=device)
coo = torch.sparse_coo_tensor(index, values, torch.Size([100, 100]), dtype=dtype, device=device)
csr = coo.to_sparse_csr()
self.assertEqual(coo.matmul(vec), csr.matmul(vec))
col_indices = torch.tensor([
31, 92, 65, 50, 34, 62, 22, 56, 74, 89
], dtype=torch.int64, device=device)
self.assertEqual(csr.col_indices(), col_indices)
values = torch.tensor([2, 1, 6, 4, 10, 3, 5, 9, 8, 7], dtype=dtype, device=device)
self.assertEqual(csr.values(), values)
@onlyCPU
@unittest.skipIf(IS_MACOS or IS_WINDOWS, "MKL doesn't work on windows or mac")
@dtypes(torch.float, torch.double)
def test_mkl_matvec_warnings(self, device, dtype):
if torch.has_mkl:
for index_dtype in [torch.int32, torch.int64]:
sp = torch.sparse_csr_tensor(torch.tensor([0, 2, 4]),
torch.tensor([0, 1, 0, 1]),
torch.tensor([1, 2, 3, 4], dtype=dtype, device=device))
vec = torch.randn((2, 1), dtype=dtype, device=device)
with warnings.catch_warnings(record=True) as w:
sp.matmul(vec)
self.assertEqual(len(w), 2)
self.assertIn("Pytorch is compiled with MKL LP64 and will convert crow_indices to int32",
str(w[0].message))
self.assertIn("Pytorch is compiled with MKL LP64 and will convert col_indices to int32",
str(w[1].message))
@dtypes(*get_all_dtypes())
def test_sparse_csr_from_dense_convert_error(self, device, dtype):
size = (4, 2, 4)
dense = make_tensor(size, dtype=dtype, device=device)
with self.assertRaisesRegex(RuntimeError, "Only 2D"):
sparse = dense.to_sparse_csr()
# TODO: Support auto generation of device check for sparse tensors
# See: https://github.com/pytorch/pytorch/issues/59058
@onlyCUDA
@dtypes(torch.double)
def test_matmul_device_mismatch(self, device, dtype):
cpu = torch.rand((10, 10))
cuda = cpu.cuda()
for s, m1, m2 in itertools.product((cpu, cuda), repeat=3):
csr = m1.to_sparse()
if s.device == csr.device == m2.device:
torch.addmm(s, csr, m2)
else:
with self.assertRaisesRegex(RuntimeError, "Expected all tensors to be on the same device"):
torch.addmm(s, csr, m2)
@skipCUDAIfNoCusparseGeneric
@dtypes(*torch.testing.floating_types())
@dtypesIfCUDA(*get_all_complex_dtypes(),
*get_all_fp_dtypes(include_half=SM53OrLater, include_bfloat16=SM80OrLater))
def test_csr_matvec(self, device, dtype):
side = 100
for index_dtype in [torch.int32, torch.int64]:
csr = self.genSparseCSRTensor((side, side), 1000, device=device, dtype=dtype, index_dtype=index_dtype)
vec = torch.randn(side, dtype=dtype, device=device)
res = csr.matmul(vec)
expected = csr.to_dense().matmul(vec)
self.assertEqual(res, expected)
bad_vec = torch.randn(side + 10, dtype=dtype, device=device)
err_msg = "mv: expected"
# CUDA path now uses generic meta/structured implementation
# TODO: move CPU path to not use `mv_sparse` function
if self.device_type == 'cuda':
err_msg = "size mismatch, got"
with self.assertRaisesRegex(RuntimeError, err_msg):
csr.matmul(bad_vec)
@dtypes(torch.double)
def test_mm(self, device, dtype):
def test_shape(di, dj, dk, nnz):
for index_dtype in [torch.int32, torch.int64]:
x = self.genSparseCSRTensor((di, dj), nnz, device=device, dtype=dtype, index_dtype=index_dtype)
t = torch.randn(di, dk, dtype=dtype, device=device)
y = torch.randn(dj, dk, dtype=dtype, device=device)
alpha = random.random()
beta = random.random()
# res = beta * t + alpha * (x @ y)
res = torch.addmm(t, x, y, beta=beta, alpha=alpha)
expected = torch.addmm(t, x.to_dense(), y, beta=beta, alpha=alpha)
self.assertEqual(res, expected)
res = torch.addmm(t, x, y)
expected = torch.addmm(t, x.to_dense(), y)
self.assertEqual(res, expected)
res = torch.mm(x, y)
expected = torch.mm(x.to_dense(), y)
self.assertEqual(res, expected)
for i in range(2, 5):
for j in range(2, 8):
for k in range(2, 8):
test_shape(i, j, k, i * j // 2)
test_shape(4, 4, 4, 0)
@dtypes(*floating_types())
@dtypesIfCUDA(*get_all_complex_dtypes(),
*get_all_fp_dtypes(include_half=SM53OrLater and TEST_CUSPARSE_GENERIC,
include_bfloat16=SM80OrLater and TEST_CUSPARSE_GENERIC))
@precisionOverride({torch.bfloat16: 1e-2, torch.float16: 1e-2})
def test_sparse_mm(self, device, dtype):
def test_shape(d1, d2, d3, nnz, transposed, index_dtype):
if transposed:
D = torch.randn(d3, d2, dtype=dtype, device=device).t_()
else:
D = torch.randn(d2, d3, dtype=dtype, device=device)
S = self.genSparseCSRTensor((d1, d2), nnz, device=device, dtype=dtype, index_dtype=index_dtype)
S_dense = S.to_dense()
self.assertEqual(torch.sparse.mm(S, D), torch.mm(S_dense, D))
for index_dtype in [torch.int32, torch.int64]:
test_shape(7, 8, 9, 20, False, index_dtype)
test_shape(7, 8, 9, 20, True, index_dtype)
@dtypes(*floating_types())
@dtypesIfCUDA(*get_all_complex_dtypes(),
*get_all_fp_dtypes(include_half=SM53OrLater and TEST_CUSPARSE_GENERIC,
include_bfloat16=SM80OrLater and TEST_CUSPARSE_GENERIC))
@precisionOverride({torch.bfloat16: 1e-2, torch.float16: 1e-2})
def test_sparse_addmm(self, device, dtype):
def test_shape(m, n, p, nnz, broadcast, index_dtype, alpha_beta=None):
if alpha_beta is None:
alpha = random.random()
beta = random.random()
else:
alpha, beta = alpha_beta
if broadcast:
D1 = make_tensor((), dtype=dtype, device=device)
else:
D1 = make_tensor([n, p], dtype=dtype, device=device)
D2 = make_tensor([m, p], dtype=dtype, device=device)
S = self.genSparseCSRTensor([n, m], nnz, dtype=dtype, device=device, index_dtype=index_dtype)
S_dense = S.to_dense()
Y = torch.sparse.addmm(D1, S, D2, beta=beta, alpha=alpha)
Y_dense = torch.addmm(D1, S_dense, D2, beta=beta, alpha=alpha)
self.assertEqual(Y, Y_dense)
for index_dtype in [torch.int32, torch.int64]:
test_shape(7, 8, 9, 20, False, index_dtype, None)
test_shape(7, 8, 9, 20, True, index_dtype, None)
test_shape(7, 8, 9, 20, False, index_dtype, (1, 0))
test_shape(7, 8, 9, 20, True, index_dtype, (1, 0))
test_shape(7, 8, 9, 20, False, index_dtype, (1, 1))
test_shape(7, 8, 9, 20, True, index_dtype, (1, 1))
@onlyCUDA
@dtypes(torch.float)
def test_addmm_errors(self, device, dtype):
# test that the errors are the same for dense and sparse versions
import re
def test1(*, is_sparse):
# shapes must be compatible for matrix multiplication
a = make_tensor((2, 3), dtype=dtype, device=device)
if is_sparse:
a_sparse = a.to_sparse_csr()
return torch.addmm(a, a_sparse, a)
else:
return torch.addmm(a, a, a)
def test2(*, is_sparse):
# mat2 must be a matrix
a = make_tensor((2, 3), dtype=dtype, device=device)
if is_sparse:
a_sparse = a.to_sparse_csr()
return torch.addmm(a, a_sparse, a.unsqueeze(0))
else:
return torch.addmm(a, a, a.unsqueeze(0))
def test3(*, is_sparse):
# the first input needs to be 1D or 2D
a = make_tensor((3, 3), dtype=dtype, device=device)
if is_sparse:
a_sparse = a.to_sparse_csr()
return torch.addmm(a.unsqueeze(0), a_sparse, a)
else:
return torch.addmm(a.unsqueeze(0), a, a)
for test in (test1, test2, test3):
try:
test(is_sparse=False)
except RuntimeError as msg:
with self.assertRaisesRegex(RuntimeError, re.escape(str(msg))):
test(is_sparse=True)
@onlyCUDA
@dtypes(torch.float)
def test_mm_errors(self, device, dtype):
# test that the errors are the same for dense and sparse versions
import re
def test1(*, is_sparse):
# shapes must be compatible for matrix multiplication
a = make_tensor((2, 3), dtype=dtype, device=device)
if is_sparse:
a_sparse = a.to_sparse_csr()
return torch.mm(a_sparse, a)
else:
return torch.mm(a, a)
def test2(*, is_sparse):
# mat2 must be a matrix
a = make_tensor((2, 3), dtype=dtype, device=device)
if is_sparse:
a_sparse = a.to_sparse_csr()
return torch.mm(a_sparse, a.unsqueeze(0))
else:
return torch.mm(a, a.unsqueeze(0))
for test in (test1, test2):
try:
test(is_sparse=False)
except RuntimeError as msg:
with self.assertRaisesRegex(RuntimeError, re.escape(str(msg))):
test(is_sparse=True)
@dtypes(torch.float, torch.double)
def test_add(self, device, dtype):
def _test_spadd_shape(nnz, shape):
x = self.genSparseCSRTensor(shape, nnz, dtype=dtype, device=device, index_dtype=torch.int32)
y = torch.randn(*shape, dtype=dtype, device=device)
r = random.random()
res = torch.add(y, x, alpha=r)
expected = y + r * x.to_dense()
self.assertEqual(res, expected)
# Non contiguous dense tensor
s = list(shape)
s[0] = shape[-1]
s[-1] = shape[0]
y = torch.randn(*s, dtype=torch.double, device=device)
y.transpose_(0, len(s) - 1)
r = random.random()
res = torch.add(y, x, alpha=r)
expected = y + r * x.to_dense()
self.assertEqual(res, expected)
_test_spadd_shape(10, [100, 100])
_test_spadd_shape(0, [100, 100])
_test_spadd_shape(10, [100, 1])
_test_spadd_shape(10, [1, 100])
@dtypes(*get_all_dtypes())
def test_coo_csr_conversion(self, device, dtype):
for m, n in itertools.product([5, 2, 0], [5, 2, 0]):
size = (m, n)
dense = make_tensor(size, dtype=dtype, device=device)
coo_sparse = dense.to_sparse()
csr_sparse = coo_sparse.to_sparse_csr()
self.assertEqual(csr_sparse.to_dense(), dense)
