def testHistogramExecution(self):
rs = np.random.RandomState(0)
raw = rs.randint(10, size=(20,))
a = tensor(raw, chunk_size=3)
raw_weights = rs.random(20)
weights = tensor(raw_weights, chunk_size=4)
# range provided
for range_ in [(0, 10), (3, 11), (3, 7)]:
bin_edges = histogram(a, range=range_)[0]
result = self.executor.execute_tensor(bin_edges)[0]
expected = np.histogram(raw, range=range_)[0]
np.testing.assert_array_equal(result, expected)
for wt in (raw_weights, weights):
for density in (True, False):
bins = [1, 4, 6, 9]
bin_edges = histogram(a, bins=bins, weights=wt, density=density)[0]
result = self.executor.execute_tensor(bin_edges)[0]
expected = np.histogram(
raw, bins=bins, weights=raw_weights, density=density)[0]
np.testing.assert_almost_equal(result, expected)
this = self
class MockSession:
def __init__(self):
self.executor = this.executor
ctx = LocalContext(MockSession())
executor = ExecutorForTest('numpy', storage=ctx)
with ctx:
raw2 = rs.randint(10, size=(1,))
b = tensor(raw2)
raw3 = rs.randint(10, size=(0,))
c = tensor(raw3)
for t, r in [(a, raw), (b, raw2), (c, raw3), (sort(a), raw)]:
for density in (True, False):
test_bins = [10, 'stone', 'auto', 'doane', 'fd',
'rice', 'scott', 'sqrt', 'sturges']
for bins in test_bins:
hist = histogram(t, bins=bins, density=density)[0]
if r.size > 0:
with self.assertRaises(TilesError):
executor.execute_tensor(hist)
result = executor.execute_tensors([hist])[0]
expected = np.histogram(r, bins=bins, density=density)[0]
np.testing.assert_array_equal(result, expected)
test_bins = [[0, 4, 8], tensor([0, 4, 8], chunk_size=2)]
for bins in test_bins:
hist = histogram(t, bins=bins, density=density)[0]
result = executor.execute_tensors([hist])[0]
expected = np.histogram(r, bins=[0, 4, 8], density=density)[0]
np.testing.assert_array_equal(result, expected)
def testHistogramBinEdgesExecution(self):
rs = np.random.RandomState(0)
raw = rs.randint(10, size=(20,))
a = tensor(raw, chunk_size=3)
# range provided
for range_ in [(0, 10), (3, 11), (3, 7)]:
bin_edges = histogram_bin_edges(a, range=range_)
result = self.executor.execute_tensor(bin_edges)[0]
expected = np.histogram_bin_edges(raw, range=range_)
np.testing.assert_array_equal(result, expected)
this = self
class MockSession:
def __init__(self):
self.executor = this.executor
ctx = LocalContext(MockSession())
executor = ExecutorForTest('numpy', storage=ctx)
with ctx:
raw2 = rs.randint(10, size=(1,))
b = tensor(raw2)
raw3 = rs.randint(10, size=(0,))
c = tensor(raw3)
for t, r in [(a, raw), (b, raw2), (c, raw3), (sort(a), raw)]:
test_bins = [10, 'stone', 'auto', 'doane', 'fd',
'rice', 'scott', 'sqrt', 'sturges']
for bins in test_bins:
bin_edges = histogram_bin_edges(t, bins=bins)
if r.size > 0:
with self.assertRaises(TilesError):
executor.execute_tensor(bin_edges)
result = executor.execute_tensors([bin_edges])[0]
expected = np.histogram_bin_edges(r, bins=bins)
np.testing.assert_array_equal(result, expected)
test_bins = [[0, 4, 8], tensor([0, 4, 8], chunk_size=2)]
for bins in test_bins:
bin_edges = histogram_bin_edges(t, bins=bins)
result = executor.execute_tensors([bin_edges])[0]
expected = np.histogram_bin_edges(r, bins=[0, 4, 8])
np.testing.assert_array_equal(result, expected)
raw = np.arange(5)
a = tensor(raw, chunk_size=3)
bin_edges = histogram_bin_edges(a)
result = executor.execute_tensors([bin_edges])[0]
expected = np.histogram_bin_edges(raw)
self.assertEqual(bin_edges.shape, expected.shape)
np.testing.assert_array_equal(result, expected)
def testPercentileExecution(self):
raw = np.random.rand(20, 10)
q = np.random.RandomState(0).randint(100, size=11)
a = tensor(raw, chunk_size=7)
r = percentile(a, q)
result = self.executor.execute_tensor(r, concat=True)[0]
expected = np.percentile(raw, q)
np.testing.assert_array_equal(result, expected)
mq = tensor(q)
this = self
class MockSession:
def __init__(self):
self.executor = this.executor
ctx = LocalContext(MockSession())
executor = ExecutorForTest('numpy', storage=ctx)
with ctx:
r = percentile(a, mq)
result = executor.execute_tensors([r])[0]
np.testing.assert_array_equal(result, expected)
class Test(unittest.TestCase):
def setUp(self):
self.executor = ExecutorForTest('numpy')
def testQRExecution(self):
data = np.random.randn(18, 6)
a = tensor(data, chunk_size=(3, 6))
q, r = qr(a)
t = q.dot(r)
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, data))
a = tensor(data, chunk_size=(9, 6))
q, r = qr(a)
t = q.dot(r)
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, data))
a = tensor(data, chunk_size=3)
q, r = qr(a)
t = q.dot(r)
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, data))
# test for Short-and-Fat QR
data = np.random.randn(6, 18)
a = tensor(data, chunk_size=(6, 9))
q, r = qr(a, method='sfqr')
t = q.dot(r)
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, data))
a = tensor(data, chunk_size=(3, 3))
q, r = qr(a, method='sfqr')
t = q.dot(r)
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, data))
a = tensor(data, chunk_size=(6, 3))
q, r = qr(a, method='sfqr')
t = q.dot(r)
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, data))
def testSVDExecution(self):
data = np.random.randn(18, 6) + 1j * np.random.randn(18, 6)
a = tensor(data, chunk_size=(9, 6))
U, s, V = svd(a)
t = U.dot(diag(s).dot(V))
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, data))
a = tensor(data, chunk_size=(18, 6))
U, s, V = svd(a)
t = U.dot(diag(s).dot(V))
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, data))
a = tensor(data, chunk_size=(2, 6))
U, s, V = svd(a)
t = U.dot(diag(s).dot(V))
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, data))
data = np.random.randn(6, 18) + 1j * np.random.randn(6, 18)
a = tensor(data)
U, s, V = svd(a)
t = U.dot(diag(s).dot(V))
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, data))
# test for matrix of ones
data = np.ones((20, 10))
a = tensor(data, chunk_size=10)
s = svd(a)[1]
res = self.executor.execute_tensor(s, concat=True)[0]
expected = np.linalg.svd(a)[1]
np.testing.assert_array_almost_equal(res, expected)
def testRandomizedSVDExecution(self):
n_samples = 100
n_features = 500
rank = 5
k = 10
for dtype in (np.int32, np.int64, np.float32, np.float64):
# generate a matrix X of approximate effective rank `rank` and no noise
# component (very structured signal):
X = make_low_rank_matrix(n_samples=n_samples,
n_features=n_features,
effective_rank=rank,
tail_strength=0.0,
random_state=0).astype(dtype, copy=False)
self.assertEqual(X.shape, (n_samples, n_features))
dtype = np.dtype(dtype)
decimal = 5 if dtype == np.float32 else 7
# compute the singular values of X using the slow exact method
X_res = self.executor.execute_tensor(X, concat=True)[0]
U, s, V = np.linalg.svd(X_res, full_matrices=False)
# Convert the singular values to the specific dtype
U = U.astype(dtype, copy=False)
s = s.astype(dtype, copy=False)
V = V.astype(dtype, copy=False)
for normalizer in ['auto', 'LU',
'QR']: # 'none' would not be stable
# compute the singular values of X using the fast approximate method
Ua, sa, Va = randomized_svd(
X,
k,
power_iteration_normalizer=normalizer,
random_state=0)
# If the input dtype is float, then the output dtype is float of the
# same bit size (f32 is not upcast to f64)
# But if the input dtype is int, the output dtype is float64
if dtype.kind == 'f':
self.assertEqual(Ua.dtype, dtype)
self.assertEqual(sa.dtype, dtype)
self.assertEqual(Va.dtype, dtype)
else:
self.assertEqual(Ua.dtype, np.float64)
self.assertEqual(sa.dtype, np.float64)
self.assertEqual(Va.dtype, np.float64)
self.assertEqual(Ua.shape, (n_samples, k))
self.assertEqual(sa.shape, (k, ))
self.assertEqual(Va.shape, (k, n_features))
# ensure that the singular values of both methods are equal up to the
# real rank of the matrix
sa_res = self.executor.execute_tensor(sa, concat=True)[0]
np.testing.assert_almost_equal(s[:k], sa_res, decimal=decimal)
# check the singular vectors too (while not checking the sign)
dot_res = self.executor.execute_tensor(dot(Ua, Va),
concat=True)[0]
np.testing.assert_almost_equal(np.dot(U[:, :k], V[:k, :]),
dot_res,
decimal=decimal)
def testCholeskyExecution(self):
data = np.random.randint(1, 10, (10, 10))
symmetric_data = data.dot(data.T)
a = tensor(symmetric_data, chunk_size=5)
U = cholesky(a)
t = U.T.dot(U)
res_u = self.executor.execute_tensor(U, concat=True)[0]
np.testing.assert_allclose(np.triu(res_u), res_u)
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, symmetric_data))
L = cholesky(a, lower=True)
U = cholesky(a)
t = L.dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, symmetric_data))
a = tensor(symmetric_data, chunk_size=2)
L = cholesky(a, lower=True)
U = cholesky(a)
t = L.dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, symmetric_data)
a = tensor(symmetric_data, chunk_size=(1, 2))
L = cholesky(a, lower=True)
U = cholesky(a)
t = L.dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, symmetric_data)
a = tensor(symmetric_data, chunk_size=4)
L = cholesky(a, lower=True)
U = cholesky(a)
t = L.dot(U)
res_u = self.executor.execute_tensor(U, concat=True)[0]
np.testing.assert_allclose(np.triu(res_u), res_u)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, symmetric_data)
a = tensor(symmetric_data, chunk_size=3)
L = cholesky(a, lower=True)
U = cholesky(a)
t = L.dot(U)
res_u = self.executor.execute_tensor(U, concat=True)[0]
np.testing.assert_allclose(np.triu(res_u), res_u)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, symmetric_data)
def testLUExecution(self):
np.random.seed(1)
# square matrix
data = np.random.randint(1, 10, (6, 6))
a = tensor(data)
P, L, U = lu(a)
# check lower and upper triangular matrix
result_l = self.executor.execute_tensor(L, concat=True)[0]
result_u = self.executor.execute_tensor(U, concat=True)[0]
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data)
a = tensor(data, chunk_size=2)
P, L, U = lu(a)
# check lower and upper triangular matrix
result_l = self.executor.execute_tensor(L, concat=True)[0]
result_u = self.executor.execute_tensor(U, concat=True)[0]
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data)
a = tensor(data, chunk_size=(2, 3))
P, L, U = lu(a)
# check lower and upper triangular matrix
result_l = self.executor.execute_tensor(L, concat=True)[0]
result_u = self.executor.execute_tensor(U, concat=True)[0]
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data)
a = tensor(data, chunk_size=4)
P, L, U = lu(a)
# check lower and upper triangular matrix
result_l = self.executor.execute_tensor(L, concat=True)[0]
result_u = self.executor.execute_tensor(U, concat=True)[0]
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data)
# shape[0] > shape[1]
data = np.random.randint(1, 10, (10, 6))
a = tensor(data)
P, L, U = lu(a)
# check lower and upper triangular matrix
result_l = self.executor.execute_tensor(L, concat=True)[0]
result_u = self.executor.execute_tensor(U, concat=True)[0]
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data)
a = tensor(data, chunk_size=2)
P, L, U = lu(a)
# check lower and upper triangular matrix
result_l = self.executor.execute_tensor(L, concat=True)[0]
result_u = self.executor.execute_tensor(U, concat=True)[0]
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data)
a = tensor(data, chunk_size=(2, 3))
P, L, U = lu(a)
# check lower and upper triangular matrix
result_l = self.executor.execute_tensor(L, concat=True)[0]
result_u = self.executor.execute_tensor(U, concat=True)[0]
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data)
a = tensor(data, chunk_size=4)
P, L, U = lu(a)
# check lower and upper triangular matrix
result_l, result_u = self.executor.execute_tensors([L, U])
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data)
# shape[0] < shape[1]
data = np.random.randint(1, 10, (6, 10))
a = tensor(data)
P, L, U = lu(a)
# check lower and upper triangular matrix
result_l = self.executor.execute_tensor(L, concat=True)[0]
result_u = self.executor.execute_tensor(U, concat=True)[0]
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data)
a = tensor(data, chunk_size=2)
P, L, U = lu(a)
# check lower and upper triangular matrix
result_l = self.executor.execute_tensor(L, concat=True)[0]
result_u = self.executor.execute_tensor(U, concat=True)[0]
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data)
a = tensor(data, chunk_size=(2, 3))
P, L, U = lu(a)
# check lower and upper triangular matrix
result_l = self.executor.execute_tensor(L, concat=True)[0]
result_u = self.executor.execute_tensor(U, concat=True)[0]
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data)
a = tensor(data, chunk_size=4)
P, L, U = lu(a)
# check lower and upper triangular matrix
result_l, result_u = self.executor.execute_tensors([L, U])
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data)
# test for sparse
data = sps.csr_matrix([[2, 0, 0, 0, 5, 2], [0, 6, 1, 0, 0, 6],
[8, 0, 9, 0, 0, 2], [0, 6, 0, 8, 7, 3],
[7, 0, 6, 1, 7, 0], [0, 0, 0, 7, 0, 8]])
a = tensor(data)
P, L, U = lu(a)
result_l = self.executor.execute_tensor(L, concat=True)[0]
result_u = self.executor.execute_tensor(U, concat=True)[0]
# check lower and upper triangular matrix
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
self.assertIsInstance(result_l, SparseNDArray)
self.assertIsInstance(result_u, SparseNDArray)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_array_almost_equal(data.A, res)
a = tensor(data, chunk_size=2)
P, L, U = lu(a)
result_l = self.executor.execute_tensor(L, concat=True)[0]
result_u = self.executor.execute_tensor(U, concat=True)[0]
# check lower and upper triangular matrix
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
self.assertIsInstance(result_l, SparseNDArray)
self.assertIsInstance(result_u, SparseNDArray)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_array_almost_equal(data.A, res)
a = tensor(data, chunk_size=(2, 3))
P, L, U = lu(a)
result_l = self.executor.execute_tensor(L, concat=True)[0]
result_u = self.executor.execute_tensor(U, concat=True)[0]
# check lower and upper triangular matrix
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
self.assertIsInstance(result_l, SparseNDArray)
self.assertIsInstance(result_u, SparseNDArray)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_array_almost_equal(data.A, res)
a = tensor(data, chunk_size=4)
P, L, U = lu(a)
result_l = self.executor.execute_tensor(L, concat=True)[0]
result_u = self.executor.execute_tensor(U, concat=True)[0]
# check lower and upper triangular matrix
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
self.assertIsInstance(result_l, SparseNDArray)
self.assertIsInstance(result_u, SparseNDArray)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_array_almost_equal(data.A, res)
def testSolveTriangular(self):
from mars.tensor import tril, triu
np.random.seed(1)
data1 = np.random.randint(1, 10, (20, 20))
data2 = np.random.randint(1, 10, (20, ))
A = tensor(data1, chunk_size=20)
b = tensor(data2, chunk_size=20)
x = solve_triangular(A, b)
t = triu(A).dot(x)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data2)
x = solve_triangular(A, b, lower=True)
t = tril(A).dot(x)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data2)
A = tensor(data1, chunk_size=10)
b = tensor(data2, chunk_size=10)
x = solve_triangular(A, b)
t = triu(A).dot(x)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data2)
x = solve_triangular(A, b, lower=True)
t = tril(A).dot(x)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data2)
data1 = np.random.randint(1, 10, (10, 10))
data2 = np.random.randint(1, 10, (10, 5))
A = tensor(data1, chunk_size=10)
b = tensor(data2, chunk_size=10)
x = solve_triangular(A, b)
t = triu(A).dot(x)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data2)
x = solve_triangular(A, b, lower=True)
t = tril(A).dot(x)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data2)
A = tensor(data1, chunk_size=3)
b = tensor(data2, chunk_size=3)
x = solve_triangular(A, b)
t = triu(A).dot(x)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data2)
x = solve_triangular(A, b, lower=True)
t = tril(A).dot(x)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data2)
# test sparse
data1 = sps.csr_matrix(np.triu(np.random.randint(1, 10, (10, 10))))
data2 = np.random.random((10, ))
A = tensor(data1, chunk_size=5)
b = tensor(data2, chunk_size=5)
x = solve_triangular(A, b)
result_x = self.executor.execute_tensor(x, concat=True)[0]
result_b = data1.dot(result_x)
self.assertIsInstance(result_x, SparseNDArray)
np.testing.assert_allclose(result_b, data2)
data1 = sps.csr_matrix(np.triu(np.random.randint(1, 10, (10, 10))))
data2 = np.random.random((10, 2))
A = tensor(data1, chunk_size=5)
b = tensor(data2, chunk_size=5)
x = solve_triangular(A, b)
result_x = self.executor.execute_tensor(x, concat=True)[0]
result_b = data1.dot(result_x)
self.assertIsInstance(result_x, SparseNDArray)
np.testing.assert_allclose(result_b, data2)
def testSolve(self):
import scipy.linalg
np.random.seed(1)
data1 = np.random.randint(1, 10, (20, 20))
data2 = np.random.randint(1, 10, (20, ))
A = tensor(data1, chunk_size=5)
b = tensor(data2, chunk_size=5)
x = solve(A, b)
res = self.executor.execute_tensor(x, concat=True)[0]
np.testing.assert_allclose(res, scipy.linalg.solve(data1, data2))
res = self.executor.execute_tensor(A.dot(x), concat=True)[0]
np.testing.assert_allclose(res, data2)
data2 = np.random.randint(1, 10, (20, 5))
A = tensor(data1, chunk_size=5)
b = tensor(data2, chunk_size=5)
x = solve(A, b)
res = self.executor.execute_tensor(x, concat=True)[0]
np.testing.assert_allclose(res, scipy.linalg.solve(data1, data2))
res = self.executor.execute_tensor(A.dot(x), concat=True)[0]
np.testing.assert_allclose(res, data2)
data2 = np.random.randint(1, 10, (20, 20))
A = tensor(data1, chunk_size=5)
b = tensor(data2, chunk_size=5)
x = solve(A, b)
res = self.executor.execute_tensor(x, concat=True)[0]
np.testing.assert_allclose(res, scipy.linalg.solve(data1, data2))
res = self.executor.execute_tensor(A.dot(x), concat=True)[0]
np.testing.assert_allclose(res, data2)
# test for not all chunks are square in matrix A
data2 = np.random.randint(1, 10, (20, ))
A = tensor(data1, chunk_size=6)
b = tensor(data2, chunk_size=6)
x = solve(A, b)
res = self.executor.execute_tensor(x, concat=True)[0]
np.testing.assert_allclose(res, scipy.linalg.solve(data1, data2))
res = self.executor.execute_tensor(A.dot(x), concat=True)[0]
np.testing.assert_allclose(res, data2)
A = tensor(data1, chunk_size=(7, 6))
b = tensor(data2, chunk_size=6)
x = solve(A, b)
res = self.executor.execute_tensor(x, concat=True)[0]
np.testing.assert_allclose(res, scipy.linalg.solve(data1, data2))
res = self.executor.execute_tensor(A.dot(x), concat=True)[0]
np.testing.assert_allclose(res, data2)
# test sparse
data1 = sps.csr_matrix(np.random.randint(1, 10, (20, 20)))
data2 = np.random.randint(1, 10, (20, ))
A = tensor(data1, chunk_size=5)
b = tensor(data2, chunk_size=5)
x = solve(A, b)
res = self.executor.execute_tensor(x, concat=True)[0]
self.assertIsInstance(res, SparseNDArray)
np.testing.assert_allclose(data1.dot(res), data2)
data2 = np.random.randint(1, 10, (20, 5))
A = tensor(data1, chunk_size=5)
b = tensor(data2, chunk_size=5)
x = solve(A, b)
res = self.executor.execute_tensor(A.dot(x), concat=True)[0]
self.assertIsInstance(res, SparseNDArray)
np.testing.assert_allclose(res, data2)
data2 = np.random.randint(1, 10, (20, 20))
A = tensor(data1, chunk_size=5)
b = tensor(data2, chunk_size=5)
x = solve(A, b)
res = self.executor.execute_tensor(A.dot(x), concat=True)[0]
self.assertIsInstance(res, SparseNDArray)
np.testing.assert_allclose(res, data2)
# test for not all chunks are square in matrix A
data2 = np.random.randint(1, 10, (20, ))
A = tensor(data1, chunk_size=6)
b = tensor(data2, chunk_size=6)
x = solve(A, b)
res = self.executor.execute_tensor(A.dot(x), concat=True)[0]
np.testing.assert_allclose(res, data2)
def testSolveSymPos(self):
import scipy.linalg
np.random.seed(1)
data = np.random.randint(1, 10, (20, 20))
data_l = np.tril(data)
data1 = data_l.dot(data_l.T)
data2 = np.random.randint(1, 10, (20, ))
A = tensor(data1, chunk_size=5)
b = tensor(data2, chunk_size=5)
x = solve(A, b, sym_pos=True)
res = self.executor.execute_tensor(x, concat=True)[0]
np.testing.assert_allclose(res, scipy.linalg.solve(data1, data2))
res = self.executor.execute_tensor(A.dot(x), concat=True)[0]
np.testing.assert_allclose(res, data2)
def testInv(self):
import scipy.linalg
np.random.seed(1)
data = np.random.randint(1, 10, (20, 20))
A = tensor(data)
inv_A = inv(A)
res = self.executor.execute_tensor(inv_A, concat=True)[0]
self.assertTrue(np.allclose(res, scipy.linalg.inv(data)))
res = self.executor.execute_tensor(A.dot(inv_A), concat=True)[0]
self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float)))
A = tensor(data, chunk_size=5)
inv_A = inv(A)
res = self.executor.execute_tensor(inv_A, concat=True)[0]
self.assertTrue(np.allclose(res, scipy.linalg.inv(data)))
res = self.executor.execute_tensor(A.dot(inv_A), concat=True)[0]
self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float)))
# test 1 chunk
A = tensor(data, chunk_size=20)
inv_A = inv(A)
res = self.executor.execute_tensor(inv_A, concat=True)[0]
self.assertTrue(np.allclose(res, scipy.linalg.inv(data)))
res = self.executor.execute_tensor(A.dot(inv_A), concat=True)[0]
self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float)))
B = A.T.dot(A)
inv_B = inv(B)
res = self.executor.execute_tensor(inv_B, concat=True)[0]
self.assertTrue(np.allclose(res, scipy.linalg.inv(data.T.dot(data))))
res = self.executor.execute_tensor(B.dot(inv_B), concat=True)[0]
self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float)))
# test for not all chunks are square in matrix A
A = tensor(data, chunk_size=8)
inv_A = inv(A)
res = self.executor.execute_tensor(inv_A, concat=True)[0]
self.assertTrue(np.allclose(res, scipy.linalg.inv(data)))
res = self.executor.execute_tensor(A.dot(inv_A), concat=True)[0]
self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float)))
# test sparse
data = np.random.randint(1, 10, (20, 20))
sp_data = sps.csr_matrix(data)
A = tensor(sp_data, chunk_size=5)
inv_A = inv(A)
res = self.executor.execute_tensor(inv_A, concat=True)[0]
self.assertIsInstance(res, SparseNDArray)
self.assertTrue(np.allclose(res, scipy.linalg.inv(data)))
res = self.executor.execute_tensor(A.dot(inv_A), concat=True)[0]
self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float)))
# test for not all chunks are square in matrix A
A = tensor(sp_data, chunk_size=8)
inv_A = inv(A)
res = self.executor.execute_tensor(inv_A, concat=True)[0]
self.assertIsInstance(res, SparseNDArray)
self.assertTrue(np.allclose(res, scipy.linalg.inv(data)))
res = self.executor.execute_tensor(A.dot(inv_A), concat=True)[0]
self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float)))
@ignore_warning
def testNormExecution(self):
d = np.arange(9) - 4
d2 = d.reshape(3, 3)
ma = [
tensor(d, chunk_size=2),
tensor(d, chunk_size=9),
tensor(d2, chunk_size=(2, 3)),
tensor(d2, chunk_size=3)
]
for i, a in enumerate(ma):
data = d if i < 2 else d2
for ord in (None, 'nuc', np.inf, -np.inf, 0, 1, -1, 2, -2):
for axis in (0, 1, (0, 1)):
for keepdims in (True, False):
try:
expected = np.linalg.norm(data,
ord=ord,
axis=axis,
keepdims=keepdims)
t = norm(a, ord=ord, axis=axis, keepdims=keepdims)
concat = t.ndim > 0
res = self.executor.execute_tensor(
t, concat=concat)[0]
np.testing.assert_allclose(res,
expected,
atol=.0001)
except ValueError:
continue
m = norm(tensor(d))
expected = self.executor.execute_tensor(m)[0]
res = np.linalg.norm(d)
self.assertEqual(expected, res)
d = uniform(-0.5, 0.5, size=(500, 2), chunk_size=50)
inside = (norm(d, axis=1) < 0.5).sum().astype(float)
t = inside / 500 * 4
res = self.executor.execute_tensor(t)[0]
self.assertAlmostEqual(res, 3.14, delta=1)
raw = np.random.RandomState(0).rand(10, 10)
d = norm(tensor(raw, chunk_size=5))
expected = self.executor.execute_tensor(d, concat=True)[0]
result = np.linalg.norm(raw)
np.testing.assert_allclose(expected, result)
def testTensordotExecution(self):
size_executor = ExecutorForTest(
sync_provider_type=ExecutorForTest.SyncProviderType.MOCK)
a_data = np.arange(60).reshape(3, 4, 5)
a = tensor(a_data, chunk_size=2)
b_data = np.arange(24).reshape(4, 3, 2)
b = tensor(b_data, chunk_size=2)
axes = ([1, 0], [0, 1])
c = tensordot(a, b, axes=axes)
size_res = size_executor.execute_tensor(c, mock=True)
self.assertEqual(sum(s[0] for s in size_res), c.nbytes)
self.assertEqual(sum(s[1] for s in size_res), c.nbytes)
res = self.executor.execute_tensor(c)
expected = np.tensordot(a_data, b_data, axes=axes)
self.assertTrue(np.array_equal(res[0], expected[:2, :]))
self.assertTrue(np.array_equal(res[1], expected[2:4, :]))
self.assertTrue(np.array_equal(res[2], expected[4:, :]))
a = ones((1000, 2000), chunk_size=500)
b = ones((2000, 100), chunk_size=500)
c = dot(a, b)
res = self.executor.execute_tensor(c)
expected = np.dot(np.ones((1000, 2000)), np.ones((2000, 100)))
self.assertEqual(len(res), 2)
self.assertTrue(np.array_equal(res[0], expected[:500, :]))
self.assertTrue(np.array_equal(res[1], expected[500:, :]))
a = ones((10, 8), chunk_size=2)
b = ones((8, 10), chunk_size=2)
c = a.dot(b)
res = self.executor.execute_tensor(c)
self.assertEqual(len(res), 25)
for r in res:
self.assertTrue(np.array_equal(r, np.tile([8], [2, 2])))
a = ones((500, 500), chunk_size=500)
b = ones((500, 100), chunk_size=500)
c = a.dot(b)
res = self.executor.execute_tensor(c)
self.assertTrue(np.array_equal(res[0], np.tile([500], [500, 100])))
raw_a = np.random.random((100, 200, 50))
raw_b = np.random.random((200, 10, 100))
a = tensor(raw_a, chunk_size=50)
b = tensor(raw_b, chunk_size=33)
c = tensordot(a, b, axes=((0, 1), (2, 0)))
res = self.executor.execute_tensor(c, concat=True)
expected = np.tensordot(raw_a, raw_b, axes=(c.op.a_axes, c.op.b_axes))
self.assertTrue(np.allclose(res[0], expected))
a = ones((1000, 2000), chunk_size=500)
b = ones((100, 2000), chunk_size=500)
c = inner(a, b)
res = self.executor.execute_tensor(c)
expected = np.inner(np.ones((1000, 2000)), np.ones((100, 2000)))
self.assertEqual(len(res), 2)
self.assertTrue(np.array_equal(res[0], expected[:500, :]))
self.assertTrue(np.array_equal(res[1], expected[500:, :]))
a = ones((100, 100), chunk_size=30)
b = ones((100, 100), chunk_size=30)
c = a.dot(b)
res = self.executor.execute_tensor(c, concat=True)[0]
np.testing.assert_array_equal(res, np.ones((100, 100)) * 100)
def testSparseDotSizeExecution(self):
from mars.tensor.linalg.tensordot import TensorTensorDot
from mars.executor import register, register_default
chunk_sizes = dict()
chunk_nbytes = dict()
chunk_input_sizes = dict()
chunk_input_nbytes = dict()
def execute_size(t):
def _tensordot_size_recorder(ctx, op):
TensorTensorDot.estimate_size(ctx, op)
chunk_key = op.outputs[0].key
chunk_sizes[chunk_key] = ctx[chunk_key]
chunk_nbytes[chunk_key] = op.outputs[0].nbytes
input_sizes = dict(
(inp.op.key, ctx[inp.key][0]) for inp in op.inputs)
chunk_input_sizes[chunk_key] = sum(input_sizes.values())
input_nbytes = dict(
(inp.op.key, inp.nbytes) for inp in op.inputs)
chunk_input_nbytes[chunk_key] = sum(input_nbytes.values())
size_executor = ExecutorForTest(
sync_provider_type=ExecutorForTest.SyncProviderType.MOCK)
try:
chunk_sizes.clear()
chunk_nbytes.clear()
chunk_input_sizes.clear()
chunk_input_nbytes.clear()
register(TensorTensorDot,
size_estimator=_tensordot_size_recorder)
size_executor.execute_tensor(t, mock=True)
finally:
register_default(TensorTensorDot)
a_data = sps.random(5, 9, density=.1)
b_data = sps.random(9, 10, density=.2)
a = tensor(a_data, chunk_size=2)
b = tensor(b_data, chunk_size=3)
c = dot(a, b)
execute_size(c)
for key in chunk_input_sizes.keys():
self.assertGreaterEqual(chunk_sizes[key][1],
chunk_input_sizes[key])
c2 = dot(a, b, sparse=False)
execute_size(c2)
for key in chunk_input_sizes.keys():
self.assertEqual(chunk_sizes[key][0], chunk_nbytes[key])
self.assertEqual(chunk_sizes[key][1],
chunk_input_nbytes[key] + chunk_nbytes[key])
def testSparseDotExecution(self):
a_data = sps.random(5, 9, density=.1)
b_data = sps.random(9, 10, density=.2)
a = tensor(a_data, chunk_size=2)
b = tensor(b_data, chunk_size=3)
c = dot(a, b)
res = self.executor.execute_tensor(c, concat=True)[0]
self.assertTrue(issparse(res))
np.testing.assert_allclose(res.toarray(), a_data.dot(b_data).toarray())
c2 = dot(a, b, sparse=False)
res = self.executor.execute_tensor(c2, concat=True)[0]
self.assertFalse(issparse(res))
np.testing.assert_allclose(res, a_data.dot(b_data).toarray())
c3 = tensordot(a, b.T, (-1, -1), sparse=False)
res = self.executor.execute_tensor(c3, concat=True)[0]
self.assertFalse(issparse(res))
np.testing.assert_allclose(res, a_data.dot(b_data).toarray())
c = inner(a, b.T)
res = self.executor.execute_tensor(c, concat=True)[0]
self.assertTrue(issparse(res))
np.testing.assert_allclose(res.toarray(), a_data.dot(b_data).toarray())
c = inner(a, b.T, sparse=False)
res = self.executor.execute_tensor(c, concat=True)[0]
self.assertFalse(issparse(res))
np.testing.assert_allclose(res, a_data.dot(b_data).toarray())
# test vector inner
a_data = np.random.rand(5)
b_data = np.random.rand(5)
a = tensor(a_data, chunk_size=2).tosparse()
b = tensor(b_data, chunk_size=2).tosparse()
c = inner(a, b)
res = self.executor.execute_tensor(c, concat=True)[0]
self.assertTrue(np.isscalar(res))
np.testing.assert_allclose(res, np.inner(a_data, b_data))
def testVdotExecution(self):
a_data = np.array([1 + 2j, 3 + 4j])
b_data = np.array([5 + 6j, 7 + 8j])
a = tensor(a_data, chunk_size=1)
b = tensor(b_data, chunk_size=1)
t = vdot(a, b)
res = self.executor.execute_tensor(t)[0]
expected = np.vdot(a_data, b_data)
np.testing.assert_equal(res, expected)
a_data = np.array([[1, 4], [5, 6]])
b_data = np.array([[4, 1], [2, 2]])
a = tensor(a_data, chunk_size=1)
b = tensor(b_data, chunk_size=1)
t = vdot(a, b)
res = self.executor.execute_tensor(t)[0]
expected = np.vdot(a_data, b_data)
np.testing.assert_equal(res, expected)
def testMatmulExecution(self):
data_a = np.random.randn(10, 20)
data_b = np.random.randn(20)
a = tensor(data_a, chunk_size=2)
b = tensor(data_b, chunk_size=3)
c = matmul(a, b)
res = self.executor.execute_tensor(c, concat=True)[0]
expected = np.matmul(data_a, data_b)
np.testing.assert_allclose(res, expected)
data_a = np.random.randn(10, 20)
data_b = np.random.randn(10)
a = tensor(data_a, chunk_size=2)
b = tensor(data_b, chunk_size=3)
c = matmul(b, a)
res = self.executor.execute_tensor(c, concat=True)[0]
expected = np.matmul(data_b, data_a)
np.testing.assert_allclose(res, expected)
data_a = np.random.randn(15, 1, 20, 30)
data_b = np.random.randn(1, 11, 30, 20)
a = tensor(data_a, chunk_size=12)
b = tensor(data_b, chunk_size=13)
c = matmul(a, b)
res = self.executor.execute_tensor(c, concat=True)[0]
expected = np.matmul(data_a, data_b)
np.testing.assert_allclose(res, expected, atol=.0001)
a = arange(2 * 2 * 4, chunk_size=1).reshape((2, 2, 4))
b = arange(2 * 2 * 4, chunk_size=1).reshape((2, 4, 2))
c = matmul(a, b)
res = self.executor.execute_tensor(c, concat=True)[0]
expected = np.matmul(
np.arange(2 * 2 * 4).reshape(2, 2, 4),
np.arange(2 * 2 * 4).reshape(2, 4, 2))
np.testing.assert_allclose(res, expected, atol=.0001)
data_a = sps.random(10, 20)
data_b = sps.random(20, 5)
a = tensor(data_a, chunk_size=2)
b = tensor(data_b, chunk_size=3)
c = matmul(a, b)
res = self.executor.execute_tensor(c, concat=True)[0]
expected = np.matmul(data_a.toarray(), data_b.toarray())
np.testing.assert_allclose(res.toarray(), expected)
# test order
data_a = np.asfortranarray(np.random.randn(10, 20))
data_b = np.asfortranarray(np.random.randn(20, 30))
a = tensor(data_a, chunk_size=12)
b = tensor(data_b, chunk_size=13)
c = matmul(a, b)
res = self.executor.execute_tensor(c, concat=True)[0]
expected = np.matmul(data_a, data_b)
np.testing.assert_allclose(res, expected)
self.assertEqual(res.flags['C_CONTIGUOUS'],
expected.flags['C_CONTIGUOUS'])
self.assertEqual(res.flags['F_CONTIGUOUS'],
expected.flags['F_CONTIGUOUS'])
c = matmul(a, b, order='A')
res = self.executor.execute_tensor(c, concat=True)[0]
expected = np.matmul(data_a, data_b, order='A')
np.testing.assert_allclose(res, expected)
self.assertEqual(res.flags['C_CONTIGUOUS'],
expected.flags['C_CONTIGUOUS'])
self.assertEqual(res.flags['F_CONTIGUOUS'],
expected.flags['F_CONTIGUOUS'])
c = matmul(a, b, order='C')
res = self.executor.execute_tensor(c, concat=True)[0]
expected = np.matmul(data_a, data_b, order='C')
np.testing.assert_allclose(res, expected)
self.assertEqual(res.flags['C_CONTIGUOUS'],
expected.flags['C_CONTIGUOUS'])
self.assertEqual(res.flags['F_CONTIGUOUS'],
expected.flags['F_CONTIGUOUS'])
class Test(unittest.TestCase):
def setUp(self) -> None:
self.executor = ExecutorForTest('numpy')
def testManualBuildFaissIndex(self):
d = 8
n = 50
n_test = 10
x = np.random.RandomState(0).rand(n, d).astype(np.float32)
y = np.random.RandomState(0).rand(n_test, d).astype(np.float32)
nn = NearestNeighbors(algorithm='kd_tree')
nn.fit(x)
_, expected_indices = nn.kneighbors(y, 5)
for index_type in ['object', 'filename', 'bytes']:
# test brute-force search
X = mt.tensor(x, chunk_size=10)
index = build_faiss_index(X, 'Flat', None, random_state=0,
same_distribution=True, return_index_type=index_type)
faiss_index = self.executor.execute_tileable(index)
index_shards = faiss.IndexShards(d)
for ind in faiss_index:
shard = _load_index(None, index.op, ind, -1)
index_shards.add_shard(shard)
faiss_index = index_shards
faiss_index.nprob = 10
_, indices = faiss_index.search(y, k=5)
np.testing.assert_array_equal(indices, expected_indices.fetch())
# test one chunk, brute force
X = mt.tensor(x, chunk_size=50)
index = build_faiss_index(X, 'Flat', None, random_state=0,
same_distribution=True, return_index_type='object')
faiss_index = self.executor.execute_tileable(index)[0]
faiss_index.nprob = 10
_, indices = faiss_index.search(y, k=5)
np.testing.assert_array_equal(indices, expected_indices.fetch())
# test train, same distribution
X = mt.tensor(x, chunk_size=10)
index = build_faiss_index(X, 'IVF30,Flat', 30, random_state=0,
same_distribution=True, return_index_type='object')
faiss_index = self.executor.execute_tileable(index)[0]
self.assertIsInstance(faiss_index, faiss.IndexIVFFlat)
self.assertEqual(faiss_index.ntotal, n)
self.assertEqual(len(get_tiled(index).chunks), 1)
# test train, distributions are variant
X = mt.tensor(x, chunk_size=10)
index = build_faiss_index(X, 'IVF10,Flat', None, random_state=0,
same_distribution=False, return_index_type='object')
faiss_index = self.executor.execute_tileable(index)
self.assertEqual(len(faiss_index), 5)
for ind in faiss_index:
self.assertIsInstance(ind, faiss.IndexIVFFlat)
self.assertEqual(ind.ntotal, 10)
# test one chunk, train
X = mt.tensor(x, chunk_size=50)
index = build_faiss_index(X, 'IVF30,Flat', 30, random_state=0,
same_distribution=True, return_index_type='object')
faiss_index = self.executor.execute_tileable(index)[0]
self.assertIsInstance(faiss_index, faiss.IndexIVFFlat)
self.assertEqual(faiss_index.ntotal, n)
# test wrong index
with self.assertRaises(ValueError):
build_faiss_index(X, 'unknown_index', None)
# test unknown metric
with self.assertRaises(ValueError):
build_faiss_index(X, 'Flat', None, metric='unknown_metric')
def testFaissQuery(self):
d = 8
n = 50
n_test = 10
x = np.random.RandomState(0).rand(n, d).astype(np.float32)
y = np.random.RandomState(1).rand(n_test, d).astype(np.float32)
test_tensors = [
# multi chunks
(mt.tensor(x, chunk_size=(20, 5)), mt.tensor(y, chunk_size=5)),
# one chunk
(mt.tensor(x, chunk_size=50), mt.tensor(y, chunk_size=10))
]
for X, Y in test_tensors:
for metric in ['l2', 'cosine']:
faiss_index = build_faiss_index(X, 'Flat', None, metric=metric,
random_state=0, return_index_type='object')
d, i = faiss_query(faiss_index, Y, 5, nprobe=10)
distance, indices = self.executor.execute_tensors([d, i])
nn = NearestNeighbors(metric=metric)
nn.fit(x)
expected_distance, expected_indices = nn.kneighbors(y, 5)
np.testing.assert_array_equal(indices, expected_indices.fetch())
np.testing.assert_almost_equal(distance, expected_distance.fetch())
def testGenIndexStringAndSampleCount(self):
d = 32
# accuracy=True, could be Flat only
ret = _gen_index_string_and_sample_count((10 ** 9, d), None, True, 'minimum')
self.assertEqual(ret, ('Flat', None))
# no memory concern
ret = _gen_index_string_and_sample_count((10 ** 5, d), None, False, 'maximum')
self.assertEqual(ret, ('HNSW32', None))
index = faiss.index_factory(d, ret[0])
self.assertTrue(index.is_trained)
# memory concern not much
ret = _gen_index_string_and_sample_count((10 ** 5, d), None, False, 'high')
self.assertEqual(ret, ('IVF1580,Flat', 47400))
index = faiss.index_factory(d, ret[0])
self.assertFalse(index.is_trained)
# memory quite important
ret = _gen_index_string_and_sample_count((5 * 10 ** 6, d), None, False, 'low')
self.assertEqual(ret, ('PCAR16,IVF65536_HNSW32,SQ8', 32 * 65536))
index = faiss.index_factory(d, ret[0])
self.assertFalse(index.is_trained)
# memory very important
ret = _gen_index_string_and_sample_count((10 ** 8, d), None, False, 'minimum')
self.assertEqual(ret, ('OPQ16_32,IVF1048576_HNSW32,PQ16', 64 * 65536))
index = faiss.index_factory(d, ret[0])
self.assertFalse(index.is_trained)
ret = _gen_index_string_and_sample_count((10 ** 10, d), None, False, 'low')
self.assertEqual(ret, ('PCAR16,IVF1048576_HNSW32,SQ8', 64 * 65536))
index = faiss.index_factory(d, ret[0])
self.assertFalse(index.is_trained)
with self.assertRaises(ValueError):
# M > 64 raise error
_gen_index_string_and_sample_count((10 ** 5, d), None, False, 'maximum', M=128)
with self.assertRaises(ValueError):
# M > 64
_gen_index_string_and_sample_count((10 ** 5, d), None, False, 'minimum', M=128)
with self.assertRaises(ValueError):
# dim should be multiple of M
_gen_index_string_and_sample_count((10 ** 5, d), None, False, 'minimum', M=16, dim=17)
with self.assertRaises(ValueError):
_gen_index_string_and_sample_count((10 ** 5, d), None, False, 'low', k=5)
def testAutoIndex(self):
d = 8
n = 50
n_test = 10
x = np.random.RandomState(0).rand(n, d).astype(np.float32)
y = np.random.RandomState(1).rand(n_test, d).astype(np.float32)
for chunk_size in (50, 20):
X = mt.tensor(x, chunk_size=chunk_size)
faiss_index = build_faiss_index(X, random_state=0, return_index_type='object')
d, i = faiss_query(faiss_index, y, 5, nprobe=10)
indices = self.executor.execute_tensor(i, concat=True)[0]
nn = NearestNeighbors()
nn.fit(x)
expected_indices = nn.kneighbors(y, 5, return_distance=False)
np.testing.assert_array_equal(indices, expected_indices)
def testQuantileExecution(self):
# test 1 chunk, 1-d
raw = np.random.rand(20)
a = tensor(raw, chunk_size=20)
raw2 = raw.copy()
raw2[np.random.RandomState(0).randint(raw.size, size=3)] = np.nan
a2 = tensor(raw2, chunk_size=20)
for q in [np.random.RandomState(0).rand(), np.random.RandomState(0).rand(5)]:
for interpolation in INTERPOLATION_TYPES:
for keepdims in [True, False]:
r = quantile(a, q, interpolation=interpolation, keepdims=keepdims)
result = self.executor.execute_tensor(r, concat=True)[0]
expected = np.quantile(
raw, q, interpolation=interpolation, keepdims=keepdims)
np.testing.assert_array_equal(result, expected)
r2 = quantile(a2, q, interpolation=interpolation, keepdims=keepdims)
result = self.executor.execute_tensor(r2, concat=True)[0]
expected = np.quantile(
raw2, q, interpolation=interpolation, keepdims=keepdims)
np.testing.assert_array_equal(result, expected)
# test 1 chunk, 2-d
raw = np.random.rand(20, 10)
a = tensor(raw, chunk_size=20)
raw2 = raw.copy()
raw2.flat[np.random.RandomState(0).randint(raw.size, size=3)] = np.nan
a2 = tensor(raw2, chunk_size=20)
for q in [np.random.RandomState(0).rand(), np.random.RandomState(0).rand(5)]:
for interpolation in INTERPOLATION_TYPES:
for keepdims in [True, False]:
for axis in [None, 0, 1]:
r = quantile(a, q, axis=axis, interpolation=interpolation, keepdims=keepdims)
result = self.executor.execute_tensor(r, concat=True)[0]
expected = np.quantile(
raw, q, axis=axis, interpolation=interpolation, keepdims=keepdims)
np.testing.assert_array_equal(result, expected)
r2 = quantile(a2, q, axis=axis, interpolation=interpolation, keepdims=keepdims)
result = self.executor.execute_tensor(r2, concat=True)[0]
expected = np.quantile(
raw2, q, axis=axis, interpolation=interpolation, keepdims=keepdims)
np.testing.assert_array_equal(result, expected)
# test multi chunks, 1-d
raw = np.random.rand(20)
a = tensor(raw, chunk_size=3)
raw2 = raw.copy()
raw2[np.random.RandomState(0).randint(raw.size, size=3)] = np.nan
a2 = tensor(raw2, chunk_size=20)
for q in [np.random.RandomState(0).rand(), np.random.RandomState(0).rand(5)]:
for interpolation in INTERPOLATION_TYPES:
for keepdims in [True, False]:
r = quantile(a, q, interpolation=interpolation, keepdims=keepdims)
result = self.executor.execute_tensor(r, concat=True)[0]
expected = np.quantile(
raw, q, interpolation=interpolation, keepdims=keepdims)
np.testing.assert_array_equal(result, expected)
r2 = quantile(a2, q, interpolation=interpolation, keepdims=keepdims)
result = self.executor.execute_tensor(r2, concat=True)[0]
expected = np.quantile(
raw2, q, interpolation=interpolation, keepdims=keepdims)
np.testing.assert_array_equal(result, expected)
# test multi chunk, 2-d
raw = np.random.rand(20, 10)
a = tensor(raw, chunk_size=(3, 4))
raw2 = raw.copy()
raw2.flat[np.random.RandomState(0).randint(raw.size, size=3)] = np.nan
a2 = tensor(raw2, chunk_size=(3, 4))
for q in [np.random.RandomState(0).rand(), np.random.RandomState(0).rand(5)]:
for interpolation in INTERPOLATION_TYPES:
for keepdims in [True, False]:
for axis in [None, 0, 1]:
r = quantile(a, q, axis=axis, interpolation=interpolation, keepdims=keepdims)
result = self.executor.execute_tensor(r, concat=True)[0]
expected = np.quantile(
raw, q, axis=axis, interpolation=interpolation, keepdims=keepdims)
np.testing.assert_array_equal(result, expected)
r2 = quantile(a2, q, axis=axis, interpolation=interpolation, keepdims=keepdims)
result = self.executor.execute_tensor(r2, concat=True)[0]
expected = np.quantile(
raw2, q, axis=axis, interpolation=interpolation, keepdims=keepdims)
np.testing.assert_array_equal(result, expected)
# test out, 1 chunk
raw = np.random.rand(20)
q = np.random.rand(11)
a = tensor(raw, chunk_size=20)
out = empty((5, 11))
quantile(a, q, out=out)
result = self.executor.execute_tensor(out, concat=True)[0]
expected = np.quantile(raw, q, out=np.empty((5, 11)))
np.testing.assert_array_equal(result, expected)
# test out, multi chunks
raw = np.random.rand(20)
q = np.random.rand(11)
a = tensor(raw, chunk_size=3)
out = empty((5, 11))
quantile(a, q, out=out)
result = self.executor.execute_tensor(out, concat=True)[0]
expected = np.quantile(raw, q, out=np.empty((5, 11)))
np.testing.assert_array_equal(result, expected)
# test q which is a tensor
q_raw = np.random.RandomState(0).rand(5)
q = tensor(q_raw, chunk_size=3)
this = self
class MockSession:
def __init__(self):
self.executor = this.executor
ctx = LocalContext(MockSession())
executor = ExecutorForTest('numpy', storage=ctx)
with ctx:
r = quantile(a, q, axis=None)
result = executor.execute_tensors([r])[0]
expected = np.quantile(raw, q_raw, axis=None)
np.testing.assert_array_equal(result, expected)
with self.assertRaises(ValueError):
q[0] = 1.1
r = quantile(a, q, axis=None)
_ = executor.execute_tensors(r)[0]
class Test(unittest.TestCase):
def setUp(self) -> None:
self.executor = ExecutorForTest('numpy')
def testNormalizeOp(self):
with self.assertRaises(ValueError):
normalize(mt.random.random(10, 3), norm='unknown')
with self.assertRaises(ValueError):
normalize(mt.random.random(10, 3), axis=-1)
with self.assertRaises(ValueError):
normalize(mt.random.rand(10, 3, 3))
def testNormalizeExecution(self):
raw_dense = np.random.rand(10, 10)
raw_sparse = sps.random(10, 10, density=0.4, format='csr')
for chunk_size in [10, 6, (10, 6), (6, 10)]:
for raw, x in [
(raw_dense, mt.tensor(raw_dense, chunk_size=chunk_size)),
(raw_sparse, mt.tensor(raw_sparse, chunk_size=chunk_size))
]:
for norm in ['l1', 'l2', 'max']:
for axis in (0, 1):
for use_sklearn in [True, False]:
n = normalize(x,
norm=norm,
axis=axis,
return_norm=False)
n.op._use_sklearn = use_sklearn
result = self.executor.execute_tensor(
n, concat=True)[0]
expected = sk_normalize(raw,
norm=norm,
axis=axis,
return_norm=False)
if sps.issparse(expected):
expected = expected.A
np.testing.assert_almost_equal(
np.asarray(result), expected)
raw_dense = np.random.rand(10, 10)
raw_sparse = sps.random(10, 10, density=0.4, format='csr')
# test copy and return_normalize
for axis in (0, 1):
for chunk_size in (10, 6, (6, 10)):
for raw in (raw_dense, raw_sparse):
x = mt.tensor(raw, chunk_size=chunk_size)
n = normalize(x, axis=axis, copy=False, return_norm=True)
results = self.executor.execute_tensors(n)
raw_copy = raw.copy()
try:
expects = sk_normalize(raw_copy,
axis=axis,
copy=False,
return_norm=True)
except NotImplementedError:
continue
if sps.issparse(expects[0]):
expected = expects[0].A
else:
expected = expects[0]
np.testing.assert_almost_equal(np.asarray(results[0]),
expected)
np.testing.assert_almost_equal(results[1], expects[1])
class Test(unittest.TestCase):
def setUp(self):
self.executor = ExecutorForTest('numpy')
def testRechunkExecution(self):
raw = np.random.random((11, 8))
arr = tensor(raw, chunk_size=3)
arr2 = arr.rechunk(4)
res = self.executor.execute_tensor(arr2)
self.assertTrue(np.array_equal(res[0], raw[:4, :4]))
self.assertTrue(np.array_equal(res[1], raw[:4, 4:]))
self.assertTrue(np.array_equal(res[2], raw[4:8, :4]))
self.assertTrue(np.array_equal(res[3], raw[4:8, 4:]))
self.assertTrue(np.array_equal(res[4], raw[8:, :4]))
self.assertTrue(np.array_equal(res[5], raw[8:, 4:]))
def testCopytoExecution(self):
a = ones((2, 3), chunk_size=1)
b = tensor([3, -1, 3], chunk_size=2)
copyto(a, b, where=b > 1)
res = self.executor.execute_tensor(a, concat=True)[0]
expected = np.array([[3, 1, 3], [3, 1, 3]])
np.testing.assert_equal(res, expected)
a = ones((2, 3), chunk_size=1)
b = tensor(np.asfortranarray(np.random.rand(2, 3)), chunk_size=2)
copyto(b, a)
res = self.executor.execute_tensor(b, concat=True)[0]
expected = np.asfortranarray(np.ones((2, 3)))
np.testing.assert_array_equal(res, expected)
self.assertTrue(res.flags['F_CONTIGUOUS'])
self.assertFalse(res.flags['C_CONTIGUOUS'])
def testAstypeExecution(self):
raw = np.random.random((10, 5))
arr = tensor(raw, chunk_size=3)
arr2 = arr.astype('i8')
res = self.executor.execute_tensor(arr2, concat=True)
np.testing.assert_array_equal(res[0], raw.astype('i8'))
raw = sps.random(10, 5, density=.2)
arr = tensor(raw, chunk_size=3)
arr2 = arr.astype('i8')
res = self.executor.execute_tensor(arr2, concat=True)
self.assertTrue(
np.array_equal(res[0].toarray(),
raw.astype('i8').toarray()))
raw = np.asfortranarray(np.random.random((10, 5)))
arr = tensor(raw, chunk_size=3)
arr2 = arr.astype('i8', order='C')
res = self.executor.execute_tensor(arr2, concat=True)[0]
np.testing.assert_array_equal(res, raw.astype('i8'))
self.assertTrue(res.flags['C_CONTIGUOUS'])
self.assertFalse(res.flags['F_CONTIGUOUS'])
def testTransposeExecution(self):
raw = np.random.random((11, 8, 5))
arr = tensor(raw, chunk_size=3)
arr2 = transpose(arr)
res = self.executor.execute_tensor(arr2, concat=True)
np.testing.assert_array_equal(res[0], raw.T)
arr3 = transpose(arr, axes=(-2, -1, -3))
res = self.executor.execute_tensor(arr3, concat=True)
np.testing.assert_array_equal(res[0], raw.transpose(1, 2, 0))
raw = sps.random(11, 8)
arr = tensor(raw, chunk_size=3)
arr2 = transpose(arr)
self.assertTrue(arr2.issparse())
res = self.executor.execute_tensor(arr2, concat=True)
np.testing.assert_array_equal(res[0].toarray(), raw.T.toarray())
# test order
raw = np.asfortranarray(np.random.random((11, 8, 5)))
arr = tensor(raw, chunk_size=3)
arr2 = transpose(arr)
res = self.executor.execute_tensor(arr2, concat=True)[0]
expected = np.transpose(raw).copy(order='A')
np.testing.assert_array_equal(res, expected)
self.assertEqual(res.flags['C_CONTIGUOUS'],
expected.flags['C_CONTIGUOUS'])
self.assertEqual(res.flags['F_CONTIGUOUS'],
expected.flags['F_CONTIGUOUS'])
arr = tensor(raw, chunk_size=3)
arr2 = transpose(arr, (1, 2, 0))
res = self.executor.execute_tensor(arr2, concat=True)[0]
expected = np.transpose(raw, (1, 2, 0)).copy(order='A')
np.testing.assert_array_equal(res, expected)
self.assertEqual(res.flags['C_CONTIGUOUS'],
expected.flags['C_CONTIGUOUS'])
self.assertEqual(res.flags['F_CONTIGUOUS'],
expected.flags['F_CONTIGUOUS'])
def testSwapaxesExecution(self):
raw = np.random.random((11, 8, 5))
arr = tensor(raw, chunk_size=3)
arr2 = arr.swapaxes(2, 0)
res = self.executor.execute_tensor(arr2, concat=True)
np.testing.assert_array_equal(res[0], raw.swapaxes(2, 0))
raw = sps.random(11, 8, density=.2)
arr = tensor(raw, chunk_size=3)
arr2 = arr.swapaxes(1, 0)
res = self.executor.execute_tensor(arr2, concat=True)
np.testing.assert_array_equal(res[0].toarray(),
raw.toarray().swapaxes(1, 0))
# test order
raw = np.asfortranarray(np.random.rand(11, 8, 5))
arr = tensor(raw, chunk_size=3)
arr2 = arr.swapaxes(2, 0)
res = self.executor.execute_tensor(arr2, concat=True)[0]
expected = raw.swapaxes(2, 0).copy(order='A')
np.testing.assert_array_equal(res, expected)
self.assertEqual(res.flags['C_CONTIGUOUS'],
expected.flags['C_CONTIGUOUS'])
self.assertEqual(res.flags['F_CONTIGUOUS'],
expected.flags['F_CONTIGUOUS'])
arr = tensor(raw, chunk_size=3)
arr2 = arr.swapaxes(0, 2)
res = self.executor.execute_tensor(arr2, concat=True)[0]
expected = raw.swapaxes(0, 2).copy(order='A')
np.testing.assert_array_equal(res, expected)
self.assertEqual(res.flags['C_CONTIGUOUS'],
expected.flags['C_CONTIGUOUS'])
self.assertEqual(res.flags['F_CONTIGUOUS'],
expected.flags['F_CONTIGUOUS'])
arr = tensor(raw, chunk_size=3)
arr2 = arr.swapaxes(1, 0)
res = self.executor.execute_tensor(arr2, concat=True)[0]
expected = raw.swapaxes(1, 0).copy(order='A')
np.testing.assert_array_equal(res, expected)
self.assertEqual(res.flags['C_CONTIGUOUS'],
expected.flags['C_CONTIGUOUS'])
self.assertEqual(res.flags['F_CONTIGUOUS'],
expected.flags['F_CONTIGUOUS'])
def testMoveaxisExecution(self):
x = zeros((3, 4, 5), chunk_size=2)
t = moveaxis(x, 0, -1)
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertEqual(res.shape, (4, 5, 3))
t = moveaxis(x, -1, 0)
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertEqual(res.shape, (5, 3, 4))
t = moveaxis(x, [0, 1], [-1, -2])
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertEqual(res.shape, (5, 4, 3))
t = moveaxis(x, [0, 1, 2], [-1, -2, -3])
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertEqual(res.shape, (5, 4, 3))
def testBroadcastToExecution(self):
raw = np.random.random((10, 5, 1))
arr = tensor(raw, chunk_size=2)
arr2 = broadcast_to(arr, (5, 10, 5, 6))
res = self.executor.execute_tensor(arr2, concat=True)[0]
np.testing.assert_array_equal(res, np.broadcast_to(raw, (5, 10, 5, 6)))
# test chunk with unknown shape
arr1 = mt.random.rand(3, 4, chunk_size=2)
arr2 = mt.random.permutation(arr1)
arr3 = broadcast_to(arr2, (2, 3, 4))
res = self.executor.execute_tensor(arr3, concat=True)[0]
self.assertEqual(res.shape, (2, 3, 4))
def testBroadcastArraysExecutions(self):
x_data = [[1, 2, 3]]
x = tensor(x_data, chunk_size=1)
y_data = [[1], [2], [3]]
y = tensor(y_data, chunk_size=2)
a = broadcast_arrays(x, y)
res = [self.executor.execute_tensor(arr, concat=True)[0] for arr in a]
expected = np.broadcast_arrays(x_data, y_data)
for r, e in zip(res, expected):
np.testing.assert_equal(r, e)
def testWhereExecution(self):
raw_cond = np.random.randint(0, 2, size=(4, 4), dtype='?')
raw_x = np.random.rand(4, 1)
raw_y = np.random.rand(4, 4)
cond, x, y = tensor(raw_cond, chunk_size=2), tensor(
raw_x, chunk_size=2), tensor(raw_y, chunk_size=2)
arr = where(cond, x, y)
res = self.executor.execute_tensor(arr, concat=True)
self.assertTrue(
np.array_equal(res[0], np.where(raw_cond, raw_x, raw_y)))
raw_cond = sps.csr_matrix(
np.random.randint(0, 2, size=(4, 4), dtype='?'))
raw_x = sps.random(4, 1, density=.1)
raw_y = sps.random(4, 4, density=.1)
cond, x, y = tensor(raw_cond, chunk_size=2), tensor(
raw_x, chunk_size=2), tensor(raw_y, chunk_size=2)
arr = where(cond, x, y)
res = self.executor.execute_tensor(arr, concat=True)[0]
self.assertTrue(
np.array_equal(
res.toarray(),
np.where(raw_cond.toarray(), raw_x.toarray(),
raw_y.toarray())))
def testReshapeExecution(self):
raw_data = np.random.rand(10, 20, 30)
x = tensor(raw_data, chunk_size=6)
y = x.reshape(-1, 30)
res = self.executor.execute_tensor(y, concat=True)
np.testing.assert_array_equal(res[0], raw_data.reshape(-1, 30))
y2 = x.reshape(10, -1)
res = self.executor.execute_tensor(y2, concat=True)
np.testing.assert_array_equal(res[0], raw_data.reshape(10, -1))
y3 = x.reshape(-1)
res = self.executor.execute_tensor(y3, concat=True)
np.testing.assert_array_equal(res[0], raw_data.reshape(-1))
y4 = x.ravel()
res = self.executor.execute_tensor(y4, concat=True)
np.testing.assert_array_equal(res[0], raw_data.ravel())
raw_data = np.random.rand(30, 100, 20)
x = tensor(raw_data, chunk_size=6)
y = x.reshape(-1, 20, 5, 5, 4)
res = self.executor.execute_tensor(y, concat=True)
np.testing.assert_array_equal(res[0],
raw_data.reshape(-1, 20, 5, 5, 4))
y2 = x.reshape(3000, 10, 2)
res = self.executor.execute_tensor(y2, concat=True)
np.testing.assert_array_equal(res[0], raw_data.reshape(3000, 10, 2))
y3 = x.reshape(60, 25, 40)
res = self.executor.execute_tensor(y3, concat=True)
np.testing.assert_array_equal(res[0], raw_data.reshape(60, 25, 40))
y4 = x.reshape(60, 25, 40)
y4.op.extra_params['_reshape_with_shuffle'] = True
size_res = self.executor.execute_tensor(y4, mock=True)
res = self.executor.execute_tensor(y4, concat=True)
self.assertEqual(res[0].nbytes, sum(v[0] for v in size_res))
self.assertTrue(np.array_equal(res[0], raw_data.reshape(60, 25, 40)))
y5 = x.ravel(order='F')
res = self.executor.execute_tensor(y5, concat=True)[0]
expected = raw_data.ravel(order='F')
np.testing.assert_array_equal(res, expected)
self.assertEqual(res.flags['C_CONTIGUOUS'],
expected.flags['C_CONTIGUOUS'])
self.assertEqual(res.flags['F_CONTIGUOUS'],
expected.flags['F_CONTIGUOUS'])
def testExpandDimsExecution(self):
raw_data = np.random.rand(10, 20, 30)
x = tensor(raw_data, chunk_size=6)
y = expand_dims(x, 1)
res = self.executor.execute_tensor(y, concat=True)
self.assertTrue(np.array_equal(res[0], np.expand_dims(raw_data, 1)))
y = expand_dims(x, 0)
res = self.executor.execute_tensor(y, concat=True)
self.assertTrue(np.array_equal(res[0], np.expand_dims(raw_data, 0)))
y = expand_dims(x, 3)
res = self.executor.execute_tensor(y, concat=True)
self.assertTrue(np.array_equal(res[0], np.expand_dims(raw_data, 3)))
y = expand_dims(x, -1)
res = self.executor.execute_tensor(y, concat=True)
self.assertTrue(np.array_equal(res[0], np.expand_dims(raw_data, -1)))
y = expand_dims(x, -4)
res = self.executor.execute_tensor(y, concat=True)
self.assertTrue(np.array_equal(res[0], np.expand_dims(raw_data, -4)))
with self.assertRaises(np.AxisError):
expand_dims(x, -5)
with self.assertRaises(np.AxisError):
expand_dims(x, 4)
def testRollAxisExecution(self):
x = ones((3, 4, 5, 6), chunk_size=1)
y = rollaxis(x, 3, 1)
res = self.executor.execute_tensor(y, concat=True)
self.assertTrue(
np.array_equal(res[0], np.rollaxis(np.ones((3, 4, 5, 6)), 3, 1)))
def testAtleast1dExecution(self):
x = 1
y = ones(3, chunk_size=2)
z = ones((3, 4), chunk_size=2)
t = atleast_1d(x, y, z)
res = [self.executor.execute_tensor(i, concat=True)[0] for i in t]
self.assertTrue(np.array_equal(res[0], np.array([1])))
self.assertTrue(np.array_equal(res[1], np.ones(3)))
self.assertTrue(np.array_equal(res[2], np.ones((3, 4))))
def testAtleast2dExecution(self):
x = 1
y = ones(3, chunk_size=2)
z = ones((3, 4), chunk_size=2)
t = atleast_2d(x, y, z)
res = [self.executor.execute_tensor(i, concat=True)[0] for i in t]
self.assertTrue(np.array_equal(res[0], np.array([[1]])))
self.assertTrue(np.array_equal(res[1], np.atleast_2d(np.ones(3))))
self.assertTrue(np.array_equal(res[2], np.ones((3, 4))))
def testAtleast3dExecution(self):
x = 1
y = ones(3, chunk_size=2)
z = ones((3, 4), chunk_size=2)
t = atleast_3d(x, y, z)
res = [self.executor.execute_tensor(i, concat=True)[0] for i in t]
self.assertTrue(np.array_equal(res[0], np.atleast_3d(x)))
self.assertTrue(np.array_equal(res[1], np.atleast_3d(np.ones(3))))
self.assertTrue(np.array_equal(res[2], np.atleast_3d(np.ones((3, 4)))))
def testArgwhereExecution(self):
x = arange(6, chunk_size=2).reshape(2, 3)
t = argwhere(x > 1)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.argwhere(np.arange(6).reshape(2, 3) > 1)
np.testing.assert_array_equal(res, expected)
data = np.asfortranarray(np.random.rand(10, 20))
x = tensor(data, chunk_size=10)
t = argwhere(x > 0.5)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.argwhere(data > 0.5)
np.testing.assert_array_equal(res, expected)
self.assertTrue(res.flags['F_CONTIGUOUS'])
self.assertFalse(res.flags['C_CONTIGUOUS'])
def testArraySplitExecution(self):
x = arange(48, chunk_size=3).reshape(2, 3, 8)
ss = array_split(x, 3, axis=2)
res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss]
expected = np.array_split(np.arange(48).reshape(2, 3, 8), 3, axis=2)
self.assertEqual(len(res), len(expected))
[np.testing.assert_equal(r, e) for r, e in zip(res, expected)]
ss = array_split(x, [3, 5, 6, 10], axis=2)
res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss]
expected = np.array_split(np.arange(48).reshape(2, 3, 8),
[3, 5, 6, 10],
axis=2)
self.assertEqual(len(res), len(expected))
[np.testing.assert_equal(r, e) for r, e in zip(res, expected)]
def testSplitExecution(self):
x = arange(48, chunk_size=3).reshape(2, 3, 8)
ss = split(x, 4, axis=2)
res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss]
expected = np.split(np.arange(48).reshape(2, 3, 8), 4, axis=2)
self.assertEqual(len(res), len(expected))
[np.testing.assert_equal(r, e) for r, e in zip(res, expected)]
ss = split(x, [3, 5, 6, 10], axis=2)
res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss]
expected = np.split(np.arange(48).reshape(2, 3, 8), [3, 5, 6, 10],
axis=2)
self.assertEqual(len(res), len(expected))
[np.testing.assert_equal(r, e) for r, e in zip(res, expected)]
# hsplit
x = arange(120, chunk_size=3).reshape(2, 12, 5)
ss = hsplit(x, 4)
res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss]
expected = np.hsplit(np.arange(120).reshape(2, 12, 5), 4)
self.assertEqual(len(res), len(expected))
[np.testing.assert_equal(r, e) for r, e in zip(res, expected)]
# vsplit
x = arange(48, chunk_size=3).reshape(8, 3, 2)
ss = vsplit(x, 4)
res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss]
expected = np.vsplit(np.arange(48).reshape(8, 3, 2), 4)
self.assertEqual(len(res), len(expected))
[np.testing.assert_equal(r, e) for r, e in zip(res, expected)]
# dsplit
x = arange(48, chunk_size=3).reshape(2, 3, 8)
ss = dsplit(x, 4)
res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss]
expected = np.dsplit(np.arange(48).reshape(2, 3, 8), 4)
self.assertEqual(len(res), len(expected))
[np.testing.assert_equal(r, e) for r, e in zip(res, expected)]
x_data = sps.random(12, 8, density=.1)
x = tensor(x_data, chunk_size=3)
ss = split(x, 4, axis=0)
res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss]
expected = np.split(x_data.toarray(), 4, axis=0)
self.assertEqual(len(res), len(expected))
[
np.testing.assert_equal(r.toarray(), e)
for r, e in zip(res, expected)
]
def testRollExecution(self):
x = arange(10, chunk_size=2)
t = roll(x, 2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.roll(np.arange(10), 2)
np.testing.assert_equal(res, expected)
x2 = x.reshape(2, 5)
t = roll(x2, 1)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.roll(np.arange(10).reshape(2, 5), 1)
np.testing.assert_equal(res, expected)
t = roll(x2, 1, axis=0)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.roll(np.arange(10).reshape(2, 5), 1, axis=0)
np.testing.assert_equal(res, expected)
t = roll(x2, 1, axis=1)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.roll(np.arange(10).reshape(2, 5), 1, axis=1)
np.testing.assert_equal(res, expected)
def testSqueezeExecution(self):
data = np.array([[[0], [1], [2]]])
x = tensor(data, chunk_size=1)
t = squeeze(x)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.squeeze(data)
np.testing.assert_equal(res, expected)
t = squeeze(x, axis=2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.squeeze(data, axis=2)
np.testing.assert_equal(res, expected)
def testDiffExecution(self):
data = np.array([1, 2, 4, 7, 0])
x = tensor(data, chunk_size=2)
t = diff(x)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.diff(data)
np.testing.assert_equal(res, expected)
t = diff(x, n=2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.diff(data, n=2)
np.testing.assert_equal(res, expected)
data = np.array([[1, 3, 6, 10], [0, 5, 6, 8]])
x = tensor(data, chunk_size=2)
t = diff(x)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.diff(data)
np.testing.assert_equal(res, expected)
t = diff(x, axis=0)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.diff(data, axis=0)
np.testing.assert_equal(res, expected)
x = mt.arange('1066-10-13', '1066-10-16', dtype=mt.datetime64)
t = diff(x)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.diff(
np.arange('1066-10-13', '1066-10-16', dtype=np.datetime64))
np.testing.assert_equal(res, expected)
def testEdiff1d(self):
data = np.array([1, 2, 4, 7, 0])
x = tensor(data, chunk_size=2)
t = ediff1d(x)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.ediff1d(data)
np.testing.assert_equal(res, expected)
to_begin = tensor(-99, chunk_size=2)
to_end = tensor([88, 99], chunk_size=2)
t = ediff1d(x, to_begin=to_begin, to_end=to_end)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.ediff1d(data, to_begin=-99, to_end=np.array([88, 99]))
np.testing.assert_equal(res, expected)
data = [[1, 2, 4], [1, 6, 24]]
t = ediff1d(tensor(data, chunk_size=2))
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.ediff1d(data)
np.testing.assert_equal(res, expected)
def testFlipExecution(self):
a = arange(8, chunk_size=2).reshape((2, 2, 2))
t = flip(a, 0)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.flip(np.arange(8).reshape(2, 2, 2), 0)
np.testing.assert_equal(res, expected)
t = flip(a, 1)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.flip(np.arange(8).reshape(2, 2, 2), 1)
np.testing.assert_equal(res, expected)
t = flipud(a)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.flipud(np.arange(8).reshape(2, 2, 2))
np.testing.assert_equal(res, expected)
t = fliplr(a)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.fliplr(np.arange(8).reshape(2, 2, 2))
np.testing.assert_equal(res, expected)
def testRepeatExecution(self):
a = repeat(3, 4)
res = self.executor.execute_tensor(a)[0]
expected = np.repeat(3, 4)
np.testing.assert_equal(res, expected)
x_data = np.random.randn(20, 30)
x = tensor(x_data, chunk_size=(3, 4))
t = repeat(x, 2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.repeat(x_data, 2)
np.testing.assert_equal(res, expected)
t = repeat(x, 3, axis=1)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.repeat(x_data, 3, axis=1)
np.testing.assert_equal(res, expected)
t = repeat(x, np.arange(20), axis=0)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.repeat(x_data, np.arange(20), axis=0)
np.testing.assert_equal(res, expected)
t = repeat(x, arange(20, chunk_size=5), axis=0)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.repeat(x_data, np.arange(20), axis=0)
np.testing.assert_equal(res, expected)
x_data = sps.random(20, 30, density=.1)
x = tensor(x_data, chunk_size=(3, 4))
t = repeat(x, 2, axis=1)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.repeat(x_data.toarray(), 2, axis=1)
np.testing.assert_equal(res.toarray(), expected)
def testTileExecution(self):
a_data = np.array([0, 1, 2])
a = tensor(a_data, chunk_size=2)
t = tile(a, 2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.tile(a_data, 2)
np.testing.assert_equal(res, expected)
t = tile(a, (2, 2))
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.tile(a_data, (2, 2))
np.testing.assert_equal(res, expected)
t = tile(a, (2, 1, 2))
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.tile(a_data, (2, 1, 2))
np.testing.assert_equal(res, expected)
b_data = np.array([[1, 2], [3, 4]])
b = tensor(b_data, chunk_size=1)
t = tile(b, 2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.tile(b_data, 2)
np.testing.assert_equal(res, expected)
t = tile(b, (2, 1))
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.tile(b_data, (2, 1))
np.testing.assert_equal(res, expected)
c_data = np.array([1, 2, 3, 4])
c = tensor(c_data, chunk_size=3)
t = tile(c, (4, 1))
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.tile(c_data, (4, 1))
np.testing.assert_equal(res, expected)
def testIsInExecution(self):
element = 2 * arange(4, chunk_size=1).reshape((2, 2))
test_elements = [1, 2, 4, 8]
mask = isin(element, test_elements)
res = self.executor.execute_tensor(mask, concat=True)[0]
expected = np.isin(2 * np.arange(4).reshape((2, 2)), test_elements)
np.testing.assert_equal(res, expected)
res = self.executor.execute_tensor(element[mask], concat=True)[0]
expected = np.array([2, 4])
np.testing.assert_equal(res, expected)
mask = isin(element, test_elements, invert=True)
res = self.executor.execute_tensor(mask, concat=True)[0]
expected = np.isin(2 * np.arange(4).reshape((2, 2)),
test_elements,
invert=True)
np.testing.assert_equal(res, expected)
res = self.executor.execute_tensor(element[mask], concat=True)[0]
expected = np.array([0, 6])
np.testing.assert_equal(res, expected)
test_set = {1, 2, 4, 8}
mask = isin(element, test_set)
res = self.executor.execute_tensor(mask, concat=True)[0]
expected = np.isin(2 * np.arange(4).reshape((2, 2)), test_set)
np.testing.assert_equal(res, expected)
def testRavelExecution(self):
arr = ones((10, 5), chunk_size=2)
flat_arr = mt.ravel(arr)
res = self.executor.execute_tensor(flat_arr, concat=True)[0]
self.assertEqual(len(res), 50)
np.testing.assert_equal(res, np.ones(50))
def testSearchsortedExecution(self):
raw = np.sort(np.random.randint(100, size=(16, )))
# test different chunk_size, 3 will have combine, 6 will skip combine
for chunk_size in (3, 6):
arr = tensor(raw, chunk_size=chunk_size)
# test scalar, with value in the middle
t1 = searchsorted(arr, 20)
res = self.executor.execute_tensor(t1, concat=True)[0]
expected = np.searchsorted(raw, 20)
np.testing.assert_array_equal(res, expected)
# test scalar, with value larger than 100
t2 = searchsorted(arr, 200)
res = self.executor.execute_tensor(t2, concat=True)[0]
expected = np.searchsorted(raw, 200)
np.testing.assert_array_equal(res, expected)
# test scalar, side left, with value exact in the middle of the array
t3 = searchsorted(arr, raw[10], side='left')
res = self.executor.execute_tensor(t3, concat=True)[0]
expected = np.searchsorted(raw, raw[10], side='left')
np.testing.assert_array_equal(res, expected)
# test scalar, side right, with value exact in the middle of the array
t4 = searchsorted(arr, raw[10], side='right')
res = self.executor.execute_tensor(t4, concat=True)[0]
expected = np.searchsorted(raw, raw[10], side='right')
np.testing.assert_array_equal(res, expected)
# test scalar, side left, with value exact in the end of the array
t5 = searchsorted(arr, raw[15], side='left')
res = self.executor.execute_tensor(t5, concat=True)[0]
expected = np.searchsorted(raw, raw[15], side='left')
np.testing.assert_array_equal(res, expected)
# test scalar, side right, with value exact in the end of the array
t6 = searchsorted(arr, raw[15], side='right')
res = self.executor.execute_tensor(t6, concat=True)[0]
expected = np.searchsorted(raw, raw[15], side='right')
np.testing.assert_array_equal(res, expected)
# test scalar, side left, with value exact in the start of the array
t7 = searchsorted(arr, raw[0], side='left')
res = self.executor.execute_tensor(t7, concat=True)[0]
expected = np.searchsorted(raw, raw[0], side='left')
np.testing.assert_array_equal(res, expected)
# test scalar, side right, with value exact in the start of the array
t8 = searchsorted(arr, raw[0], side='right')
res = self.executor.execute_tensor(t8, concat=True)[0]
expected = np.searchsorted(raw, raw[0], side='right')
np.testing.assert_array_equal(res, expected)
raw2 = np.random.randint(100, size=(3, 4))
# test tensor, side left
t9 = searchsorted(arr, tensor(raw2, chunk_size=2), side='left')
res = self.executor.execute_tensor(t9, concat=True)[0]
expected = np.searchsorted(raw, raw2, side='left')
np.testing.assert_array_equal(res, expected)
# test tensor, side right
t10 = searchsorted(arr, tensor(raw2, chunk_size=2), side='right')
res = self.executor.execute_tensor(t10, concat=True)[0]
expected = np.searchsorted(raw, raw2, side='right')
np.testing.assert_array_equal(res, expected)
# test one chunk
arr = tensor(raw, chunk_size=16)
# test scalar, tensor to search has 1 chunk
t11 = searchsorted(arr, 20)
res = self.executor.execute_tensor(t11, concat=True)[0]
expected = np.searchsorted(raw, 20)
np.testing.assert_array_equal(res, expected)
# test tensor with 1 chunk, tensor to search has 1 chunk
t12 = searchsorted(arr, tensor(raw2, chunk_size=4))
res = self.executor.execute_tensor(t12, concat=True)[0]
expected = np.searchsorted(raw, raw2)
np.testing.assert_array_equal(res, expected)
# test tensor with more than 1 chunk, tensor to search has 1 chunk
t13 = searchsorted(arr, tensor(raw2, chunk_size=2))
res = self.executor.execute_tensor(t13, concat=True)[0]
expected = np.searchsorted(raw, raw2)
np.testing.assert_array_equal(res, expected)
# test sorter
raw3 = np.random.randint(100, size=(16, ))
arr = tensor(raw3, chunk_size=3)
order = np.argsort(raw3)
order_arr = tensor(order, chunk_size=4)
t14 = searchsorted(arr, 20, sorter=order_arr)
res = self.executor.execute_tensor(t14, concat=True)[0]
expected = np.searchsorted(raw3, 20, sorter=order)
np.testing.assert_array_equal(res, expected)
def testUniqueExecution(self):
rs = np.random.RandomState(0)
raw = rs.randint(10, size=(10, ))
for chunk_size in (10, 3):
x = tensor(raw, chunk_size=chunk_size)
y = unique(x)
res = self.executor.execute_tensor(y, concat=True)[0]
expected = np.unique(raw)
np.testing.assert_array_equal(res, expected)
y, indices = unique(x, return_index=True)
res = self.executor.execute_tensors([y, indices])
expected = np.unique(raw, return_index=True)
self.assertEqual(len(res), 2)
self.assertEqual(len(expected), 2)
np.testing.assert_array_equal(res[0], expected[0])
np.testing.assert_array_equal(res[1], expected[1])
y, inverse = unique(x, return_inverse=True)
res = self.executor.execute_tensors([y, inverse])
expected = np.unique(raw, return_inverse=True)
self.assertEqual(len(res), 2)
self.assertEqual(len(expected), 2)
np.testing.assert_array_equal(res[0], expected[0])
np.testing.assert_array_equal(res[1], expected[1])
y, counts = unique(x, return_counts=True)
res = self.executor.execute_tensors([y, counts])
expected = np.unique(raw, return_counts=True)
self.assertEqual(len(res), 2)
self.assertEqual(len(expected), 2)
np.testing.assert_array_equal(res[0], expected[0])
np.testing.assert_array_equal(res[1], expected[1])
y, indices, inverse, counts = unique(x,
return_index=True,
return_inverse=True,
return_counts=True)
res = self.executor.execute_tensors([y, indices, inverse, counts])
expected = np.unique(raw,
return_index=True,
return_inverse=True,
return_counts=True)
self.assertEqual(len(res), 4)
self.assertEqual(len(expected), 4)
np.testing.assert_array_equal(res[0], expected[0])
np.testing.assert_array_equal(res[1], expected[1])
np.testing.assert_array_equal(res[2], expected[2])
np.testing.assert_array_equal(res[3], expected[3])
y, indices, counts = unique(x,
return_index=True,
return_counts=True)
res = self.executor.execute_tensors([y, indices, counts])
expected = np.unique(raw, return_index=True, return_counts=True)
self.assertEqual(len(res), 3)
self.assertEqual(len(expected), 3)
np.testing.assert_array_equal(res[0], expected[0])
np.testing.assert_array_equal(res[1], expected[1])
np.testing.assert_array_equal(res[2], expected[2])
raw2 = rs.randint(10, size=(4, 5, 6))
x2 = tensor(raw2, chunk_size=chunk_size)
y2 = unique(x2)
res = self.executor.execute_tensor(y2, concat=True)[0]
expected = np.unique(raw2)
np.testing.assert_array_equal(res, expected)
y2 = unique(x2, axis=1)
res = self.executor.execute_tensor(y2, concat=True)[0]
expected = np.unique(raw2, axis=1)
np.testing.assert_array_equal(res, expected)
y2 = unique(x2, axis=2)
res = self.executor.execute_tensor(y2, concat=True)[0]
expected = np.unique(raw2, axis=2)
np.testing.assert_array_equal(res, expected)
raw = rs.randint(10, size=(10, 20))
raw[:, 0] = raw[:, 11] = rs.randint(10, size=(10, ))
x = tensor(raw, chunk_size=2)
y, ind, inv, counts = unique(x,
aggregate_size=3,
axis=1,
return_index=True,
return_inverse=True,
return_counts=True)
res_unique, res_ind, res_inv, res_counts = self.executor.execute_tensors(
(y, ind, inv, counts))
exp_unique, exp_ind, exp_counts = np.unique(raw,
axis=1,
return_index=True,
return_counts=True)
raw_res_unique = res_unique
res_unique_df = pd.DataFrame(res_unique)
res_unique_ind = np.asarray(
res_unique_df.sort_values(list(range(res_unique.shape[0])),
axis=1).columns)
res_unique = res_unique[:, res_unique_ind]
res_ind = res_ind[res_unique_ind]
res_counts = res_counts[res_unique_ind]
np.testing.assert_array_equal(res_unique, exp_unique)
np.testing.assert_array_equal(res_ind, exp_ind)
np.testing.assert_array_equal(raw_res_unique[:, res_inv], raw)
np.testing.assert_array_equal(res_counts, exp_counts)
x = (mt.random.RandomState(0).rand(1000, chunk_size=20) > 0.5).astype(
np.int32)
y = unique(x)
res = np.sort(self.executor.execute_tensor(y, concat=True)[0])
np.testing.assert_array_equal(res, np.array([0, 1]))
@require_cupy
def testToGPUExecution(self):
raw = np.random.rand(10, 10)
x = tensor(raw, chunk_size=3)
gx = to_gpu(x)
res = self.executor.execute_tensor(gx, concat=True)[0]
np.testing.assert_array_equal(res.get(), raw)
@require_cupy
def testToCPUExecution(self):
raw = np.random.rand(10, 10)
x = tensor(raw, chunk_size=3, gpu=True)
cx = to_cpu(x)
res = self.executor.execute_tensor(cx, concat=True)[0]
np.testing.assert_array_equal(res, raw)
def testSortExecution(self):
# only 1 chunk when axis = -1
raw = np.random.rand(100, 10)
x = tensor(raw, chunk_size=10)
sx = sort(x)
res = self.executor.execute_tensor(sx, concat=True)[0]
np.testing.assert_array_equal(res, np.sort(raw))
# 1-d chunk
raw = np.random.rand(100)
x = tensor(raw, chunk_size=10)
sx = sort(x)
res = self.executor.execute_tensor(sx, concat=True)[0]
np.testing.assert_array_equal(res, np.sort(raw))
# test force need_align=True
sx = sort(x)
sx.op._need_align = True
res = self.executor.execute_tensor(sx, concat=True)[0]
self.assertEqual(get_tiled(sx).nsplits, get_tiled(x).nsplits)
np.testing.assert_array_equal(res, np.sort(raw))
# test psrs_kinds
sx = sort(x, psrs_kinds=[None, None, 'quicksort'])
res = self.executor.execute_tensor(sx, concat=True)[0]
np.testing.assert_array_equal(res, np.sort(raw))
# structured dtype
raw = np.empty(100, dtype=[('id', np.int32), ('size', np.int64)])
raw['id'] = np.random.randint(1000, size=100, dtype=np.int32)
raw['size'] = np.random.randint(1000, size=100, dtype=np.int64)
x = tensor(raw, chunk_size=10)
sx = sort(x, order=['size', 'id'])
res = self.executor.execute_tensor(sx, concat=True)[0]
np.testing.assert_array_equal(res, np.sort(raw, order=['size', 'id']))
# test psrs_kinds with structured dtype
sx = sort(x,
order=['size', 'id'],
psrs_kinds=[None, None, 'quicksort'])
res = self.executor.execute_tensor(sx, concat=True)[0]
np.testing.assert_array_equal(res, np.sort(raw, order=['size', 'id']))
# test flatten case
raw = np.random.rand(10, 10)
x = tensor(raw, chunk_size=5)
sx = sort(x, axis=None)
res = self.executor.execute_tensor(sx, concat=True)[0]
np.testing.assert_array_equal(res, np.sort(raw, axis=None))
# test multi-dimension
raw = np.random.rand(10, 100)
x = tensor(raw, chunk_size=(2, 10))
sx = sort(x, psrs_kinds=['quicksort'] * 3)
res = self.executor.execute_tensor(sx, concat=True)[0]
np.testing.assert_array_equal(res, np.sort(raw))
sx = sort(x, psrs_kinds=[None, None, 'quicksort'])
res = self.executor.execute_tensor(sx, concat=True)[0]
np.testing.assert_array_equal(res, np.sort(raw))
raw = np.random.rand(10, 99)
x = tensor(raw, chunk_size=(2, 10))
sx = sort(x)
res = self.executor.execute_tensor(sx, concat=True)[0]
np.testing.assert_array_equal(res, np.sort(raw))
# test 3-d
raw = np.random.rand(20, 25, 28)
x = tensor(raw, chunk_size=(10, 5, 7))
sx = sort(x)
res = self.executor.execute_tensor(sx, concat=True)[0]
np.testing.assert_array_equal(res, np.sort(raw))
sx = sort(x, psrs_kinds=[None, None, 'quicksort'])
res = self.executor.execute_tensor(sx, concat=True)[0]
np.testing.assert_array_equal(res, np.sort(raw))
sx = sort(x, axis=0)
res = self.executor.execute_tensor(sx, concat=True)[0]
np.testing.assert_array_equal(res, np.sort(raw, axis=0))
sx = sort(x, axis=0, psrs_kinds=[None, None, 'quicksort'])
res = self.executor.execute_tensor(sx, concat=True)[0]
np.testing.assert_array_equal(res, np.sort(raw, axis=0))
sx = sort(x, axis=1)
res = self.executor.execute_tensor(sx, concat=True)[0]
np.testing.assert_array_equal(res, np.sort(raw, axis=1))
sx = sort(x, axis=1, psrs_kinds=[None, None, 'quicksort'])
res = self.executor.execute_tensor(sx, concat=True)[0]
np.testing.assert_array_equal(res, np.sort(raw, axis=1))
# test multi-dimension with structured type
raw = np.empty((10, 100), dtype=[('id', np.int32), ('size', np.int64)])
raw['id'] = np.random.randint(1000, size=(10, 100), dtype=np.int32)
raw['size'] = np.random.randint(1000, size=(10, 100), dtype=np.int64)
x = tensor(raw, chunk_size=(3, 10))
sx = sort(x)
res = self.executor.execute_tensor(sx, concat=True)[0]
np.testing.assert_array_equal(res, np.sort(raw))
sx = sort(x, order=['size', 'id'])
res = self.executor.execute_tensor(sx, concat=True)[0]
np.testing.assert_array_equal(res, np.sort(raw, order=['size', 'id']))
sx = sort(x, order=['size'])
res = self.executor.execute_tensor(sx, concat=True)[0]
np.testing.assert_array_equal(res, np.sort(raw, order=['size']))
sx = sort(x, axis=0, order=['size', 'id'])
res = self.executor.execute_tensor(sx, concat=True)[0]
np.testing.assert_array_equal(
res, np.sort(raw, axis=0, order=['size', 'id']))
sx = sort(x,
axis=0,
order=['size', 'id'],
psrs_kinds=[None, None, 'quicksort'])
res = self.executor.execute_tensor(sx, concat=True)[0]
np.testing.assert_array_equal(
res, np.sort(raw, axis=0, order=['size', 'id']))
raw = np.random.rand(10, 12)
a = tensor(raw, chunk_size=(5, 4))
a.sort(axis=1)
res = self.executor.execute_tensor(a, concat=True)[0]
np.testing.assert_array_equal(res, np.sort(raw, axis=1))
a.sort(axis=0)
res = self.executor.execute_tensor(a, concat=True)[0]
np.testing.assert_array_equal(res, np.sort(np.sort(raw, axis=1),
axis=0))
def testPartitionExecution(self):
# only 1 chunk when axis = -1
raw = np.random.rand(100, 10)
x = tensor(raw, chunk_size=10)
px = partition(x, [1, 8])
res = self.executor.execute_tensor(px, concat=True)[0]
np.testing.assert_array_equal(res, np.partition(raw, [1, 8]))
# 1-d chunk
raw = np.random.rand(100)
x = tensor(raw, chunk_size=10)
kth = np.random.RandomState(0).randint(-100, 100, size=(10, ))
px = partition(x, kth)
res = self.executor.execute_tensor(px, concat=True)[0]
np.testing.assert_array_equal(res[kth], np.partition(raw, kth)[kth])
# structured dtype
raw = np.empty(100, dtype=[('id', np.int32), ('size', np.int64)])
raw['id'] = np.random.randint(1000, size=100, dtype=np.int32)
raw['size'] = np.random.randint(1000, size=100, dtype=np.int64)
x = tensor(raw, chunk_size=10)
px = partition(x, kth, order=['size', 'id'])
res = self.executor.execute_tensor(px, concat=True)[0]
np.testing.assert_array_equal(
res[kth],
np.partition(raw, kth, order=['size', 'id'])[kth])
# test flatten case
raw = np.random.rand(10, 10)
x = tensor(raw, chunk_size=5)
px = partition(x, kth, axis=None)
res = self.executor.execute_tensor(px, concat=True)[0]
np.testing.assert_array_equal(res[kth],
np.partition(raw, kth, axis=None)[kth])
# test multi-dimension
raw = np.random.rand(10, 100)
x = tensor(raw, chunk_size=(2, 10))
kth = np.random.RandomState(0).randint(-10, 10, size=(3, ))
px = partition(x, kth)
res = self.executor.execute_tensor(px, concat=True)[0]
np.testing.assert_array_equal(res[:, kth],
np.partition(raw, kth)[:, kth])
raw = np.random.rand(10, 99)
x = tensor(raw, chunk_size=(2, 10))
px = partition(x, kth)
res = self.executor.execute_tensor(px, concat=True)[0]
np.testing.assert_array_equal(res[:, kth],
np.partition(raw, kth)[:, kth])
# test 3-d
raw = np.random.rand(20, 25, 28)
x = tensor(raw, chunk_size=(10, 5, 7))
kth = np.random.RandomState(0).randint(-28, 28, size=(3, ))
px = partition(x, kth)
res = self.executor.execute_tensor(px, concat=True)[0]
np.testing.assert_array_equal(res[:, :, kth],
np.partition(raw, kth)[:, :, kth])
kth = np.random.RandomState(0).randint(-20, 20, size=(3, ))
px = partition(x, kth, axis=0)
res = self.executor.execute_tensor(px, concat=True)[0]
np.testing.assert_array_equal(res[kth],
np.partition(raw, kth, axis=0)[kth])
kth = np.random.RandomState(0).randint(-25, 25, size=(3, ))
px = partition(x, kth, axis=1)
res = self.executor.execute_tensor(px, concat=True)[0]
np.testing.assert_array_equal(res[:, kth],
np.partition(raw, kth, axis=1)[:, kth])
# test multi-dimension with structured type
raw = np.empty((10, 100), dtype=[('id', np.int32), ('size', np.int64)])
raw['id'] = np.random.randint(1000, size=(10, 100), dtype=np.int32)
raw['size'] = np.random.randint(1000, size=(10, 100), dtype=np.int64)
x = tensor(raw, chunk_size=(3, 10))
kth = np.random.RandomState(0).randint(-100, 100, size=(10, ))
px = partition(x, kth)
res = self.executor.execute_tensor(px, concat=True)[0]
np.testing.assert_array_equal(res[:, kth],
np.partition(raw, kth)[:, kth])
px = partition(x, kth, order=['size', 'id'])
res = self.executor.execute_tensor(px, concat=True)[0]
np.testing.assert_array_equal(
res[:, kth],
np.partition(raw, kth, order=['size', 'id'])[:, kth])
px = partition(x, kth, order=['size'])
res = self.executor.execute_tensor(px, concat=True)[0]
np.testing.assert_array_equal(
res[:, kth],
np.partition(raw, kth, order=['size'])[:, kth])
kth = np.random.RandomState(0).randint(-10, 10, size=(5, ))
px = partition(x, kth, axis=0, order=['size', 'id'])
res = self.executor.execute_tensor(px, concat=True)[0]
np.testing.assert_array_equal(
res[kth],
np.partition(raw, kth, axis=0, order=['size', 'id'])[kth])
raw = np.random.rand(10, 12)
a = tensor(raw, chunk_size=(5, 4))
kth = np.random.RandomState(0).randint(-12, 12, size=(2, ))
a.partition(kth, axis=1)
res = self.executor.execute_tensor(a, concat=True)[0]
np.testing.assert_array_equal(res[:, kth],
np.partition(raw, kth, axis=1)[:, kth])
kth = np.random.RandomState(0).randint(-10, 10, size=(2, ))
a.partition(kth, axis=0)
raw_base = res
res = self.executor.execute_tensor(a, concat=True)[0]
np.testing.assert_array_equal(res[kth],
np.partition(raw_base, kth, axis=0)[kth])
# test kth which is tensor
raw = np.random.rand(10, 12)
a = tensor(raw, chunk_size=(3, 5))
kth = (mt.random.rand(5) * 24 - 12).astype(int)
px = partition(a, kth)
sx = sort(a)
res = self.executor.execute_tensor(px, concat=True)[0]
kth_res = self.executor.execute_tensor(kth, concat=True)[0]
sort_res = self.executor.execute_tensor(sx, concat=True)[0]
np.testing.assert_array_equal(res[:, kth_res], sort_res[:, kth_res])
a = tensor(raw, chunk_size=(10, 12))
kth = (mt.random.rand(5) * 24 - 12).astype(int)
px = partition(a, kth)
sx = sort(a)
res = self.executor.execute_tensor(px, concat=True)[0]
kth_res = self.executor.execute_tensor(kth, concat=True)[0]
sort_res = self.executor.execute_tensor(sx, concat=True)[0]
np.testing.assert_array_equal(res[:, kth_res], sort_res[:, kth_res])
