def testAllAnyExecution(self):
raw1 = np.zeros((10, 15))
raw2 = np.ones((10, 15))
raw3 = np.array([[True, False, True, False], [True, True, True, True],
[False, False, False, False],
[False, True, False, True]])
arr1 = tensor(raw1, chunks=3)
arr2 = tensor(raw2, chunks=3)
arr3 = tensor(raw3, chunks=4)
self.assertFalse(self.executor.execute_tensor(arr1.all())[0])
self.assertTrue(self.executor.execute_tensor(arr2.all())[0])
self.assertFalse(self.executor.execute_tensor(arr1.any())[0])
self.assertTrue(self.executor.execute_tensor(arr1.any()))
self.assertTrue(
np.array_equal(raw3.all(axis=1),
self.executor.execute_tensor(arr3.all(axis=1))[0]))
self.assertTrue(
np.array_equal(raw3.any(axis=0),
self.executor.execute_tensor(arr3.any(axis=0))[0]))
raw = sps.random(10, 10, density=.5) > .5
arr = tensor(raw, chunks=3)
self.assertEqual(raw.A.all(),
self.executor.execute_tensor(arr.all())[0])
self.assertEqual(raw.A.any(),
self.executor.execute_tensor(arr.any())[0])
def testAverageExecution(self):
data = arange(1, 5, chunk_size=1)
t = average(data)
res = self.executor.execute_tensor(t)[0]
expected = np.average(np.arange(1, 5))
self.assertEqual(res, expected)
t = average(arange(1, 11, chunk_size=2),
weights=arange(10, 0, -1, chunk_size=2))
res = self.executor.execute_tensor(t)[0]
expected = np.average(range(1, 11), weights=range(10, 0, -1))
self.assertEqual(res, expected)
data = arange(6, chunk_size=2).reshape((3, 2))
t = average(data,
axis=1,
weights=tensor([1. / 4, 3. / 4], chunk_size=2))
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.average(np.arange(6).reshape(3, 2),
axis=1,
weights=(1. / 4, 3. / 4))
np.testing.assert_equal(res, expected)
with self.assertRaises(TypeError):
average(data, weights=tensor([1. / 4, 3. / 4], chunk_size=2))
def testMaxMinExecution(self):
raw = np.random.randint(10000, size=(10, 10, 10))
arr = tensor(raw, chunks=3)
self.assertEqual([raw.max()], self.executor.execute_tensor(arr.max()))
self.assertEqual([raw.min()], self.executor.execute_tensor(arr.min()))
self.assertTrue(
np.array_equal(
raw.max(axis=0),
self.executor.execute_tensor(arr.max(axis=0), concat=True)[0]))
self.assertTrue(
np.array_equal(
raw.min(axis=0),
self.executor.execute_tensor(arr.min(axis=0), concat=True)[0]))
self.assertTrue(
np.array_equal(
raw.max(axis=(1, 2)),
self.executor.execute_tensor(arr.max(axis=(1, 2)),
concat=True)[0]))
self.assertTrue(
np.array_equal((raw.min(axis=(1, 2))),
self.executor.execute_tensor(arr.min(axis=(1, 2)),
concat=True)[0]))
raw = sps.random(10, 10, density=.5)
arr = tensor(raw, chunks=3)
self.assertEqual([raw.max()], self.executor.execute_tensor(arr.max()))
self.assertEqual([raw.min()], self.executor.execute_tensor(arr.min()))
def testBinExecution(self):
from mars.tensor.expressions.arithmetic import BIN_UFUNC, mod, fmod, \
bitand, bitor, bitxor, lshift, rshift, ldexp
_sp_bin_ufunc = [mod, fmod, bitand, bitor, bitxor, lshift, rshift]
_new_bin_ufunc = list(BIN_UFUNC - set(_sp_bin_ufunc) - set([ldexp]))
executor_numexpr = Executor()
for i, j in itertools.permutations(range(len(_new_bin_ufunc)), 2):
raw = np.random.random((9, 9, 9))
arr1 = tensor(raw, chunks=5)
func1 = _new_bin_ufunc[i]
func2 = _new_bin_ufunc[j]
arr2 = func1(1, func2(2, arr1))
res = executor_numexpr.execute_tensor(arr2, concat=True)
res_cmp = self.executor.execute_tensor(arr2, concat=True)
self.assertTrue(np.allclose(res[0], res_cmp[0]))
for i, j in itertools.permutations(range(len(_sp_bin_ufunc)), 2):
raw = np.random.randint(1, 100, size=(10, 10, 10))
arr1 = tensor(raw, chunks=3)
func1 = _sp_bin_ufunc[i]
func2 = _sp_bin_ufunc[j]
arr2 = func1(10, func2(arr1, 5))
res = executor_numexpr.execute_tensor(arr2, concat=True)
res_cmp = self.executor.execute_tensor(arr2, concat=True)
self.assertTrue(np.allclose(res[0], res_cmp[0]))
def testCreateSparseExecution(self):
mat = sps.csr_matrix([[0, 0, 2], [2, 0, 0]])
t = tensor(mat, dtype='f8', chunk_size=2)
res = self.executor.execute_tensor(t)
self.assertIsInstance(res[0], SparseNDArray)
self.assertEqual(res[0].dtype, np.float64)
np.testing.assert_array_equal(res[0].toarray(), mat[..., :2].toarray())
np.testing.assert_array_equal(res[1].toarray(), mat[..., 2:].toarray())
t2 = ones_like(t, dtype='f4')
res = self.executor.execute_tensor(t2)
expected = sps.csr_matrix([[0, 0, 1], [1, 0, 0]])
self.assertIsInstance(res[0], SparseNDArray)
self.assertEqual(res[0].dtype, np.float32)
np.testing.assert_array_equal(res[0].toarray(), expected[..., :2].toarray())
np.testing.assert_array_equal(res[1].toarray(), expected[..., 2:].toarray())
t3 = tensor(np.array([[0, 0, 2], [2, 0, 0]]), chunk_size=2).tosparse()
res = self.executor.execute_tensor(t3)
self.assertIsInstance(res[0], SparseNDArray)
self.assertEqual(res[0].dtype, np.int_)
np.testing.assert_array_equal(res[0].toarray(), mat[..., :2].toarray())
np.testing.assert_array_equal(res[1].toarray(), mat[..., 2:].toarray())
def testTensordotExecution(self):
a_data = np.arange(60).reshape(3, 4, 5)
a = tensor(a_data, chunk_size=2)
b_data = np.arange(24).reshape(4, 3, 2)
b = tensor(b_data, chunk_size=2)
axes = ([1, 0], [0, 1])
c = tensordot(a, b, axes=axes)
res = self.executor.execute_tensor(c)
expected = np.tensordot(a_data, b_data, axes=axes)
self.assertTrue(np.array_equal(res[0], expected[:2, :]))
self.assertTrue(np.array_equal(res[1], expected[2:4, :]))
self.assertTrue(np.array_equal(res[2], expected[4:, :]))
a = ones((1000, 2000), chunk_size=500)
b = ones((2000, 100), chunk_size=500)
c = dot(a, b)
res = self.executor.execute_tensor(c)
expected = np.dot(np.ones((1000, 2000)), np.ones((2000, 100)))
self.assertEqual(len(res), 2)
self.assertTrue(np.array_equal(res[0], expected[:500, :]))
self.assertTrue(np.array_equal(res[1], expected[500:, :]))
a = ones((10, 8), chunk_size=2)
b = ones((8, 10), chunk_size=2)
c = a.dot(b)
res = self.executor.execute_tensor(c)
self.assertEqual(len(res), 25)
for r in res:
self.assertTrue(np.array_equal(r, np.tile([8], [2, 2])))
a = ones((500, 500), chunk_size=500)
b = ones((500, 100), chunk_size=500)
c = a.dot(b)
res = self.executor.execute_tensor(c)
self.assertTrue(np.array_equal(res[0], np.tile([500], [500, 100])))
raw_a = np.random.random((100, 200, 50))
raw_b = np.random.random((200, 10, 100))
a = tensor(raw_a, chunk_size=50)
b = tensor(raw_b, chunk_size=33)
c = tensordot(a, b, axes=((0, 1), (2, 0)))
res = self.executor.execute_tensor(c, concat=True)
expected = np.tensordot(raw_a, raw_b, axes=(c.op.a_axes, c.op.b_axes))
self.assertTrue(np.allclose(res[0], expected))
a = ones((1000, 2000), chunk_size=500)
b = ones((100, 2000), chunk_size=500)
c = inner(a, b)
res = self.executor.execute_tensor(c)
expected = np.inner(np.ones((1000, 2000)), np.ones((100, 2000)))
self.assertEqual(len(res), 2)
self.assertTrue(np.array_equal(res[0], expected[:500, :]))
self.assertTrue(np.array_equal(res[1], expected[500:, :]))
a = ones((100, 100), chunk_size=30)
b = ones((100, 100), chunk_size=30)
c = a.dot(b)
res = self.executor.execute_tensor(c, concat=True)[0]
np.testing.assert_array_equal(res, np.ones((100, 100)) * 100)
def testNanCumReduction(self):
raw = np.random.randint(5, size=(8, 8, 8))
raw[:2, 2:4, 4:6] = np.nan
arr = tensor(raw, chunks=3)
res1 = self.executor.execute_tensor(nancumsum(arr, axis=1),
concat=True)
res2 = self.executor.execute_tensor(nancumprod(arr, axis=1),
concat=True)
expected1 = np.nancumsum(raw, axis=1)
expected2 = np.nancumprod(raw, axis=1)
self.assertTrue(np.array_equal(res1[0], expected1))
self.assertTrue(np.array_equal(res2[0], expected2))
raw = sps.random(8, 8, density=.1, format='lil')
raw[:2, 2:4] = np.nan
arr = tensor(raw, chunks=3)
res1 = self.executor.execute_tensor(nancumsum(arr, axis=1),
concat=True)[0]
res2 = self.executor.execute_tensor(nancumprod(arr, axis=1),
concat=True)[0]
expected1 = np.nancumsum(raw.A, axis=1)
expected2 = np.nancumprod(raw.A, axis=1)
np.testing.assert_equal(res1, expected1)
np.testing.assert_equal(res2, expected2)
def testArgReduction(self):
raw = np.random.random((20, 20, 20))
arr = tensor(raw, chunks=3)
self.assertEqual(raw.argmax(),
self.executor.execute_tensor(arr.argmax())[0])
self.assertEqual(raw.argmin(),
self.executor.execute_tensor(arr.argmin())[0])
self.assertTrue(
np.array_equal(
raw.argmax(axis=0),
self.executor.execute_tensor(arr.argmax(axis=0),
concat=True)[0]))
self.assertTrue(
np.array_equal(
raw.argmin(axis=0),
self.executor.execute_tensor(arr.argmin(axis=0),
concat=True)[0]))
raw = sps.random(20, 20, density=.1)
arr = tensor(raw, chunks=3)
self.assertEqual(raw.argmax(),
self.executor.execute_tensor(arr.argmax())[0])
self.assertEqual(raw.argmin(),
self.executor.execute_tensor(arr.argmin())[0])
def testNormExecution(self):
d = np.arange(9) - 4
d2 = d.reshape(3, 3)
ma = [tensor(d, chunk_size=2), tensor(d, chunk_size=9),
tensor(d2, chunk_size=(2, 3)), tensor(d2, chunk_size=3)]
for i, a in enumerate(ma):
data = d if i < 2 else d2
for ord in (None, 'nuc', np.inf, -np.inf, 0, 1, -1, 2, -2):
for axis in (0, 1, (0, 1)):
for keepdims in (True, False):
try:
expected = np.linalg.norm(data, ord=ord, axis=axis, keepdims=keepdims)
t = norm(a, ord=ord, axis=axis, keepdims=keepdims)
concat = t.ndim > 0
res = self.executor.execute_tensor(t, concat=concat)[0]
np.testing.assert_allclose(res, expected, atol=.0001)
except ValueError:
continue
m = norm(tensor(d))
expected = self.executor.execute_tensor(m)[0]
res = np.linalg.norm(d)
self.assertEqual(expected, res)
d = uniform(-0.5, 0.5, size=(500, 2), chunk_size=50)
inside = (norm(d, axis=1) < 0.5).sum().astype(float)
t = inside / 500 * 4
res = self.executor.execute_tensor(t)[0]
self.assertAlmostEqual(res, 3.14, delta=1)
def testCholeskyExecution(self):
data = np.array([[1, -2j], [2j, 5]])
a = tensor(data, chunk_size=1)
L = cholesky(a, lower=True)
t = L.dot(L.T.conj())
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, data))
L = cholesky(a, lower=True)
U = cholesky(a)
t = L.dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, data))
a = tensor(data, chunk_size=2)
L = cholesky(a, lower=True)
U = cholesky(a)
t = L.dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data)
a = tensor(data, chunk_size=(1, 2))
L = cholesky(a, lower=True)
U = cholesky(a)
t = L.dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data)
def testInv(self):
import scipy.linalg
np.random.seed(1)
data = np.random.randint(1, 10, (20, 20))
A = tensor(data)
inv_A = inv(A)
res = self.executor.execute_tensor(inv_A, concat=True)[0]
self.assertTrue(np.allclose(res, scipy.linalg.inv(data)))
res = self.executor.execute_tensor(A.dot(inv_A), concat=True)[0]
self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float)))
A = tensor(data, chunk_size=5)
inv_A = inv(A)
res = self.executor.execute_tensor(inv_A, concat=True)[0]
self.assertTrue(np.allclose(res, scipy.linalg.inv(data)))
res = self.executor.execute_tensor(A.dot(inv_A), concat=True)[0]
self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float)))
B = A.T.dot(A)
inv_B = inv(B)
res = self.executor.execute_tensor(inv_B, concat=True)[0]
self.assertTrue(np.allclose(res, scipy.linalg.inv(data.T.dot(data))))
res = self.executor.execute_tensor(B.dot(inv_B), concat=True)[0]
self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float)))
# test for not all chunks are square in matrix A
A = tensor(data, chunk_size=8)
inv_A = inv(A)
res = self.executor.execute_tensor(inv_A, concat=True)[0]
self.assertTrue(np.allclose(res, scipy.linalg.inv(data)))
res = self.executor.execute_tensor(A.dot(inv_A), concat=True)[0]
self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float)))
# test sparse
data = np.random.randint(1, 10, (20, 20))
sp_data = sps.csr_matrix(data)
A = tensor(sp_data, chunk_size=5)
inv_A = inv(A)
res = self.executor.execute_tensor(inv_A, concat=True)[0]
self.assertIsInstance(res, SparseNDArray)
self.assertTrue(np.allclose(res, scipy.linalg.inv(data)))
res = self.executor.execute_tensor(A.dot(inv_A), concat=True)[0]
self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float)))
# test for not all chunks are square in matrix A
A = tensor(sp_data, chunk_size=8)
inv_A = inv(A)
res = self.executor.execute_tensor(inv_A, concat=True)[0]
self.assertIsInstance(res, SparseNDArray)
self.assertTrue(np.allclose(res, scipy.linalg.inv(data)))
res = self.executor.execute_tensor(A.dot(inv_A), concat=True)[0]
self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float)))
def testSparseDotExecution(self):
a_data = sps.random(5, 9, density=.1)
b_data = sps.random(9, 10, density=.2)
a = tensor(a_data, chunks=2)
b = tensor(b_data, chunks=3)
c = dot(a, b)
res = self.executor.execute_tensor(c, concat=True)[0]
self.assertTrue(issparse(res))
np.testing.assert_allclose(res.toarray(), a_data.dot(b_data).toarray())
c2 = dot(a, b, sparse=False)
res = self.executor.execute_tensor(c2, concat=True)[0]
self.assertFalse(issparse(res))
np.testing.assert_allclose(res, a_data.dot(b_data).toarray())
c3 = tensordot(a, b.T, (-1, -1), sparse=False)
res = self.executor.execute_tensor(c3, concat=True)[0]
self.assertFalse(issparse(res))
np.testing.assert_allclose(res, a_data.dot(b_data).toarray())
c = inner(a, b.T)
res = self.executor.execute_tensor(c, concat=True)[0]
self.assertTrue(issparse(res))
np.testing.assert_allclose(res.toarray(), a_data.dot(b_data).toarray())
c = inner(a, b.T, sparse=False)
res = self.executor.execute_tensor(c, concat=True)[0]
self.assertFalse(issparse(res))
np.testing.assert_allclose(res, a_data.dot(b_data).toarray())
def testMultiply(self):
t1 = tensor([[0, 1, 0], [1, 0, 0]], chunk_size=2).tosparse()
t = t1 * 10
self.assertTrue(t.issparse())
self.assertIs(type(t), SparseTensor)
t.tiles()
self.assertTrue(t.chunks[0].op.sparse)
t2 = tensor([[1, 0, 0]], chunk_size=2).tosparse()
t = t1 * t2
self.assertTrue(t.issparse())
self.assertIs(type(t), SparseTensor)
t.tiles()
self.assertTrue(t.chunks[0].op.sparse)
t3 = tensor([1, 1, 1], chunk_size=2)
t = t1 * t3
self.assertTrue(t.issparse())
self.assertIs(type(t), SparseTensor)
t.tiles()
self.assertTrue(t.chunks[0].op.sparse)
def testEdiff1d(self):
data = np.array([1, 2, 4, 7, 0])
x = tensor(data, chunks=2)
t = ediff1d(x)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.ediff1d(data)
np.testing.assert_equal(res, expected)
to_begin = tensor(-99, chunks=2)
to_end = tensor([88, 99], chunks=2)
t = ediff1d(x, to_begin=to_begin, to_end=to_end)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.ediff1d(data, to_begin=-99, to_end=np.array([88, 99]))
np.testing.assert_equal(res, expected)
data = [[1, 2, 4], [1, 6, 24]]
t = ediff1d(tensor(data, chunks=2))
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.ediff1d(data)
np.testing.assert_equal(res, expected)
def testLUExecution(self):
np.random.seed(1)
data = np.random.randint(1, 10, (6, 6))
a = tensor(data, chunks=2)
P, L, U = lu(a)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data)
a = tensor(data, chunks=1)
P, L, U = lu(a)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data)
a = tensor(data, chunks=(1, 2))
P, L, U = lu(a)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data)
a = tensor(data, chunks=3)
P, L, U = lu(a)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data)
def testRepeat(self):
a = arange(10, chunk_size=2).reshape(2, 5)
t = repeat(a, 3)
self.assertEqual(t.shape, (30,))
t = repeat(a, 3, axis=0)
self.assertEqual(t.shape, (6, 5))
t = repeat(a, 3, axis=1)
self.assertEqual(t.shape, (2, 15))
t = repeat(a, [3, 4], axis=0)
self.assertEqual(t.shape, (7, 5))
with self.assertRaises(ValueError):
repeat(a, [3, 4], axis=1)
a = tensor(np.random.randn(10), chunk_size=5)
t = repeat(a, 3)
t.tiles()
self.assertEqual(sum(t.nsplits[0]), 30)
a = tensor(np.random.randn(100), chunk_size=10)
t = repeat(a, 3)
t.tiles()
self.assertEqual(sum(t.nsplits[0]), 300)
def testModfExecution(self):
data1 = np.random.random((5, 9))
arr1 = tensor(data1.copy(), chunk_size=3)
o1, o2 = modf(arr1)
o = o1 + o2
res = self.executor.execute_tensor(o, concat=True)[0]
expected = sum(np.modf(data1))
self.assertTrue(np.allclose(res, expected))
arr1 = tensor(data1.copy(), chunk_size=3)
o1 = zeros(data1.shape, chunk_size=3)
o2 = zeros(data1.shape, chunk_size=3)
modf(arr1, o1, o2)
o = o1 + o2
res = self.executor.execute_tensor(o, concat=True)[0]
expected = sum(np.modf(data1))
self.assertTrue(np.allclose(res, expected))
data1 = sps.random(5, 9, density=.1)
arr1 = tensor(data1.copy(), chunk_size=3)
o1, o2 = modf(arr1)
o = o1 + o2
res = self.executor.execute_tensor(o, concat=True)[0]
expected = sum(np.modf(data1.toarray()))
np.testing.assert_equal(res.toarray(), expected)
def testDivide(self):
t1 = tensor([[0, 1, 0], [1, 0, 0]], chunks=2).tosparse()
t = t1 / 10
self.assertTrue(t.issparse())
self.assertIs(type(t), SparseTensor)
t.tiles()
self.assertTrue(t.chunks[0].op.sparse)
t2 = tensor([[1, 0, 0]], chunks=2).tosparse()
t = t1 / t2
self.assertFalse(t.issparse())
self.assertIs(type(t), Tensor)
t.tiles()
self.assertFalse(t.chunks[0].op.sparse)
t3 = tensor([1, 1, 1], chunks=2)
t = t1 / t3
self.assertFalse(t.issparse())
self.assertIs(type(t), Tensor)
t.tiles()
self.assertFalse(t.chunks[0].op.sparse)
t = t3 / t1
self.assertFalse(t.issparse())
self.assertIs(type(t), Tensor)
t.tiles()
self.assertFalse(t.chunks[0].op.sparse)
def testWhere(self):
cond = tensor([[True, False], [False, True]], chunk_size=1)
x = tensor([1, 2], chunk_size=1)
y = tensor([3, 4], chunk_size=1)
arr = where(cond, x, y)
arr.tiles()
self.assertEqual(len(arr.chunks), 4)
self.assertTrue(np.array_equal(arr.chunks[0].inputs[0].op.data, [[True]]))
self.assertTrue(np.array_equal(arr.chunks[0].inputs[1].op.data, [1]))
self.assertTrue(np.array_equal(arr.chunks[0].inputs[2].op.data, [3]))
self.assertTrue(np.array_equal(arr.chunks[1].inputs[0].op.data, [[False]]))
self.assertTrue(np.array_equal(arr.chunks[1].inputs[1].op.data, [2]))
self.assertTrue(np.array_equal(arr.chunks[1].inputs[2].op.data, [4]))
self.assertTrue(np.array_equal(arr.chunks[2].inputs[0].op.data, [[False]]))
self.assertTrue(np.array_equal(arr.chunks[2].inputs[1].op.data, [1]))
self.assertTrue(np.array_equal(arr.chunks[2].inputs[2].op.data, [3]))
self.assertTrue(np.array_equal(arr.chunks[3].inputs[0].op.data, [[True]]))
self.assertTrue(np.array_equal(arr.chunks[3].inputs[1].op.data, [2]))
self.assertTrue(np.array_equal(arr.chunks[3].inputs[2].op.data, [4]))
with self.assertRaises(ValueError):
where(cond, x)
x = arange(9.).reshape(3, 3)
y = where(x < 5, x, -1)
self.assertEqual(y.dtype, np.float64)
def testUnaryExecution(self):
from mars.tensor.expressions.arithmetic import UNARY_UFUNC, arccosh, invert, sin, conj
_sp_unary_ufunc = set([arccosh, invert, conj])
_new_unary_ufunc = list(UNARY_UFUNC - _sp_unary_ufunc)
executor_numexpr = Executor()
def _normalize_by_sin(func1, func2, arr):
return func1(abs(sin((func2(arr)))))
for i, j in itertools.permutations(range(len(_new_unary_ufunc)), 2):
raw = np.random.random((8, 8, 8))
arr1 = tensor(raw, chunks=4)
func1 = _new_unary_ufunc[i]
func2 = _new_unary_ufunc[j]
arr2 = _normalize_by_sin(func1, func2, arr1)
res = executor_numexpr.execute_tensor(arr2, concat=True)
res_cmp = self.executor.execute_tensor(arr2, concat=True)
np.testing.assert_allclose(res[0], res_cmp[0])
raw = np.random.randint(100, size=(8, 8, 8))
arr1 = tensor(raw, chunks=4)
arr2 = arccosh(1 + abs(invert(arr1)))
res = executor_numexpr.execute_tensor(arr2, concat=True)
res_cmp = self.executor.execute_tensor(arr2, concat=True)
self.assertTrue(np.allclose(res[0], res_cmp[0]))
def testReductionExecution(self):
raw1 = np.random.randint(5, size=(8, 8, 8))
raw2 = np.random.randint(5, size=(8, 8, 8))
arr1 = tensor(raw1, chunks=3)
arr2 = tensor(raw2, chunks=3)
res1 = self.executor.execute_tensor((arr1 + 1).sum(keepdims=True))
res2 = self.executor.execute_tensor((arr1 + 1).prod(keepdims=True))
self.assertTrue(np.array_equal((raw1 + 1).sum(keepdims=True), res1[0]))
self.assertTrue(np.array_equal((raw1 + 1).prod(keepdims=True), res2[0]))
res1 = self.executor.execute_tensor((arr1 + 1).sum(axis=1), concat=True)
res2 = self.executor.execute_tensor((arr1 + 1).prod(axis=1), concat=True)
res3 = self.executor.execute_tensor((arr1 + 1).max(axis=1), concat=True)
res4 = self.executor.execute_tensor((arr1 + 1).min(axis=1), concat=True)
self.assertTrue(np.array_equal((raw1 + 1).sum(axis=1), res1[0]))
self.assertTrue(np.array_equal((raw1 + 1).prod(axis=1), res2[0]))
self.assertTrue(np.array_equal((raw1 + 1).max(axis=1), res3[0]))
self.assertTrue(np.array_equal((raw1 + 1).min(axis=1), res4[0]))
raw3 = raw2 - raw1 + 10
arr3 = -arr1 + arr2 + 10
res1 = self.executor.execute_tensor(arr3.sum(axis=(0, 1)), concat=True)
res2 = self.executor.execute_tensor(arr3.prod(axis=(0, 1)), concat=True)
res3 = self.executor.execute_tensor(arr3.max(axis=(0, 1)), concat=True)
res4 = self.executor.execute_tensor(arr3.min(axis=(0, 1)), concat=True)
self.assertTrue(np.array_equal(raw3.sum(axis=(0, 1)), res1[0]))
self.assertTrue(np.array_equal(raw3.prod(axis=(0, 1)), res2[0]))
self.assertTrue(np.array_equal(raw3.max(axis=(0, 1)), res3[0]))
self.assertTrue(np.array_equal(raw3.min(axis=(0, 1)), res4[0]))
def testSVDExecution(self):
data = np.random.randn(18, 6) + 1j * np.random.randn(18, 6)
a = tensor(data, chunk_size=(9, 6))
U, s, V = svd(a)
t = U.dot(diag(s).dot(V))
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, data))
a = tensor(data, chunk_size=(18, 6))
U, s, V = svd(a)
t = U.dot(diag(s).dot(V))
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, data))
a = tensor(data, chunk_size=(2, 6))
U, s, V = svd(a)
t = U.dot(diag(s).dot(V))
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, data))
data = np.random.randn(6, 18) + 1j * np.random.randn(6, 18)
a = tensor(data)
U, s, V = svd(a)
t = U.dot(diag(s).dot(V))
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, data))
def testDiagflatExecution(self):
a = diagflat([[1, 2], [3, 4]], chunk_size=1)
res = self.executor.execute_tensor(a, concat=True)[0]
expected = np.diagflat([[1, 2], [3, 4]])
np.testing.assert_equal(res, expected)
d = tensor([[1, 2], [3, 4]], chunk_size=1)
a = diagflat(d)
res = self.executor.execute_tensor(a, concat=True)[0]
expected = np.diagflat([[1, 2], [3, 4]])
np.testing.assert_equal(res, expected)
a = diagflat([1, 2], 1, chunk_size=1)
res = self.executor.execute_tensor(a, concat=True)[0]
expected = np.diagflat([1, 2], 1)
np.testing.assert_equal(res, expected)
d = tensor([[1, 2]], chunk_size=1)
a = diagflat(d, 1)
res = self.executor.execute_tensor(a, concat=True)[0]
expected = np.diagflat([1, 2], 1)
np.testing.assert_equal(res, expected)
def testSetItemExecution(self):
raw = data = np.random.randint(0, 10, size=(11, 8, 12, 13))
arr = tensor(raw.copy(), chunk_size=3)
raw = raw.copy()
idx = slice(2, 9, 2), slice(3, 7), slice(-1, -9, -2), 2
arr[idx] = 20
res = self.executor.execute_tensor(arr, concat=True)
raw[idx] = 20
np.testing.assert_array_equal(res[0], raw)
raw = data
shape = raw[idx].shape
arr2 = tensor(raw.copy(), chunk_size=3)
raw = raw.copy()
replace = np.random.randint(10, 20,
size=shape[:-1] + (1, )).astype('f4')
arr2[idx] = tensor(replace, chunk_size=4)
res = self.executor.execute_tensor(arr2, concat=True)
raw[idx] = replace
np.testing.assert_array_equal(res[0], raw)
def testSparseDotExecution(self):
size_executor = Executor(
sync_provider_type=Executor.SyncProviderType.MOCK)
a_data = sps.random(5, 9, density=.1)
b_data = sps.random(9, 10, density=.2)
a = tensor(a_data, chunk_size=2)
b = tensor(b_data, chunk_size=3)
c = dot(a, b)
size_res = size_executor.execute_tensor(c, mock=True)
res = self.executor.execute_tensor(c, concat=True)[0]
self.assertEqual(sum(s[0] for s in size_res), 0)
self.assertGreaterEqual(sum(s[1] for s in size_res), 0)
self.assertTrue(issparse(res))
np.testing.assert_allclose(res.toarray(), a_data.dot(b_data).toarray())
c2 = dot(a, b, sparse=False)
size_res = size_executor.execute_tensor(c2, mock=True)
res = self.executor.execute_tensor(c2, concat=True)[0]
self.assertEqual(sum(s[0] for s in size_res), c2.nbytes)
self.assertFalse(issparse(res))
np.testing.assert_allclose(res, a_data.dot(b_data).toarray())
c3 = tensordot(a, b.T, (-1, -1), sparse=False)
res = self.executor.execute_tensor(c3, concat=True)[0]
self.assertFalse(issparse(res))
np.testing.assert_allclose(res, a_data.dot(b_data).toarray())
c = inner(a, b.T)
res = self.executor.execute_tensor(c, concat=True)[0]
self.assertTrue(issparse(res))
np.testing.assert_allclose(res.toarray(), a_data.dot(b_data).toarray())
c = inner(a, b.T, sparse=False)
res = self.executor.execute_tensor(c, concat=True)[0]
self.assertFalse(issparse(res))
np.testing.assert_allclose(res, a_data.dot(b_data).toarray())
# test vector inner
a_data = np.random.rand(5)
b_data = np.random.rand(5)
a = tensor(a_data, chunk_size=2).tosparse()
b = tensor(b_data, chunk_size=2).tosparse()
c = inner(a, b)
res = self.executor.execute_tensor(c, concat=True)[0]
self.assertTrue(np.isscalar(res))
np.testing.assert_allclose(res, np.inner(a_data, b_data))
def testCorrcoefExecution(self):
data_x = [-2.1, -1, 4.3]
data_y = [3, 1.1, 0.12]
x = tensor(data_x, chunk_size=1)
y = tensor(data_y, chunk_size=1)
t = corrcoef(x, y)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.corrcoef(data_x, data_y)
np.testing.assert_equal(res, expected)
def testStdExecution(self):
raw1 = np.random.random((20, 25))
raw2 = np.random.randint(10, size=(20, 25))
arr1 = tensor(raw1, chunk_size=3)
res1 = self.executor.execute_tensor(arr1.std())
expected1 = raw1.std()
self.assertTrue(np.allclose(res1[0], expected1))
res2 = self.executor.execute_tensor(arr1.std(axis=0))
expected2 = raw1.std(axis=0)
self.assertTrue(np.allclose(np.concatenate(res2), expected2))
res3 = self.executor.execute_tensor(arr1.std(axis=1, keepdims=True))
expected3 = raw1.std(axis=1, keepdims=True)
self.assertTrue(np.allclose(np.concatenate(res3), expected3))
arr2 = tensor(raw2, chunk_size=3)
res1 = self.executor.execute_tensor(arr2.std())
expected1 = raw2.std()
self.assertAlmostEqual(res1[0], expected1)
res2 = self.executor.execute_tensor(arr2.std(axis=0))
expected2 = raw2.std(axis=0)
self.assertTrue(np.allclose(np.concatenate(res2), expected2))
res3 = self.executor.execute_tensor(arr2.std(axis=1, keepdims=True))
expected3 = raw2.std(axis=1, keepdims=True)
self.assertTrue(np.allclose(np.concatenate(res3), expected3))
res4 = self.executor.execute_tensor(arr2.std(ddof=1))
expected4 = raw2.std(ddof=1)
self.assertAlmostEqual(res4[0], expected4)
raw1 = sps.random(20, 25, density=.1)
arr1 = tensor(raw1, chunk_size=3)
res1 = self.executor.execute_tensor(arr1.std())
expected1 = raw1.toarray().std()
self.assertTrue(np.allclose(res1[0], expected1))
arr2 = tensor(raw1, chunk_size=30)
res1 = self.executor.execute_tensor(arr2.std())
expected1 = raw1.toarray().std()
self.assertTrue(np.allclose(res1[0], expected1))
arr = std(1)
self.assertEqual(self.executor.execute_tensor(arr)[0], 0)
def testAddWithOutExecution(self):
data1 = np.random.random((5, 9, 4))
data2 = np.random.random((9, 4))
arr1 = tensor(data1.copy(), chunk_size=3)
arr2 = tensor(data2.copy(), chunk_size=3)
add(arr1, arr2, out=arr1)
res = self.executor.execute_tensor(arr1, concat=True)[0]
self.assertTrue(np.array_equal(res, data1 + data2))
arr1 = tensor(data1.copy(), chunk_size=3)
arr2 = tensor(data2.copy(), chunk_size=3)
arr3 = add(arr1, arr2, out=arr1.astype('i4'), casting='unsafe')
res = self.executor.execute_tensor(arr3, concat=True)[0]
np.testing.assert_array_equal(res, (data1 + data2).astype('i4'))
arr1 = tensor(data1.copy(), chunk_size=3)
arr2 = tensor(data2.copy(), chunk_size=3)
arr3 = truediv(arr1, arr2, out=arr1, where=arr2 > .5)
res = self.executor.execute_tensor(arr3, concat=True)[0]
self.assertTrue(
np.array_equal(
res,
np.true_divide(data1,
data2,
out=data1.copy(),
where=data2 > .5)))
arr1 = tensor(data1.copy(), chunk_size=4)
arr2 = tensor(data2.copy(), chunk_size=4)
arr3 = add(arr1, arr2, where=arr1 > .5)
res = self.executor.execute_tensor(arr3, concat=True)[0]
expected = np.add(data1, data2, where=data1 > .5)
self.assertTrue(np.array_equal(res[data1 > .5], expected[data1 > .5]))
arr1 = tensor(data1.copy(), chunk_size=4)
arr3 = add(arr1, 1, where=arr1 > .5)
res = self.executor.execute_tensor(arr3, concat=True)[0]
expected = np.add(data1, 1, where=data1 > .5)
self.assertTrue(np.array_equal(res[data1 > .5], expected[data1 > .5]))
arr1 = tensor(data2.copy(), chunk_size=3)
arr3 = add(arr1[:5, :], 1, out=arr1[-5:, :])
res = self.executor.execute_tensor(arr3, concat=True)[0]
expected = np.add(data2[:5, :], 1)
self.assertTrue(np.array_equal(res, expected))
def testBroadcastArraysExecutions(self):
x_data = [[1, 2, 3]]
x = tensor(x_data, chunk_size=1)
y_data = [[1], [2], [3]]
y = tensor(y_data, chunk_size=2)
a = broadcast_arrays(x, y)
res = [self.executor.execute_tensor(arr, concat=True)[0] for arr in a]
expected = np.broadcast_arrays(x_data, y_data)
for r, e in zip(res, expected):
np.testing.assert_equal(r, e)
def testRepeat(self):
a = arange(10, chunk_size=2).reshape(2, 5)
t = repeat(a, 3)
self.assertEqual(t.shape, (30, ))
self.assertEqual(calc_shape(t), t.shape)
t = repeat(a, 3, axis=0)
self.assertEqual(t.shape, (6, 5))
self.assertEqual(calc_shape(t), t.shape)
t = repeat(a, 3, axis=1)
self.assertEqual(t.shape, (2, 15))
self.assertEqual(calc_shape(t), t.shape)
t = repeat(a, [3], axis=1)
self.assertEqual(t.shape, (2, 15))
self.assertEqual(calc_shape(t), t.shape)
t = repeat(a, [3, 4], axis=0)
self.assertEqual(t.shape, (7, 5))
self.assertEqual(calc_shape(t), t.shape)
with self.assertRaises(ValueError):
repeat(a, [3, 4], axis=1)
a = tensor(np.random.randn(10), chunk_size=5)
t = repeat(a, 3)
t.tiles()
self.assertEqual(sum(t.nsplits[0]), 30)
self.assertEqual(calc_shape(t.chunks[0]), t.chunks[0].shape)
a = tensor(np.random.randn(100), chunk_size=10)
t = repeat(a, 3)
t.tiles()
self.assertEqual(sum(t.nsplits[0]), 300)
self.assertEqual(calc_shape(t.chunks[0]), t.chunks[0].shape)
a = tensor(np.random.randn(4))
b = tensor((4, ))
t = repeat(a, b)
self.assertEqual(calc_shape(t), t.shape)
self.assertEqual(t.rough_shape, (4, ))
t.tiles()
self.assertTrue(np.isnan(t.nsplits[0]))
self.assertEqual(calc_shape(t.chunks[0]), t.chunks[0].shape)
self.assertEqual(t.chunks[0].rough_shape, (4, ))
