def testAllAnyExecution(self): raw1 = np.zeros((10, 15)) raw2 = np.ones((10, 15)) raw3 = np.array([[True, False, True, False], [True, True, True, True], [False, False, False, False], [False, True, False, True]]) arr1 = tensor(raw1, chunks=3) arr2 = tensor(raw2, chunks=3) arr3 = tensor(raw3, chunks=4) self.assertFalse(self.executor.execute_tensor(arr1.all())[0]) self.assertTrue(self.executor.execute_tensor(arr2.all())[0]) self.assertFalse(self.executor.execute_tensor(arr1.any())[0]) self.assertTrue(self.executor.execute_tensor(arr1.any())) self.assertTrue( np.array_equal(raw3.all(axis=1), self.executor.execute_tensor(arr3.all(axis=1))[0])) self.assertTrue( np.array_equal(raw3.any(axis=0), self.executor.execute_tensor(arr3.any(axis=0))[0])) raw = sps.random(10, 10, density=.5) > .5 arr = tensor(raw, chunks=3) self.assertEqual(raw.A.all(), self.executor.execute_tensor(arr.all())[0]) self.assertEqual(raw.A.any(), self.executor.execute_tensor(arr.any())[0])
def testAverageExecution(self): data = arange(1, 5, chunk_size=1) t = average(data) res = self.executor.execute_tensor(t)[0] expected = np.average(np.arange(1, 5)) self.assertEqual(res, expected) t = average(arange(1, 11, chunk_size=2), weights=arange(10, 0, -1, chunk_size=2)) res = self.executor.execute_tensor(t)[0] expected = np.average(range(1, 11), weights=range(10, 0, -1)) self.assertEqual(res, expected) data = arange(6, chunk_size=2).reshape((3, 2)) t = average(data, axis=1, weights=tensor([1. / 4, 3. / 4], chunk_size=2)) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.average(np.arange(6).reshape(3, 2), axis=1, weights=(1. / 4, 3. / 4)) np.testing.assert_equal(res, expected) with self.assertRaises(TypeError): average(data, weights=tensor([1. / 4, 3. / 4], chunk_size=2))
def testMaxMinExecution(self): raw = np.random.randint(10000, size=(10, 10, 10)) arr = tensor(raw, chunks=3) self.assertEqual([raw.max()], self.executor.execute_tensor(arr.max())) self.assertEqual([raw.min()], self.executor.execute_tensor(arr.min())) self.assertTrue( np.array_equal( raw.max(axis=0), self.executor.execute_tensor(arr.max(axis=0), concat=True)[0])) self.assertTrue( np.array_equal( raw.min(axis=0), self.executor.execute_tensor(arr.min(axis=0), concat=True)[0])) self.assertTrue( np.array_equal( raw.max(axis=(1, 2)), self.executor.execute_tensor(arr.max(axis=(1, 2)), concat=True)[0])) self.assertTrue( np.array_equal((raw.min(axis=(1, 2))), self.executor.execute_tensor(arr.min(axis=(1, 2)), concat=True)[0])) raw = sps.random(10, 10, density=.5) arr = tensor(raw, chunks=3) self.assertEqual([raw.max()], self.executor.execute_tensor(arr.max())) self.assertEqual([raw.min()], self.executor.execute_tensor(arr.min()))
def testBinExecution(self): from mars.tensor.expressions.arithmetic import BIN_UFUNC, mod, fmod, \ bitand, bitor, bitxor, lshift, rshift, ldexp _sp_bin_ufunc = [mod, fmod, bitand, bitor, bitxor, lshift, rshift] _new_bin_ufunc = list(BIN_UFUNC - set(_sp_bin_ufunc) - set([ldexp])) executor_numexpr = Executor() for i, j in itertools.permutations(range(len(_new_bin_ufunc)), 2): raw = np.random.random((9, 9, 9)) arr1 = tensor(raw, chunks=5) func1 = _new_bin_ufunc[i] func2 = _new_bin_ufunc[j] arr2 = func1(1, func2(2, arr1)) res = executor_numexpr.execute_tensor(arr2, concat=True) res_cmp = self.executor.execute_tensor(arr2, concat=True) self.assertTrue(np.allclose(res[0], res_cmp[0])) for i, j in itertools.permutations(range(len(_sp_bin_ufunc)), 2): raw = np.random.randint(1, 100, size=(10, 10, 10)) arr1 = tensor(raw, chunks=3) func1 = _sp_bin_ufunc[i] func2 = _sp_bin_ufunc[j] arr2 = func1(10, func2(arr1, 5)) res = executor_numexpr.execute_tensor(arr2, concat=True) res_cmp = self.executor.execute_tensor(arr2, concat=True) self.assertTrue(np.allclose(res[0], res_cmp[0]))
def testCreateSparseExecution(self): mat = sps.csr_matrix([[0, 0, 2], [2, 0, 0]]) t = tensor(mat, dtype='f8', chunk_size=2) res = self.executor.execute_tensor(t) self.assertIsInstance(res[0], SparseNDArray) self.assertEqual(res[0].dtype, np.float64) np.testing.assert_array_equal(res[0].toarray(), mat[..., :2].toarray()) np.testing.assert_array_equal(res[1].toarray(), mat[..., 2:].toarray()) t2 = ones_like(t, dtype='f4') res = self.executor.execute_tensor(t2) expected = sps.csr_matrix([[0, 0, 1], [1, 0, 0]]) self.assertIsInstance(res[0], SparseNDArray) self.assertEqual(res[0].dtype, np.float32) np.testing.assert_array_equal(res[0].toarray(), expected[..., :2].toarray()) np.testing.assert_array_equal(res[1].toarray(), expected[..., 2:].toarray()) t3 = tensor(np.array([[0, 0, 2], [2, 0, 0]]), chunk_size=2).tosparse() res = self.executor.execute_tensor(t3) self.assertIsInstance(res[0], SparseNDArray) self.assertEqual(res[0].dtype, np.int_) np.testing.assert_array_equal(res[0].toarray(), mat[..., :2].toarray()) np.testing.assert_array_equal(res[1].toarray(), mat[..., 2:].toarray())
def testTensordotExecution(self): a_data = np.arange(60).reshape(3, 4, 5) a = tensor(a_data, chunk_size=2) b_data = np.arange(24).reshape(4, 3, 2) b = tensor(b_data, chunk_size=2) axes = ([1, 0], [0, 1]) c = tensordot(a, b, axes=axes) res = self.executor.execute_tensor(c) expected = np.tensordot(a_data, b_data, axes=axes) self.assertTrue(np.array_equal(res[0], expected[:2, :])) self.assertTrue(np.array_equal(res[1], expected[2:4, :])) self.assertTrue(np.array_equal(res[2], expected[4:, :])) a = ones((1000, 2000), chunk_size=500) b = ones((2000, 100), chunk_size=500) c = dot(a, b) res = self.executor.execute_tensor(c) expected = np.dot(np.ones((1000, 2000)), np.ones((2000, 100))) self.assertEqual(len(res), 2) self.assertTrue(np.array_equal(res[0], expected[:500, :])) self.assertTrue(np.array_equal(res[1], expected[500:, :])) a = ones((10, 8), chunk_size=2) b = ones((8, 10), chunk_size=2) c = a.dot(b) res = self.executor.execute_tensor(c) self.assertEqual(len(res), 25) for r in res: self.assertTrue(np.array_equal(r, np.tile([8], [2, 2]))) a = ones((500, 500), chunk_size=500) b = ones((500, 100), chunk_size=500) c = a.dot(b) res = self.executor.execute_tensor(c) self.assertTrue(np.array_equal(res[0], np.tile([500], [500, 100]))) raw_a = np.random.random((100, 200, 50)) raw_b = np.random.random((200, 10, 100)) a = tensor(raw_a, chunk_size=50) b = tensor(raw_b, chunk_size=33) c = tensordot(a, b, axes=((0, 1), (2, 0))) res = self.executor.execute_tensor(c, concat=True) expected = np.tensordot(raw_a, raw_b, axes=(c.op.a_axes, c.op.b_axes)) self.assertTrue(np.allclose(res[0], expected)) a = ones((1000, 2000), chunk_size=500) b = ones((100, 2000), chunk_size=500) c = inner(a, b) res = self.executor.execute_tensor(c) expected = np.inner(np.ones((1000, 2000)), np.ones((100, 2000))) self.assertEqual(len(res), 2) self.assertTrue(np.array_equal(res[0], expected[:500, :])) self.assertTrue(np.array_equal(res[1], expected[500:, :])) a = ones((100, 100), chunk_size=30) b = ones((100, 100), chunk_size=30) c = a.dot(b) res = self.executor.execute_tensor(c, concat=True)[0] np.testing.assert_array_equal(res, np.ones((100, 100)) * 100)
def testNanCumReduction(self): raw = np.random.randint(5, size=(8, 8, 8)) raw[:2, 2:4, 4:6] = np.nan arr = tensor(raw, chunks=3) res1 = self.executor.execute_tensor(nancumsum(arr, axis=1), concat=True) res2 = self.executor.execute_tensor(nancumprod(arr, axis=1), concat=True) expected1 = np.nancumsum(raw, axis=1) expected2 = np.nancumprod(raw, axis=1) self.assertTrue(np.array_equal(res1[0], expected1)) self.assertTrue(np.array_equal(res2[0], expected2)) raw = sps.random(8, 8, density=.1, format='lil') raw[:2, 2:4] = np.nan arr = tensor(raw, chunks=3) res1 = self.executor.execute_tensor(nancumsum(arr, axis=1), concat=True)[0] res2 = self.executor.execute_tensor(nancumprod(arr, axis=1), concat=True)[0] expected1 = np.nancumsum(raw.A, axis=1) expected2 = np.nancumprod(raw.A, axis=1) np.testing.assert_equal(res1, expected1) np.testing.assert_equal(res2, expected2)
def testArgReduction(self): raw = np.random.random((20, 20, 20)) arr = tensor(raw, chunks=3) self.assertEqual(raw.argmax(), self.executor.execute_tensor(arr.argmax())[0]) self.assertEqual(raw.argmin(), self.executor.execute_tensor(arr.argmin())[0]) self.assertTrue( np.array_equal( raw.argmax(axis=0), self.executor.execute_tensor(arr.argmax(axis=0), concat=True)[0])) self.assertTrue( np.array_equal( raw.argmin(axis=0), self.executor.execute_tensor(arr.argmin(axis=0), concat=True)[0])) raw = sps.random(20, 20, density=.1) arr = tensor(raw, chunks=3) self.assertEqual(raw.argmax(), self.executor.execute_tensor(arr.argmax())[0]) self.assertEqual(raw.argmin(), self.executor.execute_tensor(arr.argmin())[0])
def testNormExecution(self): d = np.arange(9) - 4 d2 = d.reshape(3, 3) ma = [tensor(d, chunk_size=2), tensor(d, chunk_size=9), tensor(d2, chunk_size=(2, 3)), tensor(d2, chunk_size=3)] for i, a in enumerate(ma): data = d if i < 2 else d2 for ord in (None, 'nuc', np.inf, -np.inf, 0, 1, -1, 2, -2): for axis in (0, 1, (0, 1)): for keepdims in (True, False): try: expected = np.linalg.norm(data, ord=ord, axis=axis, keepdims=keepdims) t = norm(a, ord=ord, axis=axis, keepdims=keepdims) concat = t.ndim > 0 res = self.executor.execute_tensor(t, concat=concat)[0] np.testing.assert_allclose(res, expected, atol=.0001) except ValueError: continue m = norm(tensor(d)) expected = self.executor.execute_tensor(m)[0] res = np.linalg.norm(d) self.assertEqual(expected, res) d = uniform(-0.5, 0.5, size=(500, 2), chunk_size=50) inside = (norm(d, axis=1) < 0.5).sum().astype(float) t = inside / 500 * 4 res = self.executor.execute_tensor(t)[0] self.assertAlmostEqual(res, 3.14, delta=1)
def testCholeskyExecution(self): data = np.array([[1, -2j], [2j, 5]]) a = tensor(data, chunk_size=1) L = cholesky(a, lower=True) t = L.dot(L.T.conj()) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, data)) L = cholesky(a, lower=True) U = cholesky(a) t = L.dot(U) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, data)) a = tensor(data, chunk_size=2) L = cholesky(a, lower=True) U = cholesky(a) t = L.dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data) a = tensor(data, chunk_size=(1, 2)) L = cholesky(a, lower=True) U = cholesky(a) t = L.dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data)
def testInv(self): import scipy.linalg np.random.seed(1) data = np.random.randint(1, 10, (20, 20)) A = tensor(data) inv_A = inv(A) res = self.executor.execute_tensor(inv_A, concat=True)[0] self.assertTrue(np.allclose(res, scipy.linalg.inv(data))) res = self.executor.execute_tensor(A.dot(inv_A), concat=True)[0] self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float))) A = tensor(data, chunk_size=5) inv_A = inv(A) res = self.executor.execute_tensor(inv_A, concat=True)[0] self.assertTrue(np.allclose(res, scipy.linalg.inv(data))) res = self.executor.execute_tensor(A.dot(inv_A), concat=True)[0] self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float))) B = A.T.dot(A) inv_B = inv(B) res = self.executor.execute_tensor(inv_B, concat=True)[0] self.assertTrue(np.allclose(res, scipy.linalg.inv(data.T.dot(data)))) res = self.executor.execute_tensor(B.dot(inv_B), concat=True)[0] self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float))) # test for not all chunks are square in matrix A A = tensor(data, chunk_size=8) inv_A = inv(A) res = self.executor.execute_tensor(inv_A, concat=True)[0] self.assertTrue(np.allclose(res, scipy.linalg.inv(data))) res = self.executor.execute_tensor(A.dot(inv_A), concat=True)[0] self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float))) # test sparse data = np.random.randint(1, 10, (20, 20)) sp_data = sps.csr_matrix(data) A = tensor(sp_data, chunk_size=5) inv_A = inv(A) res = self.executor.execute_tensor(inv_A, concat=True)[0] self.assertIsInstance(res, SparseNDArray) self.assertTrue(np.allclose(res, scipy.linalg.inv(data))) res = self.executor.execute_tensor(A.dot(inv_A), concat=True)[0] self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float))) # test for not all chunks are square in matrix A A = tensor(sp_data, chunk_size=8) inv_A = inv(A) res = self.executor.execute_tensor(inv_A, concat=True)[0] self.assertIsInstance(res, SparseNDArray) self.assertTrue(np.allclose(res, scipy.linalg.inv(data))) res = self.executor.execute_tensor(A.dot(inv_A), concat=True)[0] self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float)))
def testSparseDotExecution(self): a_data = sps.random(5, 9, density=.1) b_data = sps.random(9, 10, density=.2) a = tensor(a_data, chunks=2) b = tensor(b_data, chunks=3) c = dot(a, b) res = self.executor.execute_tensor(c, concat=True)[0] self.assertTrue(issparse(res)) np.testing.assert_allclose(res.toarray(), a_data.dot(b_data).toarray()) c2 = dot(a, b, sparse=False) res = self.executor.execute_tensor(c2, concat=True)[0] self.assertFalse(issparse(res)) np.testing.assert_allclose(res, a_data.dot(b_data).toarray()) c3 = tensordot(a, b.T, (-1, -1), sparse=False) res = self.executor.execute_tensor(c3, concat=True)[0] self.assertFalse(issparse(res)) np.testing.assert_allclose(res, a_data.dot(b_data).toarray()) c = inner(a, b.T) res = self.executor.execute_tensor(c, concat=True)[0] self.assertTrue(issparse(res)) np.testing.assert_allclose(res.toarray(), a_data.dot(b_data).toarray()) c = inner(a, b.T, sparse=False) res = self.executor.execute_tensor(c, concat=True)[0] self.assertFalse(issparse(res)) np.testing.assert_allclose(res, a_data.dot(b_data).toarray())
def testMultiply(self): t1 = tensor([[0, 1, 0], [1, 0, 0]], chunk_size=2).tosparse() t = t1 * 10 self.assertTrue(t.issparse()) self.assertIs(type(t), SparseTensor) t.tiles() self.assertTrue(t.chunks[0].op.sparse) t2 = tensor([[1, 0, 0]], chunk_size=2).tosparse() t = t1 * t2 self.assertTrue(t.issparse()) self.assertIs(type(t), SparseTensor) t.tiles() self.assertTrue(t.chunks[0].op.sparse) t3 = tensor([1, 1, 1], chunk_size=2) t = t1 * t3 self.assertTrue(t.issparse()) self.assertIs(type(t), SparseTensor) t.tiles() self.assertTrue(t.chunks[0].op.sparse)
def testEdiff1d(self): data = np.array([1, 2, 4, 7, 0]) x = tensor(data, chunks=2) t = ediff1d(x) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.ediff1d(data) np.testing.assert_equal(res, expected) to_begin = tensor(-99, chunks=2) to_end = tensor([88, 99], chunks=2) t = ediff1d(x, to_begin=to_begin, to_end=to_end) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.ediff1d(data, to_begin=-99, to_end=np.array([88, 99])) np.testing.assert_equal(res, expected) data = [[1, 2, 4], [1, 6, 24]] t = ediff1d(tensor(data, chunks=2)) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.ediff1d(data) np.testing.assert_equal(res, expected)
def testLUExecution(self): np.random.seed(1) data = np.random.randint(1, 10, (6, 6)) a = tensor(data, chunks=2) P, L, U = lu(a) t = P.dot(L).dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data) a = tensor(data, chunks=1) P, L, U = lu(a) t = P.dot(L).dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data) a = tensor(data, chunks=(1, 2)) P, L, U = lu(a) t = P.dot(L).dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data) a = tensor(data, chunks=3) P, L, U = lu(a) t = P.dot(L).dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data)
def testRepeat(self): a = arange(10, chunk_size=2).reshape(2, 5) t = repeat(a, 3) self.assertEqual(t.shape, (30,)) t = repeat(a, 3, axis=0) self.assertEqual(t.shape, (6, 5)) t = repeat(a, 3, axis=1) self.assertEqual(t.shape, (2, 15)) t = repeat(a, [3, 4], axis=0) self.assertEqual(t.shape, (7, 5)) with self.assertRaises(ValueError): repeat(a, [3, 4], axis=1) a = tensor(np.random.randn(10), chunk_size=5) t = repeat(a, 3) t.tiles() self.assertEqual(sum(t.nsplits[0]), 30) a = tensor(np.random.randn(100), chunk_size=10) t = repeat(a, 3) t.tiles() self.assertEqual(sum(t.nsplits[0]), 300)
def testModfExecution(self): data1 = np.random.random((5, 9)) arr1 = tensor(data1.copy(), chunk_size=3) o1, o2 = modf(arr1) o = o1 + o2 res = self.executor.execute_tensor(o, concat=True)[0] expected = sum(np.modf(data1)) self.assertTrue(np.allclose(res, expected)) arr1 = tensor(data1.copy(), chunk_size=3) o1 = zeros(data1.shape, chunk_size=3) o2 = zeros(data1.shape, chunk_size=3) modf(arr1, o1, o2) o = o1 + o2 res = self.executor.execute_tensor(o, concat=True)[0] expected = sum(np.modf(data1)) self.assertTrue(np.allclose(res, expected)) data1 = sps.random(5, 9, density=.1) arr1 = tensor(data1.copy(), chunk_size=3) o1, o2 = modf(arr1) o = o1 + o2 res = self.executor.execute_tensor(o, concat=True)[0] expected = sum(np.modf(data1.toarray())) np.testing.assert_equal(res.toarray(), expected)
def testDivide(self): t1 = tensor([[0, 1, 0], [1, 0, 0]], chunks=2).tosparse() t = t1 / 10 self.assertTrue(t.issparse()) self.assertIs(type(t), SparseTensor) t.tiles() self.assertTrue(t.chunks[0].op.sparse) t2 = tensor([[1, 0, 0]], chunks=2).tosparse() t = t1 / t2 self.assertFalse(t.issparse()) self.assertIs(type(t), Tensor) t.tiles() self.assertFalse(t.chunks[0].op.sparse) t3 = tensor([1, 1, 1], chunks=2) t = t1 / t3 self.assertFalse(t.issparse()) self.assertIs(type(t), Tensor) t.tiles() self.assertFalse(t.chunks[0].op.sparse) t = t3 / t1 self.assertFalse(t.issparse()) self.assertIs(type(t), Tensor) t.tiles() self.assertFalse(t.chunks[0].op.sparse)
def testWhere(self): cond = tensor([[True, False], [False, True]], chunk_size=1) x = tensor([1, 2], chunk_size=1) y = tensor([3, 4], chunk_size=1) arr = where(cond, x, y) arr.tiles() self.assertEqual(len(arr.chunks), 4) self.assertTrue(np.array_equal(arr.chunks[0].inputs[0].op.data, [[True]])) self.assertTrue(np.array_equal(arr.chunks[0].inputs[1].op.data, [1])) self.assertTrue(np.array_equal(arr.chunks[0].inputs[2].op.data, [3])) self.assertTrue(np.array_equal(arr.chunks[1].inputs[0].op.data, [[False]])) self.assertTrue(np.array_equal(arr.chunks[1].inputs[1].op.data, [2])) self.assertTrue(np.array_equal(arr.chunks[1].inputs[2].op.data, [4])) self.assertTrue(np.array_equal(arr.chunks[2].inputs[0].op.data, [[False]])) self.assertTrue(np.array_equal(arr.chunks[2].inputs[1].op.data, [1])) self.assertTrue(np.array_equal(arr.chunks[2].inputs[2].op.data, [3])) self.assertTrue(np.array_equal(arr.chunks[3].inputs[0].op.data, [[True]])) self.assertTrue(np.array_equal(arr.chunks[3].inputs[1].op.data, [2])) self.assertTrue(np.array_equal(arr.chunks[3].inputs[2].op.data, [4])) with self.assertRaises(ValueError): where(cond, x) x = arange(9.).reshape(3, 3) y = where(x < 5, x, -1) self.assertEqual(y.dtype, np.float64)
def testUnaryExecution(self): from mars.tensor.expressions.arithmetic import UNARY_UFUNC, arccosh, invert, sin, conj _sp_unary_ufunc = set([arccosh, invert, conj]) _new_unary_ufunc = list(UNARY_UFUNC - _sp_unary_ufunc) executor_numexpr = Executor() def _normalize_by_sin(func1, func2, arr): return func1(abs(sin((func2(arr))))) for i, j in itertools.permutations(range(len(_new_unary_ufunc)), 2): raw = np.random.random((8, 8, 8)) arr1 = tensor(raw, chunks=4) func1 = _new_unary_ufunc[i] func2 = _new_unary_ufunc[j] arr2 = _normalize_by_sin(func1, func2, arr1) res = executor_numexpr.execute_tensor(arr2, concat=True) res_cmp = self.executor.execute_tensor(arr2, concat=True) np.testing.assert_allclose(res[0], res_cmp[0]) raw = np.random.randint(100, size=(8, 8, 8)) arr1 = tensor(raw, chunks=4) arr2 = arccosh(1 + abs(invert(arr1))) res = executor_numexpr.execute_tensor(arr2, concat=True) res_cmp = self.executor.execute_tensor(arr2, concat=True) self.assertTrue(np.allclose(res[0], res_cmp[0]))
def testReductionExecution(self): raw1 = np.random.randint(5, size=(8, 8, 8)) raw2 = np.random.randint(5, size=(8, 8, 8)) arr1 = tensor(raw1, chunks=3) arr2 = tensor(raw2, chunks=3) res1 = self.executor.execute_tensor((arr1 + 1).sum(keepdims=True)) res2 = self.executor.execute_tensor((arr1 + 1).prod(keepdims=True)) self.assertTrue(np.array_equal((raw1 + 1).sum(keepdims=True), res1[0])) self.assertTrue(np.array_equal((raw1 + 1).prod(keepdims=True), res2[0])) res1 = self.executor.execute_tensor((arr1 + 1).sum(axis=1), concat=True) res2 = self.executor.execute_tensor((arr1 + 1).prod(axis=1), concat=True) res3 = self.executor.execute_tensor((arr1 + 1).max(axis=1), concat=True) res4 = self.executor.execute_tensor((arr1 + 1).min(axis=1), concat=True) self.assertTrue(np.array_equal((raw1 + 1).sum(axis=1), res1[0])) self.assertTrue(np.array_equal((raw1 + 1).prod(axis=1), res2[0])) self.assertTrue(np.array_equal((raw1 + 1).max(axis=1), res3[0])) self.assertTrue(np.array_equal((raw1 + 1).min(axis=1), res4[0])) raw3 = raw2 - raw1 + 10 arr3 = -arr1 + arr2 + 10 res1 = self.executor.execute_tensor(arr3.sum(axis=(0, 1)), concat=True) res2 = self.executor.execute_tensor(arr3.prod(axis=(0, 1)), concat=True) res3 = self.executor.execute_tensor(arr3.max(axis=(0, 1)), concat=True) res4 = self.executor.execute_tensor(arr3.min(axis=(0, 1)), concat=True) self.assertTrue(np.array_equal(raw3.sum(axis=(0, 1)), res1[0])) self.assertTrue(np.array_equal(raw3.prod(axis=(0, 1)), res2[0])) self.assertTrue(np.array_equal(raw3.max(axis=(0, 1)), res3[0])) self.assertTrue(np.array_equal(raw3.min(axis=(0, 1)), res4[0]))
def testSVDExecution(self): data = np.random.randn(18, 6) + 1j * np.random.randn(18, 6) a = tensor(data, chunk_size=(9, 6)) U, s, V = svd(a) t = U.dot(diag(s).dot(V)) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, data)) a = tensor(data, chunk_size=(18, 6)) U, s, V = svd(a) t = U.dot(diag(s).dot(V)) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, data)) a = tensor(data, chunk_size=(2, 6)) U, s, V = svd(a) t = U.dot(diag(s).dot(V)) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, data)) data = np.random.randn(6, 18) + 1j * np.random.randn(6, 18) a = tensor(data) U, s, V = svd(a) t = U.dot(diag(s).dot(V)) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, data))
def testDiagflatExecution(self): a = diagflat([[1, 2], [3, 4]], chunk_size=1) res = self.executor.execute_tensor(a, concat=True)[0] expected = np.diagflat([[1, 2], [3, 4]]) np.testing.assert_equal(res, expected) d = tensor([[1, 2], [3, 4]], chunk_size=1) a = diagflat(d) res = self.executor.execute_tensor(a, concat=True)[0] expected = np.diagflat([[1, 2], [3, 4]]) np.testing.assert_equal(res, expected) a = diagflat([1, 2], 1, chunk_size=1) res = self.executor.execute_tensor(a, concat=True)[0] expected = np.diagflat([1, 2], 1) np.testing.assert_equal(res, expected) d = tensor([[1, 2]], chunk_size=1) a = diagflat(d, 1) res = self.executor.execute_tensor(a, concat=True)[0] expected = np.diagflat([1, 2], 1) np.testing.assert_equal(res, expected)
def testSetItemExecution(self): raw = data = np.random.randint(0, 10, size=(11, 8, 12, 13)) arr = tensor(raw.copy(), chunk_size=3) raw = raw.copy() idx = slice(2, 9, 2), slice(3, 7), slice(-1, -9, -2), 2 arr[idx] = 20 res = self.executor.execute_tensor(arr, concat=True) raw[idx] = 20 np.testing.assert_array_equal(res[0], raw) raw = data shape = raw[idx].shape arr2 = tensor(raw.copy(), chunk_size=3) raw = raw.copy() replace = np.random.randint(10, 20, size=shape[:-1] + (1, )).astype('f4') arr2[idx] = tensor(replace, chunk_size=4) res = self.executor.execute_tensor(arr2, concat=True) raw[idx] = replace np.testing.assert_array_equal(res[0], raw)
def testSparseDotExecution(self): size_executor = Executor( sync_provider_type=Executor.SyncProviderType.MOCK) a_data = sps.random(5, 9, density=.1) b_data = sps.random(9, 10, density=.2) a = tensor(a_data, chunk_size=2) b = tensor(b_data, chunk_size=3) c = dot(a, b) size_res = size_executor.execute_tensor(c, mock=True) res = self.executor.execute_tensor(c, concat=True)[0] self.assertEqual(sum(s[0] for s in size_res), 0) self.assertGreaterEqual(sum(s[1] for s in size_res), 0) self.assertTrue(issparse(res)) np.testing.assert_allclose(res.toarray(), a_data.dot(b_data).toarray()) c2 = dot(a, b, sparse=False) size_res = size_executor.execute_tensor(c2, mock=True) res = self.executor.execute_tensor(c2, concat=True)[0] self.assertEqual(sum(s[0] for s in size_res), c2.nbytes) self.assertFalse(issparse(res)) np.testing.assert_allclose(res, a_data.dot(b_data).toarray()) c3 = tensordot(a, b.T, (-1, -1), sparse=False) res = self.executor.execute_tensor(c3, concat=True)[0] self.assertFalse(issparse(res)) np.testing.assert_allclose(res, a_data.dot(b_data).toarray()) c = inner(a, b.T) res = self.executor.execute_tensor(c, concat=True)[0] self.assertTrue(issparse(res)) np.testing.assert_allclose(res.toarray(), a_data.dot(b_data).toarray()) c = inner(a, b.T, sparse=False) res = self.executor.execute_tensor(c, concat=True)[0] self.assertFalse(issparse(res)) np.testing.assert_allclose(res, a_data.dot(b_data).toarray()) # test vector inner a_data = np.random.rand(5) b_data = np.random.rand(5) a = tensor(a_data, chunk_size=2).tosparse() b = tensor(b_data, chunk_size=2).tosparse() c = inner(a, b) res = self.executor.execute_tensor(c, concat=True)[0] self.assertTrue(np.isscalar(res)) np.testing.assert_allclose(res, np.inner(a_data, b_data))
def testCorrcoefExecution(self): data_x = [-2.1, -1, 4.3] data_y = [3, 1.1, 0.12] x = tensor(data_x, chunk_size=1) y = tensor(data_y, chunk_size=1) t = corrcoef(x, y) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.corrcoef(data_x, data_y) np.testing.assert_equal(res, expected)
def testStdExecution(self): raw1 = np.random.random((20, 25)) raw2 = np.random.randint(10, size=(20, 25)) arr1 = tensor(raw1, chunk_size=3) res1 = self.executor.execute_tensor(arr1.std()) expected1 = raw1.std() self.assertTrue(np.allclose(res1[0], expected1)) res2 = self.executor.execute_tensor(arr1.std(axis=0)) expected2 = raw1.std(axis=0) self.assertTrue(np.allclose(np.concatenate(res2), expected2)) res3 = self.executor.execute_tensor(arr1.std(axis=1, keepdims=True)) expected3 = raw1.std(axis=1, keepdims=True) self.assertTrue(np.allclose(np.concatenate(res3), expected3)) arr2 = tensor(raw2, chunk_size=3) res1 = self.executor.execute_tensor(arr2.std()) expected1 = raw2.std() self.assertAlmostEqual(res1[0], expected1) res2 = self.executor.execute_tensor(arr2.std(axis=0)) expected2 = raw2.std(axis=0) self.assertTrue(np.allclose(np.concatenate(res2), expected2)) res3 = self.executor.execute_tensor(arr2.std(axis=1, keepdims=True)) expected3 = raw2.std(axis=1, keepdims=True) self.assertTrue(np.allclose(np.concatenate(res3), expected3)) res4 = self.executor.execute_tensor(arr2.std(ddof=1)) expected4 = raw2.std(ddof=1) self.assertAlmostEqual(res4[0], expected4) raw1 = sps.random(20, 25, density=.1) arr1 = tensor(raw1, chunk_size=3) res1 = self.executor.execute_tensor(arr1.std()) expected1 = raw1.toarray().std() self.assertTrue(np.allclose(res1[0], expected1)) arr2 = tensor(raw1, chunk_size=30) res1 = self.executor.execute_tensor(arr2.std()) expected1 = raw1.toarray().std() self.assertTrue(np.allclose(res1[0], expected1)) arr = std(1) self.assertEqual(self.executor.execute_tensor(arr)[0], 0)
def testAddWithOutExecution(self): data1 = np.random.random((5, 9, 4)) data2 = np.random.random((9, 4)) arr1 = tensor(data1.copy(), chunk_size=3) arr2 = tensor(data2.copy(), chunk_size=3) add(arr1, arr2, out=arr1) res = self.executor.execute_tensor(arr1, concat=True)[0] self.assertTrue(np.array_equal(res, data1 + data2)) arr1 = tensor(data1.copy(), chunk_size=3) arr2 = tensor(data2.copy(), chunk_size=3) arr3 = add(arr1, arr2, out=arr1.astype('i4'), casting='unsafe') res = self.executor.execute_tensor(arr3, concat=True)[0] np.testing.assert_array_equal(res, (data1 + data2).astype('i4')) arr1 = tensor(data1.copy(), chunk_size=3) arr2 = tensor(data2.copy(), chunk_size=3) arr3 = truediv(arr1, arr2, out=arr1, where=arr2 > .5) res = self.executor.execute_tensor(arr3, concat=True)[0] self.assertTrue( np.array_equal( res, np.true_divide(data1, data2, out=data1.copy(), where=data2 > .5))) arr1 = tensor(data1.copy(), chunk_size=4) arr2 = tensor(data2.copy(), chunk_size=4) arr3 = add(arr1, arr2, where=arr1 > .5) res = self.executor.execute_tensor(arr3, concat=True)[0] expected = np.add(data1, data2, where=data1 > .5) self.assertTrue(np.array_equal(res[data1 > .5], expected[data1 > .5])) arr1 = tensor(data1.copy(), chunk_size=4) arr3 = add(arr1, 1, where=arr1 > .5) res = self.executor.execute_tensor(arr3, concat=True)[0] expected = np.add(data1, 1, where=data1 > .5) self.assertTrue(np.array_equal(res[data1 > .5], expected[data1 > .5])) arr1 = tensor(data2.copy(), chunk_size=3) arr3 = add(arr1[:5, :], 1, out=arr1[-5:, :]) res = self.executor.execute_tensor(arr3, concat=True)[0] expected = np.add(data2[:5, :], 1) self.assertTrue(np.array_equal(res, expected))
def testBroadcastArraysExecutions(self): x_data = [[1, 2, 3]] x = tensor(x_data, chunk_size=1) y_data = [[1], [2], [3]] y = tensor(y_data, chunk_size=2) a = broadcast_arrays(x, y) res = [self.executor.execute_tensor(arr, concat=True)[0] for arr in a] expected = np.broadcast_arrays(x_data, y_data) for r, e in zip(res, expected): np.testing.assert_equal(r, e)
def testRepeat(self): a = arange(10, chunk_size=2).reshape(2, 5) t = repeat(a, 3) self.assertEqual(t.shape, (30, )) self.assertEqual(calc_shape(t), t.shape) t = repeat(a, 3, axis=0) self.assertEqual(t.shape, (6, 5)) self.assertEqual(calc_shape(t), t.shape) t = repeat(a, 3, axis=1) self.assertEqual(t.shape, (2, 15)) self.assertEqual(calc_shape(t), t.shape) t = repeat(a, [3], axis=1) self.assertEqual(t.shape, (2, 15)) self.assertEqual(calc_shape(t), t.shape) t = repeat(a, [3, 4], axis=0) self.assertEqual(t.shape, (7, 5)) self.assertEqual(calc_shape(t), t.shape) with self.assertRaises(ValueError): repeat(a, [3, 4], axis=1) a = tensor(np.random.randn(10), chunk_size=5) t = repeat(a, 3) t.tiles() self.assertEqual(sum(t.nsplits[0]), 30) self.assertEqual(calc_shape(t.chunks[0]), t.chunks[0].shape) a = tensor(np.random.randn(100), chunk_size=10) t = repeat(a, 3) t.tiles() self.assertEqual(sum(t.nsplits[0]), 300) self.assertEqual(calc_shape(t.chunks[0]), t.chunks[0].shape) a = tensor(np.random.randn(4)) b = tensor((4, )) t = repeat(a, b) self.assertEqual(calc_shape(t), t.shape) self.assertEqual(t.rough_shape, (4, )) t.tiles() self.assertTrue(np.isnan(t.nsplits[0])) self.assertEqual(calc_shape(t.chunks[0]), t.chunks[0].shape) self.assertEqual(t.chunks[0].rough_shape, (4, ))