def testUnaryExecution(self): from mars.tensor.expressions.arithmetic import UNARY_UFUNC, arccosh, invert, sin, conj _sp_unary_ufunc = set([arccosh, invert, conj]) _new_unary_ufunc = list(UNARY_UFUNC - _sp_unary_ufunc) executor_numexpr = Executor() def _normalize_by_sin(func1, func2, arr): return func1(abs(sin((func2(arr))))) for i, j in itertools.permutations(range(len(_new_unary_ufunc)), 2): raw = np.random.random((8, 8, 8)) arr1 = tensor(raw, chunk_size=4) func1 = _new_unary_ufunc[i] func2 = _new_unary_ufunc[j] arr2 = _normalize_by_sin(func1, func2, arr1) res = executor_numexpr.execute_tensor(arr2, concat=True) res_cmp = self.executor.execute_tensor(arr2, concat=True) np.testing.assert_allclose(res[0], res_cmp[0]) raw = np.random.randint(100, size=(8, 8, 8)) arr1 = tensor(raw, chunk_size=4) arr2 = arccosh(1 + abs(invert(arr1))) res = executor_numexpr.execute_tensor(arr2, concat=True) res_cmp = self.executor.execute_tensor(arr2, concat=True) self.assertTrue(np.allclose(res[0], res_cmp[0]))
def testBinExecution(self): from mars.tensor.expressions.arithmetic import BIN_UFUNC, mod, fmod, \ bitand, bitor, bitxor, lshift, rshift, ldexp _sp_bin_ufunc = [mod, fmod, bitand, bitor, bitxor, lshift, rshift] _new_bin_ufunc = list(BIN_UFUNC - set(_sp_bin_ufunc) - set([ldexp])) executor_numexpr = Executor() for i, j in itertools.permutations(range(len(_new_bin_ufunc)), 2): raw = np.random.random((9, 9, 9)) arr1 = tensor(raw, chunk_size=5) func1 = _new_bin_ufunc[i] func2 = _new_bin_ufunc[j] arr2 = func1(1, func2(2, arr1)) res = executor_numexpr.execute_tensor(arr2, concat=True) res_cmp = self.executor.execute_tensor(arr2, concat=True) self.assertTrue(np.allclose(res[0], res_cmp[0])) for i, j in itertools.permutations(range(len(_sp_bin_ufunc)), 2): raw = np.random.randint(1, 100, size=(10, 10, 10)) arr1 = tensor(raw, chunk_size=3) func1 = _sp_bin_ufunc[i] func2 = _sp_bin_ufunc[j] arr2 = func1(10, func2(arr1, 5)) res = executor_numexpr.execute_tensor(arr2, concat=True) res_cmp = self.executor.execute_tensor(arr2, concat=True) self.assertTrue(np.allclose(res[0], res_cmp[0]))
class Test(unittest.TestCase): def setUp(self): self.executor = Executor('numpy', storage=MockStorage(), prefetch=True) def testPrefetch(self): t1 = ones((10, 8), chunk_size=10) t2 = ones((1, 8), chunk_size=10) t3 = t1 + t2 self.executor.execute_tensor(t3)
def testBaseExecution(self): executor_numpy = Executor('numpy') raw1 = np.random.randint(10, size=(10, 10, 10)) raw2 = np.random.randint(10, size=(10, 10, 10)) arr1 = tensor(raw1, chunk_size=3) arr2 = tensor(raw2, chunk_size=3) arr3 = arr1 + arr2 + 10 arr4 = 10 + arr1 + arr2 res3 = executor_numpy.execute_tensor(arr3, concat=True) res3_cmp = self.executor.execute_tensor(arr4, concat=True) self.assertTrue(np.array_equal(res3[0], res3_cmp[0])) res5 = executor_numpy.execute_tensor((arr1 + arr1), concat=True) res5_cmp = self.executor.execute_tensor((arr1 + arr1), concat=True) self.assertTrue(np.array_equal(res5[0], res5_cmp[0]))
class Test(unittest.TestCase): def setUp(self): self.executor = Executor('cupy') def testElementwise(self): t1 = ones((10000, 5000), chunk_size=500, gpu=True) t2 = ones(5000, chunk_size=500, gpu=True) t = (t1 - t2) / sqrt(t2 * (1 - t2) * len(t2)) g = t.build_graph(tiled=True) graph = self.executor._preprocess(g, []) self.assertTrue( any(n.op.__class__.__name__ == 'TensorCpFuseChunk' for n in graph)) c = next(n for n in graph if n.op.__class__.__name__ == 'TensorCpFuseChunk') print(_evaluate(c))
class Test(unittest.TestCase): def setUp(self): self.executor = Executor('numpy') def testRandExecution(self): arr = tensor.random.rand(10, 20, chunk_size=3, dtype='f4') res = self.executor.execute_tensor(arr, concat=True)[0] self.assertEqual(res.shape, (10, 20)) self.assertTrue(np.all(res < 1)) self.assertTrue(np.all(res > 0)) self.assertEqual(res.dtype, np.float32) def testRandnExecution(self): arr = tensor.random.randn(10, 20, chunk_size=3) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (10, 20)) arr = tensor.random.randn(10, 20, chunk_size=5).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).randn(5, 5))) def testRandintExecution(self): arr = tensor.random.randint(0, 2, size=(10, 30), chunk_size=3) res = self.executor.execute_tensor(arr, concat=True)[0] self.assertEqual(res.shape, (10, 30)) self.assertTrue(np.all(res >= 0)) self.assertTrue(np.all(res < 2)) def testRandomIntegersExecution(self): arr = tensor.random.random_integers(0, 10, size=(10, 20), chunk_size=3) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (10, 20)) arr = tensor.random.random_integers(0, 10, size=(10, 20), chunk_size=5).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): np.testing.assert_equal( res, np.random.RandomState(0).random_integers(0, 10, size=(5, 5))) def testRandomSampleExecution(self): arr = tensor.random.random_sample(size=(10, 20), chunk_size=3) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (10, 20)) arr = tensor.random.random_sample(size=(10, 20), chunk_size=5).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal( res, np.random.RandomState(0).random_sample(size=(5, 5)))) def testRandomExecution(self): arr = tensor.random.random(size=(10, 20), chunk_size=3) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (10, 20)) arr = tensor.random.random(size=(10, 20), chunk_size=5).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal( res, np.random.RandomState(0).random_sample(size=(5, 5)))) def testRandfExecution(self): arr = tensor.random.ranf(size=(10, 20), chunk_size=3) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (10, 20)) arr = tensor.random.ranf(size=(10, 20), chunk_size=5).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal( res, np.random.RandomState(0).random_sample(size=(5, 5)))) def testSampleExecution(self): arr = tensor.random.sample(size=(10, 20), chunk_size=3) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (10, 20)) arr = tensor.random.sample(size=(10, 20), chunk_size=5).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal( res, np.random.RandomState(0).random_sample(size=(5, 5)))) def testChoiceExecution(self): arr = tensor.random.choice(5, size=3, chunk_size=1) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (3, )) arr = tensor.random.choice(5, size=(15, ), chunk_size=5).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).choice(5, size=(5, )))) arr = tensor.random.choice([1, 4, 9], size=3, chunk_size=1) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (3, )) arr = tensor.random.choice([1, 4, 9], size=(15, ), chunk_size=5).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal( res, np.random.RandomState(0).choice([1, 4, 9], size=(5, )))) with self.assertRaises(ValueError): tensor.random.choice([1, 3, 4], size=5, replace=False, chunk_size=2) arr = tensor.random.choice([1, 4, 9], size=3, replace=False, chunk_size=1) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (3, )) arr = tensor.random.choice([1, 4, 9], size=(3, ), replace=False, chunk_size=1).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal( res, np.random.RandomState(0).choice([1, 4, 9], size=(1, ), replace=False))) arr = tensor.random.choice([1, 4, 9], size=3, p=[.2, .5, .3], chunk_size=1) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (3, )) arr = tensor.random.choice([1, 4, 9], size=(15, ), chunk_size=5, p=[.2, .5, .3]).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal( res, np.random.RandomState(0).choice([1, 4, 9], size=(5, ), p=[.2, .5, .3]))) def testSparseRandintExecution(self): arr = tensor.random.randint(1, 2, size=(30, 50), density=.1, chunk_size=10, dtype='f4') res = self.executor.execute_tensor(arr, concat=True)[0] self.assertTrue(issparse(res)) self.assertEqual(res.shape, (30, 50)) self.assertTrue(np.all(res.data >= 1)) self.assertTrue(np.all(res.data < 2)) self.assertAlmostEqual((res >= 1).toarray().sum(), 30 * 50 * .1, delta=20) def testBetaExecute(self): arr = tensor.random.beta(1, 2, chunk_size=2).tiles() arr.chunks[0].op._state = State(np.random.RandomState(0)) self.assertEqual( self.executor.execute_tensor(arr)[0], np.random.RandomState(0).beta(1, 2)) arr = tensor.random.beta([1, 2], [3, 4], chunk_size=2).tiles() arr.chunks[0].op._state = State(np.random.RandomState(0)) self.assertTrue( np.array_equal( self.executor.execute_tensor(arr)[0], np.random.RandomState(0).beta([1, 2], [3, 4]))) arr = tensor.random.beta([[2, 3]], from_ndarray([[4, 6], [5, 2]], chunk_size=2), chunk_size=1, size=(3, 2, 2)).tiles() for c in arr.chunks: c.op._state = State(np.random.RandomState(0)) res = self.executor.execute_tensor(arr, concat=True)[0] self.assertEqual(res[0, 0, 0], np.random.RandomState(0).beta(2, 4)) self.assertEqual(res[0, 0, 1], np.random.RandomState(0).beta(3, 6)) self.assertEqual(res[0, 1, 0], np.random.RandomState(0).beta(2, 5)) self.assertEqual(res[0, 1, 1], np.random.RandomState(0).beta(3, 2)) arr = tensor.random.RandomState(0).beta([[3, 4]], [[1], [2]], chunk_size=1) tensor.random.seed(0) arr2 = tensor.random.beta([[3, 4]], [[1], [2]], chunk_size=1) self.assertTrue( np.array_equal( self.executor.execute_tensor(arr, concat=True)[0], self.executor.execute_tensor(arr2, concat=True)[0])) def testBinomialExecute(self): arr = tensor.random.binomial(10, .5, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.binomial(10, .5, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).binomial(10, .5, 10))) def testChisquareExecute(self): arr = tensor.random.chisquare(2, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.chisquare(2, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).chisquare(2, 10))) def testDirichletExecute(self): arr = tensor.random.dirichlet((10, 5, 3), 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, 3)) arr = tensor.random.dirichlet((10, 5, 3), 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal( res, np.random.RandomState(0).dirichlet((10, 5, 3), 10))) def testExponentialExecute(self): arr = tensor.random.exponential(1.0, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.exponential(1.0, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).exponential(1.0, 10))) def testFExecute(self): arr = tensor.random.f(1.0, 2.0, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.f(1.0, 2.0, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).f(1.0, 2.0, 10))) def testGammaExecute(self): arr = tensor.random.gamma(1.0, 2.0, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.gamma(1.0, 2.0, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).gamma(1.0, 2.0, 10))) def testGeometricExecution(self): arr = tensor.random.geometric(1.0, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.geometric(1.0, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).geometric(1.0, 10))) def testGumbelExecution(self): arr = tensor.random.gumbel(.5, 1.0, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.gumbel(.5, 1.0, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).gumbel(.5, 1.0, 10))) def testHypergeometricExecution(self): arr = tensor.random.hypergeometric(10, 20, 15, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.hypergeometric(10, 20, 15, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal( res, np.random.RandomState(0).hypergeometric(10, 20, 15, 10))) def testLaplaceExecution(self): arr = tensor.random.laplace(.5, 1.0, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.laplace(.5, 1.0, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).laplace(.5, 1.0, 10))) def testLogisticExecution(self): arr = tensor.random.logistic(.5, 1.0, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.logistic(.5, 1.0, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): np.testing.assert_equal( res, np.random.RandomState(0).logistic(.5, 1.0, 10)) def testLognormalExecution(self): arr = tensor.random.lognormal(.5, 1.0, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.lognormal(.5, 1.0, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).lognormal(.5, 1.0, 10))) def testLogseriesExecution(self): arr = tensor.random.logseries(.5, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.logseries(.5, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).logseries(.5, 10))) def testMultinomialExecution(self): arr = tensor.random.multinomial(10, [.2, .5, .3], 100, chunk_size=10) self.assertEqual(arr.shape, (100, 3)) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, 3)) arr = tensor.random.multinomial(10, [.2, .5, .3], 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal( res, np.random.RandomState(0).multinomial(10, [.2, .5, .3], 10))) def testMultivariateNormalExecution(self): arr = tensor.random.multivariate_normal([1, 2], [[1, 0], [0, 1]], 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, 2)) arr = tensor.random.multivariate_normal([1, 2], [[1, 0], [0, 1]], 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal( res, np.random.RandomState(0).multivariate_normal( [1, 2], [[1, 0], [0, 1]], 10))) def testNegativeBinomialExecution(self): arr = tensor.random.negative_binomial(5, 1.0, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.negative_binomial(5, 1.0, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal( res, np.random.RandomState(0).negative_binomial(5, 1.0, 10))) def testNoncentralChisquareExecution(self): arr = tensor.random.noncentral_chisquare(.5, 1.0, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.noncentral_chisquare(.5, 1.0, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal( res, np.random.RandomState(0).noncentral_chisquare(.5, 1.0, 10))) def testNoncentralFExecution(self): arr = tensor.random.noncentral_f(1.5, 1.0, 1.1, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.noncentral_f(1.5, 1.0, 1.1, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal( res, np.random.RandomState(0).noncentral_f(1.5, 1.0, 1.1, 10))) def testNormalExecute(self): arr = tensor.random.normal(10, 1.0, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.normal(10, 1.0, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).normal(10, 1.0, 10))) def testParetoExecute(self): arr = tensor.random.pareto(1.0, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.pareto(1.0, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).pareto(1.0, 10))) def testPoissonExecute(self): arr = tensor.random.poisson(1.0, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.poisson(1.0, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).poisson(1.0, 10))) def testPowerExecute(self): arr = tensor.random.power(1.0, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.power(1.0, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).power(1.0, 10))) def testRayleighExecute(self): arr = tensor.random.rayleigh(1.0, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.rayleigh(1.0, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).rayleigh(1.0, 10))) def testStandardCauchyExecute(self): arr = tensor.random.standard_cauchy(100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.standard_cauchy(100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).standard_cauchy(10))) def testStandardExponentialExecute(self): arr = tensor.random.standard_exponential(100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.standard_exponential(100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal( res, np.random.RandomState(0).standard_exponential(10))) def testStandardGammaExecute(self): arr = tensor.random.standard_gamma(.1, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.standard_gamma(.1, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).standard_gamma(.1, 10))) def testStandardNormalExecute(self): arr = tensor.random.standard_normal(100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.standard_normal(100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).standard_normal(10))) def testStandardTExecute(self): arr = tensor.random.standard_t(.1, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.standard_t(.1, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).standard_t(.1, 10))) def testTriangularExecute(self): arr = tensor.random.triangular(.1, .2, .3, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.triangular(.1, .2, .3, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal( res, np.random.RandomState(0).triangular(.1, .2, .3, 10))) def testUniformExecute(self): arr = tensor.random.uniform(.1, .2, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.uniform(.1, .2, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).uniform(.1, .2, 10))) def testVonmisesExecute(self): arr = tensor.random.vonmises(.1, .2, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.vonmises(.1, .2, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).vonmises(.1, .2, 10))) def testWaldExecute(self): arr = tensor.random.wald(.1, .2, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.wald(.1, .2, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).wald(.1, .2, 10))) def testWeibullExecute(self): arr = tensor.random.weibull(.1, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.weibull(.1, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).weibull(.1, 10))) def testZipfExecute(self): arr = tensor.random.zipf(1.1, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.zipf(1.1, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._state = State(np.random.RandomState(0)) for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).zipf(1.1, 10)))
def setUp(self): super(Test, self).setUp() self.executor = Executor()
def setUp(self): self.executor = Executor()
def setUp(self): self.executor = Executor('numpy', storage=MockStorage(), prefetch=True)
class Test(unittest.TestCase): def setUp(self): self.executor = Executor('numpy') def _nan_equal(self, a, b): try: np.testing.assert_equal(a, b) except AssertionError: return False return True def testBaseExecution(self): arr = ones((10, 8), chunk_size=2) arr2 = arr + 1 res = self.executor.execute_tensor(arr2) self.assertTrue((res[0] == np.ones((2, 2)) + 1).all()) data = np.random.random((10, 8, 3)) arr = tensor(data, chunk_size=2) arr2 = arr + 1 res = self.executor.execute_tensor(arr2) self.assertTrue((res[0] == data[:2, :2, :2] + 1).all()) @staticmethod def _get_func(op): if isinstance(op, six.string_types): return getattr(np, op) return op def testUfuncExecution(self): from mars.tensor.expressions.arithmetic import UNARY_UFUNC, BIN_UFUNC, arccosh, \ invert, mod, fmod, bitand, bitor, bitxor, lshift, rshift, ldexp from mars.tensor.execution.arithmetic import OP_TO_HANDLER _sp_unary_ufunc = set([arccosh, invert]) _sp_bin_ufunc = set( [mod, fmod, bitand, bitor, bitxor, lshift, rshift, ldexp]) data1 = np.random.random((5, 9, 4)) data2 = np.random.random((5, 9, 4)) rand = np.random.random() arr1 = tensor(data1, chunk_size=3) arr2 = tensor(data2, chunk_size=3) _new_unary_ufunc = UNARY_UFUNC - _sp_unary_ufunc for func in _new_unary_ufunc: res_tensor = func(arr1) res = self.executor.execute_tensor(res_tensor, concat=True) expected = self._get_func(OP_TO_HANDLER[type( res_tensor.op)])(data1) self.assertTrue(np.allclose(res[0], expected)) _new_bin_ufunc = BIN_UFUNC - _sp_bin_ufunc for func in _new_bin_ufunc: res_tensor1 = func(arr1, arr2) res_tensor2 = func(arr1, rand) res_tensor3 = func(rand, arr1) res1 = self.executor.execute_tensor(res_tensor1, concat=True) res2 = self.executor.execute_tensor(res_tensor2, concat=True) res3 = self.executor.execute_tensor(res_tensor3, concat=True) expected1 = self._get_func(OP_TO_HANDLER[type(res_tensor1.op)])( data1, data2) expected2 = self._get_func(OP_TO_HANDLER[type(res_tensor1.op)])( data1, rand) expected3 = self._get_func(OP_TO_HANDLER[type(res_tensor1.op)])( rand, data1) self.assertTrue(np.allclose(res1[0], expected1)) self.assertTrue(np.allclose(res2[0], expected2)) self.assertTrue(np.allclose(res3[0], expected3)) data1 = np.random.randint(2, 10, size=(10, 10, 10)) data2 = np.random.randint(2, 10, size=(10, 10, 10)) rand = np.random.randint(1, 10) arr1 = tensor(data1, chunk_size=3) arr2 = tensor(data2, chunk_size=3) for func in _sp_unary_ufunc: res_tensor = func(arr1) res = self.executor.execute_tensor(res_tensor, concat=True) expected = self._get_func(OP_TO_HANDLER[type( res_tensor.op)])(data1) self.assertTrue(np.allclose(res[0], expected)) for func in _sp_bin_ufunc: res_tensor1 = func(arr1, arr2) res_tensor2 = func(arr1, rand) res_tensor3 = func(rand, arr1) res1 = self.executor.execute_tensor(res_tensor1, concat=True) res2 = self.executor.execute_tensor(res_tensor2, concat=True) res3 = self.executor.execute_tensor(res_tensor3, concat=True) expected1 = self._get_func(OP_TO_HANDLER[type(res_tensor1.op)])( data1, data2) expected2 = self._get_func(OP_TO_HANDLER[type(res_tensor1.op)])( data1, rand) expected3 = self._get_func(OP_TO_HANDLER[type(res_tensor1.op)])( rand, data1) self.assertTrue(np.allclose(res1[0], expected1)) self.assertTrue(np.allclose(res2[0], expected2)) self.assertTrue(np.allclose(res3[0], expected3)) @staticmethod def _get_sparse_func(op): from mars.lib.sparse.core import issparse if isinstance(op, six.string_types): op = getattr(np, op) def func(*args): new_args = [] for arg in args: if issparse(arg): new_args.append(arg.toarray()) else: new_args.append(arg) return op(*new_args) return func @staticmethod def toarray(x): if hasattr(x, 'toarray'): return x.toarray() return x def testSparseUfuncExexution(self): from mars.tensor.expressions.arithmetic import UNARY_UFUNC, BIN_UFUNC, arccosh, \ invert, mod, fmod, bitand, bitor, bitxor, lshift, rshift, ldexp from mars.tensor.execution.arithmetic import OP_TO_HANDLER _sp_unary_ufunc = set([arccosh, invert]) _sp_bin_ufunc = set( [mod, fmod, bitand, bitor, bitxor, lshift, rshift, ldexp]) data1 = sps.random(5, 9, density=.1) data2 = sps.random(5, 9, density=.2) rand = np.random.random() arr1 = tensor(data1, chunk_size=3) arr2 = tensor(data2, chunk_size=3) _new_unary_ufunc = UNARY_UFUNC - _sp_unary_ufunc for func in _new_unary_ufunc: res_tensor = func(arr1) res = self.executor.execute_tensor(res_tensor, concat=True) expected = self._get_sparse_func(OP_TO_HANDLER[type( res_tensor.op)])(data1) self._nan_equal(self.toarray(res[0]), expected) _new_bin_ufunc = BIN_UFUNC - _sp_bin_ufunc for func in _new_bin_ufunc: res_tensor1 = func(arr1, arr2) res_tensor2 = func(arr1, rand) res_tensor3 = func(rand, arr1) res1 = self.executor.execute_tensor(res_tensor1, concat=True) res2 = self.executor.execute_tensor(res_tensor2, concat=True) res3 = self.executor.execute_tensor(res_tensor3, concat=True) expected1 = self._get_sparse_func(OP_TO_HANDLER[type( res_tensor1.op)])(data1, data2) expected2 = self._get_sparse_func(OP_TO_HANDLER[type( res_tensor1.op)])(data1, rand) expected3 = self._get_sparse_func(OP_TO_HANDLER[type( res_tensor1.op)])(rand, data1) self._nan_equal(self.toarray(res1[0]), expected1) self._nan_equal(self.toarray(res2[0]), expected2) self._nan_equal(self.toarray(res3[0]), expected3) data1 = np.random.randint(2, 10, size=(10, 10)) data2 = np.random.randint(2, 10, size=(10, 10)) rand = np.random.randint(1, 10) arr1 = tensor(data1, chunk_size=3).tosparse() arr2 = tensor(data2, chunk_size=3).tosparse() for func in _sp_unary_ufunc: res_tensor = func(arr1) res = self.executor.execute_tensor(res_tensor, concat=True) expected = self._get_sparse_func(OP_TO_HANDLER[type( res_tensor.op)])(data1) self._nan_equal(self.toarray(res[0]), expected) for func in _sp_bin_ufunc: res_tensor1 = func(arr1, arr2) res_tensor2 = func(arr1, rand) res_tensor3 = func(rand, arr1) res1 = self.executor.execute_tensor(res_tensor1, concat=True) res2 = self.executor.execute_tensor(res_tensor2, concat=True) res3 = self.executor.execute_tensor(res_tensor3, concat=True) expected1 = self._get_sparse_func(OP_TO_HANDLER[type( res_tensor1.op)])(data1, data2) expected2 = self._get_sparse_func(OP_TO_HANDLER[type( res_tensor1.op)])(data1, rand) expected3 = self._get_sparse_func(OP_TO_HANDLER[type( res_tensor1.op)])(rand, data1) self._nan_equal(self.toarray(res1[0]), expected1) self._nan_equal(self.toarray(res2[0]), expected2) self._nan_equal(self.toarray(res3[0]), expected3) def testAddWithOutExecution(self): data1 = np.random.random((5, 9, 4)) data2 = np.random.random((9, 4)) arr1 = tensor(data1.copy(), chunk_size=3) arr2 = tensor(data2.copy(), chunk_size=3) add(arr1, arr2, out=arr1) res = self.executor.execute_tensor(arr1, concat=True)[0] self.assertTrue(np.array_equal(res, data1 + data2)) arr1 = tensor(data1.copy(), chunk_size=3) arr2 = tensor(data2.copy(), chunk_size=3) arr3 = add(arr1, arr2, out=arr1.astype('i4'), casting='unsafe') res = self.executor.execute_tensor(arr3, concat=True)[0] np.testing.assert_array_equal(res, (data1 + data2).astype('i4')) arr1 = tensor(data1.copy(), chunk_size=3) arr2 = tensor(data2.copy(), chunk_size=3) arr3 = truediv(arr1, arr2, out=arr1, where=arr2 > .5) res = self.executor.execute_tensor(arr3, concat=True)[0] self.assertTrue( np.array_equal( res, np.true_divide(data1, data2, out=data1.copy(), where=data2 > .5))) def testFrexpExecution(self): data1 = np.random.random((5, 9, 4)) arr1 = tensor(data1.copy(), chunk_size=3) o1, o2 = frexp(arr1) o = o1 + o2 res = self.executor.execute_tensor(o, concat=True)[0] expected = sum(np.frexp(data1)) self.assertTrue(np.allclose(res, expected)) arr1 = tensor(data1.copy(), chunk_size=3) o1 = zeros(data1.shape, chunk_size=3) o2 = zeros(data1.shape, dtype='i8', chunk_size=3) frexp(arr1, o1, o2) o = o1 + o2 res = self.executor.execute_tensor(o, concat=True)[0] expected = sum(np.frexp(data1)) self.assertTrue(np.allclose(res, expected)) data1 = sps.random(5, 9, density=.1) arr1 = tensor(data1.copy(), chunk_size=3) o1, o2 = frexp(arr1) o = o1 + o2 res = self.executor.execute_tensor(o, concat=True)[0] expected = sum(np.frexp(data1.toarray())) np.testing.assert_equal(res.toarray(), expected) def testModfExecution(self): data1 = np.random.random((5, 9)) arr1 = tensor(data1.copy(), chunk_size=3) o1, o2 = modf(arr1) o = o1 + o2 res = self.executor.execute_tensor(o, concat=True)[0] expected = sum(np.modf(data1)) self.assertTrue(np.allclose(res, expected)) arr1 = tensor(data1.copy(), chunk_size=3) o1 = zeros(data1.shape, chunk_size=3) o2 = zeros(data1.shape, chunk_size=3) modf(arr1, o1, o2) o = o1 + o2 res = self.executor.execute_tensor(o, concat=True)[0] expected = sum(np.modf(data1)) self.assertTrue(np.allclose(res, expected)) data1 = sps.random(5, 9, density=.1) arr1 = tensor(data1.copy(), chunk_size=3) o1, o2 = modf(arr1) o = o1 + o2 res = self.executor.execute_tensor(o, concat=True)[0] expected = sum(np.modf(data1.toarray())) np.testing.assert_equal(res.toarray(), expected) def testClipExecution(self): a_data = np.arange(10) a = tensor(a_data.copy(), chunk_size=3) b = clip(a, 1, 8) res = self.executor.execute_tensor(b, concat=True)[0] expected = np.clip(a_data, 1, 8) self.assertTrue(np.array_equal(res, expected)) a = tensor(a_data.copy(), chunk_size=3) clip(a, 3, 6, out=a) res = self.executor.execute_tensor(a, concat=True)[0] expected = np.clip(a_data, 3, 6) self.assertTrue(np.array_equal(res, expected)) with option_context() as options: options.tensor.chunk_size = 3 a = tensor(a_data.copy(), chunk_size=3) b = clip(a, [3, 4, 1, 1, 1, 4, 4, 4, 4, 4], 8) res = self.executor.execute_tensor(b, concat=True)[0] expected = np.clip(a_data, [3, 4, 1, 1, 1, 4, 4, 4, 4, 4], 8) self.assertTrue(np.array_equal(res, expected)) # test sparse clip a_data = sps.csr_matrix([[0, 2, 8], [0, 0, -1]]) a = tensor(a_data, chunk_size=3) b_data = sps.csr_matrix([[0, 3, 0], [1, 0, -2]]) c = clip(a, b_data, 4) res = self.executor.execute_tensor(c, concat=True)[0] expected = np.clip(a_data.toarray(), b_data.toarray(), 4) self.assertTrue(np.array_equal(res.toarray(), expected)) def testAroundExecution(self): data = np.random.randn(10, 20) x = tensor(data, chunk_size=3) t = x.round(2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.around(data, decimals=2) np.testing.assert_allclose(res, expected) data = sps.random(10, 20, density=.2) x = tensor(data, chunk_size=3) t = x.round(2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.around(data.toarray(), decimals=2) np.testing.assert_allclose(res.toarray(), expected) def testIsCloseExecution(self): data = np.array([1.05, 1.0, 1.01, np.nan]) data2 = np.array([1.04, 1.0, 1.03, np.nan]) x = tensor(data, chunk_size=2) y = tensor(data2, chunk_size=3) z = isclose(x, y, atol=.01) res = self.executor.execute_tensor(z, concat=True)[0] expected = np.isclose(data, data2, atol=.01) np.testing.assert_equal(res, expected) z = isclose(x, y, atol=.01, equal_nan=True) res = self.executor.execute_tensor(z, concat=True)[0] expected = np.isclose(data, data2, atol=.01, equal_nan=True) np.testing.assert_equal(res, expected) # test sparse data = sps.csr_matrix(np.array([0, 1.0, 1.01, np.nan])) data2 = sps.csr_matrix(np.array([0, 1.0, 1.03, np.nan])) x = tensor(data, chunk_size=2) y = tensor(data2, chunk_size=3) z = isclose(x, y, atol=.01) res = self.executor.execute_tensor(z, concat=True)[0] expected = np.isclose(data.toarray(), data2.toarray(), atol=.01) np.testing.assert_equal(res, expected) z = isclose(x, y, atol=.01, equal_nan=True) res = self.executor.execute_tensor(z, concat=True)[0] expected = np.isclose(data.toarray(), data2.toarray(), atol=.01, equal_nan=True) np.testing.assert_equal(res, expected) def testDtypeExecution(self): a = ones((10, 20), dtype='f4', chunk_size=5) c = truediv(a, 2, dtype='f8') res = self.executor.execute_tensor(c, concat=True)[0] self.assertEqual(res.dtype, np.float64) c = truediv(a, 0, dtype='f8') res = self.executor.execute_tensor(c, concat=True)[0] self.assertTrue(np.isinf(res[0, 0])) with self.assertRaises(FloatingPointError): with np.errstate(divide='raise'): c = truediv(a, 0, dtype='f8') _ = self.executor.execute_tensor(c, concat=True)[0] # noqa: F841 def testSetGetRealExecution(self): a_data = np.array([1 + 2j, 3 + 4j, 5 + 6j]) a = tensor(a_data, chunk_size=2) res = self.executor.execute_tensor(a.real, concat=True)[0] expected = a_data.real np.testing.assert_equal(res, expected) a.real = 9 res = self.executor.execute_tensor(a, concat=True)[0] expected = a_data.copy() expected.real = 9 np.testing.assert_equal(res, expected) a.real = np.array([9, 8, 7]) res = self.executor.execute_tensor(a, concat=True)[0] expected = a_data.copy() expected.real = np.array([9, 8, 7]) np.testing.assert_equal(res, expected) # test sparse a_data = np.array([[1 + 2j, 3 + 4j, 0], [0, 0, 0]]) a = tensor(sps.csr_matrix(a_data)) res = self.executor.execute_tensor(a.real, concat=True)[0].toarray() expected = a_data.real np.testing.assert_equal(res, expected) a.real = 9 res = self.executor.execute_tensor(a, concat=True)[0].toarray() expected = a_data.copy() expected.real = 9 np.testing.assert_equal(res, expected) a.real = np.array([9, 8, 7]) res = self.executor.execute_tensor(a, concat=True)[0].toarray() expected = a_data.copy() expected.real = np.array([9, 8, 7]) np.testing.assert_equal(res, expected) def testSetGetImagExecution(self): a_data = np.array([1 + 2j, 3 + 4j, 5 + 6j]) a = tensor(a_data, chunk_size=2) res = self.executor.execute_tensor(a.imag, concat=True)[0] expected = a_data.imag np.testing.assert_equal(res, expected) a.imag = 9 res = self.executor.execute_tensor(a, concat=True)[0] expected = a_data.copy() expected.imag = 9 np.testing.assert_equal(res, expected) a.imag = np.array([9, 8, 7]) res = self.executor.execute_tensor(a, concat=True)[0] expected = a_data.copy() expected.imag = np.array([9, 8, 7]) np.testing.assert_equal(res, expected) # test sparse a_data = np.array([[1 + 2j, 3 + 4j, 0], [0, 0, 0]]) a = tensor(sps.csr_matrix(a_data)) res = self.executor.execute_tensor(a.imag, concat=True)[0].toarray() expected = a_data.imag np.testing.assert_equal(res, expected) a.imag = 9 res = self.executor.execute_tensor(a, concat=True)[0].toarray() expected = a_data.copy() expected.imag = 9 np.testing.assert_equal(res, expected) a.imag = np.array([9, 8, 7]) res = self.executor.execute_tensor(a, concat=True)[0].toarray() expected = a_data.copy() expected.imag = np.array([9, 8, 7]) np.testing.assert_equal(res, expected)
class Test(unittest.TestCase): def setUp(self): self.executor = Executor() def testBaseExecution(self): executor_numpy = Executor('numpy') raw1 = np.random.randint(10, size=(10, 10, 10)) raw2 = np.random.randint(10, size=(10, 10, 10)) arr1 = tensor(raw1, chunk_size=3) arr2 = tensor(raw2, chunk_size=3) arr3 = arr1 + arr2 + 10 arr4 = 10 + arr1 + arr2 res3 = executor_numpy.execute_tensor(arr3, concat=True) res3_cmp = self.executor.execute_tensor(arr4, concat=True) self.assertTrue(np.array_equal(res3[0], res3_cmp[0])) res5 = executor_numpy.execute_tensor((arr1 + arr1), concat=True) res5_cmp = self.executor.execute_tensor((arr1 + arr1), concat=True) self.assertTrue(np.array_equal(res5[0], res5_cmp[0])) def testUnaryExecution(self): from mars.tensor.expressions.arithmetic import UNARY_UFUNC, arccosh, invert, sin, conj _sp_unary_ufunc = set([arccosh, invert, conj]) _new_unary_ufunc = list(UNARY_UFUNC - _sp_unary_ufunc) executor_numexpr = Executor() def _normalize_by_sin(func1, func2, arr): return func1(abs(sin((func2(arr))))) for i, j in itertools.permutations(range(len(_new_unary_ufunc)), 2): raw = np.random.random((8, 8, 8)) arr1 = tensor(raw, chunk_size=4) func1 = _new_unary_ufunc[i] func2 = _new_unary_ufunc[j] arr2 = _normalize_by_sin(func1, func2, arr1) res = executor_numexpr.execute_tensor(arr2, concat=True) res_cmp = self.executor.execute_tensor(arr2, concat=True) np.testing.assert_allclose(res[0], res_cmp[0]) raw = np.random.randint(100, size=(8, 8, 8)) arr1 = tensor(raw, chunk_size=4) arr2 = arccosh(1 + abs(invert(arr1))) res = executor_numexpr.execute_tensor(arr2, concat=True) res_cmp = self.executor.execute_tensor(arr2, concat=True) self.assertTrue(np.allclose(res[0], res_cmp[0])) def testBinExecution(self): from mars.tensor.expressions.arithmetic import BIN_UFUNC, mod, fmod, \ bitand, bitor, bitxor, lshift, rshift, ldexp _sp_bin_ufunc = [mod, fmod, bitand, bitor, bitxor, lshift, rshift] _new_bin_ufunc = list(BIN_UFUNC - set(_sp_bin_ufunc) - set([ldexp])) executor_numexpr = Executor() for i, j in itertools.permutations(range(len(_new_bin_ufunc)), 2): raw = np.random.random((9, 9, 9)) arr1 = tensor(raw, chunk_size=5) func1 = _new_bin_ufunc[i] func2 = _new_bin_ufunc[j] arr2 = func1(1, func2(2, arr1)) res = executor_numexpr.execute_tensor(arr2, concat=True) res_cmp = self.executor.execute_tensor(arr2, concat=True) self.assertTrue(np.allclose(res[0], res_cmp[0])) for i, j in itertools.permutations(range(len(_sp_bin_ufunc)), 2): raw = np.random.randint(1, 100, size=(10, 10, 10)) arr1 = tensor(raw, chunk_size=3) func1 = _sp_bin_ufunc[i] func2 = _sp_bin_ufunc[j] arr2 = func1(10, func2(arr1, 5)) res = executor_numexpr.execute_tensor(arr2, concat=True) res_cmp = self.executor.execute_tensor(arr2, concat=True) self.assertTrue(np.allclose(res[0], res_cmp[0])) def testReductionExecution(self): raw1 = np.random.randint(5, size=(8, 8, 8)) raw2 = np.random.randint(5, size=(8, 8, 8)) arr1 = tensor(raw1, chunk_size=3) arr2 = tensor(raw2, chunk_size=3) res1 = self.executor.execute_tensor((arr1 + 1).sum(keepdims=True)) res2 = self.executor.execute_tensor((arr1 + 1).prod(keepdims=True)) self.assertTrue(np.array_equal((raw1 + 1).sum(keepdims=True), res1[0])) self.assertTrue(np.array_equal((raw1 + 1).prod(keepdims=True), res2[0])) res1 = self.executor.execute_tensor((arr1 + 1).sum(axis=1), concat=True) res2 = self.executor.execute_tensor((arr1 + 1).prod(axis=1), concat=True) res3 = self.executor.execute_tensor((arr1 + 1).max(axis=1), concat=True) res4 = self.executor.execute_tensor((arr1 + 1).min(axis=1), concat=True) self.assertTrue(np.array_equal((raw1 + 1).sum(axis=1), res1[0])) self.assertTrue(np.array_equal((raw1 + 1).prod(axis=1), res2[0])) self.assertTrue(np.array_equal((raw1 + 1).max(axis=1), res3[0])) self.assertTrue(np.array_equal((raw1 + 1).min(axis=1), res4[0])) raw3 = raw2 - raw1 + 10 arr3 = -arr1 + arr2 + 10 res1 = self.executor.execute_tensor(arr3.sum(axis=(0, 1)), concat=True) res2 = self.executor.execute_tensor(arr3.prod(axis=(0, 1)), concat=True) res3 = self.executor.execute_tensor(arr3.max(axis=(0, 1)), concat=True) res4 = self.executor.execute_tensor(arr3.min(axis=(0, 1)), concat=True) self.assertTrue(np.array_equal(raw3.sum(axis=(0, 1)), res1[0])) self.assertTrue(np.array_equal(raw3.prod(axis=(0, 1)), res2[0])) self.assertTrue(np.array_equal(raw3.max(axis=(0, 1)), res3[0])) self.assertTrue(np.array_equal(raw3.min(axis=(0, 1)), res4[0])) def testBoolReductionExecution(self): raw = np.random.randint(5, size=(8, 8, 8)) arr = tensor(raw, chunk_size=2) res = self.executor.execute_tensor((arr > 3).sum(axis=1), concat=True) np.testing.assert_array_equal(res[0], (raw > 3).sum(axis=1)) res = self.executor.execute_tensor((arr > 3).sum()) np.testing.assert_array_equal(res, (raw > 3).sum())
class Test(unittest.TestCase): def setUp(self): self.executor = Executor('numpy') def testSumProdExecution(self): arr = ones((10, 8), chunk_size=3) self.assertEqual([80], self.executor.execute_tensor(arr.sum())) self.assertEqual( (10, ) * 8, tuple(np.concatenate(self.executor.execute_tensor( arr.sum(axis=0))))) arr = ones((3, 3), chunk_size=2) self.assertEqual([512], self.executor.execute_tensor((arr * 2).prod())) self.assertEqual((8, ) * 3, tuple( np.concatenate( self.executor.execute_tensor( (arr * 2).prod(axis=0))))) raw = sps.random(10, 20, density=.1) arr = tensor(raw, chunk_size=3) res = self.executor.execute_tensor(arr.sum())[0] self.assertAlmostEqual(res, raw.sum()) def testMaxMinExecution(self): raw = np.random.randint(10000, size=(10, 10, 10)) arr = tensor(raw, chunk_size=3) self.assertEqual([raw.max()], self.executor.execute_tensor(arr.max())) self.assertEqual([raw.min()], self.executor.execute_tensor(arr.min())) np.testing.assert_array_equal( raw.max(axis=0), self.executor.execute_tensor(arr.max(axis=0), concat=True)[0]) np.testing.assert_array_equal( raw.min(axis=0), self.executor.execute_tensor(arr.min(axis=0), concat=True)[0]) np.testing.assert_array_equal( raw.max(axis=(1, 2)), self.executor.execute_tensor(arr.max(axis=(1, 2)), concat=True)[0]) np.testing.assert_array_equal( raw.min(axis=(1, 2)), self.executor.execute_tensor(arr.min(axis=(1, 2)), concat=True)[0]) raw = sps.random(10, 10, density=.5) arr = tensor(raw, chunk_size=3) self.assertEqual([raw.max()], self.executor.execute_tensor(arr.max())) self.assertEqual([raw.min()], self.executor.execute_tensor(arr.min())) def testAllAnyExecution(self): raw1 = np.zeros((10, 15)) raw2 = np.ones((10, 15)) raw3 = np.array([[True, False, True, False], [True, True, True, True], [False, False, False, False], [False, True, False, True]]) arr1 = tensor(raw1, chunk_size=3) arr2 = tensor(raw2, chunk_size=3) arr3 = tensor(raw3, chunk_size=4) self.assertFalse(self.executor.execute_tensor(arr1.all())[0]) self.assertTrue(self.executor.execute_tensor(arr2.all())[0]) self.assertFalse(self.executor.execute_tensor(arr1.any())[0]) self.assertTrue(self.executor.execute_tensor(arr1.any())) np.testing.assert_array_equal( raw3.all(axis=1), self.executor.execute_tensor(arr3.all(axis=1))[0]) np.testing.assert_array_equal( raw3.any(axis=0), self.executor.execute_tensor(arr3.any(axis=0))[0]) raw = sps.random(10, 10, density=.5) > .5 arr = tensor(raw, chunk_size=3) self.assertEqual(raw.A.all(), self.executor.execute_tensor(arr.all())[0]) self.assertEqual(raw.A.any(), self.executor.execute_tensor(arr.any())[0]) def testMeanExecution(self): raw1 = np.random.random((20, 25)) raw2 = np.random.randint(10, size=(20, 25)) arr1 = tensor(raw1, chunk_size=3) res1 = self.executor.execute_tensor(arr1.mean()) expected1 = raw1.mean() self.assertTrue(np.allclose(res1[0], expected1)) res2 = self.executor.execute_tensor(arr1.mean(axis=0)) expected2 = raw1.mean(axis=0) self.assertTrue(np.allclose(np.concatenate(res2), expected2)) res3 = self.executor.execute_tensor(arr1.mean(axis=1, keepdims=True)) expected3 = raw1.mean(axis=1, keepdims=True) self.assertTrue(np.allclose(np.concatenate(res3), expected3)) arr2 = tensor(raw2, chunk_size=3) res1 = self.executor.execute_tensor(arr2.mean()) expected1 = raw2.mean() self.assertEqual(res1[0], expected1) res2 = self.executor.execute_tensor(arr2.mean(axis=0)) expected2 = raw2.mean(axis=0) self.assertTrue(np.allclose(np.concatenate(res2), expected2)) res3 = self.executor.execute_tensor(arr2.mean(axis=1, keepdims=True)) expected3 = raw2.mean(axis=1, keepdims=True) self.assertTrue(np.allclose(np.concatenate(res3), expected3)) raw1 = sps.random(20, 25, density=.1) arr1 = tensor(raw1, chunk_size=3) res1 = self.executor.execute_tensor(arr1.mean()) expected1 = raw1.mean() self.assertTrue(np.allclose(res1[0], expected1)) arr2 = tensor(raw1, chunk_size=30) res1 = self.executor.execute_tensor(arr2.mean()) expected1 = raw1.mean() self.assertTrue(np.allclose(res1[0], expected1)) arr = mean(1) self.assertEqual(self.executor.execute_tensor(arr)[0], 1) def testVarExecution(self): raw1 = np.random.random((20, 25)) raw2 = np.random.randint(10, size=(20, 25)) arr1 = tensor(raw1, chunk_size=3) res1 = self.executor.execute_tensor(arr1.var()) expected1 = raw1.var() self.assertTrue(np.allclose(res1[0], expected1)) res2 = self.executor.execute_tensor(arr1.var(axis=0)) expected2 = raw1.var(axis=0) self.assertTrue(np.allclose(np.concatenate(res2), expected2)) res3 = self.executor.execute_tensor(arr1.var(axis=1, keepdims=True)) expected3 = raw1.var(axis=1, keepdims=True) self.assertTrue(np.allclose(np.concatenate(res3), expected3)) arr2 = tensor(raw2, chunk_size=3) res1 = self.executor.execute_tensor(arr2.var()) expected1 = raw2.var() self.assertAlmostEqual(res1[0], expected1) res2 = self.executor.execute_tensor(arr2.var(axis=0)) expected2 = raw2.var(axis=0) self.assertTrue(np.allclose(np.concatenate(res2), expected2)) res3 = self.executor.execute_tensor(arr2.var(axis=1, keepdims=True)) expected3 = raw2.var(axis=1, keepdims=True) self.assertTrue(np.allclose(np.concatenate(res3), expected3)) res4 = self.executor.execute_tensor(arr2.var(ddof=1)) expected4 = raw2.var(ddof=1) self.assertAlmostEqual(res4[0], expected4) raw1 = sps.random(20, 25, density=.1) arr1 = tensor(raw1, chunk_size=3) res1 = self.executor.execute_tensor(arr1.var()) expected1 = raw1.toarray().var() self.assertTrue(np.allclose(res1[0], expected1)) arr2 = tensor(raw1, chunk_size=30) res1 = self.executor.execute_tensor(arr2.var()) expected1 = raw1.toarray().var() self.assertTrue(np.allclose(res1[0], expected1)) arr = var(1) self.assertEqual(self.executor.execute_tensor(arr)[0], 0) def testStdExecution(self): raw1 = np.random.random((20, 25)) raw2 = np.random.randint(10, size=(20, 25)) arr1 = tensor(raw1, chunk_size=3) res1 = self.executor.execute_tensor(arr1.std()) expected1 = raw1.std() self.assertTrue(np.allclose(res1[0], expected1)) res2 = self.executor.execute_tensor(arr1.std(axis=0)) expected2 = raw1.std(axis=0) self.assertTrue(np.allclose(np.concatenate(res2), expected2)) res3 = self.executor.execute_tensor(arr1.std(axis=1, keepdims=True)) expected3 = raw1.std(axis=1, keepdims=True) self.assertTrue(np.allclose(np.concatenate(res3), expected3)) arr2 = tensor(raw2, chunk_size=3) res1 = self.executor.execute_tensor(arr2.std()) expected1 = raw2.std() self.assertAlmostEqual(res1[0], expected1) res2 = self.executor.execute_tensor(arr2.std(axis=0)) expected2 = raw2.std(axis=0) self.assertTrue(np.allclose(np.concatenate(res2), expected2)) res3 = self.executor.execute_tensor(arr2.std(axis=1, keepdims=True)) expected3 = raw2.std(axis=1, keepdims=True) self.assertTrue(np.allclose(np.concatenate(res3), expected3)) res4 = self.executor.execute_tensor(arr2.std(ddof=1)) expected4 = raw2.std(ddof=1) self.assertAlmostEqual(res4[0], expected4) raw1 = sps.random(20, 25, density=.1) arr1 = tensor(raw1, chunk_size=3) res1 = self.executor.execute_tensor(arr1.std()) expected1 = raw1.toarray().std() self.assertTrue(np.allclose(res1[0], expected1)) arr2 = tensor(raw1, chunk_size=30) res1 = self.executor.execute_tensor(arr2.std()) expected1 = raw1.toarray().std() self.assertTrue(np.allclose(res1[0], expected1)) arr = std(1) self.assertEqual(self.executor.execute_tensor(arr)[0], 0) def testArgReduction(self): raw = np.random.random((20, 20, 20)) arr = tensor(raw, chunk_size=3) self.assertEqual(raw.argmax(), self.executor.execute_tensor(arr.argmax())[0]) self.assertEqual(raw.argmin(), self.executor.execute_tensor(arr.argmin())[0]) np.testing.assert_array_equal( raw.argmax(axis=0), self.executor.execute_tensor(arr.argmax(axis=0), concat=True)[0]) np.testing.assert_array_equal( raw.argmin(axis=0), self.executor.execute_tensor(arr.argmin(axis=0), concat=True)[0]) raw_format = sps.random(20, 20, density=.1, format='lil') random_min = np.random.randint(0, 200) random_max = np.random.randint(200, 400) raw_format[np.unravel_index(random_min, raw_format.shape)] = -1 raw_format[np.unravel_index(random_max, raw_format.shape)] = 2 raw = raw_format.tocoo() arr = tensor(raw, chunk_size=3) self.assertEqual(raw.argmax(), self.executor.execute_tensor(arr.argmax())[0]) self.assertEqual(raw.argmin(), self.executor.execute_tensor(arr.argmin())[0]) def testNanReduction(self): raw = np.random.choice(a=[0, 1, np.nan], size=(10, 10), p=[0.3, 0.4, 0.3]) arr = tensor(raw, chunk_size=3) self.assertEqual(np.nansum(raw), self.executor.execute_tensor(nansum(arr))[0]) self.assertEqual(np.nanprod(raw), self.executor.execute_tensor(nanprod(arr))[0]) self.assertEqual(np.nanmax(raw), self.executor.execute_tensor(nanmax(arr))[0]) self.assertEqual(np.nanmin(raw), self.executor.execute_tensor(nanmin(arr))[0]) self.assertEqual(np.nanmean(raw), self.executor.execute_tensor(nanmean(arr))[0]) self.assertAlmostEqual(np.nanvar(raw), self.executor.execute_tensor(nanvar(arr))[0]) self.assertAlmostEqual( np.nanvar(raw, ddof=1), self.executor.execute_tensor(nanvar(arr, ddof=1))[0]) self.assertAlmostEqual(np.nanstd(raw), self.executor.execute_tensor(nanstd(arr))[0]) self.assertAlmostEqual( np.nanstd(raw, ddof=1), self.executor.execute_tensor(nanstd(arr, ddof=1))[0]) arr = tensor(raw, chunk_size=10) self.assertEqual(np.nansum(raw), self.executor.execute_tensor(nansum(arr))[0]) self.assertEqual(np.nanprod(raw), self.executor.execute_tensor(nanprod(arr))[0]) self.assertEqual(np.nanmax(raw), self.executor.execute_tensor(nanmax(arr))[0]) self.assertEqual(np.nanmin(raw), self.executor.execute_tensor(nanmin(arr))[0]) self.assertEqual(np.nanmean(raw), self.executor.execute_tensor(nanmean(arr))[0]) self.assertAlmostEqual(np.nanvar(raw), self.executor.execute_tensor(nanvar(arr))[0]) self.assertAlmostEqual( np.nanvar(raw, ddof=1), self.executor.execute_tensor(nanvar(arr, ddof=1))[0]) self.assertAlmostEqual(np.nanstd(raw), self.executor.execute_tensor(nanstd(arr))[0]) self.assertAlmostEqual( np.nanstd(raw, ddof=1), self.executor.execute_tensor(nanstd(arr, ddof=1))[0]) raw = np.random.random((10, 10)) raw[:3, :3] = np.nan arr = tensor(raw, chunk_size=3) self.assertEqual(np.nanargmin(raw), self.executor.execute_tensor(nanargmin(arr))[0]) self.assertEqual(np.nanargmax(raw), self.executor.execute_tensor(nanargmax(arr))[0]) raw = np.full((10, 10), np.nan) arr = tensor(raw, chunk_size=3) self.assertEqual(0, self.executor.execute_tensor(nansum(arr))[0]) self.assertEqual(1, self.executor.execute_tensor(nanprod(arr))[0]) self.assertTrue(np.isnan(self.executor.execute_tensor(nanmax(arr))[0])) self.assertTrue(np.isnan(self.executor.execute_tensor(nanmin(arr))[0])) self.assertTrue(np.isnan( self.executor.execute_tensor(nanmean(arr))[0])) self.assertRaises( ValueError, lambda: self.executor.execute_tensor(nanargmin(arr))[0]) self.assertRaises( ValueError, lambda: self.executor.execute_tensor(nanargmax(arr))[0]) raw = sps.random(10, 10, density=.1, format='csr') raw[:3, :3] = np.nan arr = tensor(raw, chunk_size=3) self.assertAlmostEqual(np.nansum(raw.A), self.executor.execute_tensor(nansum(arr))[0]) self.assertAlmostEqual(np.nanprod(raw.A), self.executor.execute_tensor(nanprod(arr))[0]) self.assertAlmostEqual(np.nanmax(raw.A), self.executor.execute_tensor(nanmax(arr))[0]) self.assertAlmostEqual(np.nanmin(raw.A), self.executor.execute_tensor(nanmin(arr))[0]) self.assertAlmostEqual(np.nanmean(raw.A), self.executor.execute_tensor(nanmean(arr))[0]) self.assertAlmostEqual(np.nanvar(raw.A), self.executor.execute_tensor(nanvar(arr))[0]) self.assertAlmostEqual( np.nanvar(raw.A, ddof=1), self.executor.execute_tensor(nanvar(arr, ddof=1))[0]) self.assertAlmostEqual(np.nanstd(raw.A), self.executor.execute_tensor(nanstd(arr))[0]) self.assertAlmostEqual( np.nanstd(raw.A, ddof=1), self.executor.execute_tensor(nanstd(arr, ddof=1))[0]) arr = nansum(1) self.assertEqual(self.executor.execute_tensor(arr)[0], 1) def testCumReduction(self): raw = np.random.randint(5, size=(8, 8, 8)) arr = tensor(raw, chunk_size=3) res1 = self.executor.execute_tensor(arr.cumsum(axis=1), concat=True) res2 = self.executor.execute_tensor(arr.cumprod(axis=1), concat=True) expected1 = raw.cumsum(axis=1) expected2 = raw.cumprod(axis=1) np.testing.assert_array_equal(res1[0], expected1) np.testing.assert_array_equal(res2[0], expected2) raw = sps.random(8, 8, density=.1) arr = tensor(raw, chunk_size=3) res1 = self.executor.execute_tensor(arr.cumsum(axis=1), concat=True) res2 = self.executor.execute_tensor(arr.cumprod(axis=1), concat=True) expected1 = raw.A.cumsum(axis=1) expected2 = raw.A.cumprod(axis=1) self.assertTrue(np.allclose(res1[0], expected1)) self.assertTrue(np.allclose(res2[0], expected2)) def testNanCumReduction(self): raw = np.random.randint(5, size=(8, 8, 8)) raw[:2, 2:4, 4:6] = np.nan arr = tensor(raw, chunk_size=3) res1 = self.executor.execute_tensor(nancumsum(arr, axis=1), concat=True) res2 = self.executor.execute_tensor(nancumprod(arr, axis=1), concat=True) expected1 = np.nancumsum(raw, axis=1) expected2 = np.nancumprod(raw, axis=1) np.testing.assert_array_equal(res1[0], expected1) np.testing.assert_array_equal(res2[0], expected2) raw = sps.random(8, 8, density=.1, format='lil') raw[:2, 2:4] = np.nan arr = tensor(raw, chunk_size=3) res1 = self.executor.execute_tensor(nancumsum(arr, axis=1), concat=True)[0] res2 = self.executor.execute_tensor(nancumprod(arr, axis=1), concat=True)[0] expected1 = np.nancumsum(raw.A, axis=1) expected2 = np.nancumprod(raw.A, axis=1) self.assertTrue(np.allclose(res1, expected1)) self.assertTrue(np.allclose(res2, expected2)) def testOutReductionExecution(self): raw = np.random.randint(5, size=(8, 8, 8)) arr = tensor(raw, chunk_size=3) arr2 = ones((8, 8), dtype='i8', chunk_size=3) arr.sum(axis=1, out=arr2) res = self.executor.execute_tensor(arr2, concat=True)[0] expected = raw.sum(axis=1) np.testing.assert_array_equal(res, expected) def testOutCumReductionExecution(self): raw = np.random.randint(5, size=(8, 8, 8)) arr = tensor(raw, chunk_size=3) arr.cumsum(axis=0, out=arr) res = self.executor.execute_tensor(arr, concat=True)[0] expected = raw.cumsum(axis=0) np.testing.assert_array_equal(res, expected) def testCountNonzeroExecution(self): raw = [[0, 1, 7, 0, 0], [3, 0, 0, 2, 19]] arr = tensor(raw, chunk_size=2) t = count_nonzero(arr) res = self.executor.execute_tensor(t)[0] expected = np.count_nonzero(raw) np.testing.assert_equal(res, expected) t = count_nonzero(arr, axis=0) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.count_nonzero(raw, axis=0) np.testing.assert_equal(res, expected) t = count_nonzero(arr, axis=1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.count_nonzero(raw, axis=1) np.testing.assert_equal(res, expected) raw = sps.csr_matrix(raw) arr = tensor(raw, chunk_size=2) t = count_nonzero(arr) res = self.executor.execute_tensor(t)[0] expected = np.count_nonzero(raw.A) np.testing.assert_equal(res, expected) t = count_nonzero(arr, axis=0) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.count_nonzero(raw.A, axis=0) np.testing.assert_equal(res, expected) t = count_nonzero(arr, axis=1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.count_nonzero(raw.A, axis=1) np.testing.assert_equal(res, expected) def testAllcloseExecution(self): a = tensor([1e10, 1e-7], chunk_size=1) b = tensor([1.00001e10, 1e-8], chunk_size=1) t = allclose(a, b) res = self.executor.execute_tensor(t)[0] self.assertFalse(res) a = tensor([1e10, 1e-8], chunk_size=1) b = tensor([1.00001e10, 1e-9], chunk_size=1) t = allclose(a, b) res = self.executor.execute_tensor(t)[0] self.assertTrue(res) a = tensor([1.0, np.nan], chunk_size=1) b = tensor([1.0, np.nan], chunk_size=1) t = allclose(a, b, equal_nan=True) res = self.executor.execute_tensor(t)[0] self.assertTrue(res) a = tensor(sps.csr_matrix([[1e10, 1e-7], [0, 0]]), chunk_size=1) b = tensor(sps.csr_matrix([[1.00001e10, 1e-8], [0, 0]]), chunk_size=1) t = allclose(a, b) res = self.executor.execute_tensor(t)[0] self.assertFalse(res) def testArrayEqual(self): a = ones((10, 5), chunk_size=1) b = ones((10, 5), chunk_size=2) c = array_equal(a, b) res = bool(self.executor.execute_tensor(c)[0]) self.assertTrue(res)
class Test(unittest.TestCase): def setUp(self): self.executor = Executor('numpy') def testRechunkExecution(self): raw = np.random.random((11, 8)) arr = tensor(raw, chunks=3) arr2 = arr.rechunk(4) res = self.executor.execute_tensor(arr2) self.assertTrue(np.array_equal(res[0], raw[:4, :4])) self.assertTrue(np.array_equal(res[1], raw[:4, 4:])) self.assertTrue(np.array_equal(res[2], raw[4:8, :4])) self.assertTrue(np.array_equal(res[3], raw[4:8, 4:])) self.assertTrue(np.array_equal(res[4], raw[8:, :4])) self.assertTrue(np.array_equal(res[5], raw[8:, 4:])) def testCopytoExecution(self): a = ones((2, 3), chunks=1) b = tensor([3, -1, 3], chunks=2) copyto(a, b, where=b > 1) res = self.executor.execute_tensor(a, concat=True)[0] expected = np.array([[3, 1, 3], [3, 1, 3]]) np.testing.assert_equal(res, expected) def testAstypeExecution(self): raw = np.random.random((10, 5)) arr = tensor(raw, chunks=3) arr2 = arr.astype('i8') res = self.executor.execute_tensor(arr2, concat=True) self.assertTrue(np.array_equal(res[0], raw.astype('i8'))) raw = sps.random(10, 5, density=.2) arr = tensor(raw, chunks=3) arr2 = arr.astype('i8') res = self.executor.execute_tensor(arr2, concat=True) self.assertTrue(np.array_equal(res[0].toarray(), raw.astype('i8').toarray())) def testTransposeExecution(self): raw = np.random.random((11, 8, 5)) arr = tensor(raw, chunks=3) arr2 = transpose(arr) res = self.executor.execute_tensor(arr2, concat=True) self.assertTrue(np.array_equal(res[0], raw.T)) arr3 = transpose(arr, axes=(-2, -1, -3)) res = self.executor.execute_tensor(arr3, concat=True) self.assertTrue(np.array_equal(res[0], raw.transpose(1, 2, 0))) raw = sps.random(11, 8) arr = tensor(raw, chunks=3) arr2 = transpose(arr) self.assertTrue(arr2.issparse()) res = self.executor.execute_tensor(arr2, concat=True) self.assertTrue(np.array_equal(res[0].toarray(), raw.T.toarray())) def testSwapaxesExecution(self): raw = np.random.random((11, 8, 5)) arr = tensor(raw, chunks=3) arr2 = arr.swapaxes(2, 0) res = self.executor.execute_tensor(arr2, concat=True) self.assertTrue(np.array_equal(res[0], raw.swapaxes(2, 0))) raw = sps.random(11, 8, density=.2) arr = tensor(raw, chunks=3) arr2 = arr.swapaxes(1, 0) res = self.executor.execute_tensor(arr2, concat=True) self.assertTrue(np.array_equal(res[0].toarray(), raw.toarray().swapaxes(1, 0))) def testMoveaxisExecution(self): x = zeros((3, 4, 5), chunks=2) t = moveaxis(x, 0, -1) res = self.executor.execute_tensor(t, concat=True)[0] self.assertEqual(res.shape, (4, 5, 3)) t = moveaxis(x, -1, 0) res = self.executor.execute_tensor(t, concat=True)[0] self.assertEqual(res.shape, (5, 3, 4)) t = moveaxis(x, [0, 1], [-1, -2]) res = self.executor.execute_tensor(t, concat=True)[0] self.assertEqual(res.shape, (5, 4, 3)) t = moveaxis(x, [0, 1, 2], [-1, -2, -3]) res = self.executor.execute_tensor(t, concat=True)[0] self.assertEqual(res.shape, (5, 4, 3)) def testBroadcastToExecution(self): raw = np.random.random((10, 5, 1)) arr = tensor(raw, chunks=2) arr2 = broadcast_to(arr, (5, 10, 5, 6)) res = self.executor.execute_tensor(arr2, concat=True) self.assertTrue(np.array_equal(res[0], np.broadcast_to(raw, (5, 10, 5, 6)))) def testBroadcastArraysExecutions(self): x_data = [[1, 2, 3]] x = tensor(x_data, chunks=1) y_data = [[1], [2], [3]] y = tensor(y_data, chunks=2) a = broadcast_arrays(x, y) res = [self.executor.execute_tensor(arr, concat=True)[0] for arr in a] expected = np.broadcast_arrays(x_data, y_data) for r, e in zip(res, expected): np.testing.assert_equal(r, e) def testWhereExecution(self): raw_cond = np.random.randint(0, 2, size=(4, 4), dtype='?') raw_x = np.random.rand(4, 1) raw_y = np.random.rand(4, 4) cond, x, y = tensor(raw_cond, chunks=2), tensor(raw_x, chunks=2), tensor(raw_y, chunks=2) arr = where(cond, x, y) res = self.executor.execute_tensor(arr, concat=True) self.assertTrue(np.array_equal(res[0], np.where(raw_cond, raw_x, raw_y))) raw_cond = sps.csr_matrix(np.random.randint(0, 2, size=(4, 4), dtype='?')) raw_x = sps.random(4, 1, density=.1) raw_y = sps.random(4, 4, density=.1) cond, x, y = tensor(raw_cond, chunks=2), tensor(raw_x, chunks=2), tensor(raw_y, chunks=2) arr = where(cond, x, y) res = self.executor.execute_tensor(arr, concat=True)[0] self.assertTrue(np.array_equal(res.toarray(), np.where(raw_cond.toarray(), raw_x.toarray(), raw_y.toarray()))) def testReshapeExecution(self): raw_data = np.random.rand(10, 20, 30) x = tensor(raw_data, chunks=6) y = x.reshape(-1, 30) res = self.executor.execute_tensor(y, concat=True) self.assertTrue(np.array_equal(res[0], raw_data.reshape(-1, 30))) y2 = x.reshape(10, -1) res = self.executor.execute_tensor(y2, concat=True) self.assertTrue(np.array_equal(res[0], raw_data.reshape(10, -1))) y3 = x.reshape(-1) res = self.executor.execute_tensor(y3, concat=True) self.assertTrue(np.array_equal(res[0], raw_data.reshape(-1))) y4 = x.ravel() res = self.executor.execute_tensor(y4, concat=True) self.assertTrue(np.array_equal(res[0], raw_data.ravel())) raw_data = np.random.rand(30, 100, 20) x = tensor(raw_data, chunks=6) y = x.reshape(-1, 20, 5, 5, 4) res = self.executor.execute_tensor(y, concat=True) self.assertTrue(np.array_equal(res[0], raw_data.reshape(-1, 20, 5, 5, 4))) y2 = x.reshape(3000, 10, 2) res = self.executor.execute_tensor(y2, concat=True) self.assertTrue(np.array_equal(res[0], raw_data.reshape(3000, 10, 2))) y3 = x.reshape(60, 25, 40) res = self.executor.execute_tensor(y3, concat=True) self.assertTrue(np.array_equal(res[0], raw_data.reshape(60, 25, 40))) def testExpandDimsExecution(self): raw_data = np.random.rand(10, 20, 30) x = tensor(raw_data, chunks=6) y = expand_dims(x, 1) res = self.executor.execute_tensor(y, concat=True) self.assertTrue(np.array_equal(res[0], np.expand_dims(raw_data, 1))) y = expand_dims(x, 0) res = self.executor.execute_tensor(y, concat=True) self.assertTrue(np.array_equal(res[0], np.expand_dims(raw_data, 0))) y = expand_dims(x, 3) res = self.executor.execute_tensor(y, concat=True) self.assertTrue(np.array_equal(res[0], np.expand_dims(raw_data, 3))) y = expand_dims(x, -1) res = self.executor.execute_tensor(y, concat=True) self.assertTrue(np.array_equal(res[0], np.expand_dims(raw_data, -1))) y = expand_dims(x, -4) res = self.executor.execute_tensor(y, concat=True) self.assertTrue(np.array_equal(res[0], np.expand_dims(raw_data, -4))) with self.assertRaises(np.AxisError): expand_dims(x, -5) with self.assertRaises(np.AxisError): expand_dims(x, 4) def testRollAxisExecution(self): x = ones((3, 4, 5, 6), chunks=1) y = rollaxis(x, 3, 1) res = self.executor.execute_tensor(y, concat=True) self.assertTrue(np.array_equal(res[0], np.rollaxis(np.ones((3, 4, 5, 6)), 3, 1))) def testAtleast1dExecution(self): x = 1 y = ones(3, chunks=2) z = ones((3, 4), chunks=2) t = atleast_1d(x, y, z) res = [self.executor.execute_tensor(i, concat=True)[0] for i in t] self.assertTrue(np.array_equal(res[0], np.array([1]))) self.assertTrue(np.array_equal(res[1], np.ones(3))) self.assertTrue(np.array_equal(res[2], np.ones((3, 4)))) def testAtleast2dExecution(self): x = 1 y = ones(3, chunks=2) z = ones((3, 4), chunks=2) t = atleast_2d(x, y, z) res = [self.executor.execute_tensor(i, concat=True)[0] for i in t] self.assertTrue(np.array_equal(res[0], np.array([[1]]))) self.assertTrue(np.array_equal(res[1], np.atleast_2d(np.ones(3)))) self.assertTrue(np.array_equal(res[2], np.ones((3, 4)))) def testAtleast3dExecution(self): x = 1 y = ones(3, chunks=2) z = ones((3, 4), chunks=2) t = atleast_3d(x, y, z) res = [self.executor.execute_tensor(i, concat=True)[0] for i in t] self.assertTrue(np.array_equal(res[0], np.atleast_3d(x))) self.assertTrue(np.array_equal(res[1], np.atleast_3d(np.ones(3)))) self.assertTrue(np.array_equal(res[2], np.atleast_3d(np.ones((3, 4))))) def testArgwhereExecution(self): x = arange(6, chunks=2).reshape(2, 3) t = argwhere(x > 1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.argwhere(np.arange(6).reshape(2, 3) > 1) self.assertTrue(np.array_equal(res, expected)) def testArraySplitExecution(self): x = arange(48, chunks=3).reshape(2, 3, 8) ss = array_split(x, 3, axis=2) res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss] expected = np.array_split(np.arange(48).reshape(2, 3, 8), 3, axis=2) self.assertEqual(len(res), len(expected)) [np.testing.assert_equal(r, e) for r, e in zip(res, expected)] ss = array_split(x, [3, 5, 6, 10], axis=2) res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss] expected = np.array_split(np.arange(48).reshape(2, 3, 8), [3, 5, 6, 10], axis=2) self.assertEqual(len(res), len(expected)) [np.testing.assert_equal(r, e) for r, e in zip(res, expected)] def testSplitExecution(self): x = arange(48, chunks=3).reshape(2, 3, 8) ss = split(x, 4, axis=2) res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss] expected = np.split(np.arange(48).reshape(2, 3, 8), 4, axis=2) self.assertEqual(len(res), len(expected)) [np.testing.assert_equal(r, e) for r, e in zip(res, expected)] ss = split(x, [3, 5, 6, 10], axis=2) res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss] expected = np.split(np.arange(48).reshape(2, 3, 8), [3, 5, 6, 10], axis=2) self.assertEqual(len(res), len(expected)) [np.testing.assert_equal(r, e) for r, e in zip(res, expected)] # hsplit x = arange(120, chunks=3).reshape(2, 12, 5) ss = hsplit(x, 4) res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss] expected = np.hsplit(np.arange(120).reshape(2, 12, 5), 4) self.assertEqual(len(res), len(expected)) [np.testing.assert_equal(r, e) for r, e in zip(res, expected)] # vsplit x = arange(48, chunks=3).reshape(8, 3, 2) ss = vsplit(x, 4) res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss] expected = np.vsplit(np.arange(48).reshape(8, 3, 2), 4) self.assertEqual(len(res), len(expected)) [np.testing.assert_equal(r, e) for r, e in zip(res, expected)] # dsplit x = arange(48, chunks=3).reshape(2, 3, 8) ss = dsplit(x, 4) res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss] expected = np.dsplit(np.arange(48).reshape(2, 3, 8), 4) self.assertEqual(len(res), len(expected)) [np.testing.assert_equal(r, e) for r, e in zip(res, expected)] x_data = sps.random(12, 8, density=.1) x = tensor(x_data, chunks=3) ss = split(x, 4, axis=0) res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss] expected = np.split(x_data.toarray(), 4, axis=0) self.assertEqual(len(res), len(expected)) [np.testing.assert_equal(r.toarray(), e) for r, e in zip(res, expected)] def testRollExecution(self): x = arange(10, chunks=2) t = roll(x, 2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.roll(np.arange(10), 2) np.testing.assert_equal(res, expected) x2 = x.reshape(2, 5) t = roll(x2, 1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.roll(np.arange(10).reshape(2, 5), 1) np.testing.assert_equal(res, expected) t = roll(x2, 1, axis=0) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.roll(np.arange(10).reshape(2, 5), 1, axis=0) np.testing.assert_equal(res, expected) t = roll(x2, 1, axis=1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.roll(np.arange(10).reshape(2, 5), 1, axis=1) np.testing.assert_equal(res, expected) def testSqueezeExecution(self): data = np.array([[[0], [1], [2]]]) x = tensor(data, chunks=1) t = squeeze(x) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.squeeze(data) np.testing.assert_equal(res, expected) t = squeeze(x, axis=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.squeeze(data, axis=2) np.testing.assert_equal(res, expected) def testPtpExecution(self): x = arange(4, chunks=1).reshape(2, 2) t = ptp(x, axis=0) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.ptp(np.arange(4).reshape(2, 2), axis=0) np.testing.assert_equal(res, expected) t = ptp(x, axis=1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.ptp(np.arange(4).reshape(2, 2), axis=1) np.testing.assert_equal(res, expected) t = ptp(x) res = self.executor.execute_tensor(t)[0] expected = np.ptp(np.arange(4).reshape(2, 2)) np.testing.assert_equal(res, expected) def testDiffExecution(self): data = np.array([1, 2, 4, 7, 0]) x = tensor(data, chunks=2) t = diff(x) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.diff(data) np.testing.assert_equal(res, expected) t = diff(x, n=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.diff(data, n=2) np.testing.assert_equal(res, expected) data = np.array([[1, 3, 6, 10], [0, 5, 6, 8]]) x = tensor(data, chunks=2) t = diff(x) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.diff(data) np.testing.assert_equal(res, expected) t = diff(x, axis=0) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.diff(data, axis=0) np.testing.assert_equal(res, expected) x = mt.arange('1066-10-13', '1066-10-16', dtype=mt.datetime64) t = diff(x) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.diff(np.arange('1066-10-13', '1066-10-16', dtype=np.datetime64)) np.testing.assert_equal(res, expected) def testEdiff1d(self): data = np.array([1, 2, 4, 7, 0]) x = tensor(data, chunks=2) t = ediff1d(x) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.ediff1d(data) np.testing.assert_equal(res, expected) to_begin = tensor(-99, chunks=2) to_end = tensor([88, 99], chunks=2) t = ediff1d(x, to_begin=to_begin, to_end=to_end) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.ediff1d(data, to_begin=-99, to_end=np.array([88, 99])) np.testing.assert_equal(res, expected) data = [[1, 2, 4], [1, 6, 24]] t = ediff1d(tensor(data, chunks=2)) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.ediff1d(data) np.testing.assert_equal(res, expected) def testDigitizeExecution(self): data = np.array([0.2, 6.4, 3.0, 1.6]) x = tensor(data, chunks=2) bins = np.array([0.0, 1.0, 2.5, 4.0, 10.0]) inds = digitize(x, bins) res = self.executor.execute_tensor(inds, concat=True)[0] expected = np.digitize(data, bins) np.testing.assert_equal(res, expected) b = tensor(bins, chunks=2) inds = digitize(x, b) res = self.executor.execute_tensor(inds, concat=True)[0] expected = np.digitize(data, bins) np.testing.assert_equal(res, expected) data = np.array([1.2, 10.0, 12.4, 15.5, 20.]) x = tensor(data, chunks=2) bins = np.array([0, 5, 10, 15, 20]) inds = digitize(x, bins, right=True) res = self.executor.execute_tensor(inds, concat=True)[0] expected = np.digitize(data, bins, right=True) np.testing.assert_equal(res, expected) inds = digitize(x, bins, right=False) res = self.executor.execute_tensor(inds, concat=True)[0] expected = np.digitize(data, bins, right=False) np.testing.assert_equal(res, expected) data = sps.random(10, 1, density=.1) * 12 x = tensor(data, chunks=2) bins = np.array([1.0, 2.0, 2.5, 4.0, 10.0]) inds = digitize(x, bins) res = self.executor.execute_tensor(inds, concat=True)[0] expected = np.digitize(data.toarray(), bins, right=False) np.testing.assert_equal(res.toarray(), expected) def testAverageExecution(self): data = arange(1, 5, chunks=1) t = average(data) res = self.executor.execute_tensor(t)[0] expected = np.average(np.arange(1, 5)) self.assertEqual(res, expected) t = average(arange(1, 11, chunks=2), weights=arange(10, 0, -1, chunks=2)) res = self.executor.execute_tensor(t)[0] expected = np.average(range(1, 11), weights=range(10, 0, -1)) self.assertEqual(res, expected) data = arange(6, chunks=2).reshape((3, 2)) t = average(data, axis=1, weights=tensor([1./4, 3./4], chunks=2)) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.average(np.arange(6).reshape(3, 2), axis=1, weights=(1./4, 3./4)) np.testing.assert_equal(res, expected) with self.assertRaises(TypeError): average(data, weights=tensor([1./4, 3./4], chunks=2)) def testCovExecution(self): data = np.array([[0, 2], [1, 1], [2, 0]]).T x = tensor(data, chunks=1) t = cov(x) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.cov(data) np.testing.assert_equal(res, expected) data_x = [-2.1, -1, 4.3] data_y = [3, 1.1, 0.12] x = tensor(data_x, chunks=1) y = tensor(data_y, chunks=1) X = stack((x, y), axis=0) t = cov(x, y) r = tall(t == cov(X)) self.assertTrue(self.executor.execute_tensor(r)[0]) def testCorrcoefExecution(self): data_x = [-2.1, -1, 4.3] data_y = [3, 1.1, 0.12] x = tensor(data_x, chunks=1) y = tensor(data_y, chunks=1) t = corrcoef(x, y) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.corrcoef(data_x, data_y) np.testing.assert_equal(res, expected) def testFlipExecution(self): a = arange(8, chunks=2).reshape((2, 2, 2)) t = flip(a, 0) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.flip(np.arange(8).reshape(2, 2, 2), 0) np.testing.assert_equal(res, expected) t = flip(a, 1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.flip(np.arange(8).reshape(2, 2, 2), 1) np.testing.assert_equal(res, expected) t = flipud(a) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.flipud(np.arange(8).reshape(2, 2, 2)) np.testing.assert_equal(res, expected) t = fliplr(a) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.fliplr(np.arange(8).reshape(2, 2, 2)) np.testing.assert_equal(res, expected) def testRepeatExecution(self): a = repeat(3, 4) res = self.executor.execute_tensor(a)[0] expected = np.repeat(3, 4) np.testing.assert_equal(res, expected) x_data = np.random.randn(20, 30) x = tensor(x_data, chunks=(3, 4)) t = repeat(x, 2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.repeat(x_data, 2) np.testing.assert_equal(res, expected) t = repeat(x, 3, axis=1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.repeat(x_data, 3, axis=1) np.testing.assert_equal(res, expected) t = repeat(x, np.arange(20), axis=0) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.repeat(x_data, np.arange(20), axis=0) np.testing.assert_equal(res, expected) t = repeat(x, arange(20, chunks=5), axis=0) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.repeat(x_data, np.arange(20), axis=0) np.testing.assert_equal(res, expected) x_data = sps.random(20, 30, density=.1) x = tensor(x_data, chunks=(3, 4)) t = repeat(x, 2, axis=1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.repeat(x_data.toarray(), 2, axis=1) np.testing.assert_equal(res.toarray(), expected) def testTileExecution(self): a_data = np.array([0, 1, 2]) a = tensor(a_data, chunks=2) t = tile(a, 2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.tile(a_data, 2) np.testing.assert_equal(res, expected) t = tile(a, (2, 2)) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.tile(a_data, (2, 2)) np.testing.assert_equal(res, expected) t = tile(a, (2, 1, 2)) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.tile(a_data, (2, 1, 2)) np.testing.assert_equal(res, expected) b_data = np.array([[1, 2], [3, 4]]) b = tensor(b_data, chunks=1) t = tile(b, 2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.tile(b_data, 2) np.testing.assert_equal(res, expected) t = tile(b, (2, 1)) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.tile(b_data, (2, 1)) np.testing.assert_equal(res, expected) c_data = np.array([1, 2, 3, 4]) c = tensor(c_data, chunks=3) t = tile(c, (4, 1)) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.tile(c_data, (4, 1)) np.testing.assert_equal(res, expected) def testIsInExecution(self): element = 2 * arange(4, chunks=1).reshape((2, 2)) test_elements = [1, 2, 4, 8] mask = isin(element, test_elements) res = self.executor.execute_tensor(mask, concat=True)[0] expected = np.isin(2 * np.arange(4).reshape((2, 2)), test_elements) np.testing.assert_equal(res, expected) res = self.executor.execute_tensor(element[mask], concat=True)[0] expected = np.array([2, 4]) np.testing.assert_equal(res, expected) mask = isin(element, test_elements, invert=True) res = self.executor.execute_tensor(mask, concat=True)[0] expected = np.isin(2 * np.arange(4).reshape((2, 2)), test_elements, invert=True) np.testing.assert_equal(res, expected) res = self.executor.execute_tensor(element[mask], concat=True)[0] expected = np.array([0, 6]) np.testing.assert_equal(res, expected) test_set = {1, 2, 4, 8} mask = isin(element, test_set) res = self.executor.execute_tensor(mask, concat=True)[0] expected = np.isin(2 * np.arange(4).reshape((2, 2)), test_set) np.testing.assert_equal(res, expected)
class Test(unittest.TestCase): def setUp(self): self.executor = Executor('numpy') def testQRExecution(self): data = np.random.randn(18, 6) a = tensor(data, chunk_size=(3, 6)) q, r = qr(a) t = q.dot(r) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, data)) a = tensor(data, chunk_size=(9, 6)) q, r = qr(a) t = q.dot(r) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, data)) a = tensor(data, chunk_size=3) q, r = qr(a) t = q.dot(r) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, data)) def testSVDExecution(self): data = np.random.randn(18, 6) + 1j * np.random.randn(18, 6) a = tensor(data, chunk_size=(9, 6)) U, s, V = svd(a) t = U.dot(diag(s).dot(V)) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, data)) a = tensor(data, chunk_size=(18, 6)) U, s, V = svd(a) t = U.dot(diag(s).dot(V)) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, data)) a = tensor(data, chunk_size=(2, 6)) U, s, V = svd(a) t = U.dot(diag(s).dot(V)) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, data)) def testCholeskyExecution(self): data = np.array([[1, -2j], [2j, 5]]) a = tensor(data, chunk_size=1) L = cholesky(a, lower=True) t = L.dot(L.T.conj()) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, data)) L = cholesky(a, lower=True) U = cholesky(a) t = L.dot(U) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, data)) a = tensor(data, chunk_size=2) L = cholesky(a, lower=True) U = cholesky(a) t = L.dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data) a = tensor(data, chunk_size=(1, 2)) L = cholesky(a, lower=True) U = cholesky(a) t = L.dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data) def testLUExecution(self): np.random.seed(1) data = np.random.randint(1, 10, (6, 6)) a = tensor(data, chunk_size=2) P, L, U = lu(a) t = P.dot(L).dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data) a = tensor(data, chunk_size=1) P, L, U = lu(a) t = P.dot(L).dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data) a = tensor(data, chunk_size=(1, 2)) P, L, U = lu(a) t = P.dot(L).dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data) a = tensor(data, chunk_size=3) P, L, U = lu(a) t = P.dot(L).dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data) def testSolveTriangular(self): from mars.tensor import tril, triu np.random.seed(1) data1 = np.random.randint(1, 10, (20, 20)) data2 = np.random.randint(1, 10, (20, )) A = tensor(data1, chunk_size=20) b = tensor(data2, chunk_size=20) x = solve_triangular(A, b) t = triu(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) x = solve_triangular(A, b, lower=True) t = tril(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) A = tensor(data1, chunk_size=10) b = tensor(data2, chunk_size=10) x = solve_triangular(A, b) t = triu(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) x = solve_triangular(A, b, lower=True) t = tril(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) data1 = np.random.randint(1, 10, (10, 10)) data2 = np.random.randint(1, 10, (10, 5)) A = tensor(data1, chunk_size=10) b = tensor(data2, chunk_size=10) x = solve_triangular(A, b) t = triu(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) x = solve_triangular(A, b, lower=True) t = tril(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) A = tensor(data1, chunk_size=3) b = tensor(data2, chunk_size=3) x = solve_triangular(A, b) t = triu(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) x = solve_triangular(A, b, lower=True) t = tril(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) def testSolve(self): import scipy.linalg np.random.seed(1) data1 = np.random.randint(1, 10, (20, 20)) data2 = np.random.randint(1, 10, (20, )) A = tensor(data1, chunk_size=5) b = tensor(data2, chunk_size=5) x = solve(A, b) res = self.executor.execute_tensor(x, concat=True)[0] np.testing.assert_allclose(res, scipy.linalg.solve(data1, data2)) res = self.executor.execute_tensor(A.dot(x), concat=True)[0] np.testing.assert_allclose(res, data2) data2 = np.random.randint(1, 10, (20, 5)) A = tensor(data1, chunk_size=5) b = tensor(data2, chunk_size=5) x = solve(A, b) res = self.executor.execute_tensor(x, concat=True)[0] np.testing.assert_allclose(res, scipy.linalg.solve(data1, data2)) res = self.executor.execute_tensor(A.dot(x), concat=True)[0] np.testing.assert_allclose(res, data2) data2 = np.random.randint(1, 10, (20, 20)) A = tensor(data1, chunk_size=5) b = tensor(data2, chunk_size=5) x = solve(A, b) res = self.executor.execute_tensor(x, concat=True)[0] np.testing.assert_allclose(res, scipy.linalg.solve(data1, data2)) res = self.executor.execute_tensor(A.dot(x), concat=True)[0] np.testing.assert_allclose(res, data2) def testSolveSymPos(self): import scipy.linalg np.random.seed(1) data = np.random.randint(1, 10, (20, 20)) data_l = np.tril(data) data1 = data_l.dot(data_l.T) data2 = np.random.randint(1, 10, (20, )) A = tensor(data1, chunk_size=5) b = tensor(data2, chunk_size=5) x = solve(A, b, sym_pos=True) res = self.executor.execute_tensor(x, concat=True)[0] np.testing.assert_allclose(res, scipy.linalg.solve(data1, data2)) res = self.executor.execute_tensor(A.dot(x), concat=True)[0] np.testing.assert_allclose(res, data2) def testInv(self): import scipy.linalg np.random.seed(1) data = np.random.randint(1, 10, (20, 20)) A = tensor(data, chunk_size=5) inv_A = inv(A) res = self.executor.execute_tensor(inv_A, concat=True)[0] self.assertTrue(np.allclose(res, scipy.linalg.inv(data))) res = self.executor.execute_tensor(A.dot(inv_A), concat=True)[0] self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float))) def testNormExecution(self): d = np.arange(9) - 4 d2 = d.reshape(3, 3) ma = [ tensor(d, chunk_size=2), tensor(d, chunk_size=9), tensor(d2, chunk_size=(2, 3)), tensor(d2, chunk_size=3) ] for i, a in enumerate(ma): data = d if i < 2 else d2 for ord in (None, 'nuc', np.inf, -np.inf, 0, 1, -1, 2, -2): for axis in (0, 1, (0, 1)): for keepdims in (True, False): try: expected = np.linalg.norm(data, ord=ord, axis=axis, keepdims=keepdims) t = norm(a, ord=ord, axis=axis, keepdims=keepdims) concat = t.ndim > 0 res = self.executor.execute_tensor( t, concat=concat)[0] np.testing.assert_allclose(res, expected, atol=.0001) except ValueError: continue m = norm(tensor(d)) expected = self.executor.execute_tensor(m)[0] res = np.linalg.norm(d) self.assertEqual(expected, res) d = uniform(-0.5, 0.5, size=(500, 2), chunk_size=50) inside = (norm(d, axis=1) < 0.5).sum().astype(float) t = inside / 500 * 4 res = self.executor.execute_tensor(t)[0] self.assertAlmostEqual(res, 3.14, delta=1) def testTensordotExecution(self): a_data = np.arange(60).reshape(3, 4, 5) a = tensor(a_data, chunk_size=2) b_data = np.arange(24).reshape(4, 3, 2) b = tensor(b_data, chunk_size=2) axes = ([1, 0], [0, 1]) c = tensordot(a, b, axes=axes) res = self.executor.execute_tensor(c) expected = np.tensordot(a_data, b_data, axes=axes) self.assertTrue(np.array_equal(res[0], expected[:2, :])) self.assertTrue(np.array_equal(res[1], expected[2:4, :])) self.assertTrue(np.array_equal(res[2], expected[4:, :])) a = ones((1000, 2000), chunk_size=500) b = ones((2000, 100), chunk_size=500) c = dot(a, b) res = self.executor.execute_tensor(c) expected = np.dot(np.ones((1000, 2000)), np.ones((2000, 100))) self.assertEqual(len(res), 2) self.assertTrue(np.array_equal(res[0], expected[:500, :])) self.assertTrue(np.array_equal(res[1], expected[500:, :])) a = ones((10, 8), chunk_size=2) b = ones((8, 10), chunk_size=2) c = a.dot(b) res = self.executor.execute_tensor(c) self.assertEqual(len(res), 25) for r in res: self.assertTrue(np.array_equal(r, np.tile([8], [2, 2]))) a = ones((500, 500), chunk_size=500) b = ones((500, 100), chunk_size=500) c = a.dot(b) res = self.executor.execute_tensor(c) self.assertTrue(np.array_equal(res[0], np.tile([500], [500, 100]))) raw_a = np.random.random((100, 200, 50)) raw_b = np.random.random((200, 10, 100)) a = tensor(raw_a, chunk_size=50) b = tensor(raw_b, chunk_size=33) c = tensordot(a, b, axes=((0, 1), (2, 0))) res = self.executor.execute_tensor(c, concat=True) expected = np.tensordot(raw_a, raw_b, axes=(c.op.a_axes, c.op.b_axes)) self.assertTrue(np.allclose(res[0], expected)) a = ones((1000, 2000), chunk_size=500) b = ones((100, 2000), chunk_size=500) c = inner(a, b) res = self.executor.execute_tensor(c) expected = np.inner(np.ones((1000, 2000)), np.ones((100, 2000))) self.assertEqual(len(res), 2) self.assertTrue(np.array_equal(res[0], expected[:500, :])) self.assertTrue(np.array_equal(res[1], expected[500:, :])) def testSparseDotExecution(self): a_data = sps.random(5, 9, density=.1) b_data = sps.random(9, 10, density=.2) a = tensor(a_data, chunk_size=2) b = tensor(b_data, chunk_size=3) c = dot(a, b) res = self.executor.execute_tensor(c, concat=True)[0] self.assertTrue(issparse(res)) np.testing.assert_allclose(res.toarray(), a_data.dot(b_data).toarray()) c2 = dot(a, b, sparse=False) res = self.executor.execute_tensor(c2, concat=True)[0] self.assertFalse(issparse(res)) np.testing.assert_allclose(res, a_data.dot(b_data).toarray()) c3 = tensordot(a, b.T, (-1, -1), sparse=False) res = self.executor.execute_tensor(c3, concat=True)[0] self.assertFalse(issparse(res)) np.testing.assert_allclose(res, a_data.dot(b_data).toarray()) c = inner(a, b.T) res = self.executor.execute_tensor(c, concat=True)[0] self.assertTrue(issparse(res)) np.testing.assert_allclose(res.toarray(), a_data.dot(b_data).toarray()) c = inner(a, b.T, sparse=False) res = self.executor.execute_tensor(c, concat=True)[0] self.assertFalse(issparse(res)) np.testing.assert_allclose(res, a_data.dot(b_data).toarray()) def testVdotExecution(self): a_data = np.array([1 + 2j, 3 + 4j]) b_data = np.array([5 + 6j, 7 + 8j]) a = tensor(a_data, chunk_size=1) b = tensor(b_data, chunk_size=1) t = vdot(a, b) res = self.executor.execute_tensor(t)[0] expected = np.vdot(a_data, b_data) np.testing.assert_equal(res, expected) a_data = np.array([[1, 4], [5, 6]]) b_data = np.array([[4, 1], [2, 2]]) a = tensor(a_data, chunk_size=1) b = tensor(b_data, chunk_size=1) t = vdot(a, b) res = self.executor.execute_tensor(t)[0] expected = np.vdot(a_data, b_data) np.testing.assert_equal(res, expected) def testMatmulExecution(self): data_a = np.random.randn(10, 20) data_b = np.random.randn(20) a = tensor(data_a, chunk_size=2) b = tensor(data_b, chunk_size=3) c = matmul(a, b) res = self.executor.execute_tensor(c, concat=True)[0] expected = np.matmul(data_a, data_b) np.testing.assert_allclose(res, expected) data_a = np.random.randn(10, 20) data_b = np.random.randn(10) a = tensor(data_a, chunk_size=2) b = tensor(data_b, chunk_size=3) c = matmul(b, a) res = self.executor.execute_tensor(c, concat=True)[0] expected = np.matmul(data_b, data_a) np.testing.assert_allclose(res, expected) data_a = np.random.randn(15, 1, 20, 30) data_b = np.random.randn(1, 11, 30, 20) a = tensor(data_a, chunk_size=12) b = tensor(data_b, chunk_size=13) c = matmul(a, b) res = self.executor.execute_tensor(c, concat=True)[0] expected = np.matmul(data_a, data_b) np.testing.assert_allclose(res, expected, atol=.0001) a = arange(2 * 2 * 4, chunk_size=1).reshape((2, 2, 4)) b = arange(2 * 2 * 4, chunk_size=1).reshape((2, 4, 2)) c = matmul(a, b) res = self.executor.execute_tensor(c, concat=True)[0] expected = np.matmul( np.arange(2 * 2 * 4).reshape(2, 2, 4), np.arange(2 * 2 * 4).reshape(2, 4, 2)) np.testing.assert_allclose(res, expected, atol=.0001) data_a = sps.random(10, 20) data_b = sps.random(20, 5) a = tensor(data_a, chunk_size=2) b = tensor(data_b, chunk_size=3) c = matmul(a, b) res = self.executor.execute_tensor(c, concat=True)[0] expected = np.matmul(data_a.toarray(), data_b.toarray()) np.testing.assert_allclose(res.toarray(), expected)
class Test(TestBase): def setUp(self): super(Test, self).setUp() self.executor = Executor() def testCreateSparseExecution(self): mat = sps.csr_matrix([[0, 0, 2], [2, 0, 0]]) t = tensor(mat, dtype='f8', chunks=2) res = self.executor.execute_tensor(t) self.assertIsInstance(res[0], SparseNDArray) self.assertEqual(res[0].dtype, np.float64) np.testing.assert_array_equal(res[0].toarray(), mat[..., :2].toarray()) np.testing.assert_array_equal(res[1].toarray(), mat[..., 2:].toarray()) t2 = ones_like(t, dtype='f4') res = self.executor.execute_tensor(t2) expected = sps.csr_matrix([[0, 0, 1], [1, 0, 0]]) self.assertIsInstance(res[0], SparseNDArray) self.assertEqual(res[0].dtype, np.float32) np.testing.assert_array_equal(res[0].toarray(), expected[..., :2].toarray()) np.testing.assert_array_equal(res[1].toarray(), expected[..., 2:].toarray()) t3 = tensor(np.array([[0, 0, 2], [2, 0, 0]]), chunks=2).tosparse() res = self.executor.execute_tensor(t3) self.assertIsInstance(res[0], SparseNDArray) self.assertEqual(res[0].dtype, np.int_) np.testing.assert_array_equal(res[0].toarray(), mat[..., :2].toarray()) np.testing.assert_array_equal(res[1].toarray(), mat[..., 2:].toarray()) def testZerosExecution(self): t = zeros((20, 30), dtype='i8', chunks=5) res = self.executor.execute_tensor(t, concat=True) self.assertTrue(np.array_equal(res[0], np.zeros((20, 30), dtype='i8'))) self.assertEqual(res[0].dtype, np.int64) t2 = zeros_like(t) res = self.executor.execute_tensor(t2, concat=True) self.assertTrue(np.array_equal(res[0], np.zeros((20, 30), dtype='i8'))) self.assertEqual(res[0].dtype, np.int64) t = zeros((20, 30), dtype='i4', chunks=5, sparse=True) res = self.executor.execute_tensor(t, concat=True) self.assertEqual(res[0].nnz, 0) def testEmptyExecution(self): t = empty((20, 30), dtype='i8', chunks=5) res = self.executor.execute_tensor(t, concat=True) self.assertEqual(res[0].shape, (20, 30)) self.assertEqual(res[0].dtype, np.int64) self.assertFalse(np.array_equal(res, np.zeros((20, 30)))) t = empty((20, 30), chunks=5) res = self.executor.execute_tensor(t, concat=True) self.assertFalse(np.allclose(res, np.zeros((20, 30)))) t2 = empty_like(t) res = self.executor.execute_tensor(t2, concat=True) self.assertEqual(res[0].shape, (20, 30)) self.assertEqual(res[0].dtype, np.float64) def testFullExecution(self): t = full((2, 2), 1, dtype='f4', chunks=1) res = self.executor.execute_tensor(t, concat=True) self.assertTrue(np.array_equal(res[0], np.full((2, 2), 1, dtype='f4'))) t = full((2, 2), [1, 2], dtype='f8', chunks=1) res = self.executor.execute_tensor(t, concat=True) self.assertTrue( np.array_equal(res[0], np.full((2, 2), [1, 2], dtype='f8'))) def testArangeExecution(self): t = arange(1, 20, 3, chunks=2) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.array_equal(res, np.arange(1, 20, 3))) t = arange(1, 20, .3, chunks=4) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.arange(1, 20, .3) self.assertTrue(np.allclose(res, expected)) t = arange(1.0, 1.8, .3, chunks=4) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.arange(1.0, 1.8, .3) self.assertTrue(np.allclose(res, expected)) t = arange('1066-10-13', '1066-10-31', dtype=np.datetime64, chunks=3) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.arange('1066-10-13', '1066-10-31', dtype=np.datetime64) self.assertTrue(np.array_equal(res, expected)) def testDiagExecution(self): # 2-d 6 * 6 a = arange(36, chunks=2).reshape(6, 6) d = diag(a) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.diag(np.arange(36).reshape(6, 6)) np.testing.assert_equal(res, expected) d = diag(a, k=1) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.diag(np.arange(36).reshape(6, 6), k=1) np.testing.assert_equal(res, expected) d = diag(a, k=3) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.diag(np.arange(36).reshape(6, 6), k=3) np.testing.assert_equal(res, expected) d = diag(a, k=-2) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.diag(np.arange(36).reshape(6, 6), k=-2) np.testing.assert_equal(res, expected) d = diag(a, k=-5) res = self.executor.execute_tensor(d)[0] expected = np.diag(np.arange(36).reshape(6, 6), k=-5) np.testing.assert_equal(res, expected) # 2-d 4 * 9 a = arange(36, chunks=2).reshape(4, 9) d = diag(a) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.diag(np.arange(36).reshape(4, 9)) np.testing.assert_equal(res, expected) d = diag(a, k=1) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.diag(np.arange(36).reshape(4, 9), k=1) np.testing.assert_equal(res, expected) d = diag(a, k=3) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.diag(np.arange(36).reshape(4, 9), k=3) np.testing.assert_equal(res, expected) d = diag(a, k=-2) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.diag(np.arange(36).reshape(4, 9), k=-2) np.testing.assert_equal(res, expected) d = diag(a, k=-3) res = self.executor.execute_tensor(d)[0] expected = np.diag(np.arange(36).reshape(4, 9), k=-3) np.testing.assert_equal(res, expected) # 1-d a = arange(5, chunks=2) d = diag(a) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.diag(np.arange(5)) np.testing.assert_equal(res, expected) d = diag(a, k=1) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.diag(np.arange(5), k=1) np.testing.assert_equal(res, expected) d = diag(a, k=3) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.diag(np.arange(5), k=3) np.testing.assert_equal(res, expected) d = diag(a, k=-2) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.diag(np.arange(5), k=-2) np.testing.assert_equal(res, expected) d = diag(a, k=-3) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.diag(np.arange(5), k=-3) np.testing.assert_equal(res, expected) d = diag(a, sparse=True) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.diag(np.arange(5)) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res.toarray(), expected) d = diag(a, k=1, sparse=True) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.diag(np.arange(5), k=1) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res.toarray(), expected) d = diag(a, k=2, sparse=True) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.diag(np.arange(5), k=2) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res.toarray(), expected) d = diag(a, k=-2, sparse=True) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.diag(np.arange(5), k=-2) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res.toarray(), expected) d = diag(a, k=-3, sparse=True) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.diag(np.arange(5), k=-3) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res.toarray(), expected) def testDiagflatExecution(self): a = diagflat([[1, 2], [3, 4]], chunks=1) res = self.executor.execute_tensor(a, concat=True)[0] expected = np.diagflat([[1, 2], [3, 4]]) np.testing.assert_equal(res, expected) d = tensor([[1, 2], [3, 4]], chunks=1) a = diagflat(d) res = self.executor.execute_tensor(a, concat=True)[0] expected = np.diagflat([[1, 2], [3, 4]]) np.testing.assert_equal(res, expected) a = diagflat([1, 2], 1, chunks=1) res = self.executor.execute_tensor(a, concat=True)[0] expected = np.diagflat([1, 2], 1) np.testing.assert_equal(res, expected) d = tensor([[1, 2]], chunks=1) a = diagflat(d, 1) res = self.executor.execute_tensor(a, concat=True)[0] expected = np.diagflat([1, 2], 1) np.testing.assert_equal(res, expected) def testEyeExecution(self): t = eye(5, chunks=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.eye(5) np.testing.assert_equal(res, expected) t = eye(5, k=1, chunks=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.eye(5, k=1) np.testing.assert_equal(res, expected) t = eye(5, k=2, chunks=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.eye(5, k=2) np.testing.assert_equal(res, expected) t = eye(5, k=-1, chunks=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.eye(5, k=-1) np.testing.assert_equal(res, expected) t = eye(5, k=-3, chunks=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.eye(5, k=-3) np.testing.assert_equal(res, expected) t = eye(5, M=3, k=1, chunks=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.eye(5, M=3, k=1) np.testing.assert_equal(res, expected) t = eye(5, M=3, k=-3, chunks=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.eye(5, M=3, k=-3) np.testing.assert_equal(res, expected) t = eye(5, M=7, k=1, chunks=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.eye(5, M=7, k=1) np.testing.assert_equal(res, expected) t = eye(5, M=8, k=-3, chunks=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.eye(5, M=8, k=-3) np.testing.assert_equal(res, expected) t = eye(2, dtype=int) res = self.executor.execute_tensor(t, concat=True)[0] self.assertEqual(res.dtype, np.int_) # test sparse t = eye(5, sparse=True, chunks=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.eye(5) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res.toarray(), expected) t = eye(5, k=1, sparse=True, chunks=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.eye(5, k=1) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res.toarray(), expected) t = eye(5, k=2, sparse=True, chunks=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.eye(5, k=2) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res.toarray(), expected) t = eye(5, k=-1, sparse=True, chunks=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.eye(5, k=-1) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res.toarray(), expected) t = eye(5, k=-3, sparse=True, chunks=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.eye(5, k=-3) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res.toarray(), expected) t = eye(5, M=3, k=1, sparse=True, chunks=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.eye(5, M=3, k=1) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res.toarray(), expected) t = eye(5, M=3, k=-3, sparse=True, chunks=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.eye(5, M=3, k=-3) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res.toarray(), expected) t = eye(5, M=7, k=1, sparse=True, chunks=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.eye(5, M=7, k=1) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res.toarray(), expected) t = eye(5, M=8, k=-3, sparse=True, chunks=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.eye(5, M=8, k=-3) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res.toarray(), expected) def testLinspaceExecution(self): a = linspace(2.0, 9.0, num=11, chunks=3) res = self.executor.execute_tensor(a, concat=True)[0] expected = np.linspace(2.0, 9.0, num=11) np.testing.assert_allclose(res, expected) a = linspace(2.0, 9.0, num=11, endpoint=False, chunks=3) res = self.executor.execute_tensor(a, concat=True)[0] expected = np.linspace(2.0, 9.0, num=11, endpoint=False) np.testing.assert_allclose(res, expected) a = linspace(2.0, 9.0, num=11, chunks=3, dtype=int) res = self.executor.execute_tensor(a, concat=True)[0] self.assertEqual(res.dtype, np.int_) def testMeshgridExecution(self): a = arange(5, chunks=2) b = arange(6, 12, chunks=3) c = arange(12, 19, chunks=4) A, B, C = meshgrid(a, b, c) A_res = self.executor.execute_tensor(A, concat=True)[0] A_expected = np.meshgrid(np.arange(5), np.arange(6, 12), np.arange(12, 19))[0] np.testing.assert_equal(A_res, A_expected) B_res = self.executor.execute_tensor(B, concat=True)[0] B_expected = np.meshgrid(np.arange(5), np.arange(6, 12), np.arange(12, 19))[1] np.testing.assert_equal(B_res, B_expected) C_res = self.executor.execute_tensor(C, concat=True)[0] C_expected = np.meshgrid(np.arange(5), np.arange(6, 12), np.arange(12, 19))[2] np.testing.assert_equal(C_res, C_expected) A, B, C = meshgrid(a, b, c, indexing='ij') A_res = self.executor.execute_tensor(A, concat=True)[0] A_expected = np.meshgrid(np.arange(5), np.arange(6, 12), np.arange(12, 19), indexing='ij')[0] np.testing.assert_equal(A_res, A_expected) B_res = self.executor.execute_tensor(B, concat=True)[0] B_expected = np.meshgrid(np.arange(5), np.arange(6, 12), np.arange(12, 19), indexing='ij')[1] np.testing.assert_equal(B_res, B_expected) C_res = self.executor.execute_tensor(C, concat=True)[0] C_expected = np.meshgrid(np.arange(5), np.arange(6, 12), np.arange(12, 19), indexing='ij')[2] np.testing.assert_equal(C_res, C_expected) A, B, C = meshgrid(a, b, c, sparse=True) A_res = self.executor.execute_tensor(A, concat=True)[0] A_expected = np.meshgrid(np.arange(5), np.arange(6, 12), np.arange(12, 19), sparse=True)[0] np.testing.assert_equal(A_res, A_expected) B_res = self.executor.execute_tensor(B, concat=True)[0] B_expected = np.meshgrid(np.arange(5), np.arange(6, 12), np.arange(12, 19), sparse=True)[1] np.testing.assert_equal(B_res, B_expected) C_res = self.executor.execute_tensor(C, concat=True)[0] C_expected = np.meshgrid(np.arange(5), np.arange(6, 12), np.arange(12, 19), sparse=True)[2] np.testing.assert_equal(C_res, C_expected) A, B, C = meshgrid(a, b, c, indexing='ij', sparse=True) A_res = self.executor.execute_tensor(A, concat=True)[0] A_expected = np.meshgrid(np.arange(5), np.arange(6, 12), np.arange(12, 19), indexing='ij', sparse=True)[0] np.testing.assert_equal(A_res, A_expected) B_res = self.executor.execute_tensor(B, concat=True)[0] B_expected = np.meshgrid(np.arange(5), np.arange(6, 12), np.arange(12, 19), indexing='ij', sparse=True)[1] np.testing.assert_equal(B_res, B_expected) C_res = self.executor.execute_tensor(C, concat=True)[0] C_expected = np.meshgrid(np.arange(5), np.arange(6, 12), np.arange(12, 19), indexing='ij', sparse=True)[2] np.testing.assert_equal(C_res, C_expected) def testIndicesExecution(self): grid = indices((2, 3), chunks=1) res = self.executor.execute_tensor(grid, concat=True)[0] expected = np.indices((2, 3)) np.testing.assert_equal(res, expected) res = self.executor.execute_tensor(grid[0], concat=True)[0] np.testing.assert_equal(res, expected[0]) res = self.executor.execute_tensor(grid[1], concat=True)[0] np.testing.assert_equal(res, expected[1]) def testTriuExecution(self): a = arange(24, chunks=2).reshape(2, 3, 4) t = triu(a) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.triu(np.arange(24).reshape(2, 3, 4)) np.testing.assert_equal(res, expected) t = triu(a, k=1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.triu(np.arange(24).reshape(2, 3, 4), k=1) np.testing.assert_equal(res, expected) t = triu(a, k=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.triu(np.arange(24).reshape(2, 3, 4), k=2) np.testing.assert_equal(res, expected) t = triu(a, k=-1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.triu(np.arange(24).reshape(2, 3, 4), k=-1) np.testing.assert_equal(res, expected) t = triu(a, k=-2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.triu(np.arange(24).reshape(2, 3, 4), k=-2) np.testing.assert_equal(res, expected) # test sparse a = arange(12, chunks=2).reshape(3, 4).tosparse() t = triu(a) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.triu(np.arange(12).reshape(3, 4)) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res, expected) t = triu(a, k=1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.triu(np.arange(12).reshape(3, 4), k=1) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res, expected) t = triu(a, k=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.triu(np.arange(12).reshape(3, 4), k=2) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res, expected) t = triu(a, k=-1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.triu(np.arange(12).reshape(3, 4), k=-1) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res, expected) t = triu(a, k=-2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.triu(np.arange(12).reshape(3, 4), k=-2) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res, expected) def testTrilExecution(self): a = arange(24, chunks=2).reshape(2, 3, 4) t = tril(a) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.tril(np.arange(24).reshape(2, 3, 4)) np.testing.assert_equal(res, expected) t = tril(a, k=1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.tril(np.arange(24).reshape(2, 3, 4), k=1) np.testing.assert_equal(res, expected) t = tril(a, k=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.tril(np.arange(24).reshape(2, 3, 4), k=2) np.testing.assert_equal(res, expected) t = tril(a, k=-1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.tril(np.arange(24).reshape(2, 3, 4), k=-1) np.testing.assert_equal(res, expected) t = tril(a, k=-2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.tril(np.arange(24).reshape(2, 3, 4), k=-2) np.testing.assert_equal(res, expected) a = arange(12, chunks=2).reshape(3, 4).tosparse() t = tril(a) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.tril(np.arange(12).reshape(3, 4)) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res, expected) t = tril(a, k=1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.tril(np.arange(12).reshape(3, 4), k=1) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res, expected) t = tril(a, k=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.tril(np.arange(12).reshape(3, 4), k=2) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res, expected) t = tril(a, k=-1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.tril(np.arange(12).reshape(3, 4), k=-1) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res, expected) t = tril(a, k=-2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.tril(np.arange(12).reshape(3, 4), k=-2) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res, expected) def testIndexTrickExecution(self): mgrid = nd_grid() t = mgrid[0:5, 0:5] res = self.executor.execute_tensor(t, concat=True)[0] expected = np.lib.index_tricks.nd_grid()[0:5, 0:5] np.testing.assert_equal(res, expected) t = mgrid[-1:1:5j] res = self.executor.execute_tensor(t, concat=True)[0] expected = np.lib.index_tricks.nd_grid()[-1:1:5j] np.testing.assert_equal(res, expected) ogrid = nd_grid(sparse=True) t = ogrid[0:5, 0:5] res = [self.executor.execute_tensor(o, concat=True)[0] for o in t] expected = np.lib.index_tricks.nd_grid(sparse=True)[0:5, 0:5] [np.testing.assert_equal(r, e) for r, e in zip(res, expected)]
class Test(unittest.TestCase): def setUp(self): self.executor = Executor('numpy') def testFFTExecution(self): raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(4, 4, 30)) r = fft(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fft(raw) self.assertTrue(np.allclose(res, expected)) r = fft(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fft(raw, norm='ortho') self.assertTrue(np.allclose(res, expected)) r = fft(t, n=11) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fft(raw, n=11) self.assertTrue(np.allclose(res, expected)) raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(4, 4, 4)) r = fft(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fft(raw) self.assertTrue(np.allclose(res, expected)) r = fft(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fft(raw, norm='ortho') self.assertTrue(np.allclose(res, expected)) r = fft(t, n=11) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fft(raw, n=11) self.assertTrue(np.allclose(res, expected)) def testIFFTExecution(self): raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(4, 4, 30)) r = ifft(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifft(raw) self.assertTrue(np.allclose(res, expected)) r = ifft(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifft(raw, norm='ortho') self.assertTrue(np.allclose(res, expected)) r = ifft(t, n=11) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifft(raw, n=11) self.assertTrue(np.allclose(res, expected)) raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(4, 4, 5)) r = ifft(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifft(raw) self.assertTrue(np.allclose(res, expected)) r = ifft(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifft(raw, norm='ortho') self.assertTrue(np.allclose(res, expected)) r = ifft(t, n=11) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifft(raw, n=11) self.assertTrue(np.allclose(res, expected)) def testFFT2Execution(self): raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(4, 20, 30)) r = fft2(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fft2(raw) self.assertTrue(np.allclose(res, expected)) r = fft2(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fft2(raw, norm='ortho') self.assertTrue(np.allclose(res, expected)) r = fft2(t, s=(11, 12)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fft2(raw, s=(11, 12)) self.assertTrue(np.allclose(res, expected)) r = fft2(t, s=(11, 12), axes=(-1, -2)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fft2(raw, s=(11, 12), axes=(-1, -2)) self.assertTrue(np.allclose(res, expected)) raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(4, 5, 6)) r = fft2(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fft2(raw) self.assertTrue(np.allclose(res, expected)) r = fft2(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fft2(raw, norm='ortho') self.assertTrue(np.allclose(res, expected)) r = fft2(t, s=(11, 12)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fft2(raw, s=(11, 12)) self.assertTrue(np.allclose(res, expected)) r = fft2(t, s=(11, 12), axes=(-1, -2)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fft2(raw, s=(11, 12), axes=(-1, -2)) self.assertTrue(np.allclose(res, expected)) def testIFFT2Execution(self): raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(4, 20, 30)) r = ifft2(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifft2(raw) self.assertTrue(np.allclose(res, expected)) r = ifft2(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifft2(raw, norm='ortho') self.assertTrue(np.allclose(res, expected)) r = ifft2(t, s=(11, 12)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifft2(raw, s=(11, 12)) self.assertTrue(np.allclose(res, expected)) r = ifft2(t, s=(11, 12), axes=(-1, -2)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifft2(raw, s=(11, 12), axes=(-1, -2)) self.assertTrue(np.allclose(res, expected)) raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(4, 3, 5)) r = ifft2(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifft2(raw) self.assertTrue(np.allclose(res, expected)) r = ifft2(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifft2(raw, norm='ortho') self.assertTrue(np.allclose(res, expected)) r = ifft2(t, s=(11, 12)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifft2(raw, s=(11, 12)) self.assertTrue(np.allclose(res, expected)) r = ifft2(t, s=(11, 12), axes=(-1, -2)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifft2(raw, s=(11, 12), axes=(-1, -2)) self.assertTrue(np.allclose(res, expected)) def testFFTNExecution(self): raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(10, 20, 30)) r = fftn(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fftn(raw) self.assertTrue(np.allclose(res, expected)) r = fftn(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fftn(raw, norm='ortho') self.assertTrue(np.allclose(res, expected)) r = fftn(t, s=(11, 12, 5)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fftn(raw, s=(11, 12, 5)) self.assertTrue(np.allclose(res, expected)) r = fftn(t, s=(11, 12, 5), axes=(-1, -2, -3)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fftn(raw, s=(11, 12, 5), axes=(-1, -2, -3)) self.assertTrue(np.allclose(res, expected)) raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(3, 3, 4)) r = fftn(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fftn(raw) self.assertTrue(np.allclose(res, expected)) r = fftn(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fftn(raw, norm='ortho') self.assertTrue(np.allclose(res, expected)) r = fftn(t, s=(11, 12, 5)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fftn(raw, s=(11, 12, 5)) self.assertTrue(np.allclose(res, expected)) r = fftn(t, s=(11, 12, 5), axes=(-1, -2, -3)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fftn(raw, s=(11, 12, 5), axes=(-1, -2, -3)) self.assertTrue(np.allclose(res, expected)) def testIFFTNExecution(self): raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(10, 20, 30)) r = ifftn(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifftn(raw) self.assertTrue(np.allclose(res, expected)) r = ifftn(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifftn(raw, norm='ortho') self.assertTrue(np.allclose(res, expected)) r = ifftn(t, s=(11, 12, 5)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifftn(raw, s=(11, 12, 5)) self.assertTrue(np.allclose(res, expected)) r = ifftn(t, s=(11, 12, 5), axes=(-1, -2, -3)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifftn(raw, s=(11, 12, 5), axes=(-1, -2, -3)) self.assertTrue(np.allclose(res, expected)) raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(3, 4, 7)) r = ifftn(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifftn(raw) self.assertTrue(np.allclose(res, expected)) r = ifftn(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifftn(raw, norm='ortho') self.assertTrue(np.allclose(res, expected)) r = ifftn(t, s=(11, 12, 5)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifftn(raw, s=(11, 12, 5)) self.assertTrue(np.allclose(res, expected)) r = ifftn(t, s=(11, 12, 5), axes=(-1, -2, -3)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifftn(raw, s=(11, 12, 5), axes=(-1, -2, -3)) self.assertTrue(np.allclose(res, expected)) def testRFFTExecution(self): raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(4, 4, 30)) r = rfft(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.rfft(raw) self.assertTrue(np.allclose(res, expected)) r = rfft(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.rfft(raw, norm='ortho') self.assertTrue(np.allclose(res, expected)) r = rfft(t, n=11) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.rfft(raw, n=11) self.assertTrue(np.allclose(res, expected)) def testIRFFTExecution(self): raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(4, 4, 30)) r = irfft(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.irfft(raw) self.assertTrue(np.allclose(res, expected)) r = irfft(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.irfft(raw, norm='ortho') self.assertTrue(np.allclose(res, expected)) r = irfft(t, n=11) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.irfft(raw, n=11) self.assertTrue(np.allclose(res, expected)) def testRFFT2Execution(self): raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(4, 20, 30)) r = rfft2(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.rfft2(raw) self.assertTrue(np.allclose(res, expected)) r = rfft2(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.rfft2(raw, norm='ortho') self.assertTrue(np.allclose(res, expected)) r = rfft2(t, s=(11, 12)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.rfft2(raw, s=(11, 12)) self.assertTrue(np.allclose(res, expected)) r = rfft2(t, s=(11, 12), axes=(-1, -2)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.rfft2(raw, s=(11, 12), axes=(-1, -2)) self.assertTrue(np.allclose(res, expected)) def testIRFFT2Execution(self): raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(4, 20, 30)) r = irfft2(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.irfft2(raw) self.assertTrue(np.allclose(res, expected)) r = irfft2(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.irfft2(raw, norm='ortho') self.assertTrue(np.allclose(res, expected)) r = irfft2(t, s=(11, 12)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.irfft2(raw, s=(11, 12)) self.assertTrue(np.allclose(res, expected)) r = irfft2(t, s=(11, 12), axes=(-1, -2)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.irfft2(raw, s=(11, 12), axes=(-1, -2)) self.assertTrue(np.allclose(res, expected)) def testRFFTNExecution(self): raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(10, 20, 30)) r = rfftn(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.rfftn(raw) self.assertTrue(np.allclose(res, expected)) r = rfftn(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.rfftn(raw, norm='ortho') self.assertTrue(np.allclose(res, expected)) r = rfftn(t, s=(11, 12, 5)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.rfftn(raw, s=(11, 12, 5)) self.assertTrue(np.allclose(res, expected)) r = rfftn(t, s=(11, 12, 11), axes=(-1, -2, -3)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.rfftn(raw, s=(11, 12, 11), axes=(-1, -2, -3)) self.assertTrue(np.allclose(res, expected)) def testIRFFTNExecution(self): raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(10, 20, 30)) r = irfftn(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.irfftn(raw) self.assertTrue(np.allclose(res, expected)) r = irfftn(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.irfftn(raw, norm='ortho') self.assertTrue(np.allclose(res, expected)) r = irfftn(t, s=(11, 21, 5)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.irfftn(raw, s=(11, 21, 5)) self.assertTrue(np.allclose(res, expected)) r = irfftn(t, s=(11, 21, 30), axes=(-1, -2, -3)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.irfftn(raw, s=(11, 21, 30), axes=(-1, -2, -3)) self.assertTrue(np.allclose(res, expected)) def testHFFTExecution(self): raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(4, 4, 30)) r = hfft(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.hfft(raw) self.assertTrue(np.allclose(res, expected)) r = hfft(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.hfft(raw, norm='ortho') self.assertTrue(np.allclose(res, expected)) r = hfft(t, n=11) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.hfft(raw, n=11) self.assertTrue(np.allclose(res, expected)) def testIHFFTExecution(self): raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(4, 4, 30)) r = ihfft(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ihfft(raw) self.assertTrue(np.allclose(res, expected)) r = ihfft(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ihfft(raw, norm='ortho') self.assertTrue(np.allclose(res, expected)) r = ihfft(t, n=11) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ihfft(raw, n=11) self.assertTrue(np.allclose(res, expected)) r = ihfft(t, n=12) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ihfft(raw, n=12) self.assertTrue(np.allclose(res, expected)) def testFFTFreqExecution(self): t = fftfreq(10, .1, chunk_size=3) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, np.fft.fftfreq(10, .1))) t = fftfreq(11, .01, chunk_size=3) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, np.fft.fftfreq(11, .01))) def testRFFTFreqExecution(self): t = rfftfreq(20, .1, chunk_size=3) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, np.fft.rfftfreq(20, .1))) t = rfftfreq(21, .01, chunk_size=3) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, np.fft.rfftfreq(21, .01))) def testFFTShiftExecution(self): t = fftfreq(10, .1, chunk_size=3) r = fftshift(t) res = self.executor.execute_tensor(r, concat=True)[0] self.assertTrue( np.allclose(res, np.fft.fftshift(np.fft.fftfreq(10, .1)))) freqs = fftfreq(9, d=1. / 9, chunk_size=2).reshape(3, 3) r = fftshift(freqs, axes=(1, )) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fftshift(np.fft.fftfreq(9, d=1. / 9).reshape(3, 3), axes=(1, )) self.assertTrue(np.allclose(res, expected)) def testIFFTShiftExecution(self): t = fftfreq(9, d=1. / 9, chunk_size=2).reshape(3, 3) r = ifftshift(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifftshift(np.fft.fftfreq(9, d=1. / 9).reshape(3, 3)) self.assertTrue(np.allclose(res, expected))
def setUp(self): self.executor = Executor('numpy') self.old_chunk = options.tensor.chunk_size options.tensor.chunk_size = 10
class Test(unittest.TestCase): def setUp(self): self.executor = Executor('numpy') self.old_chunk = options.tensor.chunk_size options.tensor.chunk_size = 10 def tearDown(self): options.tensor.chunk_size = self.old_chunk def testBoolIndexingExecution(self): raw = np.random.random((11, 8, 12, 14)) arr = tensor(raw, chunk_size=3) index = arr < .5 arr2 = arr[index] res = self.executor.execute_tensor(arr2) self.assertTrue(np.array_equal(np.sort(np.concatenate(res)), np.sort(raw[raw < .5]))) index2 = tensor(raw[:, :, 0, 0], chunk_size=3) < .5 arr3 = arr[index2] res = self.executor.execute_tensor(arr3) self.assertEqual(sum(it.size for it in res), raw[raw[:, :, 0, 0] < .5].size) def testFancyIndexingExecution(self): raw = np.random.random((11, 8, 12, 14)) arr = tensor(raw, chunk_size=(2, 3, 2, 3)) index = [8, 10, 3, 1, 9, 10] arr2 = arr[index] res = self.executor.execute_tensor(arr2, concat=True) self.assertTrue(np.array_equal(res[0], raw[index])) index = np.random.permutation(8) arr3 = arr[:2, ..., index] res = self.executor.execute_tensor(arr3, concat=True) self.assertTrue(np.array_equal(res[0], raw[:2, ..., index])) index = [1, 3, 9, 10] arr4 = arr[..., index, :5] res = self.executor.execute_tensor(arr4, concat=True) self.assertTrue(np.array_equal(res[0], raw[..., index, :5])) def testSliceExecution(self): raw = np.random.random((11, 8, 12, 14)) arr = tensor(raw, chunk_size=3) arr2 = arr[2:9:2, 3:7, -1:-9:-2, 12:-11:-4] res = self.executor.execute_tensor(arr2, concat=True) self.assertTrue(np.array_equal(res[0], raw[2:9:2, 3:7, -1:-9:-2, 12:-11:-4])) raw = sps.random(12, 14, density=.1) arr = tensor(raw, chunk_size=3) arr2 = arr[-1:-9:-2, 12:-11:-4] res = self.executor.execute_tensor(arr2, concat=True)[0] np.testing.assert_equal(res.toarray(), raw.toarray()[-1:-9:-2, 12:-11:-4]) def testMixedIndexingExecution(self): raw = np.random.random((11, 8, 12, 13)) arr = tensor(raw, chunk_size=3) raw_cond = raw[0, :, 0, 0] < .5 cond = tensor(raw[0, :, 0, 0], chunk_size=3) < .5 arr2 = arr[10::-2, cond, None, ..., :5] res = self.executor.execute_tensor(arr2, concat=True) self.assertTrue(np.array_equal(res[0], raw[10::-2, raw_cond, None, ..., :5])) b_raw = np.random.random(8) cond = tensor(b_raw, chunk_size=2) < .5 arr3 = arr[-2::-3, cond, ...] res = self.executor.execute_tensor(arr3, concat=True) self.assertTrue(np.array_equal(res[0], raw[-2::-3, b_raw < .5, ...])) def testSetItemExecution(self): raw = data = np.random.randint(0, 10, size=(11, 8, 12, 13)) arr = tensor(raw.copy(), chunk_size=3) raw = raw.copy() idx = slice(2, 9, 2), slice(3, 7), slice(-1, -9, -2), 2 arr[idx] = 20 res = self.executor.execute_tensor(arr, concat=True) raw[idx] = 20 self.assertTrue(np.array_equal(res[0], raw)) raw = data shape = raw[idx].shape arr2 = tensor(raw.copy(), chunk_size=3) raw = raw.copy() replace = np.random.randint(10, 20, size=shape[:-1] + (1,)).astype('f4') arr2[idx] = tensor(replace, chunk_size=4) res = self.executor.execute_tensor(arr2, concat=True) raw[idx] = replace self.assertTrue(np.array_equal(res[0], raw)) def testTakeExecution(self): data = np.random.rand(10, 20, 30) t = tensor(data, chunk_size=10) a = t.take([4, 1, 2, 6, 200]) res = self.executor.execute_tensor(a, concat=True)[0] expected = np.take(data, [4, 1, 2, 6, 200]) self.assertTrue(np.array_equal(res, expected)) a = take(t, [5, 19, 2, 13], axis=1) res = self.executor.execute_tensor(a, concat=True)[0] expected = np.take(data, [5, 19, 2, 13], axis=1) self.assertTrue(np.array_equal(res, expected)) def testCompressExecution(self): data = np.array([[1, 2], [3, 4], [5, 6]]) a = tensor(data, chunk_size=1) t = compress([0, 1], a, axis=0) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.compress([0, 1], data, axis=0) self.assertTrue(np.array_equal(res, expected)) t = compress([0, 1], a, axis=1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.compress([0, 1], data, axis=1) self.assertTrue(np.array_equal(res, expected)) t = a.compress([0, 1, 1]) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.compress([0, 1, 1], data) self.assertTrue(np.array_equal(res, expected)) t = compress([False, True, True], a, axis=0) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.compress([False, True, True], data, axis=0) self.assertTrue(np.array_equal(res, expected)) t = compress([False, True], a, axis=1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.compress([False, True], data, axis=1) self.assertTrue(np.array_equal(res, expected)) with self.assertRaises(np.AxisError): compress([0, 1, 1], a, axis=1) def testExtractExecution(self): data = np.arange(12).reshape((3, 4)) a = tensor(data, chunk_size=2) condition = mod(a, 3) == 0 t = extract(condition, a) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.extract(np.mod(data, 3) == 0, data) self.assertTrue(np.array_equal(res, expected)) def testChooseExecution(self): options.tensor.chunk_size = 2 choices = [[0, 1, 2, 3], [10, 11, 12, 13], [20, 21, 22, 23], [30, 31, 32, 33]] a = choose([2, 3, 1, 0], choices) res = self.executor.execute_tensor(a, concat=True)[0] expected = np.choose([2, 3, 1, 0], choices) self.assertTrue(np.array_equal(res, expected)) a = choose([2, 4, 1, 0], choices, mode='clip') # 4 goes to 3 (4-1) expected = np.choose([2, 4, 1, 0], choices, mode='clip') res = self.executor.execute_tensor(a, concat=True)[0] self.assertTrue(np.array_equal(res, expected)) a = choose([2, 4, 1, 0], choices, mode='wrap') # 4 goes to (4 mod 4) expected = np.choose([2, 4, 1, 0], choices, mode='wrap') # 4 goes to (4 mod 4) res = self.executor.execute_tensor(a, concat=True)[0] self.assertTrue(np.array_equal(res, expected)) a = [[1, 0, 1], [0, 1, 0], [1, 0, 1]] choices = [-10, 10] b = choose(a, choices) expected = np.choose(a, choices) res = self.executor.execute_tensor(b, concat=True)[0] self.assertTrue(np.array_equal(res, expected)) a = np.array([0, 1]).reshape((2, 1, 1)) c1 = np.array([1, 2, 3]).reshape((1, 3, 1)) c2 = np.array([-1, -2, -3, -4, -5]).reshape((1, 1, 5)) b = choose(a, (c1, c2)) expected = np.choose(a, (c1, c2)) res = self.executor.execute_tensor(b, concat=True)[0] self.assertTrue(np.array_equal(res, expected)) def testUnravelExecution(self): a = tensor([22, 41, 37], chunk_size=1) t = stack(unravel_index(a, (7, 6))) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.stack(np.unravel_index([22, 41, 37], (7, 6))) self.assertTrue(np.array_equal(res, expected)) def testNonzeroExecution(self): data = np.array([[1, 0, 0], [0, 2, 0], [1, 1, 0]]) x = tensor(data, chunk_size=2) t = hstack(nonzero(x)) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.hstack(np.nonzero(data)) self.assertTrue(np.array_equal(res, expected)) def testFlatnonzeroExecution(self): x = arange(-2, 3, chunk_size=2) t = flatnonzero(x) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.flatnonzero(np.arange(-2, 3)) np.testing.assert_equal(res, expected)
def setUp(self): self.executor = Executor('numpy') local_session = LocalSession() local_session._executor = self.executor self.session = Session() self.session._sess = local_session
def setUp(self): self.executor = Executor('cupy')
def setUp(self): self.executor = Executor('numpy')
class Test(unittest.TestCase): def setUp(self): self.executor = Executor('numpy') def testConcatenateExecution(self): a_data = np.random.rand(10, 20, 30) b_data = np.random.rand(10, 20, 40) c_data = np.random.rand(10, 20, 50) a = tensor(a_data, chunk_size=5) b = tensor(b_data, chunk_size=6) c = tensor(c_data, chunk_size=7) d = concatenate([a, b, c], axis=-1) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.concatenate([a_data, b_data, c_data], axis=-1) self.assertTrue(np.array_equal(res, expected)) a_data = sps.random(10, 30) b_data = sps.rand(10, 40) c_data = sps.rand(10, 50) a = tensor(a_data, chunk_size=5) b = tensor(b_data, chunk_size=6) c = tensor(c_data, chunk_size=7) d = concatenate([a, b, c], axis=-1) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.concatenate([a_data.A, b_data.A, c_data.A], axis=-1) self.assertTrue(np.array_equal(res.toarray(), expected)) def testStackExecution(self): raw = [np.random.randn(3, 4) for _ in range(10)] arrs = [tensor(a, chunk_size=3) for a in raw] arr2 = stack(arrs) res = self.executor.execute_tensor(arr2, concat=True) self.assertTrue(np.array_equal(res[0], np.stack(raw))) arr3 = stack(arrs, axis=1) res = self.executor.execute_tensor(arr3, concat=True) self.assertTrue(np.array_equal(res[0], np.stack(raw, axis=1))) arr4 = stack(arrs, axis=2) res = self.executor.execute_tensor(arr4, concat=True) self.assertTrue(np.array_equal(res[0], np.stack(raw, axis=2))) def testHStackExecution(self): a_data = np.random.rand(10) b_data = np.random.rand(20) a = tensor(a_data, chunk_size=4) b = tensor(b_data, chunk_size=4) c = hstack([a, b]) res = self.executor.execute_tensor(c, concat=True)[0] expected = np.hstack([a_data, b_data]) self.assertTrue(np.array_equal(res, expected)) a_data = np.random.rand(10, 20) b_data = np.random.rand(10, 5) a = tensor(a_data, chunk_size=3) b = tensor(b_data, chunk_size=4) c = hstack([a, b]) res = self.executor.execute_tensor(c, concat=True)[0] expected = np.hstack([a_data, b_data]) self.assertTrue(np.array_equal(res, expected)) def testVStackExecution(self): a_data = np.random.rand(10) b_data = np.random.rand(10) a = tensor(a_data, chunk_size=4) b = tensor(b_data, chunk_size=4) c = vstack([a, b]) res = self.executor.execute_tensor(c, concat=True)[0] expected = np.vstack([a_data, b_data]) self.assertTrue(np.array_equal(res, expected)) a_data = np.random.rand(10, 20) b_data = np.random.rand(5, 20) a = tensor(a_data, chunk_size=3) b = tensor(b_data, chunk_size=4) c = vstack([a, b]) res = self.executor.execute_tensor(c, concat=True)[0] expected = np.vstack([a_data, b_data]) self.assertTrue(np.array_equal(res, expected)) def testDStackExecution(self): a_data = np.random.rand(10) b_data = np.random.rand(10) a = tensor(a_data, chunk_size=4) b = tensor(b_data, chunk_size=4) c = dstack([a, b]) res = self.executor.execute_tensor(c, concat=True)[0] expected = np.dstack([a_data, b_data]) self.assertTrue(np.array_equal(res, expected)) a_data = np.random.rand(10, 20) b_data = np.random.rand(10, 20) a = tensor(a_data, chunk_size=3) b = tensor(b_data, chunk_size=4) c = dstack([a, b]) res = self.executor.execute_tensor(c, concat=True)[0] expected = np.dstack([a_data, b_data]) self.assertTrue(np.array_equal(res, expected)) def testColumnStackExecution(self): a_data = np.array((1, 2, 3)) b_data = np.array((2, 3, 4)) a = tensor(a_data, chunk_size=1) b = tensor(b_data, chunk_size=2) c = column_stack((a, b)) res = self.executor.execute_tensor(c, concat=True)[0] expected = np.column_stack((a_data, b_data)) np.testing.assert_equal(res, expected) a_data = np.random.rand(4, 2, 3) b_data = np.random.rand(4, 2, 3) a = tensor(a_data, chunk_size=1) b = tensor(b_data, chunk_size=2) c = column_stack((a, b)) res = self.executor.execute_tensor(c, concat=True)[0] expected = np.column_stack((a_data, b_data)) np.testing.assert_equal(res, expected)