def testMockExecuteSize(self): import mars.tensor as mt from mars.core.graph import DAG from mars.tensor.fetch import TensorFetch from mars.tensor.arithmetic import TensorTreeAdd graph_add = DAG() input_chunks = [] for _ in range(2): fetch_op = TensorFetch(dtype=np.dtype('int64')) inp_chunk = fetch_op.new_chunk(None, shape=(100, 100)).data input_chunks.append(inp_chunk) add_op = TensorTreeAdd(args=input_chunks, dtype=np.dtype('int64')) add_chunk = add_op.new_chunk(input_chunks, shape=(100, 100), dtype=np.dtype('int64')).data graph_add.add_node(add_chunk) for inp_chunk in input_chunks: graph_add.add_node(inp_chunk) graph_add.add_edge(inp_chunk, add_chunk) executor = Executor() res = executor.execute_graph(graph_add, [add_chunk.key], compose=False, mock=True)[0] self.assertEqual(res, (80000, 80000)) self.assertEqual(executor.mock_max_memory, 80000) for _ in range(3): new_add_op = TensorTreeAdd(args=[add_chunk], dtype=np.dtype('int64')) new_add_chunk = new_add_op.new_chunk([add_chunk], shape=(100, 100), dtype=np.dtype('int64')).data graph_add.add_node(new_add_chunk) graph_add.add_edge(add_chunk, new_add_chunk) add_chunk = new_add_chunk executor = Executor() res = executor.execute_graph(graph_add, [add_chunk.key], compose=False, mock=True)[0] self.assertEqual(res, (80000, 80000)) self.assertEqual(executor.mock_max_memory, 160000) a = mt.random.rand(10, 10, chunk_size=10) b = a[:, mt.newaxis, :] - a r = mt.triu(mt.sqrt(b**2).sum(axis=2)) executor = Executor() res = executor.execute_tensor(r, concat=False, mock=True) # larger than maximal memory size in calc procedure self.assertGreaterEqual(res[0][0], 800) self.assertGreaterEqual(executor.mock_max_memory, 8000)
def testMockExecuteSize(self): a = mt.random.rand(10, 10, chunk_size=10) b = a[:, mt.newaxis, :] - a r = mt.triu(mt.sqrt(b**2).sum(axis=2)) executor = Executor() res = executor.execute_tensor(r, concat=False, mock=True) # larger than maximal memory size in calc procedure self.assertGreaterEqual(res[0][0], 800) self.assertGreaterEqual(res[0][1], 8000)
def test_triu_tril(): a_data = np.arange(12).reshape(4, 3) a = tensor(a_data, chunk_size=2) t = triu(a) assert t.op.gpu is False t = tile(t) assert len(t.chunks) == 4 assert isinstance(t.chunks[0].op, TensorTriu) assert isinstance(t.chunks[1].op, TensorTriu) assert isinstance(t.chunks[2].op, TensorZeros) assert isinstance(t.chunks[3].op, TensorTriu) t = triu(a, k=1) t = tile(t) assert len(t.chunks) == 4 assert isinstance(t.chunks[0].op, TensorTriu) assert isinstance(t.chunks[1].op, TensorTriu) assert isinstance(t.chunks[2].op, TensorZeros) assert isinstance(t.chunks[3].op, TensorZeros) t = triu(a, k=2) t = tile(t) assert len(t.chunks) == 4 assert isinstance(t.chunks[0].op, TensorZeros) assert isinstance(t.chunks[1].op, TensorTriu) assert isinstance(t.chunks[2].op, TensorZeros) assert isinstance(t.chunks[3].op, TensorZeros) t = triu(a, k=-1) t = tile(t) assert len(t.chunks) == 4 assert isinstance(t.chunks[0].op, TensorTriu) assert isinstance(t.chunks[1].op, TensorTriu) assert isinstance(t.chunks[2].op, TensorTriu) assert isinstance(t.chunks[3].op, TensorTriu) t = tril(a) assert t.op.gpu is False t = tile(t) assert len(t.chunks) == 4 assert isinstance(t.chunks[0].op, TensorTril) assert isinstance(t.chunks[1].op, TensorZeros) assert isinstance(t.chunks[2].op, TensorTril) assert isinstance(t.chunks[3].op, TensorTril) t = tril(a, k=1) t = tile(t) assert len(t.chunks) == 4 assert isinstance(t.chunks[0].op, TensorTril) assert isinstance(t.chunks[1].op, TensorTril) assert isinstance(t.chunks[2].op, TensorTril) assert isinstance(t.chunks[3].op, TensorTril) t = tril(a, k=-1) t = tile(t) assert len(t.chunks) == 4 assert isinstance(t.chunks[0].op, TensorTril) assert isinstance(t.chunks[1].op, TensorZeros) assert isinstance(t.chunks[2].op, TensorTril) assert isinstance(t.chunks[3].op, TensorTril) t = tril(a, k=-2) t = tile(t) assert len(t.chunks) == 4 assert isinstance(t.chunks[0].op, TensorZeros) assert isinstance(t.chunks[1].op, TensorZeros) assert isinstance(t.chunks[2].op, TensorTril) assert isinstance(t.chunks[3].op, TensorZeros)
def testTriuTril(self): a_data = np.arange(12).reshape(4, 3) a = tensor(a_data, chunk_size=2) t = triu(a) self.assertFalse(t.op.gpu) t = t.tiles() self.assertEqual(len(t.chunks), 4) self.assertIsInstance(t.chunks[0].op, TensorTriu) self.assertIsInstance(t.chunks[1].op, TensorTriu) self.assertIsInstance(t.chunks[2].op, TensorZeros) self.assertIsInstance(t.chunks[3].op, TensorTriu) t = triu(a, k=1) t = t.tiles() self.assertEqual(len(t.chunks), 4) self.assertIsInstance(t.chunks[0].op, TensorTriu) self.assertIsInstance(t.chunks[1].op, TensorTriu) self.assertIsInstance(t.chunks[2].op, TensorZeros) self.assertIsInstance(t.chunks[3].op, TensorZeros) t = triu(a, k=2) t = t.tiles() self.assertEqual(len(t.chunks), 4) self.assertIsInstance(t.chunks[0].op, TensorZeros) self.assertIsInstance(t.chunks[1].op, TensorTriu) self.assertIsInstance(t.chunks[2].op, TensorZeros) self.assertIsInstance(t.chunks[3].op, TensorZeros) t = triu(a, k=-1) t = t.tiles() self.assertEqual(len(t.chunks), 4) self.assertIsInstance(t.chunks[0].op, TensorTriu) self.assertIsInstance(t.chunks[1].op, TensorTriu) self.assertIsInstance(t.chunks[2].op, TensorTriu) self.assertIsInstance(t.chunks[3].op, TensorTriu) t = tril(a) self.assertFalse(t.op.gpu) t = t.tiles() self.assertEqual(len(t.chunks), 4) self.assertIsInstance(t.chunks[0].op, TensorTril) self.assertIsInstance(t.chunks[1].op, TensorZeros) self.assertIsInstance(t.chunks[2].op, TensorTril) self.assertIsInstance(t.chunks[3].op, TensorTril) t = tril(a, k=1) t = t.tiles() self.assertEqual(len(t.chunks), 4) self.assertIsInstance(t.chunks[0].op, TensorTril) self.assertIsInstance(t.chunks[1].op, TensorTril) self.assertIsInstance(t.chunks[2].op, TensorTril) self.assertIsInstance(t.chunks[3].op, TensorTril) t = tril(a, k=-1) t = t.tiles() self.assertEqual(len(t.chunks), 4) self.assertIsInstance(t.chunks[0].op, TensorTril) self.assertIsInstance(t.chunks[1].op, TensorZeros) self.assertIsInstance(t.chunks[2].op, TensorTril) self.assertIsInstance(t.chunks[3].op, TensorTril) t = tril(a, k=-2) t = t.tiles() self.assertEqual(len(t.chunks), 4) self.assertIsInstance(t.chunks[0].op, TensorZeros) self.assertIsInstance(t.chunks[1].op, TensorZeros) self.assertIsInstance(t.chunks[2].op, TensorTril) self.assertIsInstance(t.chunks[3].op, TensorZeros)
def testSolveTriangular(self): from mars.tensor import tril, triu np.random.seed(1) data1 = np.random.randint(1, 10, (20, 20)) data2 = np.random.randint(1, 10, (20, )) A = tensor(data1, chunk_size=20) b = tensor(data2, chunk_size=20) x = solve_triangular(A, b) t = triu(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) x = solve_triangular(A, b, lower=True) t = tril(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) A = tensor(data1, chunk_size=10) b = tensor(data2, chunk_size=10) x = solve_triangular(A, b) t = triu(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) x = solve_triangular(A, b, lower=True) t = tril(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) data1 = np.random.randint(1, 10, (10, 10)) data2 = np.random.randint(1, 10, (10, 5)) A = tensor(data1, chunk_size=10) b = tensor(data2, chunk_size=10) x = solve_triangular(A, b) t = triu(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) x = solve_triangular(A, b, lower=True) t = tril(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) A = tensor(data1, chunk_size=3) b = tensor(data2, chunk_size=3) x = solve_triangular(A, b) t = triu(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) x = solve_triangular(A, b, lower=True) t = tril(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) # test sparse data1 = sps.csr_matrix(np.triu(np.random.randint(1, 10, (10, 10)))) data2 = np.random.random((10, )) A = tensor(data1, chunk_size=5) b = tensor(data2, chunk_size=5) x = solve_triangular(A, b) result_x = self.executor.execute_tensor(x, concat=True)[0] result_b = data1.dot(result_x) self.assertIsInstance(result_x, SparseNDArray) np.testing.assert_allclose(result_b, data2) data1 = sps.csr_matrix(np.triu(np.random.randint(1, 10, (10, 10)))) data2 = np.random.random((10, 2)) A = tensor(data1, chunk_size=5) b = tensor(data2, chunk_size=5) x = solve_triangular(A, b) result_x = self.executor.execute_tensor(x, concat=True)[0] result_b = data1.dot(result_x) self.assertIsInstance(result_x, SparseNDArray) np.testing.assert_allclose(result_b, data2)
def test_solve_triangular(setup): from mars.tensor import tril, triu rs = np.random.RandomState(0) data1 = rs.randint(1, 10, (20, 20)) data2 = rs.randint(1, 10, (20, )) A = tensor(data1, chunk_size=20) b = tensor(data2, chunk_size=20) x = solve_triangular(A, b) t = triu(A).dot(x) res = t.execute().fetch() np.testing.assert_allclose(res, data2) x = solve_triangular(A, b, lower=True) t = tril(A).dot(x) res = t.execute().fetch() np.testing.assert_allclose(res, data2) A = tensor(data1, chunk_size=10) b = tensor(data2, chunk_size=10) x = solve_triangular(A, b) t = triu(A).dot(x) res = t.execute().fetch() np.testing.assert_allclose(res, data2) x = solve_triangular(A, b, lower=True) t = tril(A).dot(x) res = t.execute().fetch() np.testing.assert_allclose(res, data2) data1 = rs.randint(1, 10, (10, 10)) data2 = rs.randint(1, 10, (10, 5)) A = tensor(data1, chunk_size=10) b = tensor(data2, chunk_size=10) x = solve_triangular(A, b) t = triu(A).dot(x) res = t.execute().fetch() np.testing.assert_allclose(res, data2) x = solve_triangular(A, b, lower=True) t = tril(A).dot(x) res = t.execute().fetch() np.testing.assert_allclose(res, data2) # test sparse data1 = sps.csr_matrix(np.triu(rs.randint(1, 10, (10, 10)))) data2 = rs.random((10, )) A = tensor(data1, chunk_size=5) b = tensor(data2, chunk_size=5) x = solve_triangular(A, b) result_x = x.execute().fetch() result_b = data1.dot(result_x) assert isinstance(result_x, SparseNDArray) np.testing.assert_allclose(result_b, data2) data1 = sps.csr_matrix(np.triu(rs.randint(1, 10, (10, 10)))) data2 = rs.random((10, 2)) A = tensor(data1, chunk_size=5) b = tensor(data2, chunk_size=5) x = solve_triangular(A, b) result_x = x.execute().fetch() result_b = data1.dot(result_x) assert isinstance(result_x, SparseNDArray) np.testing.assert_allclose(result_b, data2)
def testSolveTriangular(self): from mars.tensor import tril, triu np.random.seed(1) data1 = np.random.randint(1, 10, (20, 20)) data2 = np.random.randint(1, 10, (20, )) A = tensor(data1, chunk_size=20) b = tensor(data2, chunk_size=20) x = solve_triangular(A, b) t = triu(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) x = solve_triangular(A, b, lower=True) t = tril(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) A = tensor(data1, chunk_size=10) b = tensor(data2, chunk_size=10) x = solve_triangular(A, b) t = triu(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) x = solve_triangular(A, b, lower=True) t = tril(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) data1 = np.random.randint(1, 10, (10, 10)) data2 = np.random.randint(1, 10, (10, 5)) A = tensor(data1, chunk_size=10) b = tensor(data2, chunk_size=10) x = solve_triangular(A, b) t = triu(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) x = solve_triangular(A, b, lower=True) t = tril(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) A = tensor(data1, chunk_size=3) b = tensor(data2, chunk_size=3) x = solve_triangular(A, b) t = triu(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) x = solve_triangular(A, b, lower=True) t = tril(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2)