def testRegister(self): from mars.graph import DAG fake_result = np.random.rand(10, 10) fake_size = (fake_result.nbytes * 2, fake_result.nbytes * 2) def fake_execute(ctx, op): ctx[op.outputs[0].key] = fake_result def fake_estimate(ctx, op): ctx[op.outputs[0].key] = fake_size register(FakeOperand, fake_execute, fake_estimate) graph = DAG() chunk = FakeOperand().new_chunk(None, shape=(10, 10)) graph.add_node(chunk.data) executor = Executor() res = executor.execute_graph(graph, keys=[chunk.key])[0] np.testing.assert_array_equal(res, fake_result) size = executor.execute_graph(graph, keys=[chunk.key], mock=True)[0] self.assertEqual(size, fake_size) graph = DAG() chunk = SubFakeOperand().new_chunk(None, shape=(10, 10)) graph.add_node(chunk.data) executor = Executor() res = executor.execute_graph(graph, keys=[chunk.key])[0] np.testing.assert_array_equal(res, fake_result)
def testMockExecuteSize(self): import mars.tensor as mt from mars.core.graph import DAG from mars.tensor.fetch import TensorFetch from mars.tensor.arithmetic import TensorTreeAdd graph_add = DAG() input_chunks = [] for _ in range(2): fetch_op = TensorFetch(dtype=np.dtype('int64')) inp_chunk = fetch_op.new_chunk(None, shape=(100, 100)).data input_chunks.append(inp_chunk) add_op = TensorTreeAdd(args=input_chunks, dtype=np.dtype('int64')) add_chunk = add_op.new_chunk(input_chunks, shape=(100, 100), dtype=np.dtype('int64')).data graph_add.add_node(add_chunk) for inp_chunk in input_chunks: graph_add.add_node(inp_chunk) graph_add.add_edge(inp_chunk, add_chunk) executor = Executor() res = executor.execute_graph(graph_add, [add_chunk.key], compose=False, mock=True)[0] self.assertEqual(res, (80000, 80000)) self.assertEqual(executor.mock_max_memory, 80000) for _ in range(3): new_add_op = TensorTreeAdd(args=[add_chunk], dtype=np.dtype('int64')) new_add_chunk = new_add_op.new_chunk([add_chunk], shape=(100, 100), dtype=np.dtype('int64')).data graph_add.add_node(new_add_chunk) graph_add.add_edge(add_chunk, new_add_chunk) add_chunk = new_add_chunk executor = Executor() res = executor.execute_graph(graph_add, [add_chunk.key], compose=False, mock=True)[0] self.assertEqual(res, (80000, 80000)) self.assertEqual(executor.mock_max_memory, 160000) a = mt.random.rand(10, 10, chunk_size=10) b = a[:, mt.newaxis, :] - a r = mt.triu(mt.sqrt(b**2).sum(axis=2)) executor = Executor() res = executor.execute_tensor(r, concat=False, mock=True) # larger than maximal memory size in calc procedure self.assertGreaterEqual(res[0][0], 800) self.assertGreaterEqual(executor.mock_max_memory, 8000)
class ExecutorActor(Actor): def __init__(self): self._executor = Executor(sync_provider_type=Executor.SyncProviderType.GEVENT) def post_create(self): register(FakeOperand, fake_execution_maker(self.ctx)) def on_receive(self, message): op = FakeOperand(_num=message) chunk = op.new_chunk(None, ()) graph = DirectedGraph() graph.add_node(chunk.data) res = self._executor.execute_graph(graph, [chunk.key]) assert res[0] == message + 1
class Test(TestBase): def setUp(self): super(Test, self).setUp() self.executor = Executor() def testPandasExecution(self): pdf = pd.DataFrame(np.random.rand(20, 30), index=[np.arange(20), np.arange(20, 0, -1)]) df = from_pandas(pdf, chunk_size=(13, 21)) graph = df.build_graph(tiled=True) results = self.executor.execute_graph(graph, keys=[c.key for c in df.chunks]) self.assertEqual(len(results), 4) pd.testing.assert_frame_equal(pdf.iloc[:13, :21], results[0]) pd.testing.assert_frame_equal(pdf.iloc[:13, 21:], results[1]) pd.testing.assert_frame_equal(pdf.iloc[13:, :21], results[2]) pd.testing.assert_frame_equal(pdf.iloc[13:, 21:], results[3])