def testSameKeyAssign(self): import numpy as np from mars.tensor.random import TensorRandint from mars.tensor.arithmetic import TensorTreeAdd graph = DAG() r""" Proper initial allocation should divide the graph like U U | U U | U U | | | | | | | | | | | | | | U U | U U | U U """ inputs = [ tuple( TensorRandint(_key=str(i), dtype=np.float32()).new_chunk( None, shape=(10, 10)) for _ in range(2)) for i in range(6) ] results = [ TensorTreeAdd(dtype=np.float32()).new_chunk(None, shape=(10, 10)) for _ in range(6) ] for inp, r in zip(inputs, results): r.op._inputs = list(inp) graph.add_node(r) for n in inp: graph.add_node(n) graph.add_edge(n, r) analyzer = GraphAnalyzer(graph, dict(w1=24, w2=24, w3=24)) assignments = analyzer.calc_operand_assignments( analyzer.get_initial_operand_keys()) self.assertEqual(len(assignments), 6)
def testInitialAssignsWithInputs(self): import numpy as np from mars.tensor.random import TensorRandint from mars.tensor.arithmetic import TensorTreeAdd n1 = TensorRandint(state=np.random.RandomState(0), dtype=np.float32()).new_chunk(None, shape=(10, 10)) n2 = TensorRandint(state=np.random.RandomState(1), dtype=np.float32()).new_chunk(None, shape=(10, 10)) n3 = TensorTreeAdd(dtype=np.float32()).new_chunk(None, shape=(10, 10)) n3.op._inputs = [n1, n2] n4 = TensorTreeAdd(dtype=np.float32()).new_chunk(None, shape=(10, 10)) n4.op._inputs = [n3] graph = DAG() graph.add_node(n1) graph.add_node(n3) graph.add_node(n4) graph.add_edge(n1, n3) graph.add_edge(n3, n4) analyzer = GraphAnalyzer(graph, {}) ext_chunks = analyzer.collect_external_input_chunks(initial=False) self.assertListEqual(ext_chunks[n3.op.key], [n2.key]) self.assertEqual( len(analyzer.collect_external_input_chunks(initial=True)), 0)
def _build_chunk_dag(node_str, edge_str): from mars.tensor.random import TensorRandint from mars.tensor.arithmetic import TensorTreeAdd char_dag = DAG() for s in node_str.split(','): char_dag.add_node(s.strip()) for s in edge_str.split(','): l, r = s.split('->') char_dag.add_edge(l.strip(), r.strip()) chunk_dag = DAG() str_to_chunk = dict() for s in char_dag.topological_iter(): if char_dag.count_predecessors(s): c = TensorTreeAdd(_key=s, dtype=np.float32()).new_chunk(None, shape=(10, 10)) inputs = c.op._inputs = [ str_to_chunk[ps] for ps in char_dag.predecessors(s) ] else: c = TensorRandint(_key=s, dtype=np.float32()).new_chunk(None, shape=(10, 10)) inputs = [] str_to_chunk[s] = c chunk_dag.add_node(c) for inp in inputs: chunk_dag.add_edge(inp, c) return chunk_dag, str_to_chunk
def testRegister(self): from mars.graph import DAG fake_result = np.random.rand(10, 10) fake_size = (fake_result.nbytes * 2, fake_result.nbytes * 2) def fake_execute(ctx, op): ctx[op.outputs[0].key] = fake_result def fake_estimate(ctx, op): ctx[op.outputs[0].key] = fake_size register(FakeOperand, fake_execute, fake_estimate) graph = DAG() chunk = FakeOperand().new_chunk(None, shape=(10, 10)) graph.add_node(chunk.data) executor = Executor() res = executor.execute_graph(graph, keys=[chunk.key])[0] np.testing.assert_array_equal(res, fake_result) size = executor.execute_graph(graph, keys=[chunk.key], mock=True)[0] self.assertEqual(size, fake_size) graph = DAG() chunk = SubFakeOperand().new_chunk(None, shape=(10, 10)) graph.add_node(chunk.data) executor = Executor() res = executor.execute_graph(graph, keys=[chunk.key])[0] np.testing.assert_array_equal(res, fake_result)
def testFullInitialAssign(self): import numpy as np from mars.tensor.expressions.random import TensorRandint from mars.tensor.expressions.arithmetic import TensorTreeAdd graph = DAG() r""" Proper initial allocation should divide the graph like U U U U | U U U U | U U U U \ / \ / | \ / \ / | \ / \ / U U | U U | U U """ inputs = [ tuple( TensorRandint(dtype=np.float32()).new_chunk(None, (10, 10)) for _ in range(2)) for _ in range(6) ] results = [ TensorTreeAdd(dtype=np.float32()).new_chunk(None, (10, 10)) for _ in range(6) ] for inp, r in zip(inputs, results): r.op._inputs = list(inp) graph.add_node(r) for n in inp: graph.add_node(n) graph.add_edge(n, r) analyzer = GraphAnalyzer(graph, dict(w1=24, w2=24, w3=24)) assignments = analyzer.calc_initial_assignments() for inp in inputs: self.assertEqual(1, len(set(assignments[n.op.key] for n in inp)))
def testDataFrameGraphSerialize(self): df = from_pandas_df(pd.DataFrame(np.random.rand(10, 10), columns=pd.timedelta_range(start='1 day', periods=10), index=pd.date_range('2020-1-1', periods=10))) graph = df.build_graph(tiled=False) pb = graph.to_pb() graph2 = DAG.from_pb(pb) self.assertEqual(len(graph), len(graph2)) t = next(iter(graph)) t2 = next(iter(graph2)) self.assertTrue(t2.op.outputs[0], ReferenceType) # make sure outputs are all weak reference self.assertBaseEqual(t.op, t2.op) self.assertEqual(t.shape, t2.shape) self.assertEqual(sorted(i.key for i in t.inputs), sorted(i.key for i in t2.inputs)) pd.testing.assert_index_equal(t2.index_value.to_pandas(), t.index_value.to_pandas()) pd.testing.assert_index_equal(t2.columns_value.to_pandas(), t.columns_value.to_pandas()) jsn = graph.to_json() graph2 = DAG.from_json(jsn) self.assertEqual(len(graph), len(graph2)) t = next(iter(graph)) t2 = next(iter(graph2)) self.assertTrue(t2.op.outputs[0], ReferenceType) # make sure outputs are all weak reference self.assertBaseEqual(t.op, t2.op) self.assertEqual(t.shape, t2.shape) self.assertEqual(sorted(i.key for i in t.inputs), sorted(i.key for i in t2.inputs)) pd.testing.assert_index_equal(t2.index_value.to_pandas(), t.index_value.to_pandas()) pd.testing.assert_index_equal(t2.columns_value.to_pandas(), t.columns_value.to_pandas()) # test graph with tiled DataFrame t2 = from_pandas_df(pd.DataFrame(np.random.rand(10, 10)), chunk_size=(5, 4)).tiles() graph = DAG() graph.add_node(t2) pb = graph.to_pb() graph2 = DAG.from_pb(pb) self.assertEqual(len(graph), len(graph2)) chunks = next(iter(graph2)).chunks self.assertEqual(len(chunks), 6) self.assertIsInstance(chunks[0], DataFrameChunk) self.assertEqual(chunks[0].index, t2.chunks[0].index) self.assertBaseEqual(chunks[0].op, t2.chunks[0].op) pd.testing.assert_index_equal(chunks[0].index_value.to_pandas(), t2.chunks[0].index_value.to_pandas()) pd.testing.assert_index_equal(chunks[0].columns_value.to_pandas(), t2.chunks[0].columns_value.to_pandas()) jsn = graph.to_json() graph2 = DAG.from_json(jsn) self.assertEqual(len(graph), len(graph2)) chunks = next(iter(graph2)).chunks self.assertEqual(len(chunks), 6) self.assertIsInstance(chunks[0], DataFrameChunk) self.assertEqual(chunks[0].index, t2.chunks[0].index) self.assertBaseEqual(chunks[0].op, t2.chunks[0].op) pd.testing.assert_index_equal(chunks[0].index_value.to_pandas(), t2.chunks[0].index_value.to_pandas()) pd.testing.assert_index_equal(chunks[0].columns_value.to_pandas(), t2.chunks[0].columns_value.to_pandas())
def testTensorGraphSerialize(self): t = ones((10, 3), chunk_size=(5, 2)) + tensor(np.random.random( (10, 3)), chunk_size=(5, 2)) graph = t.build_graph(tiled=False) pb = graph.to_pb() graph2 = DAG.from_pb(pb) self.assertEqual(len(graph), len(graph2)) t = next(c for c in graph if c.inputs) t2 = next(c for c in graph2 if c.key == t.key) self.assertTrue( t2.op.outputs[0], ReferenceType) # make sure outputs are all weak reference self.assertBaseEqual(t.op, t2.op) self.assertEqual(t.shape, t2.shape) self.assertEqual(sorted(i.key for i in t.inputs), sorted(i.key for i in t2.inputs)) jsn = graph.to_json() graph2 = DAG.from_json(jsn) self.assertEqual(len(graph), len(graph2)) t = next(c for c in graph if c.inputs) t2 = next(c for c in graph2 if c.key == t.key) self.assertTrue( t2.op.outputs[0], ReferenceType) # make sure outputs are all weak reference self.assertBaseEqual(t.op, t2.op) self.assertEqual(t.shape, t2.shape) self.assertEqual(sorted(i.key for i in t.inputs), sorted(i.key for i in t2.inputs)) # test graph with tiled tensor t2 = ones((10, 10), chunk_size=(5, 4)).tiles() graph = DAG() graph.add_node(t2) pb = graph.to_pb() graph2 = DAG.from_pb(pb) self.assertEqual(len(graph), len(graph2)) chunks = next(iter(graph2)).chunks self.assertEqual(len(chunks), 6) self.assertIsInstance(chunks[0], TensorChunk) self.assertEqual(chunks[0].index, t2.chunks[0].index) self.assertBaseEqual(chunks[0].op, t2.chunks[0].op) jsn = graph.to_json() graph2 = DAG.from_json(jsn) self.assertEqual(len(graph), len(graph2)) chunks = next(iter(graph2)).chunks self.assertEqual(len(chunks), 6) self.assertIsInstance(chunks[0], TensorChunk) self.assertEqual(chunks[0].index, t2.chunks[0].index) self.assertBaseEqual(chunks[0].op, t2.chunks[0].op)
def testDataFrameGraphSerialize(self): df = from_pandas(pd.DataFrame(np.random.rand(10, 10))) graph = df.build_graph(tiled=False) pb = graph.to_pb() graph2 = DAG.from_pb(pb) self.assertEqual(len(graph), len(graph2)) t = next(iter(graph)) t2 = next(iter(graph2)) self.assertTrue( t2.op.outputs[0], ReferenceType) # make sure outputs are all weak reference self.assertBaseEqual(t.op, t2.op) self.assertEqual(t.shape, t2.shape) self.assertEqual(sorted(i.key for i in t.inputs), sorted(i.key for i in t2.inputs)) jsn = graph.to_json() graph2 = DAG.from_json(jsn) self.assertEqual(len(graph), len(graph2)) t = next(iter(graph)) t2 = next(iter(graph2)) self.assertTrue( t2.op.outputs[0], ReferenceType) # make sure outputs are all weak reference self.assertBaseEqual(t.op, t2.op) self.assertEqual(t.shape, t2.shape) self.assertEqual(sorted(i.key for i in t.inputs), sorted(i.key for i in t2.inputs)) # test graph with tiled DataFrame t2 = from_pandas(pd.DataFrame(np.random.rand(10, 10)), chunk_size=(5, 4)).tiles() graph = DAG() graph.add_node(t2) pb = graph.to_pb() graph2 = DAG.from_pb(pb) self.assertEqual(len(graph), len(graph2)) chunks = next(iter(graph2)).chunks self.assertEqual(len(chunks), 6) self.assertIsInstance(chunks[0], DataFrameChunk) self.assertEqual(chunks[0].index, t2.chunks[0].index) self.assertBaseEqual(chunks[0].op, t2.chunks[0].op) jsn = graph.to_json() graph2 = DAG.from_json(jsn) self.assertEqual(len(graph), len(graph2)) chunks = next(iter(graph2)).chunks self.assertEqual(len(chunks), 6) self.assertIsInstance(chunks[0], DataFrameChunk) self.assertEqual(chunks[0].index, t2.chunks[0].index) self.assertBaseEqual(chunks[0].op, t2.chunks[0].op)
def testDAG(self): """ 1 --- 4 2 --- 6 \ / 5 / 3 """ dag = DAG() [dag.add_node(i) for i in range(1, 7)] dag.add_edge(1, 4) dag.add_edge(2, 6) dag.add_edge(2, 5) dag.add_edge(5, 6) dag.add_edge(3, 5) self.assertEqual(set(dag[2]), set([5, 6])) self.assertEqual(list(dag.topological_iter()), [3, 2, 5, 6, 1, 4]) self.assertEqual(list(dag.dfs()), [3, 2, 5, 6, 1, 4]) self.assertEqual(list(dag.bfs()), [1, 2, 3, 4, 5, 6]) dag.add_edge(6, 1) dag.add_edge(1, 2) self.assertRaises(GraphContainsCycleError, lambda: list(dag.topological_iter()))
def _build_test_graph(data_list): from mars.tensor.fetch import TensorFetch from mars.tensor.arithmetic import TensorTreeAdd inputs = [] for idx, d in enumerate(data_list): chunk_key = 'chunk-%d' % idx fetch_chunk = TensorFetch(to_fetch_key=chunk_key, dtype=d.dtype) \ .new_chunk([], shape=d.shape, _key=chunk_key) inputs.append(fetch_chunk) add_chunk = TensorTreeAdd(data_list[0].dtype).new_chunk(inputs, shape=data_list[0].shape) exec_graph = DAG() exec_graph.add_node(add_chunk) for input_chunk in inputs: exec_graph.add_node(input_chunk) exec_graph.add_edge(input_chunk, add_chunk) return exec_graph, inputs, add_chunk
def testMockExecuteSize(self): import mars.tensor as mt from mars.graph import DAG from mars.tensor.fetch import TensorFetch from mars.tensor.arithmetic import TensorTreeAdd graph_add = DAG() input_chunks = [] for _ in range(2): fetch_op = TensorFetch(dtype=np.dtype('int64')) inp_chunk = fetch_op.new_chunk(None, shape=(100, 100)).data input_chunks.append(inp_chunk) add_op = TensorTreeAdd(args=input_chunks, dtype=np.dtype('int64')) add_chunk = add_op.new_chunk(input_chunks, shape=(100, 100), dtype=np.dtype('int64')).data graph_add.add_node(add_chunk) for inp_chunk in input_chunks: graph_add.add_node(inp_chunk) graph_add.add_edge(inp_chunk, add_chunk) executor = Executor() res = executor.execute_graph(graph_add, [add_chunk.key], compose=False, mock=True)[0] self.assertEqual(res, (80000, 80000)) self.assertEqual(executor.mock_max_memory, 80000) for _ in range(3): new_add_op = TensorTreeAdd(args=[add_chunk], dtype=np.dtype('int64')) new_add_chunk = new_add_op.new_chunk([add_chunk], shape=(100, 100), dtype=np.dtype('int64')).data graph_add.add_node(new_add_chunk) graph_add.add_edge(add_chunk, new_add_chunk) add_chunk = new_add_chunk executor = Executor() res = executor.execute_graph(graph_add, [add_chunk.key], compose=False, mock=True)[0] self.assertEqual(res, (80000, 80000)) self.assertEqual(executor.mock_max_memory, 160000) a = mt.random.rand(10, 10, chunk_size=10) b = a[:, mt.newaxis, :] - a r = mt.triu(mt.sqrt(b**2).sum(axis=2)) executor = Executor() res = executor.execute_tensor(r, concat=False, mock=True) # larger than maximal memory size in calc procedure self.assertGreaterEqual(res[0][0], 800) self.assertGreaterEqual(executor.mock_max_memory, 8000)
def testAssignWithPreviousData(self): import numpy as np from mars.scheduler.chunkmeta import WorkerMeta from mars.tensor.random import TensorRandint from mars.tensor.arithmetic import TensorTreeAdd graph = DAG() r""" Proper initial allocation should divide the graph like U U | U U | U U \ / | \ / | \ / U | U | U """ inputs = [ tuple( TensorRandint(_key=str(i * 2 + j), dtype=np.float32()).new_chunk( None, shape=(10, 10)) for j in range(2)) for i in range(3) ] results = [ TensorTreeAdd(dtype=np.float32()).new_chunk(None, shape=(10, 10)) for _ in range(3) ] for inp, r in zip(inputs, results): r.op._inputs = list(inp) graph.add_node(r) for n in inp: graph.add_node(n) graph.add_edge(n, r) # assign with partial mismatch data_dist = { '0': dict(c00=WorkerMeta(chunk_size=5, workers=('w1', )), c01=WorkerMeta(chunk_size=5, workers=('w2', ))), '1': dict(c10=WorkerMeta(chunk_size=10, workers=('w1', ))), '2': dict(c20=WorkerMeta(chunk_size=10, workers=('w3', ))), '3': dict(c30=WorkerMeta(chunk_size=10, workers=('w3', ))), '4': dict(c40=WorkerMeta(chunk_size=7, workers=('w3', ))), } analyzer = GraphAnalyzer(graph, dict(w1=24, w2=24, w3=24)) assignments = analyzer.calc_operand_assignments( analyzer.get_initial_operand_keys(), input_chunk_metas=data_dist) self.assertEqual(len(assignments), 6) # explanation of the result: # for '1', all data are in w1, hence assigned to w1 # '0' assigned to w1 according to connectivity # '2' and '3' assigned to w3 according to connectivity # '4' assigned to w2 because it has fewer data, and the slots of w3 is used up self.assertEqual(assignments['0'], 'w1') self.assertEqual(assignments['1'], 'w1') self.assertEqual(assignments['2'], 'w3') self.assertEqual(assignments['3'], 'w3') self.assertEqual(assignments['4'], 'w2') self.assertEqual(assignments['5'], 'w2') # assign with full mismatch data_dist = { '0': dict(c00=WorkerMeta(chunk_size=5, workers=('w1', )), c01=WorkerMeta(chunk_size=5, workers=( 'w1', 'w2', ))), '1': dict(c10=WorkerMeta(chunk_size=10, workers=('w1', ))), '2': dict(c20=WorkerMeta(chunk_size=10, workers=('w3', ))), '3': dict(c30=WorkerMeta(chunk_size=10, workers=('w3', ))), '4': dict(c40=WorkerMeta(chunk_size=7, workers=('w2', ))), '5': dict(c50=WorkerMeta(chunk_size=7, workers=('w2', ))), } analyzer = GraphAnalyzer(graph, dict(w1=24, w2=24, w3=24)) assignments = analyzer.calc_operand_assignments( analyzer.get_initial_operand_keys(), input_chunk_metas=data_dist) self.assertEqual(len(assignments), 6) self.assertEqual(assignments['0'], 'w1') self.assertEqual(assignments['1'], 'w1') self.assertEqual(assignments['2'], 'w3') self.assertEqual(assignments['3'], 'w3') self.assertEqual(assignments['4'], 'w2') self.assertEqual(assignments['5'], 'w2')
def testPrepushGraph(self): import mars.tensor as mt from mars.graph import DAG from mars.tensor.expressions.datasource import TensorFetch data_inputs = [np.random.random((4, )) for _ in range(2)] arr_inputs = [mt.tensor(di, chunk_size=4) for di in data_inputs] arr_add = arr_inputs[0] + arr_inputs[1] graph_inputs = [a.build_graph(tiled=True) for a in arr_inputs] graph_input_op_keys = [a.chunks[0].op.key for a in arr_inputs] arr_add.build_graph(tiled=True) graph_add = DAG() input_chunks = [] for a in arr_inputs: fetch_op = TensorFetch(dtype=a.dtype) inp_chunk = fetch_op.new_chunk(None, a.shape, _key=a.chunks[0].key).data input_chunks.append(inp_chunk) new_op = arr_add.chunks[0].op.copy() new_add_chunk = new_op.new_chunk(input_chunks, arr_add.shape, index=arr_add.chunks[0].index, dtype=arr_add.dtype, _key=arr_add.chunks[0].key) graph_add.add_node(new_add_chunk) for inp_chunk in input_chunks: graph_add.add_node(inp_chunk) graph_add.add_edge(inp_chunk, new_add_chunk) graph_add_key = arr_add.chunks[0].op.key pool_address = '127.0.0.1:%d' % get_next_port() session_id = str(uuid.uuid4()) def _validate(_): data = test_actor._chunk_store.get(session_id, arr_add.chunks[0].key) assert_array_equal(data, data_inputs[0] + data_inputs[1]) options.worker.spill_directory = tempfile.mkdtemp( 'mars_worker_prep_spilled-') # register when all predecessors unfinished with create_actor_pool(n_process=1, backend='gevent', address=pool_address) as pool: self.create_standard_actors(pool, pool_address, with_daemon=False, with_status=False) pool.create_actor(SpillActor) pool.create_actor(CpuCalcActor) with self.run_actor_test(pool) as test_actor: execution_ref = test_actor.promise_ref( ExecutionActor.default_name()) execution_ref.enqueue_graph( session_id, graph_add_key, serialize_graph(graph_add), dict(chunks=[new_add_chunk.key]), None, pred_keys=graph_input_op_keys, _promise=True) \ .then(lambda *_: execution_ref.start_execution(session_id, graph_add_key, _promise=True)) \ .then(_validate) \ .then(lambda *_: test_actor.set_result(None)) \ .catch(lambda *exc: test_actor.set_result(exc, False)) for ginput, op_key, gtensor in zip(graph_inputs, graph_input_op_keys, arr_inputs): def _start_exec_promise(session_id, op_key, *_): return execution_ref.start_execution(session_id, op_key, _promise=True) execution_ref.enqueue_graph( session_id, op_key, serialize_graph(ginput), dict(chunks=[gtensor.chunks[0].key]), None, succ_keys=[new_add_chunk.op.key], _promise=True) \ .then(functools.partial(_start_exec_promise, session_id, op_key)) self.get_result() pool_address = '127.0.0.1:%d' % get_next_port() session_id = str(uuid.uuid4()) # register when part of predecessors unfinished with create_actor_pool(n_process=1, backend='gevent', address=pool_address) as pool: self.create_standard_actors(pool, pool_address, with_daemon=False, with_status=False) pool.create_actor(SpillActor) pool.create_actor(CpuCalcActor) with self.run_actor_test(pool) as test_actor: execution_ref = test_actor.promise_ref( ExecutionActor.default_name()) execution_ref.enqueue_graph( session_id, graph_input_op_keys[0], serialize_graph(graph_inputs[0]), dict(chunks=[input_chunks[0].key]), None, succ_keys=[new_add_chunk.op.key], _promise=True) \ .then(lambda *_: execution_ref.start_execution(session_id, graph_input_op_keys[0], _promise=True)) \ .then(lambda *_: test_actor.set_result(None, destroy=False)) \ .catch(lambda *exc: test_actor.set_result(exc, False)) self.get_result() execution_ref.enqueue_graph( session_id, graph_add_key, serialize_graph(graph_add), dict(chunks=[new_add_chunk.key]), None, pred_keys=graph_input_op_keys, _promise=True) \ .then(lambda *_: execution_ref.start_execution(session_id, graph_add_key, _promise=True)) \ .then(_validate) \ .then(lambda *_: test_actor.set_result(None)) \ .catch(lambda *exc: test_actor.set_result(exc, False)) execution_ref.enqueue_graph( session_id, graph_input_op_keys[1], serialize_graph(graph_inputs[1]), dict(chunks=[input_chunks[1].key]), None, succ_keys=[new_add_chunk.op.key], _promise=True) \ .then(lambda *_: execution_ref.start_execution(session_id, graph_input_op_keys[1], _promise=True)) self.get_result()
def testDAG(self): r""" 1 --- 4 2 --- 6 \ / 5 / 3 """ dag = DAG() [dag.add_node(i) for i in range(1, 7)] dag.add_edge(1, 4) dag.add_edge(2, 6) dag.add_edge(2, 5) dag.add_edge(5, 6) dag.add_edge(3, 5) with self.assertRaises(KeyError): dag.add_edge(1, 10) with self.assertRaises(KeyError): dag.add_edge(10, 1) self.assertEqual(set(dag[2]), {5, 6}) self.assertEqual(list(dag.topological_iter()), [3, 2, 5, 6, 1, 4]) self.assertEqual(list(dag.dfs()), [3, 2, 5, 6, 1, 4]) self.assertEqual(list(dag.bfs()), [1, 2, 3, 4, 5, 6]) dag.add_edge(6, 1) dag.add_edge(1, 2) with self.assertRaises(KeyError): for _ in dag.iter_predecessors(-1): pass with self.assertRaises(KeyError): for _ in dag.iter_successors(-1): pass self.assertRaises(GraphContainsCycleError, lambda: list(dag.topological_iter())) dag.remove_edge(2, 5) self.assertFalse(dag.has_successor(2, 5)) with self.assertRaises(KeyError): dag.remove_edge(2, 5) rev_dag = dag.build_reversed() for n in dag: self.assertIn(n, rev_dag) self.assertTrue( all( rev_dag.has_successor(n, pred) for pred in dag.predecessors(n))) undigraph = dag.build_undirected() for n in dag: self.assertIn(n, undigraph) self.assertTrue( all( undigraph.has_predecessor(pred, n) for pred in dag.predecessors(n))) self.assertTrue( all( undigraph.has_successor(n, pred) for pred in dag.predecessors(n))) dag_copy = dag.copy() for n in dag: self.assertIn(n, dag_copy) self.assertTrue( all( dag_copy.has_successor(pred, n) for pred in dag_copy.predecessors(n)))
def testCompose(self): """ test compose in build graph and optimize """ r""" graph(@: node, #: composed_node): @ --> @ --> @ ========> # """ chunks = [ TensorTreeAdd(args=[], _key=str(n)).new_chunk(None, None) for n in range(3) ] graph = DAG() list(map(graph.add_node, chunks[:3])) graph.add_edge(chunks[0], chunks[1]) graph.add_edge(chunks[1], chunks[2]) composed_nodes = graph.compose() self.assertTrue(composed_nodes[0].composed == chunks[:3]) # make the middle one as result chunk, thus the graph cannot be composed composed_nodes = graph.compose(keys=[chunks[1].key]) self.assertEqual(len(composed_nodes), 0) r""" graph(@: node, #: composed_node): @ @ @ @ \ / \ / @ --> @ ========> # / \ / \ @ @ @ @ """ chunks = [ TensorTreeAdd(args=[], _key=str(n)).new_chunk(None, None) for n in range(6) ] graph = DAG() list(map(graph.add_node, chunks[:6])) chunks[2].op._inputs = [chunks[0], chunks[1]] chunks[3].op._inputs = [chunks[2]] chunks[4].op._inputs = [chunks[3]] chunks[5].op._inputs = [chunks[3]] graph.add_edge(chunks[0], chunks[2]) graph.add_edge(chunks[1], chunks[2]) graph.add_edge(chunks[2], chunks[3]) graph.add_edge(chunks[3], chunks[4]) graph.add_edge(chunks[3], chunks[5]) composed_nodes = graph.compose() self.assertTrue(composed_nodes[0].composed == chunks[2:4]) # to make sure the predecessors and successors of compose are right # 0 and 1's successors must be composed self.assertIn(composed_nodes[0], graph.successors(chunks[0])) self.assertIn(composed_nodes[0], graph.successors(chunks[1])) # check composed's inputs self.assertIn(chunks[0].data, composed_nodes[0].inputs) self.assertIn(chunks[1].data, composed_nodes[0].inputs) # check composed's predecessors self.assertIn(chunks[0], graph.predecessors(composed_nodes[0])) self.assertIn(chunks[1], graph.predecessors(composed_nodes[0])) # check 4 and 5's inputs self.assertIn(composed_nodes[0].data, graph.successors(composed_nodes[0])[0].inputs) self.assertIn(composed_nodes[0].data, graph.successors(composed_nodes[0])[0].inputs) # check 4 and 5's predecessors self.assertIn(composed_nodes[0], graph.predecessors(chunks[4])) self.assertIn(composed_nodes[0], graph.predecessors(chunks[5])) # test optimizer compose r""" graph(@: node, S: Slice Chunk, #: composed_node): @ @ @ @ \ / \ / @ --> @ --> S ========> # --> S / \ / \ @ @ @ @ compose stopped at S, because numexpr don't support Slice op """ chunks = [ TensorTreeAdd(args=[], _key=str(n)).new_chunk(None, None) for n in range(6) ] chunk_slice = TensorSlice().new_chunk([None], None) graph = DAG() list(map(graph.add_node, chunks[:6])) graph.add_node(chunk_slice) graph.add_edge(chunks[0], chunks[2]) graph.add_edge(chunks[1], chunks[2]) graph.add_edge(chunks[2], chunks[3]) graph.add_edge(chunks[3], chunk_slice) graph.add_edge(chunk_slice, chunks[4]) graph.add_edge(chunk_slice, chunks[5]) optimizer = NeRuntimeOptimizer(graph) composed_nodes = optimizer.compose() self.assertTrue(composed_nodes[0].composed == chunks[2:4]) r""" graph(@: node, S: Slice Chunk, #: composed_node): @ --> @ --> S --> @ ========> # --> S --> @ compose stopped at S, because numexpr don't support Slice op """ chunks = [ TensorTreeAdd(args=[], _key=str(n)).new_chunk(None, None) for n in range(4) ] graph = DAG() list(map(graph.add_node, chunks[:3])) graph.add_node(chunk_slice) graph.add_edge(chunks[0], chunks[1]) graph.add_edge(chunks[1], chunk_slice) graph.add_edge(chunk_slice, chunks[2]) optimizer = NeRuntimeOptimizer(graph) composed_nodes = optimizer.compose() self.assertTrue(composed_nodes[0].composed == chunks[:2]) self.assertTrue(len(composed_nodes) == 1) r""" graph(@: node, S: Slice Chunk, #: composed_node): @ --> @ --> S --> @ --> @ ========> # --> S --> # compose stopped at S, because numexpr don't support Slice op """ chunks = [ TensorTreeAdd(args=[], _key=str(n)).new_chunk(None, None) for n in range(4) ] graph = DAG() list(map(graph.add_node, chunks[:4])) graph.add_node(chunk_slice) graph.add_edge(chunks[0], chunks[1]) graph.add_edge(chunks[1], chunk_slice) graph.add_edge(chunk_slice, chunks[2]) graph.add_edge(chunks[2], chunks[3]) optimizer = NeRuntimeOptimizer(graph) composed_nodes = optimizer.compose() self.assertTrue(composed_nodes[0].composed == chunks[:2]) self.assertTrue(composed_nodes[1].composed == chunks[2:4])
def testAssignOnWorkerAdd(self): import numpy as np from mars.scheduler import OperandState from mars.tensor.random import TensorRandint from mars.tensor.arithmetic import TensorTreeAdd graph = DAG() r""" Proper initial allocation should divide the graph like F F R R | F F R R | R R R R | x | | x | | | x | | x | | | x | | x | R R U U | R R U U | U U U U U: UNSCHEDULED F: FINISHED R: READY """ inputs = [ tuple( TensorRandint( dtype=np.float32()).new_chunk(None, shape=(10, 10)) for _ in range(2)) for _ in range(6) ] results = [ tuple( TensorTreeAdd(_key='%d_%d' % (i, j), dtype=np.float32()).new_chunk( None, shape=(10, 10)) for j in range(2)) for i in range(6) ] for inp, outp in zip(inputs, results): for o in outp: o.op._inputs = list(inp) graph.add_node(o) for n in inp: graph.add_node(n) for o in outp: graph.add_edge(n, o) # mark initial assigns fixed_assigns = dict() op_states = dict() for idx in range(2): for i in range(2): fixed_assigns[inputs[idx][i].op.key] = 'w%d' % (idx + 1) op_states[results[idx][i].op.key] = OperandState.READY fixed_assigns[results[idx][i].op.key] = 'w%d' % (idx + 1) for inp in inputs: for n in inp: if n.op.key in fixed_assigns: continue op_states[n.op.key] = OperandState.READY worker_metrics = dict(w1=24, w2=24, w3=24) analyzer = GraphAnalyzer(graph, worker_metrics, fixed_assigns, op_states) assignments = analyzer.calc_operand_assignments( analyzer.get_initial_operand_keys()) for inp in inputs: if any(n.op.key in fixed_assigns for n in inp): continue self.assertEqual(1, len(set(assignments[n.op.key] for n in inp))) worker_assigns = dict((k, 0) for k in worker_metrics) for w in assignments.values(): worker_assigns[w] += 1 self.assertEqual(2, worker_assigns['w1']) self.assertEqual(2, worker_assigns['w2']) self.assertEqual(4, worker_assigns['w3'])
def testAssignOnWorkerLost(self): import numpy as np from mars.scheduler import OperandState from mars.tensor.random import TensorRandint from mars.tensor.arithmetic import TensorTreeAdd graph = DAG() r""" Proper initial allocation should divide the graph like FL FL F F R R | FL FL F F R R | x | | x | | x | | | x | | x | | x | R R R R U U | R R R R U U U: UNSCHEDULED F: FINISHED R: READY L: LOST """ op_states = dict() inputs = [ tuple( TensorRandint( dtype=np.float32()).new_chunk(None, shape=(10, 10)) for _ in range(2)) for _ in range(6) ] results = [ tuple( TensorTreeAdd(_key=f'{i}_{j}', dtype=np.float32()).new_chunk( None, shape=(10, 10)) for j in range(2)) for i in range(6) ] for inp, outp in zip(inputs, results): for o in outp: o.op._inputs = list(inp) op_states[o.op.key] = OperandState.UNSCHEDULED graph.add_node(o) for n in inp: op_states[n.op.key] = OperandState.UNSCHEDULED graph.add_node(n) for o in outp: graph.add_edge(n, o) fixed_assigns = dict() for idx in range(4): for i in range(2): fixed_assigns[inputs[idx][i].op.key] = f'w{idx % 2 + 1}' op_states[inputs[idx][i].op.key] = OperandState.FINISHED fixed_assigns[results[idx][i].op.key] = f'w{idx % 2 + 1}' op_states[results[idx][i].op.key] = OperandState.READY for inp in inputs: for n in inp: if n.op.key in fixed_assigns: continue op_states[n.op.key] = OperandState.READY lost_chunks = [c.key for inp in (inputs[0], inputs[2]) for c in inp] worker_metrics = dict(w2=24, w3=24) analyzer = GraphAnalyzer(graph, worker_metrics, fixed_assigns, op_states, lost_chunks) changed_states = analyzer.analyze_state_changes() self.assertEqual(len(changed_states), 8) self.assertTrue( all(changed_states[c.op.key] == OperandState.READY for inp in (inputs[0], inputs[2]) for c in inp)) self.assertTrue( all(changed_states[c.op.key] == OperandState.UNSCHEDULED for res in (results[0], results[2]) for c in res)) assignments = analyzer.calc_operand_assignments( analyzer.get_initial_operand_keys()) for inp in inputs: if any(n.op.key in fixed_assigns for n in inp): continue self.assertEqual(1, len(set(assignments[n.op.key] for n in inp))) worker_assigns = dict((k, 0) for k in worker_metrics) for w in assignments.values(): worker_assigns[w] += 1 self.assertEqual(2, worker_assigns['w2']) self.assertEqual(6, worker_assigns['w3'])