def testBuildTileableGraph(self): a = mt.ones((10, 10), chunk_size=8) b = mt.ones((10, 10), chunk_size=8) c = (a + 1) * 2 + b graph = utils.build_tileable_graph([c], set()) self.assertEqual(len(graph), 5) a_data = next(n for n in graph if n.key == a.key) self.assertEqual(graph.count_successors(a_data), 1) self.assertEqual(graph.count_predecessors(a_data), 0) c_data = next(n for n in graph if n.key == c.key) self.assertEqual(graph.count_successors(c_data), 0) self.assertEqual(graph.count_predecessors(c_data), 2) graph = utils.build_tileable_graph([a, b, c], set()) self.assertEqual(len(graph), 5) # test fetch replacement a = mt.ones((10, 10), chunk_size=8) b = mt.ones((10, 10), chunk_size=8) c = (a + 1) * 2 + b executed_keys = {a.key, b.key} graph = utils.build_tileable_graph([c], executed_keys) self.assertEqual(len(graph), 5) self.assertNotIn(a.data, graph) self.assertNotIn(b.data, graph) self.assertTrue(any(isinstance(n.op, TensorFetch) for n in graph)) c_data = next(n for n in graph if n.key == c.key) self.assertEqual(graph.count_successors(c_data), 0) self.assertEqual(graph.count_predecessors(c_data), 2) executed_keys = {(a + 1).key} graph = utils.build_tileable_graph([c], executed_keys) self.assertTrue(any(isinstance(n.op, TensorFetch) for n in graph)) self.assertEqual(len(graph), 4) executed_keys = {((a + 1) * 2).key} graph = utils.build_tileable_graph([c], executed_keys) self.assertTrue(any(isinstance(n.op, TensorFetch) for n in graph)) self.assertEqual(len(graph), 3) executed_keys = {c.key} graph = utils.build_tileable_graph([c], executed_keys) self.assertTrue(any(isinstance(n.op, TensorFetch) for n in graph)) self.assertEqual(len(graph), 1)
def test_build_tileable_graph(): a = mt.ones((10, 10), chunk_size=8) b = mt.ones((10, 10), chunk_size=8) c = (a + 1) * 2 + b graph = utils.build_tileable_graph([c], set()) assert len(graph) == 5 a_data = next(n for n in graph if n.key == a.key) assert graph.count_successors(a_data) == 1 assert graph.count_predecessors(a_data) == 0 c_data = next(n for n in graph if n.key == c.key) assert graph.count_successors(c_data) == 0 assert graph.count_predecessors(c_data) == 2 graph = utils.build_tileable_graph([a, b, c], set()) assert len(graph) == 5 # test fetch replacement a = mt.ones((10, 10), chunk_size=8) b = mt.ones((10, 10), chunk_size=8) c = (a + 1) * 2 + b executed_keys = {a.key, b.key} graph = utils.build_tileable_graph([c], executed_keys) assert len(graph) == 5 assert a.data not in graph assert b.data not in graph assert any(isinstance(n.op, TensorFetch) for n in graph) c_data = next(n for n in graph if n.key == c.key) assert graph.count_successors(c_data) == 0 assert graph.count_predecessors(c_data) == 2 executed_keys = {(a + 1).key} graph = utils.build_tileable_graph([c], executed_keys) assert any(isinstance(n.op, TensorFetch) for n in graph) assert len(graph) == 4 executed_keys = {((a + 1) * 2).key} graph = utils.build_tileable_graph([c], executed_keys) assert any(isinstance(n.op, TensorFetch) for n in graph) assert len(graph) == 3 executed_keys = {c.key} graph = utils.build_tileable_graph([c], executed_keys) assert any(isinstance(n.op, TensorFetch) for n in graph) assert len(graph) == 1
def testIterativeTilingWithoutEtcd(self): self.start_processes(etcd=False) session_id = uuid.uuid1() actor_client = new_client() rs = np.random.RandomState(0) session_ref = actor_client.actor_ref(self.session_manager_ref.create_session(session_id)) raw = rs.rand(100) a = mt.tensor(raw, chunk_size=10) a.sort() c = a[:5] graph = c.build_graph() targets = [c.key] graph_key = uuid.uuid1() session_ref.submit_tileable_graph(json.dumps(graph.to_json()), graph_key, target_tileables=targets) state = self.wait_for_termination(actor_client, session_ref, graph_key) self.assertEqual(state, GraphState.SUCCEEDED) result = session_ref.fetch_result(graph_key, c.key) expected = np.sort(raw)[:5] assert_allclose(loads(result), expected) with self.assertRaises(KeyError): session_ref.fetch_result(graph_key, a.key, check=False) raw1 = rs.rand(20) raw2 = rs.rand(20) a = mt.tensor(raw1, chunk_size=10) a.sort() b = mt.tensor(raw2, chunk_size=15) + 1 c = mt.concatenate([a[:10], b]) c.sort() d = c[:5] graph = d.build_graph() targets = [d.key] graph_key = uuid.uuid1() session_ref.submit_tileable_graph(json.dumps(graph.to_json()), graph_key, target_tileables=targets) state = self.wait_for_termination(actor_client, session_ref, graph_key) self.assertEqual(state, GraphState.SUCCEEDED) result = session_ref.fetch_result(graph_key, d.key) expected = np.sort(np.concatenate([np.sort(raw1)[:10], raw2 + 1]))[:5] assert_allclose(loads(result), expected) raw = rs.randint(100, size=(100,)) a = mt.tensor(raw, chunk_size=53) a.sort() b = mt.histogram(a, bins='scott') graph = build_tileable_graph(b, set()) targets = [b[0].key, b[1].key] graph_key = uuid.uuid1() session_ref.submit_tileable_graph(json.dumps(graph.to_json()), graph_key, target_tileables=targets) state = self.wait_for_termination(actor_client, session_ref, graph_key) self.assertEqual(state, GraphState.SUCCEEDED) res = session_ref.fetch_result(graph_key, b[0].key), \ session_ref.fetch_result(graph_key, b[1].key) expected = np.histogram(np.sort(raw), bins='scott') assert_allclose(loads(res[0]), expected[0]) assert_allclose(loads(res[1]), expected[1])