def testBuildTileableGraph(self):
a = mt.ones((10, 10), chunk_size=8)
b = mt.ones((10, 10), chunk_size=8)
c = (a + 1) * 2 + b
graph = utils.build_tileable_graph([c], set())
self.assertEqual(len(graph), 5)
a_data = next(n for n in graph if n.key == a.key)
self.assertEqual(graph.count_successors(a_data), 1)
self.assertEqual(graph.count_predecessors(a_data), 0)
c_data = next(n for n in graph if n.key == c.key)
self.assertEqual(graph.count_successors(c_data), 0)
self.assertEqual(graph.count_predecessors(c_data), 2)
graph = utils.build_tileable_graph([a, b, c], set())
self.assertEqual(len(graph), 5)
# test fetch replacement
a = mt.ones((10, 10), chunk_size=8)
b = mt.ones((10, 10), chunk_size=8)
c = (a + 1) * 2 + b
executed_keys = {a.key, b.key}
graph = utils.build_tileable_graph([c], executed_keys)
self.assertEqual(len(graph), 5)
self.assertNotIn(a.data, graph)
self.assertNotIn(b.data, graph)
self.assertTrue(any(isinstance(n.op, TensorFetch) for n in graph))
c_data = next(n for n in graph if n.key == c.key)
self.assertEqual(graph.count_successors(c_data), 0)
self.assertEqual(graph.count_predecessors(c_data), 2)
executed_keys = {(a + 1).key}
graph = utils.build_tileable_graph([c], executed_keys)
self.assertTrue(any(isinstance(n.op, TensorFetch) for n in graph))
self.assertEqual(len(graph), 4)
executed_keys = {((a + 1) * 2).key}
graph = utils.build_tileable_graph([c], executed_keys)
self.assertTrue(any(isinstance(n.op, TensorFetch) for n in graph))
self.assertEqual(len(graph), 3)
executed_keys = {c.key}
graph = utils.build_tileable_graph([c], executed_keys)
self.assertTrue(any(isinstance(n.op, TensorFetch) for n in graph))
self.assertEqual(len(graph), 1)
def test_build_tileable_graph():
a = mt.ones((10, 10), chunk_size=8)
b = mt.ones((10, 10), chunk_size=8)
c = (a + 1) * 2 + b
graph = utils.build_tileable_graph([c], set())
assert len(graph) == 5
a_data = next(n for n in graph if n.key == a.key)
assert graph.count_successors(a_data) == 1
assert graph.count_predecessors(a_data) == 0
c_data = next(n for n in graph if n.key == c.key)
assert graph.count_successors(c_data) == 0
assert graph.count_predecessors(c_data) == 2
graph = utils.build_tileable_graph([a, b, c], set())
assert len(graph) == 5
# test fetch replacement
a = mt.ones((10, 10), chunk_size=8)
b = mt.ones((10, 10), chunk_size=8)
c = (a + 1) * 2 + b
executed_keys = {a.key, b.key}
graph = utils.build_tileable_graph([c], executed_keys)
assert len(graph) == 5
assert a.data not in graph
assert b.data not in graph
assert any(isinstance(n.op, TensorFetch) for n in graph)
c_data = next(n for n in graph if n.key == c.key)
assert graph.count_successors(c_data) == 0
assert graph.count_predecessors(c_data) == 2
executed_keys = {(a + 1).key}
graph = utils.build_tileable_graph([c], executed_keys)
assert any(isinstance(n.op, TensorFetch) for n in graph)
assert len(graph) == 4
executed_keys = {((a + 1) * 2).key}
graph = utils.build_tileable_graph([c], executed_keys)
assert any(isinstance(n.op, TensorFetch) for n in graph)
assert len(graph) == 3
executed_keys = {c.key}
graph = utils.build_tileable_graph([c], executed_keys)
assert any(isinstance(n.op, TensorFetch) for n in graph)
assert len(graph) == 1
def testIterativeTilingWithoutEtcd(self):
self.start_processes(etcd=False)
session_id = uuid.uuid1()
actor_client = new_client()
rs = np.random.RandomState(0)
session_ref = actor_client.actor_ref(self.session_manager_ref.create_session(session_id))
raw = rs.rand(100)
a = mt.tensor(raw, chunk_size=10)
a.sort()
c = a[:5]
graph = c.build_graph()
targets = [c.key]
graph_key = uuid.uuid1()
session_ref.submit_tileable_graph(json.dumps(graph.to_json()),
graph_key, target_tileables=targets)
state = self.wait_for_termination(actor_client, session_ref, graph_key)
self.assertEqual(state, GraphState.SUCCEEDED)
result = session_ref.fetch_result(graph_key, c.key)
expected = np.sort(raw)[:5]
assert_allclose(loads(result), expected)
with self.assertRaises(KeyError):
session_ref.fetch_result(graph_key, a.key, check=False)
raw1 = rs.rand(20)
raw2 = rs.rand(20)
a = mt.tensor(raw1, chunk_size=10)
a.sort()
b = mt.tensor(raw2, chunk_size=15) + 1
c = mt.concatenate([a[:10], b])
c.sort()
d = c[:5]
graph = d.build_graph()
targets = [d.key]
graph_key = uuid.uuid1()
session_ref.submit_tileable_graph(json.dumps(graph.to_json()),
graph_key, target_tileables=targets)
state = self.wait_for_termination(actor_client, session_ref, graph_key)
self.assertEqual(state, GraphState.SUCCEEDED)
result = session_ref.fetch_result(graph_key, d.key)
expected = np.sort(np.concatenate([np.sort(raw1)[:10], raw2 + 1]))[:5]
assert_allclose(loads(result), expected)
raw = rs.randint(100, size=(100,))
a = mt.tensor(raw, chunk_size=53)
a.sort()
b = mt.histogram(a, bins='scott')
graph = build_tileable_graph(b, set())
targets = [b[0].key, b[1].key]
graph_key = uuid.uuid1()
session_ref.submit_tileable_graph(json.dumps(graph.to_json()),
graph_key, target_tileables=targets)
state = self.wait_for_termination(actor_client, session_ref, graph_key)
self.assertEqual(state, GraphState.SUCCEEDED)
res = session_ref.fetch_result(graph_key, b[0].key), \
session_ref.fetch_result(graph_key, b[1].key)
expected = np.histogram(np.sort(raw), bins='scott')
assert_allclose(loads(res[0]), expected[0])
assert_allclose(loads(res[1]), expected[1])
