def testIterativeTiling(self):
sess = new_session()
rs = np.random.RandomState(0)
raw = rs.rand(100)
a = mt.tensor(raw, chunk_size=10)
a.sort()
c = a[:5]
ret = sess.run(c)
np.testing.assert_array_equal(ret, np.sort(raw)[:5])
executor = sess._sess.executor
self.assertEqual(len(executor.chunk_result), 1)
executor.chunk_result.clear()
raw1 = rs.rand(20)
raw2 = rs.rand(20)
a = mt.tensor(raw1, chunk_size=10)
a.sort()
b = mt.tensor(raw2, chunk_size=15) + 1
c = mt.concatenate([a[:10], b])
c.sort()
d = c[:5]
ret = sess.run(d)
expected = np.sort(np.concatenate([np.sort(raw1)[:10], raw2 + 1]))[:5]
np.testing.assert_array_equal(ret, expected)
self.assertEqual(len(executor.chunk_result), len(get_tiled(d).chunks))
raw = rs.rand(100)
a = mt.tensor(raw, chunk_size=10)
a.sort()
b = a + 1
c = b[:5]
ret = sess.run([b, c])
expected = np.sort(raw + 1)[:5]
np.testing.assert_array_equal(ret[1], expected)
raw = rs.randint(100, size=(100, ))
a = mt.tensor(raw, chunk_size=23)
a.sort()
b = mt.histogram(a, bins='stone')
res = sess.run(b)
expected = np.histogram(np.sort(raw), bins='stone')
np.testing.assert_almost_equal(res[0], expected[0])
np.testing.assert_almost_equal(res[1], expected[1])
def testIterativeTilingWithoutEtcd(self):
self.start_processes(etcd=False)
sess = new_session(self.session_manager_ref.address)
actor_client = sess._api.actor_client
session_ref = actor_client.actor_ref(
self.session_manager_ref.create_session(sess.session_id))
rs = np.random.RandomState(0)
raw = rs.rand(100)
a = mt.tensor(raw, chunk_size=10)
a.sort()
r = a[:5]
result = r.execute(session=sess,
timeout=self.timeout).fetch(session=sess)
expected = np.sort(raw)[:5]
np.testing.assert_allclose(result, expected)
graph_key = sess._get_tileable_graph_key(r.key)
graph_ref = actor_client.actor_ref(
session_ref.get_graph_refs()[graph_key])
with self.assertRaises(KeyError):
_, keys, _ = graph_ref.get_tileable_metas(
[a.key],
filter_fields=['nsplits', 'chunk_keys', 'chunk_indexes'])[0]
sess._api.fetch_chunk_data(sess.session_id, keys[0])
raw1 = rs.rand(20)
raw2 = rs.rand(20)
a = mt.tensor(raw1, chunk_size=10)
a.sort()
b = mt.tensor(raw2, chunk_size=15) + 1
c = mt.concatenate([a[:10], b])
c.sort()
r = c[:5]
result = r.execute(session=sess,
timeout=self.timeout).fetch(session=sess)
expected = np.sort(np.concatenate([np.sort(raw1)[:10], raw2 + 1]))[:5]
np.testing.assert_allclose(result, expected)
raw = rs.randint(100, size=(100, ))
a = mt.tensor(raw, chunk_size=53)
a.sort()
r = mt.histogram(a, bins='scott')
result = r.execute(session=sess,
timeout=self.timeout).fetch(session=sess)
expected = np.histogram(np.sort(raw), bins='scott')
np.testing.assert_allclose(result[0], expected[0])
np.testing.assert_allclose(result[1], expected[1])
def testIterativeTilingWithoutEtcd(self):
self.start_processes(etcd=False)
session_id = uuid.uuid1()
actor_client = new_client()
rs = np.random.RandomState(0)
session_ref = actor_client.actor_ref(self.session_manager_ref.create_session(session_id))
raw = rs.rand(100)
a = mt.tensor(raw, chunk_size=10)
a.sort()
c = a[:5]
graph = c.build_graph()
targets = [c.key]
graph_key = uuid.uuid1()
session_ref.submit_tileable_graph(json.dumps(graph.to_json()),
graph_key, target_tileables=targets)
state = self.wait_for_termination(actor_client, session_ref, graph_key)
self.assertEqual(state, GraphState.SUCCEEDED)
result = session_ref.fetch_result(graph_key, c.key)
expected = np.sort(raw)[:5]
assert_allclose(loads(result), expected)
with self.assertRaises(KeyError):
session_ref.fetch_result(graph_key, a.key, check=False)
raw1 = rs.rand(20)
raw2 = rs.rand(20)
a = mt.tensor(raw1, chunk_size=10)
a.sort()
b = mt.tensor(raw2, chunk_size=15) + 1
c = mt.concatenate([a[:10], b])
c.sort()
d = c[:5]
graph = d.build_graph()
targets = [d.key]
graph_key = uuid.uuid1()
session_ref.submit_tileable_graph(json.dumps(graph.to_json()),
graph_key, target_tileables=targets)
state = self.wait_for_termination(actor_client, session_ref, graph_key)
self.assertEqual(state, GraphState.SUCCEEDED)
result = session_ref.fetch_result(graph_key, d.key)
expected = np.sort(np.concatenate([np.sort(raw1)[:10], raw2 + 1]))[:5]
assert_allclose(loads(result), expected)
raw = rs.randint(100, size=(100,))
a = mt.tensor(raw, chunk_size=53)
a.sort()
b = mt.histogram(a, bins='scott')
graph = build_tileable_graph(b, set())
targets = [b[0].key, b[1].key]
graph_key = uuid.uuid1()
session_ref.submit_tileable_graph(json.dumps(graph.to_json()),
graph_key, target_tileables=targets)
state = self.wait_for_termination(actor_client, session_ref, graph_key)
self.assertEqual(state, GraphState.SUCCEEDED)
res = session_ref.fetch_result(graph_key, b[0].key), \
session_ref.fetch_result(graph_key, b[1].key)
expected = np.histogram(np.sort(raw), bins='scott')
assert_allclose(loads(res[0]), expected[0])
assert_allclose(loads(res[1]), expected[1])
