def testIterativeTiling(self): sess = new_session() rs = np.random.RandomState(0) raw = rs.rand(100) a = mt.tensor(raw, chunk_size=10) a.sort() c = a[:5] ret = sess.run(c) np.testing.assert_array_equal(ret, np.sort(raw)[:5]) executor = sess._sess.executor self.assertEqual(len(executor.chunk_result), 1) executor.chunk_result.clear() raw1 = rs.rand(20) raw2 = rs.rand(20) a = mt.tensor(raw1, chunk_size=10) a.sort() b = mt.tensor(raw2, chunk_size=15) + 1 c = mt.concatenate([a[:10], b]) c.sort() d = c[:5] ret = sess.run(d) expected = np.sort(np.concatenate([np.sort(raw1)[:10], raw2 + 1]))[:5] np.testing.assert_array_equal(ret, expected) self.assertEqual(len(executor.chunk_result), len(get_tiled(d).chunks)) raw = rs.rand(100) a = mt.tensor(raw, chunk_size=10) a.sort() b = a + 1 c = b[:5] ret = sess.run([b, c]) expected = np.sort(raw + 1)[:5] np.testing.assert_array_equal(ret[1], expected) raw = rs.randint(100, size=(100, )) a = mt.tensor(raw, chunk_size=23) a.sort() b = mt.histogram(a, bins='stone') res = sess.run(b) expected = np.histogram(np.sort(raw), bins='stone') np.testing.assert_almost_equal(res[0], expected[0]) np.testing.assert_almost_equal(res[1], expected[1])
def testIterativeTilingWithoutEtcd(self): self.start_processes(etcd=False) sess = new_session(self.session_manager_ref.address) actor_client = sess._api.actor_client session_ref = actor_client.actor_ref( self.session_manager_ref.create_session(sess.session_id)) rs = np.random.RandomState(0) raw = rs.rand(100) a = mt.tensor(raw, chunk_size=10) a.sort() r = a[:5] result = r.execute(session=sess, timeout=self.timeout).fetch(session=sess) expected = np.sort(raw)[:5] np.testing.assert_allclose(result, expected) graph_key = sess._get_tileable_graph_key(r.key) graph_ref = actor_client.actor_ref( session_ref.get_graph_refs()[graph_key]) with self.assertRaises(KeyError): _, keys, _ = graph_ref.get_tileable_metas( [a.key], filter_fields=['nsplits', 'chunk_keys', 'chunk_indexes'])[0] sess._api.fetch_chunk_data(sess.session_id, keys[0]) raw1 = rs.rand(20) raw2 = rs.rand(20) a = mt.tensor(raw1, chunk_size=10) a.sort() b = mt.tensor(raw2, chunk_size=15) + 1 c = mt.concatenate([a[:10], b]) c.sort() r = c[:5] result = r.execute(session=sess, timeout=self.timeout).fetch(session=sess) expected = np.sort(np.concatenate([np.sort(raw1)[:10], raw2 + 1]))[:5] np.testing.assert_allclose(result, expected) raw = rs.randint(100, size=(100, )) a = mt.tensor(raw, chunk_size=53) a.sort() r = mt.histogram(a, bins='scott') result = r.execute(session=sess, timeout=self.timeout).fetch(session=sess) expected = np.histogram(np.sort(raw), bins='scott') np.testing.assert_allclose(result[0], expected[0]) np.testing.assert_allclose(result[1], expected[1])
def testIterativeTilingWithoutEtcd(self): self.start_processes(etcd=False) session_id = uuid.uuid1() actor_client = new_client() rs = np.random.RandomState(0) session_ref = actor_client.actor_ref(self.session_manager_ref.create_session(session_id)) raw = rs.rand(100) a = mt.tensor(raw, chunk_size=10) a.sort() c = a[:5] graph = c.build_graph() targets = [c.key] graph_key = uuid.uuid1() session_ref.submit_tileable_graph(json.dumps(graph.to_json()), graph_key, target_tileables=targets) state = self.wait_for_termination(actor_client, session_ref, graph_key) self.assertEqual(state, GraphState.SUCCEEDED) result = session_ref.fetch_result(graph_key, c.key) expected = np.sort(raw)[:5] assert_allclose(loads(result), expected) with self.assertRaises(KeyError): session_ref.fetch_result(graph_key, a.key, check=False) raw1 = rs.rand(20) raw2 = rs.rand(20) a = mt.tensor(raw1, chunk_size=10) a.sort() b = mt.tensor(raw2, chunk_size=15) + 1 c = mt.concatenate([a[:10], b]) c.sort() d = c[:5] graph = d.build_graph() targets = [d.key] graph_key = uuid.uuid1() session_ref.submit_tileable_graph(json.dumps(graph.to_json()), graph_key, target_tileables=targets) state = self.wait_for_termination(actor_client, session_ref, graph_key) self.assertEqual(state, GraphState.SUCCEEDED) result = session_ref.fetch_result(graph_key, d.key) expected = np.sort(np.concatenate([np.sort(raw1)[:10], raw2 + 1]))[:5] assert_allclose(loads(result), expected) raw = rs.randint(100, size=(100,)) a = mt.tensor(raw, chunk_size=53) a.sort() b = mt.histogram(a, bins='scott') graph = build_tileable_graph(b, set()) targets = [b[0].key, b[1].key] graph_key = uuid.uuid1() session_ref.submit_tileable_graph(json.dumps(graph.to_json()), graph_key, target_tileables=targets) state = self.wait_for_termination(actor_client, session_ref, graph_key) self.assertEqual(state, GraphState.SUCCEEDED) res = session_ref.fetch_result(graph_key, b[0].key), \ session_ref.fetch_result(graph_key, b[1].key) expected = np.histogram(np.sort(raw), bins='scott') assert_allclose(loads(res[0]), expected[0]) assert_allclose(loads(res[1]), expected[1])