def testConcatenateExecution(self):
a_data = np.random.rand(10, 20, 30)
b_data = np.random.rand(10, 20, 40)
c_data = np.random.rand(10, 20, 50)
a = tensor(a_data, chunk_size=5)
b = tensor(b_data, chunk_size=6)
c = tensor(c_data, chunk_size=7)
d = concatenate([a, b, c], axis=-1)
res = self.executor.execute_tensor(d, concat=True)[0]
expected = np.concatenate([a_data, b_data, c_data], axis=-1)
self.assertTrue(np.array_equal(res, expected))
a_data = sps.random(10, 30)
b_data = sps.rand(10, 40)
c_data = sps.rand(10, 50)
a = tensor(a_data, chunk_size=5)
b = tensor(b_data, chunk_size=6)
c = tensor(c_data, chunk_size=7)
d = concatenate([a, b, c], axis=-1)
res = self.executor.execute_tensor(d, concat=True)[0]
expected = np.concatenate([a_data.A, b_data.A, c_data.A], axis=-1)
self.assertTrue(np.array_equal(res.toarray(), expected))
def test_concatenate_execution(setup):
a_data = np.random.rand(10, 20, 30)
b_data = np.random.rand(10, 20, 40)
c_data = np.random.rand(10, 20, 50)
a = tensor(a_data, chunk_size=8)
b = tensor(b_data, chunk_size=10)
c = tensor(c_data, chunk_size=15)
d = concatenate([a, b, c], axis=-1)
res = d.execute().fetch()
expected = np.concatenate([a_data, b_data, c_data], axis=-1)
np.testing.assert_array_equal(res, expected)
a_data = sps.random(10, 30)
b_data = sps.rand(10, 40)
c_data = sps.rand(10, 50)
a = tensor(a_data, chunk_size=8)
b = tensor(b_data, chunk_size=10)
c = tensor(c_data, chunk_size=15)
d = concatenate([a, b, c], axis=-1)
res = d.execute().fetch()
expected = np.concatenate([a_data.A, b_data.A, c_data.A], axis=-1)
np.testing.assert_array_equal(res.toarray(), expected)
def tile(cls, op: "_TestOperand"):
data1, data2 = op.inputs
data1 = mt.sort(data1)
data2 = mt.sort(data2)
data_all = mt.concatenate([data1, data2])
s1 = mt.searchsorted(data1, data_all)
s2 = mt.searchsorted(data2, data_all)
result = yield from recursive_tile(mt.concatenate([s1, s2]))
# data1 will be yield by s1
assert not has_unknown_shape(data1)
assert not has_unknown_shape(data2)
assert not has_unknown_shape(data_all)
return result
def testOperandActorWithSameKey(self, *_):
arr = mt.ones((5, 5), chunk_size=3)
arr2 = mt.concatenate((arr, arr))
session_id = str(uuid.uuid4())
graph_key = str(uuid.uuid4())
self._run_operand_case(session_id, graph_key, arr2,
lambda pool: pool.create_actor(FakeExecutionActor))
def testIterativeTilingWithoutEtcd(self):
self.start_processes(etcd=False)
session_id = uuid.uuid1()
actor_client = new_client()
session_ref = actor_client.actor_ref(
self.session_manager_ref.create_session(session_id))
rs = np.random.RandomState(0)
raw = rs.rand(100)
a = mt.tensor(raw, chunk_size=10)
a.sort()
c = a[:5]
graph = c.build_graph()
targets = [c.key]
graph_key = uuid.uuid1()
session_ref.submit_tileable_graph(json.dumps(graph.to_json()),
graph_key,
target_tileables=targets)
state = self.wait_for_termination(actor_client, session_ref, graph_key)
self.assertEqual(state, GraphState.SUCCEEDED)
result = session_ref.fetch_result(graph_key, c.key)
expected = np.sort(raw)[:5]
assert_allclose(loads(result), expected)
with self.assertRaises(TypeError):
session_ref.fetch_result(graph_key, a.key, check=False)
raw1 = rs.rand(20)
raw2 = rs.rand(20)
a = mt.tensor(raw1, chunk_size=10)
a.sort()
b = mt.tensor(raw2, chunk_size=15) + 1
c = mt.concatenate([a[:10], b])
c.sort()
d = c[:5]
graph = d.build_graph()
targets = [d.key]
graph_key = uuid.uuid1()
session_ref.submit_tileable_graph(json.dumps(graph.to_json()),
graph_key,
target_tileables=targets)
state = self.wait_for_termination(actor_client, session_ref, graph_key)
self.assertEqual(state, GraphState.SUCCEEDED)
result = session_ref.fetch_result(graph_key, d.key)
expected = np.sort(np.concatenate([np.sort(raw1)[:10], raw2 + 1]))[:5]
assert_allclose(loads(result), expected)
def testIterativeTiling(self):
sess = new_session()
rs = np.random.RandomState(0)
raw = rs.rand(100)
a = mt.tensor(raw, chunk_size=10)
a.sort()
c = a[:5]
ret = sess.run(c)
np.testing.assert_array_equal(ret, np.sort(raw)[:5])
executor = sess._sess.executor
self.assertEqual(len(executor.chunk_result), 1)
executor.chunk_result.clear()
raw1 = rs.rand(20)
raw2 = rs.rand(20)
a = mt.tensor(raw1, chunk_size=10)
a.sort()
b = mt.tensor(raw2, chunk_size=15) + 1
c = mt.concatenate([a[:10], b])
c.sort()
d = c[:5]
ret = sess.run(d)
expected = np.sort(np.concatenate([np.sort(raw1)[:10], raw2 + 1]))[:5]
np.testing.assert_array_equal(ret, expected)
self.assertEqual(len(executor.chunk_result), len(get_tiled(d).chunks))
raw = rs.rand(100)
a = mt.tensor(raw, chunk_size=10)
a.sort()
b = a + 1
c = b[:5]
ret = sess.run([b, c])
expected = np.sort(raw + 1)[:5]
np.testing.assert_array_equal(ret[1], expected)
raw = rs.randint(100, size=(100, ))
a = mt.tensor(raw, chunk_size=23)
a.sort()
b = mt.histogram(a, bins='stone')
res = sess.run(b)
expected = np.histogram(np.sort(raw), bins='stone')
np.testing.assert_almost_equal(res[0], expected[0])
np.testing.assert_almost_equal(res[1], expected[1])
def testIterativeTilingWithoutEtcd(self):
self.start_processes(etcd=False)
sess = new_session(self.session_manager_ref.address)
actor_client = sess._api.actor_client
session_ref = actor_client.actor_ref(
self.session_manager_ref.create_session(sess.session_id))
rs = np.random.RandomState(0)
raw = rs.rand(100)
a = mt.tensor(raw, chunk_size=10)
a.sort()
r = a[:5]
result = r.execute(session=sess,
timeout=self.timeout).fetch(session=sess)
expected = np.sort(raw)[:5]
np.testing.assert_allclose(result, expected)
graph_key = sess._get_tileable_graph_key(r.key)
graph_ref = actor_client.actor_ref(
session_ref.get_graph_refs()[graph_key])
with self.assertRaises(KeyError):
_, keys, _ = graph_ref.get_tileable_metas(
[a.key],
filter_fields=['nsplits', 'chunk_keys', 'chunk_indexes'])[0]
sess._api.fetch_chunk_data(sess.session_id, keys[0])
raw1 = rs.rand(20)
raw2 = rs.rand(20)
a = mt.tensor(raw1, chunk_size=10)
a.sort()
b = mt.tensor(raw2, chunk_size=15) + 1
c = mt.concatenate([a[:10], b])
c.sort()
r = c[:5]
result = r.execute(session=sess,
timeout=self.timeout).fetch(session=sess)
expected = np.sort(np.concatenate([np.sort(raw1)[:10], raw2 + 1]))[:5]
np.testing.assert_allclose(result, expected)
raw = rs.randint(100, size=(100, ))
a = mt.tensor(raw, chunk_size=53)
a.sort()
r = mt.histogram(a, bins='scott')
result = r.execute(session=sess,
timeout=self.timeout).fetch(session=sess)
expected = np.histogram(np.sort(raw), bins='scott')
np.testing.assert_allclose(result[0], expected[0])
np.testing.assert_allclose(result[1], expected[1])
def testSameKeyPreparation(self, *_):
arr = mt.ones((5, 5), chunk_size=3)
arr2 = mt.concatenate((arr, arr))
with self.prepare_graph_in_pool(arr2, clean_io_meta=False):
pass
def testSameKey(self, *_):
session_id = str(uuid.uuid4())
graph_key = str(uuid.uuid4())
arr = mt.ones((5, 5), chunk_size=3)
arr2 = mt.concatenate((arr, arr))
graph = arr2.build_graph(compose=False)
serialized_graph = serialize_graph(graph)
chunked_graph = arr2.build_graph(compose=False, tiled=True)
addr = '127.0.0.1:%d' % get_next_port()
with create_actor_pool(n_process=1, backend='gevent',
address=addr) as pool:
pool.create_actor(ClusterInfoActor, [pool.cluster_info.address],
uid=ClusterInfoActor.default_name())
resource_ref = pool.create_actor(ResourceActor,
uid=ResourceActor.default_name())
pool.create_actor(ChunkMetaActor,
uid=ChunkMetaActor.default_name())
pool.create_actor(AssignerActor,
uid=AssignerActor.gen_name(session_id))
graph_ref = pool.create_actor(GraphActor,
session_id,
graph_key,
serialized_graph,
uid=GraphActor.gen_name(
session_id, graph_key))
graph_ref.prepare_graph(compose=False)
fetched_graph = graph_ref.get_chunk_graph()
self.assertIsNotNone(fetched_graph)
self.assertEqual(len(chunked_graph), len(fetched_graph))
graph_ref.scan_node()
op_infos = graph_ref.get_operand_info()
for n in fetched_graph:
depth = op_infos[n.op.key]['optimize']['depth']
self.assertIsNotNone(depth)
successor_size = op_infos[
n.op.key]['optimize']['successor_size']
self.assertIsNotNone(successor_size)
descendant_size = op_infos[
n.op.key]['optimize']['descendant_size']
self.assertIsNotNone(descendant_size)
resource_ref.set_worker_meta('localhost:12345',
dict(hardware=dict(cpu_total=4)))
resource_ref.set_worker_meta('localhost:23456',
dict(hardware=dict(cpu_total=4)))
graph_ref.place_initial_chunks()
op_infos = graph_ref.get_operand_info()
for n in fetched_graph:
if fetched_graph.count_predecessors(n) != 0:
continue
target_worker = op_infos[n.op.key]['target_worker']
self.assertIsNotNone(target_worker)
graph_ref.create_operand_actors(_clean_io_meta=False)
op_infos = graph_ref.get_operand_info()
for n in fetched_graph:
self.assertEqual(op_infos[n.op.key]['op_name'],
type(n.op).__name__)
io_meta = op_infos[n.op.key]['io_meta']
orig_io_meta = dict(
predecessors=list(
set(pn.op.key
for pn in fetched_graph.iter_predecessors(n))),
successors=list(
set(sn.op.key
for sn in fetched_graph.iter_successors(n))),
input_chunks=list(
set(pn.key
for pn in fetched_graph.iter_predecessors(n))),
chunks=list(c.key for c in n.op.outputs),
)
self.assertSetEqual(set(io_meta['predecessors']),
set(orig_io_meta['predecessors']))
self.assertSetEqual(set(io_meta['successors']),
set(orig_io_meta['successors']))
self.assertSetEqual(set(io_meta['input_chunks']),
set(orig_io_meta['input_chunks']))
self.assertSetEqual(set(io_meta['chunks']),
set(orig_io_meta['chunks']))
self.assertEqual(op_infos[n.op.key]['output_size'],
sum(ch.nbytes for ch in n.op.outputs))
def testSameKey(self):
arr = mt.ones((5, 5), chunk_size=3)
arr2 = mt.concatenate((arr, arr))
self.run_expr_suite(arr2)