def testRegister(self):
from mars.graph import DAG
fake_result = np.random.rand(10, 10)
fake_size = (fake_result.nbytes * 2, fake_result.nbytes * 2)
def fake_execute(ctx, op):
ctx[op.outputs[0].key] = fake_result
def fake_estimate(ctx, op):
ctx[op.outputs[0].key] = fake_size
register(FakeOperand, fake_execute, fake_estimate)
graph = DAG()
chunk = FakeOperand().new_chunk(None, shape=(10, 10))
graph.add_node(chunk.data)
executor = Executor()
res = executor.execute_graph(graph, keys=[chunk.key])[0]
np.testing.assert_array_equal(res, fake_result)
size = executor.execute_graph(graph, keys=[chunk.key], mock=True)[0]
self.assertEqual(size, fake_size)
graph = DAG()
chunk = SubFakeOperand().new_chunk(None, shape=(10, 10))
graph.add_node(chunk.data)
executor = Executor()
res = executor.execute_graph(graph, keys=[chunk.key])[0]
np.testing.assert_array_equal(res, fake_result)
def testMockExecuteSize(self):
import mars.tensor as mt
from mars.core.graph import DAG
from mars.tensor.fetch import TensorFetch
from mars.tensor.arithmetic import TensorTreeAdd
graph_add = DAG()
input_chunks = []
for _ in range(2):
fetch_op = TensorFetch(dtype=np.dtype('int64'))
inp_chunk = fetch_op.new_chunk(None, shape=(100, 100)).data
input_chunks.append(inp_chunk)
add_op = TensorTreeAdd(args=input_chunks, dtype=np.dtype('int64'))
add_chunk = add_op.new_chunk(input_chunks,
shape=(100, 100),
dtype=np.dtype('int64')).data
graph_add.add_node(add_chunk)
for inp_chunk in input_chunks:
graph_add.add_node(inp_chunk)
graph_add.add_edge(inp_chunk, add_chunk)
executor = Executor()
res = executor.execute_graph(graph_add, [add_chunk.key],
compose=False,
mock=True)[0]
self.assertEqual(res, (80000, 80000))
self.assertEqual(executor.mock_max_memory, 80000)
for _ in range(3):
new_add_op = TensorTreeAdd(args=[add_chunk],
dtype=np.dtype('int64'))
new_add_chunk = new_add_op.new_chunk([add_chunk],
shape=(100, 100),
dtype=np.dtype('int64')).data
graph_add.add_node(new_add_chunk)
graph_add.add_edge(add_chunk, new_add_chunk)
add_chunk = new_add_chunk
executor = Executor()
res = executor.execute_graph(graph_add, [add_chunk.key],
compose=False,
mock=True)[0]
self.assertEqual(res, (80000, 80000))
self.assertEqual(executor.mock_max_memory, 160000)
a = mt.random.rand(10, 10, chunk_size=10)
b = a[:, mt.newaxis, :] - a
r = mt.triu(mt.sqrt(b**2).sum(axis=2))
executor = Executor()
res = executor.execute_tensor(r, concat=False, mock=True)
# larger than maximal memory size in calc procedure
self.assertGreaterEqual(res[0][0], 800)
self.assertGreaterEqual(executor.mock_max_memory, 8000)
class ExecutorActor(Actor):
def __init__(self):
self._executor = Executor(sync_provider_type=Executor.SyncProviderType.GEVENT)
def post_create(self):
register(FakeOperand, fake_execution_maker(self.ctx))
def on_receive(self, message):
op = FakeOperand(_num=message)
chunk = op.new_chunk(None, ())
graph = DirectedGraph()
graph.add_node(chunk.data)
res = self._executor.execute_graph(graph, [chunk.key])
assert res[0] == message + 1
class Test(TestBase):
def setUp(self):
super(Test, self).setUp()
self.executor = Executor()
def testPandasExecution(self):
pdf = pd.DataFrame(np.random.rand(20, 30),
index=[np.arange(20),
np.arange(20, 0, -1)])
df = from_pandas(pdf, chunk_size=(13, 21))
graph = df.build_graph(tiled=True)
results = self.executor.execute_graph(graph,
keys=[c.key for c in df.chunks])
self.assertEqual(len(results), 4)
pd.testing.assert_frame_equal(pdf.iloc[:13, :21], results[0])
pd.testing.assert_frame_equal(pdf.iloc[:13, 21:], results[1])
pd.testing.assert_frame_equal(pdf.iloc[13:, :21], results[2])
pd.testing.assert_frame_equal(pdf.iloc[13:, 21:], results[3])
