def testRegister(self):
from mars.graph import DAG
fake_result = np.random.rand(10, 10)
fake_size = (fake_result.nbytes * 2, fake_result.nbytes * 2)
def fake_execute(ctx, op):
ctx[op.outputs[0].key] = fake_result
def fake_estimate(ctx, op):
ctx[op.outputs[0].key] = fake_size
register(FakeOperand, fake_execute, fake_estimate)
graph = DAG()
chunk = FakeOperand().new_chunk(None, shape=(10, 10))
graph.add_node(chunk.data)
executor = Executor()
res = executor.execute_graph(graph, keys=[chunk.key])[0]
np.testing.assert_array_equal(res, fake_result)
size = executor.execute_graph(graph, keys=[chunk.key], mock=True)[0]
self.assertEqual(size, fake_size)
graph = DAG()
chunk = SubFakeOperand().new_chunk(None, shape=(10, 10))
graph.add_node(chunk.data)
executor = Executor()
res = executor.execute_graph(graph, keys=[chunk.key])[0]
np.testing.assert_array_equal(res, fake_result)
def testMockExecuteSize(self):
import mars.tensor as mt
from mars.core.graph import DAG
from mars.tensor.fetch import TensorFetch
from mars.tensor.arithmetic import TensorTreeAdd
graph_add = DAG()
input_chunks = []
for _ in range(2):
fetch_op = TensorFetch(dtype=np.dtype('int64'))
inp_chunk = fetch_op.new_chunk(None, shape=(100, 100)).data
input_chunks.append(inp_chunk)
add_op = TensorTreeAdd(args=input_chunks, dtype=np.dtype('int64'))
add_chunk = add_op.new_chunk(input_chunks,
shape=(100, 100),
dtype=np.dtype('int64')).data
graph_add.add_node(add_chunk)
for inp_chunk in input_chunks:
graph_add.add_node(inp_chunk)
graph_add.add_edge(inp_chunk, add_chunk)
executor = Executor()
res = executor.execute_graph(graph_add, [add_chunk.key],
compose=False,
mock=True)[0]
self.assertEqual(res, (80000, 80000))
self.assertEqual(executor.mock_max_memory, 80000)
for _ in range(3):
new_add_op = TensorTreeAdd(args=[add_chunk],
dtype=np.dtype('int64'))
new_add_chunk = new_add_op.new_chunk([add_chunk],
shape=(100, 100),
dtype=np.dtype('int64')).data
graph_add.add_node(new_add_chunk)
graph_add.add_edge(add_chunk, new_add_chunk)
add_chunk = new_add_chunk
executor = Executor()
res = executor.execute_graph(graph_add, [add_chunk.key],
compose=False,
mock=True)[0]
self.assertEqual(res, (80000, 80000))
self.assertEqual(executor.mock_max_memory, 160000)
a = mt.random.rand(10, 10, chunk_size=10)
b = a[:, mt.newaxis, :] - a
r = mt.triu(mt.sqrt(b**2).sum(axis=2))
executor = Executor()
res = executor.execute_tensor(r, concat=False, mock=True)
# larger than maximal memory size in calc procedure
self.assertGreaterEqual(res[0][0], 800)
self.assertGreaterEqual(executor.mock_max_memory, 8000)
def testFuseSizeExecution(self):
executor_size = Executor()
executor_numpy = Executor()
raw1 = np.random.randint(10, size=(10, 10, 10))
arr1 = tensor(raw1, chunk_size=3)
arr2 = arr1 + 10
arr3 = arr2 * 3
arr4 = arr3 + 5
res4_size = executor_size.execute_tensor(arr4, mock=True)
res4 = executor_numpy.execute_tensor(arr4, concat=True)
res4_cmp = self.executor.execute_tensor(arr4, concat=True)
self.assertEqual(sum(s[0] for s in res4_size), arr4.nbytes)
self.assertTrue(np.array_equal(res4[0], res4_cmp[0]))
def testBinExecution(self):
from mars.tensor.arithmetic import BIN_UFUNC, mod, fmod, \
bitand, bitor, bitxor, lshift, rshift, ldexp
_sp_bin_ufunc = [mod, fmod, bitand, bitor, bitxor, lshift, rshift]
_new_bin_ufunc = list(BIN_UFUNC - set(_sp_bin_ufunc) - {ldexp})
executor_numexpr = Executor()
tested = set()
for func1, func2 in self._gen_pairs(_new_bin_ufunc):
raw = np.random.random((9, 9, 9))
arr1 = tensor(raw, chunk_size=5)
arr2 = func1(1, func2(2, arr1))
res = executor_numexpr.execute_tensor(arr2, concat=True)
res_cmp = self.executor.execute_tensor(arr2, concat=True)
self.assertTrue(np.allclose(res[0], res_cmp[0]))
tested.update([func1, func2])
# make sure all functions tested
self.assertEqual(tested, set(_new_bin_ufunc))
tested = set()
for func1, func2 in self._gen_pairs(_sp_bin_ufunc):
raw = np.random.randint(1, 100, size=(10, 10, 10))
arr1 = tensor(raw, chunk_size=3)
arr2 = func1(10, func2(arr1, 5))
res = executor_numexpr.execute_tensor(arr2, concat=True)
res_cmp = self.executor.execute_tensor(arr2, concat=True)
self.assertTrue(np.allclose(res[0], res_cmp[0]))
tested.update([func1, func2])
# make sure all functions tested
self.assertEqual(tested, set(_sp_bin_ufunc))
def execute_size(t):
def _tensordot_size_recorder(ctx, op):
TensorTensorDot.estimate_size(ctx, op)
chunk_key = op.outputs[0].key
chunk_sizes[chunk_key] = ctx[chunk_key]
chunk_nbytes[chunk_key] = op.outputs[0].nbytes
input_sizes = dict(
(inp.op.key, ctx[inp.key][0]) for inp in op.inputs)
chunk_input_sizes[chunk_key] = sum(input_sizes.values())
input_nbytes = dict(
(inp.op.key, inp.nbytes) for inp in op.inputs)
chunk_input_nbytes[chunk_key] = sum(input_nbytes.values())
size_executor = Executor(
sync_provider_type=Executor.SyncProviderType.MOCK)
try:
chunk_sizes.clear()
chunk_nbytes.clear()
chunk_input_sizes.clear()
chunk_input_nbytes.clear()
register(TensorTensorDot,
size_estimator=_tensordot_size_recorder)
size_executor.execute_tensor(t, mock=True)
finally:
register_default(TensorTensorDot)
def testUnaryExecution(self):
from mars.tensor.arithmetic import UNARY_UFUNC, arccosh, invert, sin, conj
_sp_unary_ufunc = {arccosh, invert, conj}
_new_unary_ufunc = list(UNARY_UFUNC - _sp_unary_ufunc)
executor_numexpr = Executor()
def _normalize_by_sin(func1, func2, arr):
return func1(abs(sin((func2(arr)))))
tested = set()
for func1, func2 in self._gen_pairs(_new_unary_ufunc):
raw = np.random.random((8, 8, 8))
arr1 = tensor(raw, chunk_size=4)
arr2 = _normalize_by_sin(func1, func2, arr1)
res = executor_numexpr.execute_tensor(arr2, concat=True)
res_cmp = self.executor.execute_tensor(arr2, concat=True)
np.testing.assert_allclose(res[0], res_cmp[0])
tested.update([func1, func2])
# make sure all functions tested
self.assertEqual(tested, set(_new_unary_ufunc))
raw = np.random.randint(100, size=(8, 8, 8))
arr1 = tensor(raw, chunk_size=4)
arr2 = arccosh(1 + abs(invert(arr1)))
res = executor_numexpr.execute_tensor(arr2, concat=True)
res_cmp = self.executor.execute_tensor(arr2, concat=True)
self.assertTrue(np.allclose(res[0], res_cmp[0]))
def testUnaryExecution(self):
from mars.tensor.arithmetic import UNARY_UFUNC, arccosh, invert, sin, conj
_sp_unary_ufunc = {arccosh, invert, conj}
_new_unary_ufunc = list(UNARY_UFUNC - _sp_unary_ufunc)
executor_numexpr = Executor()
def _normalize_by_sin(func1, func2, arr):
return func1(abs(sin((func2(arr)))))
for i, j in itertools.permutations(range(len(_new_unary_ufunc)), 2):
raw = np.random.random((8, 8, 8))
arr1 = tensor(raw, chunk_size=4)
func1 = _new_unary_ufunc[i]
func2 = _new_unary_ufunc[j]
arr2 = _normalize_by_sin(func1, func2, arr1)
res = executor_numexpr.execute_tensor(arr2, concat=True)
res_cmp = self.executor.execute_tensor(arr2, concat=True)
np.testing.assert_allclose(res[0], res_cmp[0])
raw = np.random.randint(100, size=(8, 8, 8))
arr1 = tensor(raw, chunk_size=4)
arr2 = arccosh(1 + abs(invert(arr1)))
res = executor_numexpr.execute_tensor(arr2, concat=True)
res_cmp = self.executor.execute_tensor(arr2, concat=True)
self.assertTrue(np.allclose(res[0], res_cmp[0]))
def testBinExecution(self):
from mars.tensor.arithmetic import BIN_UFUNC, mod, fmod, \
bitand, bitor, bitxor, lshift, rshift, ldexp
_sp_bin_ufunc = [mod, fmod, bitand, bitor, bitxor, lshift, rshift]
_new_bin_ufunc = list(BIN_UFUNC - set(_sp_bin_ufunc) - {ldexp})
executor_numexpr = Executor()
for i, j in itertools.permutations(range(len(_new_bin_ufunc)), 2):
raw = np.random.random((9, 9, 9))
arr1 = tensor(raw, chunk_size=5)
func1 = _new_bin_ufunc[i]
func2 = _new_bin_ufunc[j]
arr2 = func1(1, func2(2, arr1))
res = executor_numexpr.execute_tensor(arr2, concat=True)
res_cmp = self.executor.execute_tensor(arr2, concat=True)
self.assertTrue(np.allclose(res[0], res_cmp[0]))
for i, j in itertools.permutations(range(len(_sp_bin_ufunc)), 2):
raw = np.random.randint(1, 100, size=(10, 10, 10))
arr1 = tensor(raw, chunk_size=3)
func1 = _sp_bin_ufunc[i]
func2 = _sp_bin_ufunc[j]
arr2 = func1(10, func2(arr1, 5))
res = executor_numexpr.execute_tensor(arr2, concat=True)
res_cmp = self.executor.execute_tensor(arr2, concat=True)
self.assertTrue(np.allclose(res[0], res_cmp[0]))
def testExecutorWithGeventProvider(self):
executor = Executor(
sync_provider_type=Executor.SyncProviderType.GEVENT)
a = mt.ones((10, 10), chunk_size=2)
res = executor.execute_tensor(a, concat=True)[0]
np.testing.assert_array_equal(res, np.ones((10, 10)))
def testMockExecuteSize(self):
a = mt.random.rand(10, 10, chunk_size=10)
b = a[:, mt.newaxis, :] - a
r = mt.triu(mt.sqrt(b**2).sum(axis=2))
executor = Executor()
res = executor.execute_tensor(r, concat=False, mock=True)
# larger than maximal memory size in calc procedure
self.assertGreaterEqual(res[0][0], 800)
self.assertGreaterEqual(res[0][1], 8000)
def testSparseDotExecution(self):
size_executor = Executor(
sync_provider_type=Executor.SyncProviderType.MOCK)
a_data = sps.random(5, 9, density=.1)
b_data = sps.random(9, 10, density=.2)
a = tensor(a_data, chunk_size=2)
b = tensor(b_data, chunk_size=3)
c = dot(a, b)
size_res = size_executor.execute_tensor(c, mock=True)
res = self.executor.execute_tensor(c, concat=True)[0]
self.assertEqual(sum(s[0] for s in size_res), 0)
self.assertGreaterEqual(sum(s[1] for s in size_res), 0)
self.assertTrue(issparse(res))
np.testing.assert_allclose(res.toarray(), a_data.dot(b_data).toarray())
c2 = dot(a, b, sparse=False)
size_res = size_executor.execute_tensor(c2, mock=True)
res = self.executor.execute_tensor(c2, concat=True)[0]
self.assertEqual(sum(s[0] for s in size_res), c2.nbytes)
self.assertFalse(issparse(res))
np.testing.assert_allclose(res, a_data.dot(b_data).toarray())
c3 = tensordot(a, b.T, (-1, -1), sparse=False)
res = self.executor.execute_tensor(c3, concat=True)[0]
self.assertFalse(issparse(res))
np.testing.assert_allclose(res, a_data.dot(b_data).toarray())
c = inner(a, b.T)
res = self.executor.execute_tensor(c, concat=True)[0]
self.assertTrue(issparse(res))
np.testing.assert_allclose(res.toarray(), a_data.dot(b_data).toarray())
c = inner(a, b.T, sparse=False)
res = self.executor.execute_tensor(c, concat=True)[0]
self.assertFalse(issparse(res))
np.testing.assert_allclose(res, a_data.dot(b_data).toarray())
# test vector inner
a_data = np.random.rand(5)
b_data = np.random.rand(5)
a = tensor(a_data, chunk_size=2).tosparse()
b = tensor(b_data, chunk_size=2).tosparse()
c = inner(a, b)
res = self.executor.execute_tensor(c, concat=True)[0]
self.assertTrue(np.isscalar(res))
np.testing.assert_allclose(res, np.inner(a_data, b_data))
def testRandintExecution(self):
size_executor = Executor(
sync_provider_type=Executor.SyncProviderType.MOCK)
arr = tensor.random.randint(0, 2, size=(10, 30), chunk_size=3)
size_res = size_executor.execute_tensor(arr, mock=True)
self.assertEqual(arr.nbytes, sum(tp[0] for tp in size_res))
res = self.executor.execute_tensor(arr, concat=True)[0]
self.assertEqual(res.shape, (10, 30))
self.assertTrue(np.all(res >= 0))
self.assertTrue(np.all(res < 2))
def testBaseExecution(self):
executor_numpy = Executor('numpy')
raw1 = np.random.randint(10, size=(10, 10, 10))
raw2 = np.random.randint(10, size=(10, 10, 10))
arr1 = tensor(raw1, chunk_size=3)
arr2 = tensor(raw2, chunk_size=3)
arr3 = arr1 + arr2 + 10
arr4 = 10 + arr1 + arr2
res3 = executor_numpy.execute_tensor(arr3, concat=True)
res3_cmp = self.executor.execute_tensor(arr4, concat=True)
self.assertTrue(np.array_equal(res3[0], res3_cmp[0]))
res5 = executor_numpy.execute_tensor((arr1 + arr1), concat=True)
res5_cmp = self.executor.execute_tensor((arr1 + arr1), concat=True)
self.assertTrue(np.array_equal(res5[0], res5_cmp[0]))
def testSparseRandintExecution(self):
size_executor = Executor(
sync_provider_type=Executor.SyncProviderType.MOCK)
arr = tensor.random.randint(1,
2,
size=(30, 50),
density=.1,
chunk_size=10,
dtype='f4')
size_res = size_executor.execute_tensor(arr, mock=True)
self.assertAlmostEqual(arr.nbytes * 0.1, sum(tp[0] for tp in size_res))
res = self.executor.execute_tensor(arr, concat=True)[0]
self.assertTrue(issparse(res))
self.assertEqual(res.shape, (30, 50))
self.assertTrue(np.all(res.data >= 1))
self.assertTrue(np.all(res.data < 2))
self.assertAlmostEqual((res >= 1).toarray().sum(),
30 * 50 * .1,
delta=20)
def setUp(self):
self.executor = Executor('numpy', storage=MockStorage(), prefetch=True)
self._raw_tensor_ones = self.executor._op_runners[TensorOnes]
self.executor._op_runners[TensorOnes] = _slow_tensor_ones
def setUp(self):
self.executor = Executor('numpy')
self.old_chunk = options.tensor.chunk_size
options.tensor.chunk_size = 10
def setUp(self):
super(Test, self).setUp()
self.executor = Executor()
def setUp(self):
self.executor = Executor('numpy')
local_session = LocalSession()
local_session._executor = self.executor
self.session = Session()
self.session._sess = local_session
def setUp(self):
self.executor = Executor('cupy')
def testTensordotExecution(self):
size_executor = Executor(
sync_provider_type=Executor.SyncProviderType.MOCK)
a_data = np.arange(60).reshape(3, 4, 5)
a = tensor(a_data, chunk_size=2)
b_data = np.arange(24).reshape(4, 3, 2)
b = tensor(b_data, chunk_size=2)
axes = ([1, 0], [0, 1])
c = tensordot(a, b, axes=axes)
size_res = size_executor.execute_tensor(c, mock=True)
res = self.executor.execute_tensor(c)
expected = np.tensordot(a_data, b_data, axes=axes)
self.assertEqual(sum(s[0] for s in size_res), c.nbytes)
self.assertTrue(np.array_equal(res[0], expected[:2, :]))
self.assertTrue(np.array_equal(res[1], expected[2:4, :]))
self.assertTrue(np.array_equal(res[2], expected[4:, :]))
a = ones((1000, 2000), chunk_size=500)
b = ones((2000, 100), chunk_size=500)
c = dot(a, b)
res = self.executor.execute_tensor(c)
expected = np.dot(np.ones((1000, 2000)), np.ones((2000, 100)))
self.assertEqual(len(res), 2)
self.assertTrue(np.array_equal(res[0], expected[:500, :]))
self.assertTrue(np.array_equal(res[1], expected[500:, :]))
a = ones((10, 8), chunk_size=2)
b = ones((8, 10), chunk_size=2)
c = a.dot(b)
res = self.executor.execute_tensor(c)
self.assertEqual(len(res), 25)
for r in res:
self.assertTrue(np.array_equal(r, np.tile([8], [2, 2])))
a = ones((500, 500), chunk_size=500)
b = ones((500, 100), chunk_size=500)
c = a.dot(b)
res = self.executor.execute_tensor(c)
self.assertTrue(np.array_equal(res[0], np.tile([500], [500, 100])))
raw_a = np.random.random((100, 200, 50))
raw_b = np.random.random((200, 10, 100))
a = tensor(raw_a, chunk_size=50)
b = tensor(raw_b, chunk_size=33)
c = tensordot(a, b, axes=((0, 1), (2, 0)))
res = self.executor.execute_tensor(c, concat=True)
expected = np.tensordot(raw_a, raw_b, axes=(c.op.a_axes, c.op.b_axes))
self.assertTrue(np.allclose(res[0], expected))
a = ones((1000, 2000), chunk_size=500)
b = ones((100, 2000), chunk_size=500)
c = inner(a, b)
res = self.executor.execute_tensor(c)
expected = np.inner(np.ones((1000, 2000)), np.ones((100, 2000)))
self.assertEqual(len(res), 2)
self.assertTrue(np.array_equal(res[0], expected[:500, :]))
self.assertTrue(np.array_equal(res[1], expected[500:, :]))
a = ones((100, 100), chunk_size=30)
b = ones((100, 100), chunk_size=30)
c = a.dot(b)
res = self.executor.execute_tensor(c, concat=True)[0]
np.testing.assert_array_equal(res, np.ones((100, 100)) * 100)
def __init__(self):
self._executor = Executor(
sync_provider_type=Executor.SyncProviderType.GEVENT)
def setUp(self):
self.executor = Executor('numpy', storage=MockStorage(), prefetch=True)
def setUp(self):
self.executor = Executor()
def setUp(self):
self.executor = Executor('numpy')
def setUp(self):
self.executor = Executor('numexpr')
