def testMainWithoutEtcd(self):
self.start_processes()
session_id = uuid.uuid1()
actor_client = new_client()
session_ref = actor_client.actor_ref(self.session_manager_ref.create_session(session_id))
a = mt.ones((100, 100), chunk_size=30) * 2 * 1 + 1
b = mt.ones((100, 100), chunk_size=30) * 2 * 1 + 1
c = (a * b * 2 + 1).sum()
graph = c.build_graph()
targets = [c.key]
graph_key = uuid.uuid1()
session_ref.submit_tensor_graph(json.dumps(graph.to_json()),
graph_key, target_tensors=targets)
state = self.wait_for_termination(actor_client, session_ref, graph_key)
self.assertEqual(state, GraphState.SUCCEEDED)
result = session_ref.fetch_result(graph_key, c.key)
expected = (np.ones(a.shape) * 2 * 1 + 1) ** 2 * 2 + 1
assert_allclose(loads(result), expected.sum())
a = mt.ones((100, 50), chunk_size=35) * 2 + 1
b = mt.ones((50, 200), chunk_size=35) * 2 + 1
c = a.dot(b)
graph = c.build_graph()
targets = [c.key]
graph_key = uuid.uuid1()
session_ref.submit_tensor_graph(json.dumps(graph.to_json()),
graph_key, target_tensors=targets)
state = self.wait_for_termination(actor_client, session_ref, graph_key)
self.assertEqual(state, GraphState.SUCCEEDED)
result = session_ref.fetch_result(graph_key, c.key)
assert_allclose(loads(result), np.ones((100, 200)) * 450)
base_arr = np.random.random((100, 100))
a = mt.array(base_arr)
sumv = reduce(operator.add, [a[:10, :10] for _ in range(10)])
graph = sumv.build_graph()
targets = [sumv.key]
graph_key = uuid.uuid1()
session_ref.submit_tensor_graph(json.dumps(graph.to_json()),
graph_key, target_tensors=targets)
state = self.wait_for_termination(actor_client, session_ref, graph_key)
self.assertEqual(state, GraphState.SUCCEEDED)
expected = reduce(operator.add, [base_arr[:10, :10] for _ in range(10)])
result = session_ref.fetch_result(graph_key, sumv.key)
assert_allclose(loads(result), expected)
def testMultipleAdd(self):
import numpy as np
import operator
from mars.compat import reduce
base_arr = np.random.random((100, 100))
a = mt.array(base_arr)
sumv = reduce(operator.add, [a[:10, :10] for _ in range(10)])
self.run_expr_suite(sumv)
def testMultipleAddPreparation(self, *_):
import numpy as np
import operator
from mars.compat import reduce
base_arr = np.random.random((100, 100))
a = mt.array(base_arr)
sumv = reduce(operator.add, [a[:10, :10] for _ in range(10)])
with self.prepare_graph_in_pool(sumv):
pass
def testMainTensorWithoutEtcd(self):
self.start_processes()
session_id = uuid.uuid1()
actor_client = new_client()
session_ref = actor_client.actor_ref(
self.session_manager_ref.create_session(session_id))
a = mt.ones((100, 100), chunk_size=30) * 2 * 1 + 1
b = mt.ones((100, 100), chunk_size=30) * 2 * 1 + 1
c = (a * b * 2 + 1).sum()
graph = c.build_graph()
targets = [c.key]
graph_key = uuid.uuid1()
session_ref.submit_tileable_graph(json.dumps(graph.to_json()),
graph_key,
target_tileables=targets)
state = self.wait_for_termination(actor_client, session_ref, graph_key)
self.assertEqual(state, GraphState.SUCCEEDED)
result = session_ref.fetch_result(graph_key, c.key)
expected = (np.ones(a.shape) * 2 * 1 + 1)**2 * 2 + 1
assert_allclose(loads(result), expected.sum())
a = mt.ones((100, 50), chunk_size=35) * 2 + 1
b = mt.ones((50, 200), chunk_size=35) * 2 + 1
c = a.dot(b)
graph = c.build_graph()
targets = [c.key]
graph_key = uuid.uuid1()
session_ref.submit_tileable_graph(json.dumps(graph.to_json()),
graph_key,
target_tileables=targets)
state = self.wait_for_termination(actor_client, session_ref, graph_key)
self.assertEqual(state, GraphState.SUCCEEDED)
result = session_ref.fetch_result(graph_key, c.key)
assert_allclose(loads(result), np.ones((100, 200)) * 450)
base_arr = np.random.random((100, 100))
a = mt.array(base_arr)
sumv = reduce(operator.add, [a[:10, :10] for _ in range(10)])
graph = sumv.build_graph()
targets = [sumv.key]
graph_key = uuid.uuid1()
session_ref.submit_tileable_graph(json.dumps(graph.to_json()),
graph_key,
target_tileables=targets)
state = self.wait_for_termination(actor_client, session_ref, graph_key)
self.assertEqual(state, GraphState.SUCCEEDED)
expected = reduce(operator.add,
[base_arr[:10, :10] for _ in range(10)])
result = session_ref.fetch_result(graph_key, sumv.key)
assert_allclose(loads(result), expected)
a = mt.ones((31, 27), chunk_size=10)
b = a.reshape(27, 31)
b.op.extra_params['_reshape_with_shuffle'] = True
r = b.sum(axis=1)
graph = r.build_graph()
targets = [r.key]
graph_key = uuid.uuid1()
session_ref.submit_tileable_graph(json.dumps(graph.to_json()),
graph_key,
target_tileables=targets)
state = self.wait_for_termination(actor_client, session_ref, graph_key)
self.assertEqual(state, GraphState.SUCCEEDED)
result = session_ref.fetch_result(graph_key, r.key)
assert_allclose(loads(result), np.ones((27, 31)).sum(axis=1))
raw = np.random.RandomState(0).rand(10, 10)
a = mt.tensor(raw, chunk_size=(5, 4))
b = a[a.argmin(axis=1), mt.tensor(np.arange(10))]
graph = b.build_graph()
targets = [b.key]
graph_key = uuid.uuid1()
session_ref.submit_tileable_graph(json.dumps(graph.to_json()),
graph_key,
target_tileables=targets)
state = self.wait_for_termination(actor_client, session_ref, graph_key)
self.assertEqual(state, GraphState.SUCCEEDED)
result = session_ref.fetch_result(graph_key, b.key)
np.testing.assert_array_equal(loads(result), raw[raw.argmin(axis=1),
np.arange(10)])
def testMain(self):
session_id = uuid.uuid1()
scheduler_address = '127.0.0.1:' + self.scheduler_port
actor_client = new_client()
session_ref = actor_client.create_actor(SessionActor,
uid=SessionActor.gen_name(session_id),
address=scheduler_address,
session_id=session_id)
a = mt.ones((100, 100), chunk_size=30) * 2 * 1 + 1
b = mt.ones((100, 100), chunk_size=30) * 2 * 1 + 1
c = (a * b * 2 + 1).sum()
graph = c.build_graph()
targets = [c.key]
graph_key = uuid.uuid1()
session_ref.submit_tensor_graph(json.dumps(graph.to_json()),
graph_key, target_tensors=targets)
state = self.wait_for_termination(session_ref, graph_key)
self.assertEqual(state, GraphState.SUCCEEDED)
result = session_ref.fetch_result(graph_key, c.key)
expected = (np.ones(a.shape) * 2 * 1 + 1) ** 2 * 2 + 1
assert_array_equal(loads(result), expected.sum())
graph_key = uuid.uuid1()
session_ref.submit_tensor_graph(json.dumps(graph.to_json()),
graph_key, target_tensors=targets)
# todo this behavior may change when eager mode is introduced
state = self.wait_for_termination(session_ref, graph_key)
self.assertEqual(state, GraphState.FAILED)
a = mt.ones((100, 50), chunk_size=35) * 2 + 1
b = mt.ones((50, 200), chunk_size=35) * 2 + 1
c = a.dot(b)
graph = c.build_graph()
targets = [c.key]
graph_key = uuid.uuid1()
session_ref.submit_tensor_graph(json.dumps(graph.to_json()),
graph_key, target_tensors=targets)
state = self.wait_for_termination(session_ref, graph_key)
self.assertEqual(state, GraphState.SUCCEEDED)
result = session_ref.fetch_result(graph_key, c.key)
assert_array_equal(loads(result), np.ones((100, 200)) * 450)
base_arr = np.random.random((100, 100))
a = mt.array(base_arr)
sumv = reduce(operator.add, [a[:10, :10] for _ in range(10)])
graph = sumv.build_graph()
targets = [sumv.key]
graph_key = uuid.uuid1()
session_ref.submit_tensor_graph(json.dumps(graph.to_json()),
graph_key, target_tensors=targets)
state = self.wait_for_termination(session_ref, graph_key)
self.assertEqual(state, GraphState.SUCCEEDED)
expected = reduce(operator.add, [base_arr[:10, :10] for _ in range(10)])
result = session_ref.fetch_result(graph_key, sumv.key)
assert_array_equal(loads(result), expected)
