def testOperandsWithoutPrepareInputs(self):
self.start_processes(etcd=False, modules=['mars.scheduler.tests.integrated.no_prepare_op'])
session_id = uuid.uuid1()
actor_client = new_client()
session_ref = actor_client.actor_ref(self.session_manager_ref.create_session(session_id))
actor_address = self.cluster_info.get_scheduler(ResourceActor.default_uid())
resource_ref = actor_client.actor_ref(ResourceActor.default_uid(), address=actor_address)
worker_endpoints = resource_ref.get_worker_endpoints()
t1 = mt.random.rand(10)
t1.op._expect_worker = worker_endpoints[0]
t2 = mt.random.rand(10)
t2.op._expect_worker = worker_endpoints[1]
t = NoPrepareOperand().new_tileable([t1, t2])
t.op._prepare_inputs = [False, False]
graph = t.build_graph()
targets = [t.key]
graph_key = uuid.uuid1()
session_ref.submit_tileable_graph(json.dumps(graph.to_json()),
graph_key, target_tileables=targets)
state = self.wait_for_termination(actor_client, session_ref, graph_key)
self.assertEqual(state, GraphState.SUCCEEDED)
def testRemoteWithoutEtcd(self):
from mars.scheduler.resource import ResourceActor
from mars.worker.dispatcher import DispatchActor
self.start_processes(etcd=False, modules=['mars.scheduler.tests.integrated.no_prepare_op'])
sess = new_session(self.session_manager_ref.address)
resource_ref = sess._api.actor_client.actor_ref(
ResourceActor.default_uid(),
address=self.cluster_info.get_scheduler(ResourceActor.default_uid())
)
worker_ips = resource_ref.get_worker_endpoints()
rs = np.random.RandomState(0)
raw1 = rs.rand(10, 10)
raw2 = rs.rand(10, 10)
def f_none(_x):
return None
r_none = spawn(f_none, raw1)
result = r_none.execute(session=sess, timeout=self.timeout).fetch(session=sess)
self.assertIsNone(result)
def f1(x):
return x + 1
def f2(x, y, z=None):
return x * y * (z[0] + z[1])
r1 = spawn(f1, raw1)
r2 = spawn(f1, raw2)
r3 = spawn(f2, (r1, r2), {'z': [r1, r2]})
result = r3.execute(session=sess, timeout=self.timeout).fetch(session=sess)
expected = (raw1 + 1) * (raw2 + 1) * (raw1 + 1 + raw2 + 1)
np.testing.assert_allclose(result, expected)
def f(t, x):
mul = (t * x).execute()
return mul.sum().to_numpy()
rs = np.random.RandomState(0)
raw = rs.rand(5, 4)
t1 = mt.tensor(raw, chunk_size=3)
t2 = t1.sum(axis=0)
s = spawn(f, args=(t2, 3))
result = s.execute(session=sess, timeout=self.timeout).fetch(session=sess)
expected = (raw.sum(axis=0) * 3).sum()
self.assertAlmostEqual(result, expected)
time.sleep(1)
for worker_ip in worker_ips:
ref = sess._api.actor_client.actor_ref(DispatchActor.default_uid(), address=worker_ip)
self.assertEqual(len(ref.get_slots('cpu')), 1)
def testOperandsWithPureDepends(self):
self.start_processes(etcd=False, modules=['mars.scheduler.tests.integrated.no_prepare_op'])
sess = new_session(self.session_manager_ref.address)
actor_address = self.cluster_info.get_scheduler(ResourceActor.default_uid())
resource_ref = sess._api.actor_client.actor_ref(ResourceActor.default_uid(), address=actor_address)
worker_endpoints = resource_ref.get_worker_endpoints()
t1 = mt.random.rand(10)
t1.op._expect_worker = worker_endpoints[0]
t2 = mt.random.rand(10)
t2.op._expect_worker = worker_endpoints[1]
t = PureDependsOperand().new_tileable([t1, t2])
t.op._pure_depends = [True, True]
t.execute(session=sess, timeout=self.timeout)