def testLocalCluster(self, *_):
endpoint = gen_endpoint('0.0.0.0')
with LocalDistributedCluster(endpoint,
scheduler_n_process=2,
worker_n_process=3,
shared_memory='20M') as cluster:
pool = cluster.pool
self.assertTrue(
pool.has_actor(
pool.actor_ref(SchedulerClusterInfoActor.default_uid())))
self.assertTrue(
pool.has_actor(
pool.actor_ref(SessionManagerActor.default_uid())))
self.assertTrue(
pool.has_actor(pool.actor_ref(DispatchActor.default_uid())))
with new_session(endpoint) as session:
api = session._api
t = mt.ones((3, 3), chunk_size=2)
result = session.run(t, timeout=_exec_timeout)
np.testing.assert_array_equal(result, np.ones((3, 3)))
self.assertNotIn(session._session_id,
api.session_manager.get_sessions())
def testRemoteWithoutEtcd(self):
from mars.scheduler.resource import ResourceActor
from mars.worker.dispatcher import DispatchActor
self.start_processes(etcd=False, modules=['mars.scheduler.tests.integrated.no_prepare_op'])
sess = new_session(self.session_manager_ref.address)
resource_ref = sess._api.actor_client.actor_ref(
ResourceActor.default_uid(),
address=self.cluster_info.get_scheduler(ResourceActor.default_uid())
)
worker_ips = resource_ref.get_worker_endpoints()
rs = np.random.RandomState(0)
raw1 = rs.rand(10, 10)
raw2 = rs.rand(10, 10)
def f_none(_x):
return None
r_none = spawn(f_none, raw1)
result = r_none.execute(session=sess, timeout=self.timeout).fetch(session=sess)
self.assertIsNone(result)
def f1(x):
return x + 1
def f2(x, y, z=None):
return x * y * (z[0] + z[1])
r1 = spawn(f1, raw1)
r2 = spawn(f1, raw2)
r3 = spawn(f2, (r1, r2), {'z': [r1, r2]})
result = r3.execute(session=sess, timeout=self.timeout).fetch(session=sess)
expected = (raw1 + 1) * (raw2 + 1) * (raw1 + 1 + raw2 + 1)
np.testing.assert_allclose(result, expected)
def f(t, x):
mul = (t * x).execute()
return mul.sum().to_numpy()
rs = np.random.RandomState(0)
raw = rs.rand(5, 4)
t1 = mt.tensor(raw, chunk_size=3)
t2 = t1.sum(axis=0)
s = spawn(f, args=(t2, 3))
result = s.execute(session=sess, timeout=self.timeout).fetch(session=sess)
expected = (raw.sum(axis=0) * 3).sum()
self.assertAlmostEqual(result, expected)
time.sleep(1)
for worker_ip in worker_ips:
ref = sess._api.actor_client.actor_ref(DispatchActor.default_uid(), address=worker_ip)
self.assertEqual(len(ref.get_slots('cpu')), 1)
def testDistribute(self):
distributor = gen_distributor(2, 4)
idx = distributor.distribute(SessionActor.default_name())
self.assertEqual(idx, 1)
idx = distributor.distribute(
GraphActor.gen_uid('fake_session_id', 'fake_graph_key'))
self.assertEqual(idx, 0)
idx = distributor.distribute(DispatchActor.default_name())
self.assertEqual(idx, 2)
idx = distributor.distribute('w:1:mars-sender')
self.assertEqual(idx, 3)
def testDistribute(self):
distributor = gen_distributor(2, 4)
idx = distributor.distribute(SessionActor.default_uid())
self.assertEqual(idx, 1)
idx = distributor.distribute(
GraphActor.gen_uid('fake_session_id', 'fake_graph_key'))
self.assertEqual(idx, 0)
idx = distributor.distribute(DispatchActor.default_uid())
self.assertEqual(idx, 2)
idx = distributor.distribute('w:1:mars-sender')
self.assertEqual(idx, 3)
uid = distributor.make_same_process('s:0:test_uid', 's:1:SessionActor')
self.assertEqual(uid, 's:1:test_uid')
uid = distributor.make_same_process('w:0:test_uid', 'w:1:mars-sender')
self.assertEqual(uid, 'w:1:test_uid')