def testLocalCluster(self, *_): endpoint = gen_endpoint('0.0.0.0') with LocalDistributedCluster(endpoint, scheduler_n_process=2, worker_n_process=3, shared_memory='20M') as cluster: pool = cluster.pool self.assertTrue( pool.has_actor( pool.actor_ref(SchedulerClusterInfoActor.default_uid()))) self.assertTrue( pool.has_actor( pool.actor_ref(SessionManagerActor.default_uid()))) self.assertTrue( pool.has_actor(pool.actor_ref(DispatchActor.default_uid()))) with new_session(endpoint) as session: api = session._api t = mt.ones((3, 3), chunk_size=2) result = session.run(t, timeout=_exec_timeout) np.testing.assert_array_equal(result, np.ones((3, 3))) self.assertNotIn(session._session_id, api.session_manager.get_sessions())
def testRemoteWithoutEtcd(self): from mars.scheduler.resource import ResourceActor from mars.worker.dispatcher import DispatchActor self.start_processes(etcd=False, modules=['mars.scheduler.tests.integrated.no_prepare_op']) sess = new_session(self.session_manager_ref.address) resource_ref = sess._api.actor_client.actor_ref( ResourceActor.default_uid(), address=self.cluster_info.get_scheduler(ResourceActor.default_uid()) ) worker_ips = resource_ref.get_worker_endpoints() rs = np.random.RandomState(0) raw1 = rs.rand(10, 10) raw2 = rs.rand(10, 10) def f_none(_x): return None r_none = spawn(f_none, raw1) result = r_none.execute(session=sess, timeout=self.timeout).fetch(session=sess) self.assertIsNone(result) def f1(x): return x + 1 def f2(x, y, z=None): return x * y * (z[0] + z[1]) r1 = spawn(f1, raw1) r2 = spawn(f1, raw2) r3 = spawn(f2, (r1, r2), {'z': [r1, r2]}) result = r3.execute(session=sess, timeout=self.timeout).fetch(session=sess) expected = (raw1 + 1) * (raw2 + 1) * (raw1 + 1 + raw2 + 1) np.testing.assert_allclose(result, expected) def f(t, x): mul = (t * x).execute() return mul.sum().to_numpy() rs = np.random.RandomState(0) raw = rs.rand(5, 4) t1 = mt.tensor(raw, chunk_size=3) t2 = t1.sum(axis=0) s = spawn(f, args=(t2, 3)) result = s.execute(session=sess, timeout=self.timeout).fetch(session=sess) expected = (raw.sum(axis=0) * 3).sum() self.assertAlmostEqual(result, expected) time.sleep(1) for worker_ip in worker_ips: ref = sess._api.actor_client.actor_ref(DispatchActor.default_uid(), address=worker_ip) self.assertEqual(len(ref.get_slots('cpu')), 1)
def testDistribute(self): distributor = gen_distributor(2, 4) idx = distributor.distribute(SessionActor.default_name()) self.assertEqual(idx, 1) idx = distributor.distribute( GraphActor.gen_uid('fake_session_id', 'fake_graph_key')) self.assertEqual(idx, 0) idx = distributor.distribute(DispatchActor.default_name()) self.assertEqual(idx, 2) idx = distributor.distribute('w:1:mars-sender') self.assertEqual(idx, 3)
def testDistribute(self): distributor = gen_distributor(2, 4) idx = distributor.distribute(SessionActor.default_uid()) self.assertEqual(idx, 1) idx = distributor.distribute( GraphActor.gen_uid('fake_session_id', 'fake_graph_key')) self.assertEqual(idx, 0) idx = distributor.distribute(DispatchActor.default_uid()) self.assertEqual(idx, 2) idx = distributor.distribute('w:1:mars-sender') self.assertEqual(idx, 3) uid = distributor.make_same_process('s:0:test_uid', 's:1:SessionActor') self.assertEqual(uid, 's:1:test_uid') uid = distributor.make_same_process('w:0:test_uid', 'w:1:mars-sender') self.assertEqual(uid, 'w:1:test_uid')