Пример #1
0
    def testOperandsWithoutPrepareInputs(self):
        self.start_processes(etcd=False, modules=['mars.scheduler.tests.integrated.no_prepare_op'])

        session_id = uuid.uuid1()
        actor_client = new_client()

        session_ref = actor_client.actor_ref(self.session_manager_ref.create_session(session_id))

        actor_address = self.cluster_info.get_scheduler(ResourceActor.default_uid())
        resource_ref = actor_client.actor_ref(ResourceActor.default_uid(), address=actor_address)
        worker_endpoints = resource_ref.get_worker_endpoints()

        t1 = mt.random.rand(10)
        t1.op._expect_worker = worker_endpoints[0]
        t2 = mt.random.rand(10)
        t2.op._expect_worker = worker_endpoints[1]

        t = NoPrepareOperand().new_tileable([t1, t2])
        t.op._prepare_inputs = [False, False]

        graph = t.build_graph()
        targets = [t.key]
        graph_key = uuid.uuid1()
        session_ref.submit_tileable_graph(json.dumps(graph.to_json()),
                                          graph_key, target_tileables=targets)

        state = self.wait_for_termination(actor_client, session_ref, graph_key)
        self.assertEqual(state, GraphState.SUCCEEDED)
Пример #2
0
    def testRemoteWithoutEtcd(self):
        from mars.scheduler.resource import ResourceActor
        from mars.worker.dispatcher import DispatchActor

        self.start_processes(etcd=False, modules=['mars.scheduler.tests.integrated.no_prepare_op'])
        sess = new_session(self.session_manager_ref.address)
        resource_ref = sess._api.actor_client.actor_ref(
            ResourceActor.default_uid(),
            address=self.cluster_info.get_scheduler(ResourceActor.default_uid())
        )
        worker_ips = resource_ref.get_worker_endpoints()

        rs = np.random.RandomState(0)
        raw1 = rs.rand(10, 10)
        raw2 = rs.rand(10, 10)

        def f_none(_x):
            return None

        r_none = spawn(f_none, raw1)
        result = r_none.execute(session=sess, timeout=self.timeout).fetch(session=sess)
        self.assertIsNone(result)

        def f1(x):
            return x + 1

        def f2(x, y, z=None):
            return x * y * (z[0] + z[1])

        r1 = spawn(f1, raw1)
        r2 = spawn(f1, raw2)
        r3 = spawn(f2, (r1, r2), {'z': [r1, r2]})
        result = r3.execute(session=sess, timeout=self.timeout).fetch(session=sess)
        expected = (raw1 + 1) * (raw2 + 1) * (raw1 + 1 + raw2 + 1)
        np.testing.assert_allclose(result, expected)

        def f(t, x):
            mul = (t * x).execute()
            return mul.sum().to_numpy()

        rs = np.random.RandomState(0)
        raw = rs.rand(5, 4)

        t1 = mt.tensor(raw, chunk_size=3)
        t2 = t1.sum(axis=0)
        s = spawn(f, args=(t2, 3))

        result = s.execute(session=sess, timeout=self.timeout).fetch(session=sess)
        expected = (raw.sum(axis=0) * 3).sum()
        self.assertAlmostEqual(result, expected)

        time.sleep(1)
        for worker_ip in worker_ips:
            ref = sess._api.actor_client.actor_ref(DispatchActor.default_uid(), address=worker_ip)
            self.assertEqual(len(ref.get_slots('cpu')), 1)
Пример #3
0
    def testOperandsWithPureDepends(self):
        self.start_processes(etcd=False, modules=['mars.scheduler.tests.integrated.no_prepare_op'])
        sess = new_session(self.session_manager_ref.address)

        actor_address = self.cluster_info.get_scheduler(ResourceActor.default_uid())
        resource_ref = sess._api.actor_client.actor_ref(ResourceActor.default_uid(), address=actor_address)
        worker_endpoints = resource_ref.get_worker_endpoints()

        t1 = mt.random.rand(10)
        t1.op._expect_worker = worker_endpoints[0]
        t2 = mt.random.rand(10)
        t2.op._expect_worker = worker_endpoints[1]

        t = PureDependsOperand().new_tileable([t1, t2])
        t.op._pure_depends = [True, True]
        t.execute(session=sess, timeout=self.timeout)