Exemple #1
0
    def testDestroyCalcActor(self):
        import gevent.event

        with self._start_calc_pool() as (_pool, test_actor):
            calc_ref = _pool.actor_ref(CpuCalcActor.default_uid())
            calc_ref.mark_destroy()
            gevent.sleep(0.8)
            self.assertFalse(_pool.has_actor(calc_ref))

        with self._start_calc_pool() as (_pool, test_actor):
            calc_ref = test_actor.promise_ref(CpuCalcActor.default_uid())

            session_id = str(uuid.uuid4())
            data_list = [np.random.random((10, 10)) for _ in range(2)]
            exec_graph, fetch_chunks, add_chunk = self._build_test_graph(
                data_list)
            exec_graph2, fetch_chunks2, add_chunk2 = self._build_test_graph(
                data_list[::-1])

            storage_client = test_actor.storage_client

            for fetch_chunk, d in zip(fetch_chunks, data_list):
                self.waitp(
                    storage_client.put_objects(
                        session_id, [fetch_chunk.key], [d],
                        [DataStorageDevice.SHARED_MEMORY]), )
            for fetch_chunk2, d in zip(fetch_chunks2, data_list[::-1]):
                self.waitp(
                    storage_client.put_objects(
                        session_id, [fetch_chunk2.key], [d],
                        [DataStorageDevice.SHARED_MEMORY]), )

            orig_calc_results = CpuCalcActor._calc_results

            start_event = gevent.event.Event()

            def _mock_calc_delayed(actor_obj, *args, **kwargs):
                start_event.set()
                gevent.sleep(1)
                return orig_calc_results(actor_obj, *args, **kwargs)

            with patch_method(CpuCalcActor._calc_results, _mock_calc_delayed):
                p = calc_ref.calc(session_id, add_chunk.op.key, serialize_graph(exec_graph),
                                  [add_chunk.key], _promise=True) \
                    .then(lambda *_: calc_ref.store_results(
                        session_id, add_chunk.op.key, [add_chunk.key], None, _promise=True))
                start_event.wait()
                calc_ref.mark_destroy()

                p2 = calc_ref.calc(session_id, add_chunk2.op.key, serialize_graph(exec_graph2),
                                   [add_chunk2.key], _promise=True) \
                    .then(lambda *_: calc_ref.store_results(
                        session_id, add_chunk2.op.key, [add_chunk2.key], None, _promise=True))

                self.assertTrue(_pool.has_actor(calc_ref._ref))
                self.waitp(p)
                self.waitp(p2)

            gevent.sleep(0.8)
            self.assertFalse(_pool.has_actor(calc_ref._ref))
Exemple #2
0
    def _start_calc_pool(self):
        mock_addr = f'127.0.0.1:{get_next_port()}'
        with self.create_pool(n_process=1, backend='gevent',
                              address=mock_addr) as pool:
            pool.create_actor(SchedulerClusterInfoActor, [mock_addr],
                              uid=SchedulerClusterInfoActor.default_uid())
            pool.create_actor(WorkerClusterInfoActor, [mock_addr],
                              uid=WorkerClusterInfoActor.default_uid())

            pool.create_actor(ChunkMetaActor, uid=ChunkMetaActor.default_uid())
            pool.create_actor(StatusActor,
                              mock_addr,
                              uid=StatusActor.default_uid())

            pool.create_actor(PlasmaKeyMapActor,
                              uid=PlasmaKeyMapActor.default_uid())
            pool.create_actor(WorkerDaemonActor,
                              uid=WorkerDaemonActor.default_uid())
            pool.create_actor(DispatchActor, uid=DispatchActor.default_uid())
            pool.create_actor(StorageManagerActor,
                              uid=StorageManagerActor.default_uid())
            pool.create_actor(IORunnerActor)
            pool.create_actor(QuotaActor,
                              1024**2,
                              uid=MemQuotaActor.default_uid())
            shared_holder_ref = pool.create_actor(
                SharedHolderActor, uid=SharedHolderActor.default_uid())
            pool.create_actor(InProcHolderActor)
            pool.create_actor(CpuCalcActor, uid=CpuCalcActor.default_uid())

            with self.run_actor_test(pool) as test_actor:
                try:
                    yield pool, test_actor
                finally:
                    shared_holder_ref.destroy()
Exemple #3
0
    def testCpuCalcErrorInRunning(self):
        with self._start_calc_pool() as (_pool, test_actor):
            calc_ref = test_actor.promise_ref(CpuCalcActor.default_uid())

            session_id = str(uuid.uuid4())
            data_list = [np.random.random((10, 10)) for _ in range(2)]
            exec_graph, fetch_chunks, add_chunk = self._build_test_graph(
                data_list)

            storage_client = test_actor.storage_client

            for fetch_chunk, d in zip(fetch_chunks, data_list):
                self.waitp(
                    storage_client.put_objects(
                        session_id, [fetch_chunk.key], [d],
                        [DataStorageDevice.SHARED_MEMORY]), )

            def _mock_calc_results_error(*_, **__):
                raise ValueError

            with patch_method(CpuCalcActor._calc_results, _mock_calc_results_error), \
                    self.assertRaises(ValueError):
                self.waitp(
                    calc_ref.calc(
                        session_id,
                        add_chunk.op.key,
                        serialize_graph(exec_graph), [add_chunk.key],
                        _promise=True).then(lambda *_: calc_ref.store_results(
                            session_id,
                            add_chunk.op.key, [add_chunk.key],
                            None,
                            _promise=True)))
Exemple #4
0
    def testCpuCalcSingleFetches(self):
        import gc
        with self._start_calc_pool() as (_pool, test_actor):
            quota_ref = test_actor.promise_ref(MemQuotaActor.default_uid())
            calc_ref = test_actor.promise_ref(CpuCalcActor.default_uid())

            session_id = str(uuid.uuid4())
            data_list = [np.random.random((10, 10)) for _ in range(3)]
            exec_graph, fetch_chunks, add_chunk = self._build_test_graph(
                data_list)

            storage_client = test_actor.storage_client

            for fetch_chunk, d in zip(fetch_chunks, data_list):
                self.waitp(
                    storage_client.put_objects(
                        session_id, [fetch_chunk.key], [d],
                        [DataStorageDevice.SHARED_MEMORY]), )
            self.assertEqual(
                list(
                    storage_client.get_data_locations(
                        session_id, [fetch_chunks[0].key])[0]),
                [(0, DataStorageDevice.SHARED_MEMORY)])

            quota_batch = {
                build_quota_key(session_id, add_chunk.key, add_chunk.op.key):
                data_list[0].nbytes,
            }

            for idx in [1, 2]:
                quota_batch[build_quota_key(session_id, fetch_chunks[idx].key, add_chunk.op.key)] \
                    = data_list[idx].nbytes

                self.waitp(
                    storage_client.copy_to(
                        session_id, [fetch_chunks[idx].key],
                        [DataStorageDevice.DISK
                         ]).then(lambda *_: storage_client.delete(
                             session_id, [fetch_chunks[idx].key],
                             [DataStorageDevice.SHARED_MEMORY])))
                self.assertEqual(
                    list(
                        storage_client.get_data_locations(
                            session_id, [fetch_chunks[idx].key])[0]),
                    [(0, DataStorageDevice.DISK)])

            self.waitp(
                quota_ref.request_batch_quota(quota_batch, _promise=True), )

            o_create = PlasmaSharedStore.create

            def _mock_plasma_create(store, session_id, data_key, size):
                if data_key == fetch_chunks[2].key:
                    raise StorageFull
                return o_create(store, session_id, data_key, size)

            id_type_set = set()

            def _extract_value_ref(*_):
                inproc_handler = storage_client.get_storage_handler(
                    (0, DataStorageDevice.PROC_MEMORY))
                obj = inproc_handler.get_objects(session_id,
                                                 [add_chunk.key])[0]
                id_type_set.add((id(obj), type(obj)))
                del obj

            with patch_method(PlasmaSharedStore.create, _mock_plasma_create):
                self.waitp(
                    calc_ref.calc(session_id,
                                  add_chunk.op.key,
                                  serialize_graph(exec_graph), [add_chunk.key],
                                  _promise=True).then(_extract_value_ref).then(
                                      lambda *_: calc_ref.store_results(
                                          session_id,
                                          add_chunk.op.key, [add_chunk.key],
                                          None,
                                          _promise=True)))

            self.assertTrue(
                all((id(obj), type(obj)) not in id_type_set
                    for obj in gc.get_objects()))

            quota_dump = quota_ref.dump_data()
            self.assertEqual(len(quota_dump.allocations), 0)
            self.assertEqual(len(quota_dump.requests), 0)
            self.assertEqual(len(quota_dump.proc_sizes), 0)
            self.assertEqual(len(quota_dump.hold_sizes), 0)

            self.assertEqual(
                sorted(
                    storage_client.get_data_locations(
                        session_id, [fetch_chunks[0].key])[0]),
                [(0, DataStorageDevice.SHARED_MEMORY)])
            self.assertEqual(
                sorted(
                    storage_client.get_data_locations(
                        session_id, [fetch_chunks[1].key])[0]),
                [(0, DataStorageDevice.DISK)])
            self.assertEqual(
                sorted(
                    storage_client.get_data_locations(
                        session_id, [fetch_chunks[2].key])[0]),
                [(0, DataStorageDevice.DISK)])
            self.assertEqual(
                sorted(
                    storage_client.get_data_locations(session_id,
                                                      [add_chunk.key])[0]),
                [(0, DataStorageDevice.SHARED_MEMORY)])