def testHolder(self): pool_address = '127.0.0.1:%d' % get_next_port() with create_actor_pool(n_process=1, backend='gevent', address=pool_address) as pool: pool.create_actor(PlasmaKeyMapActor, uid=PlasmaKeyMapActor.default_name()) pool.create_actor(WorkerClusterInfoActor, schedulers=[pool_address], uid=WorkerClusterInfoActor.default_name()) pool.create_actor(KVStoreActor, uid=KVStoreActor.default_name()) pool.create_actor(DispatchActor, uid=DispatchActor.default_name()) pool.create_actor(QuotaActor, 1024 * 1024 * 10, uid=MemQuotaActor.default_name()) cache_ref = pool.create_actor(ChunkHolderActor, self.plasma_storage_size, uid=ChunkHolderActor.default_name()) pool.create_actor(SpillActor) try: test_ref = pool.create_actor(CacheTestActor) test_ref.run_test_cache() while not test_ref.get_exc_info()[0]: pool.sleep(0.1) exc_info = test_ref.get_exc_info()[1] if exc_info: six.reraise(*exc_info) finally: pool.destroy_actor(cache_ref)
def testExecute(self): pool_address = '127.0.0.1:%d' % get_next_port() with create_actor_pool(n_process=1, backend='gevent', address=pool_address) as pool: pool.create_actor(ClusterInfoActor, schedulers=[pool_address], uid=ClusterInfoActor.default_name()) cache_ref = pool.create_actor(ChunkHolderActor, self._plasma_helper._size, uid='ChunkHolderActor') pool.create_actor(KVStoreActor, uid=KVStoreActor.default_name()) pool.create_actor(DispatchActor, uid='DispatchActor') pool.create_actor(QuotaActor, 1024 * 1024, uid='MemQuotaActor') pool.create_actor(CpuCalcActor) pool.create_actor(ExecutionActor, uid='ExecutionActor') try: test_ref = pool.create_actor(ExecuteTestActor) test_ref.run_test() while not test_ref.get_exc_info()[0]: gevent.sleep(0.1) exc_info = test_ref.get_exc_info()[1] if exc_info: six.reraise(*exc_info) finally: pool.destroy_actor(cache_ref)
def get_results(self): if not self._results: return None if self._results[0][0]: return self._shared_store.get(self._session_id, self._array_key) else: six.reraise(*self._results[0][1])
def get_result(self): if self._accept is None: return None elif self._accept: return self._result else: six.reraise(*self._result)
def get_result(self): if not self._result: raise ValueError val, accept = self._result if not accept: six.reraise(*val) else: return val
def get_result(self, timeout=None): self._result_event.wait(timeout) self._result_event.clear() r, accept = self._result_store if accept: return r else: six.reraise(*r)
def _handle_reject(*exc): if issubclass(exc[0], NoDataToSpill): return six.reraise(*exc)
def testSimpleTransfer(self): import tempfile session_id = str(uuid.uuid4()) local_pool_addr = 'localhost:%d' % get_next_port() remote_pool_addr = 'localhost:%d' % get_next_port() remote_chunk_keys = [str(uuid.uuid4()) for _ in range(9)] msg_queue = multiprocessing.Queue() remote_plasma_socket = '/tmp/plasma_%d_%d.sock' % (os.getpid(), id(run_transfer_worker)) remote_spill_dir = os.path.join(tempfile.gettempdir(), 'mars_spill_%d_%d' % (os.getpid(), id(run_transfer_worker))) proc = multiprocessing.Process( target=run_transfer_worker, args=(remote_pool_addr, session_id, remote_plasma_socket, remote_chunk_keys, remote_spill_dir, msg_queue) ) proc.start() try: msg_queue.get(30) except: if proc.is_alive(): proc.terminate() raise with create_actor_pool(n_process=1, distributor=WorkerDistributor(3), backend='gevent', address=local_pool_addr) as pool: pool.create_actor(ClusterInfoActor, schedulers=[local_pool_addr], uid=ClusterInfoActor.default_name()) pool.create_actor(KVStoreActor, uid=KVStoreActor.default_name()) pool.create_actor(DispatchActor, uid=DispatchActor.default_name()) pool.create_actor(QuotaActor, 1024 * 1024 * 20, uid=MemQuotaActor.default_name()) cache_ref = pool.create_actor(ChunkHolderActor, self.plasma_storage_size, uid=ChunkHolderActor.default_name()) pool.create_actor(SpillActor) sender_refs = [ pool.create_actor(SenderActor, uid='w:1:%s' % str(uuid.uuid4())), pool.create_actor(SenderActor, uid='w:2:%s' % str(uuid.uuid4())), ] receiver_refs = [ pool.create_actor(ReceiverActor, uid='w:1:%s' % str(uuid.uuid4())), pool.create_actor(ReceiverActor, uid='w:1:%s' % str(uuid.uuid4())), pool.create_actor(ReceiverActor, uid='w:2:%s' % str(uuid.uuid4())), pool.create_actor(ReceiverActor, uid='w:2:%s' % str(uuid.uuid4())), ] test_ref = pool.create_actor(TransferTestActor, local_pool_addr, remote_pool_addr, remote_plasma_socket, remote_spill_dir) try: for data_id in (-1, 1): chunk_key = remote_chunk_keys[data_id] test_ref.do_transfer_test(session_id, chunk_key) check_time = time.time() while test_ref.get_results()[0] != chunk_key: gevent.sleep(0.5) if not proc.is_alive(): raise SystemError('Transfer worker dead. exit code %s' % proc.exitcode) if time.time() - check_time > 60: raise SystemError('Wait result timeout') exc = test_ref.get_results()[1] if exc: six.reraise(*exc) remote_holder_ref = pool.actor_ref('HolderActor', address=remote_pool_addr) remote_holder_ref.trigger() finally: for ref in sender_refs: pool.destroy_actor(ref) for ref in receiver_refs: pool.destroy_actor(ref) pool.destroy_actor(cache_ref) pool.destroy_actor(test_ref) os.unlink(remote_plasma_socket) if proc.is_alive(): proc.terminate()
def _handle_reject(*exc): if issubclass(exc[0], SpillExhausted): return six.reraise(*exc)