def start( cls, use_process=False, port=0, state_cache_size=0, data_buffer_time_limit_ms=-1, container_executable=None # type: Optional[str] ): # type: (...) -> Tuple[str, grpc.Server] worker_server = grpc.server(UnboundedThreadPoolExecutor()) worker_address = 'localhost:%s' % worker_server.add_insecure_port( '[::]:%s' % port) worker_pool = cls(use_process=use_process, container_executable=container_executable, state_cache_size=state_cache_size, data_buffer_time_limit_ms=data_buffer_time_limit_ms) beam_fn_api_pb2_grpc.add_BeamFnExternalWorkerPoolServicer_to_server( worker_pool, worker_server) worker_server.start() # Register to kill the subprocesses on exit. def kill_worker_processes(): for worker_process in worker_pool._worker_processes.values(): worker_process.kill() atexit.register(kill_worker_processes) return worker_address, worker_server
def start(cls, worker_threads=1): worker_server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) worker_address = 'localhost:%s' % worker_server.add_insecure_port('[::]:0') beam_fn_api_pb2_grpc.add_BeamFnExternalWorkerPoolServicer_to_server( cls(worker_threads), worker_server) worker_server.start() return worker_address, worker_server
def start(cls, worker_threads=1, use_process=False, port=0, state_cache_size=0, container_executable=None): worker_server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) worker_address = 'localhost:%s' % worker_server.add_insecure_port( '[::]:%s' % port) worker_pool = cls(worker_threads, use_process=use_process, container_executable=container_executable, state_cache_size=state_cache_size) beam_fn_api_pb2_grpc.add_BeamFnExternalWorkerPoolServicer_to_server( worker_pool, worker_server) worker_server.start() # Register to kill the subprocesses on exit. def kill_worker_processes(): for worker_process in worker_pool._worker_processes.values(): worker_process.kill() atexit.register(kill_worker_processes) return worker_address, worker_server
def start(cls, worker_threads=1, use_process=False): worker_server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) worker_address = 'localhost:%s' % worker_server.add_insecure_port('[::]:0') beam_fn_api_pb2_grpc.add_BeamFnExternalWorkerPoolServicer_to_server( cls(worker_threads, use_process=use_process), worker_server) worker_server.start() return worker_address, worker_server
def start( cls, use_process=False, port=0, state_cache_size=0, data_buffer_time_limit_ms=-1, container_executable=None # type: Optional[str] ): # type: (...) -> Tuple[str, grpc.Server] options = [("grpc.http2.max_pings_without_data", 0), ("grpc.http2.max_ping_strikes", 0)] worker_server = grpc.server( thread_pool_executor.shared_unbounded_instance(), options=options) worker_address = 'localhost:%s' % worker_server.add_insecure_port( '[::]:%s' % port) worker_pool = cls(use_process=use_process, container_executable=container_executable, state_cache_size=state_cache_size, data_buffer_time_limit_ms=data_buffer_time_limit_ms) beam_fn_api_pb2_grpc.add_BeamFnExternalWorkerPoolServicer_to_server( worker_pool, worker_server) worker_server.start() _LOGGER.info('Listening for workers at %s', worker_address) # Register to kill the subprocesses on exit. def kill_worker_processes(): for worker_process in worker_pool._worker_processes.values(): worker_process.kill() atexit.register(kill_worker_processes) return worker_address, worker_server
def start(self): if not self._worker_address: worker_server = grpc.server( thread_pool_executor.shared_unbounded_instance()) worker_address = 'localhost:%s' % worker_server.add_insecure_port( '[::]:0') beam_fn_api_pb2_grpc.add_BeamFnExternalWorkerPoolServicer_to_server( self, worker_server) worker_server.start() self._worker_address = worker_address atexit.register(functools.partial(worker_server.stop, 1)) return self._worker_address