def start(
cls,
use_process=False,
port=0,
state_cache_size=0,
data_buffer_time_limit_ms=-1,
container_executable=None # type: Optional[str]
):
# type: (...) -> Tuple[str, grpc.Server]
worker_server = grpc.server(UnboundedThreadPoolExecutor())
worker_address = 'localhost:%s' % worker_server.add_insecure_port(
'[::]:%s' % port)
worker_pool = cls(use_process=use_process,
container_executable=container_executable,
state_cache_size=state_cache_size,
data_buffer_time_limit_ms=data_buffer_time_limit_ms)
beam_fn_api_pb2_grpc.add_BeamFnExternalWorkerPoolServicer_to_server(
worker_pool, worker_server)
worker_server.start()
# Register to kill the subprocesses on exit.
def kill_worker_processes():
for worker_process in worker_pool._worker_processes.values():
worker_process.kill()
atexit.register(kill_worker_processes)
return worker_address, worker_server
def start(cls, worker_threads=1):
worker_server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
worker_address = 'localhost:%s' % worker_server.add_insecure_port('[::]:0')
beam_fn_api_pb2_grpc.add_BeamFnExternalWorkerPoolServicer_to_server(
cls(worker_threads), worker_server)
worker_server.start()
return worker_address, worker_server
def start(cls,
worker_threads=1,
use_process=False,
port=0,
state_cache_size=0,
container_executable=None):
worker_server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
worker_address = 'localhost:%s' % worker_server.add_insecure_port(
'[::]:%s' % port)
worker_pool = cls(worker_threads,
use_process=use_process,
container_executable=container_executable,
state_cache_size=state_cache_size)
beam_fn_api_pb2_grpc.add_BeamFnExternalWorkerPoolServicer_to_server(
worker_pool, worker_server)
worker_server.start()
# Register to kill the subprocesses on exit.
def kill_worker_processes():
for worker_process in worker_pool._worker_processes.values():
worker_process.kill()
atexit.register(kill_worker_processes)
return worker_address, worker_server
def start(cls, worker_threads=1, use_process=False):
worker_server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
worker_address = 'localhost:%s' % worker_server.add_insecure_port('[::]:0')
beam_fn_api_pb2_grpc.add_BeamFnExternalWorkerPoolServicer_to_server(
cls(worker_threads, use_process=use_process), worker_server)
worker_server.start()
return worker_address, worker_server
def start(
cls,
use_process=False,
port=0,
state_cache_size=0,
data_buffer_time_limit_ms=-1,
container_executable=None # type: Optional[str]
):
# type: (...) -> Tuple[str, grpc.Server]
options = [("grpc.http2.max_pings_without_data", 0),
("grpc.http2.max_ping_strikes", 0)]
worker_server = grpc.server(
thread_pool_executor.shared_unbounded_instance(), options=options)
worker_address = 'localhost:%s' % worker_server.add_insecure_port(
'[::]:%s' % port)
worker_pool = cls(use_process=use_process,
container_executable=container_executable,
state_cache_size=state_cache_size,
data_buffer_time_limit_ms=data_buffer_time_limit_ms)
beam_fn_api_pb2_grpc.add_BeamFnExternalWorkerPoolServicer_to_server(
worker_pool, worker_server)
worker_server.start()
_LOGGER.info('Listening for workers at %s', worker_address)
# Register to kill the subprocesses on exit.
def kill_worker_processes():
for worker_process in worker_pool._worker_processes.values():
worker_process.kill()
atexit.register(kill_worker_processes)
return worker_address, worker_server
def start(self):
if not self._worker_address:
worker_server = grpc.server(
thread_pool_executor.shared_unbounded_instance())
worker_address = 'localhost:%s' % worker_server.add_insecure_port(
'[::]:0')
beam_fn_api_pb2_grpc.add_BeamFnExternalWorkerPoolServicer_to_server(
self, worker_server)
worker_server.start()
self._worker_address = worker_address
atexit.register(functools.partial(worker_server.stop, 1))
return self._worker_address
