def StartWorker(
self,
start_worker_request, # type: beam_fn_api_pb2.StartWorkerRequest
unused_context):
# type: (...) -> beam_fn_api_pb2.StartWorkerResponse
try:
if self._use_process:
command = [
'python', '-c',
'from apache_beam.runners.worker.sdk_worker '
'import SdkHarness; '
'SdkHarness('
'"%s",'
'worker_id="%s",'
'state_cache_size=%d'
'data_buffer_time_limit_ms=%d'
')'
'.run()' %
(start_worker_request.control_endpoint.url,
start_worker_request.worker_id, self._state_cache_size,
self._data_buffer_time_limit_ms)
]
if self._container_executable:
# command as per container spec
# the executable is responsible to handle concurrency
# for artifact retrieval and other side effects
command = [
self._container_executable,
'--id=%s' % start_worker_request.worker_id,
'--logging_endpoint=%s' %
start_worker_request.logging_endpoint.url,
'--artifact_endpoint=%s' %
start_worker_request.artifact_endpoint.url,
'--provision_endpoint=%s' %
start_worker_request.provision_endpoint.url,
'--control_endpoint=%s' %
start_worker_request.control_endpoint.url,
]
_LOGGER.warning("Starting worker with command %s" % command)
worker_process = subprocess.Popen(command,
stdout=subprocess.PIPE,
close_fds=True)
self._worker_processes[
start_worker_request.worker_id] = worker_process
else:
worker = sdk_worker.SdkHarness(
start_worker_request.control_endpoint.url,
worker_id=start_worker_request.worker_id,
state_cache_size=self._state_cache_size,
data_buffer_time_limit_ms=self._data_buffer_time_limit_ms)
worker_thread = threading.Thread(
name='run_worker_%s' % start_worker_request.worker_id,
target=worker.run)
worker_thread.daemon = True
worker_thread.start()
return beam_fn_api_pb2.StartWorkerResponse()
except Exception as exn:
return beam_fn_api_pb2.StartWorkerResponse(error=str(exn))
def StartWorker(self,
start_worker_request: beam_fn_api_pb2.StartWorkerRequest,
unused_context):
try:
self._start_sdk_worker_main(start_worker_request)
return beam_fn_api_pb2.StartWorkerResponse()
except Exception:
return beam_fn_api_pb2.StartWorkerResponse(
error=traceback.format_exc())
def StartWorker(self,
start_worker_request: beam_fn_api_pb2.StartWorkerRequest,
unused_context):
try:
worker_thread = threading.Thread(
name='run_worker_%s' % start_worker_request.worker_id,
target=functools.partial(self._start_sdk_worker_main,
start_worker_request))
worker_thread.daemon = True
worker_thread.start()
return beam_fn_api_pb2.StartWorkerResponse()
except Exception:
return beam_fn_api_pb2.StartWorkerResponse(
error=traceback.format_exc())