def NotifyRunnerAvailable(self, start_worker_request, context):
try:
if self._use_process:
command = ['python', '-c',
'from apache_beam.runners.worker.sdk_worker '
'import SdkHarness; '
'SdkHarness("%s",worker_count=%d,worker_id="%s").run()' % (
start_worker_request.control_endpoint.url,
self._worker_threads,
start_worker_request.worker_id)]
logging.warn("Starting worker with command %s" % (command))
worker_process = subprocess.Popen(command, stdout=subprocess.PIPE)
# Register to kill the subprocess on exit.
atexit.register(worker_process.kill)
else:
worker = sdk_worker.SdkHarness(
start_worker_request.control_endpoint.url,
worker_count=self._worker_threads,
worker_id=start_worker_request.worker_id)
worker_thread = threading.Thread(
name='run_worker_%s' % start_worker_request.worker_id,
target=worker.run)
worker_thread.daemon = True
worker_thread.start()
return beam_fn_api_pb2.NotifyRunnerAvailableResponse()
except Exception as exn:
return beam_fn_api_pb2.NotifyRunnerAvailableResponse(
error=str(exn))
def NotifyRunnerAvailable(self, start_worker_request, context):
try:
worker = sdk_worker.SdkHarness(
start_worker_request.control_endpoint.url,
worker_count=self._worker_threads,
worker_id=start_worker_request.worker_id)
worker_thread = threading.Thread(name='run_worker_%s' %
start_worker_request.worker_id,
target=worker.run)
worker_thread.daemon = True
worker_thread.start()
return beam_fn_api_pb2.NotifyRunnerAvailableResponse()
except Exception as exn:
return beam_fn_api_pb2.NotifyRunnerAvailableResponse(
error=str(exn))
