def StartWorker( self, start_worker_request, # type: beam_fn_api_pb2.StartWorkerRequest unused_context): # type: (...) -> beam_fn_api_pb2.StartWorkerResponse try: if self._use_process: command = [ 'python', '-c', 'from apache_beam.runners.worker.sdk_worker ' 'import SdkHarness; ' 'SdkHarness(' '"%s",' 'worker_id="%s",' 'state_cache_size=%d' 'data_buffer_time_limit_ms=%d' ')' '.run()' % (start_worker_request.control_endpoint.url, start_worker_request.worker_id, self._state_cache_size, self._data_buffer_time_limit_ms) ] if self._container_executable: # command as per container spec # the executable is responsible to handle concurrency # for artifact retrieval and other side effects command = [ self._container_executable, '--id=%s' % start_worker_request.worker_id, '--logging_endpoint=%s' % start_worker_request.logging_endpoint.url, '--artifact_endpoint=%s' % start_worker_request.artifact_endpoint.url, '--provision_endpoint=%s' % start_worker_request.provision_endpoint.url, '--control_endpoint=%s' % start_worker_request.control_endpoint.url, ] _LOGGER.warning("Starting worker with command %s" % command) worker_process = subprocess.Popen(command, stdout=subprocess.PIPE, close_fds=True) self._worker_processes[ start_worker_request.worker_id] = worker_process else: worker = sdk_worker.SdkHarness( start_worker_request.control_endpoint.url, worker_id=start_worker_request.worker_id, state_cache_size=self._state_cache_size, data_buffer_time_limit_ms=self._data_buffer_time_limit_ms) worker_thread = threading.Thread( name='run_worker_%s' % start_worker_request.worker_id, target=worker.run) worker_thread.daemon = True worker_thread.start() return beam_fn_api_pb2.StartWorkerResponse() except Exception as exn: return beam_fn_api_pb2.StartWorkerResponse(error=str(exn))
def StartWorker(self, start_worker_request: beam_fn_api_pb2.StartWorkerRequest, unused_context): try: self._start_sdk_worker_main(start_worker_request) return beam_fn_api_pb2.StartWorkerResponse() except Exception: return beam_fn_api_pb2.StartWorkerResponse( error=traceback.format_exc())
def StartWorker(self, start_worker_request: beam_fn_api_pb2.StartWorkerRequest, unused_context): try: worker_thread = threading.Thread( name='run_worker_%s' % start_worker_request.worker_id, target=functools.partial(self._start_sdk_worker_main, start_worker_request)) worker_thread.daemon = True worker_thread.start() return beam_fn_api_pb2.StartWorkerResponse() except Exception: return beam_fn_api_pb2.StartWorkerResponse( error=traceback.format_exc())