def run(self): logging_server = grpc.server( futures.ThreadPoolExecutor(max_workers=10)) logging_port = logging_server.add_insecure_port('[::]:0') logging_server.start() logging_servicer = BeamFnLoggingServicer() beam_fn_api_pb2_grpc.add_BeamFnLoggingServicer_to_server( logging_servicer, logging_server) logging_descriptor = text_format.MessageToString( endpoints_pb2.ApiServiceDescriptor(url='localhost:%s' % logging_port)) control_descriptor = text_format.MessageToString( endpoints_pb2.ApiServiceDescriptor(url=self._control_address)) p = subprocess.Popen( self._worker_command_line, shell=True, env=dict(os.environ, CONTROL_API_SERVICE_DESCRIPTOR=control_descriptor, LOGGING_API_SERVICE_DESCRIPTOR=logging_descriptor)) try: p.wait() if p.returncode: raise RuntimeError( 'Worker subprocess exited with return code %s' % p.returncode) finally: if p.poll() is None: p.kill() logging_server.stop(0)
def run(self): logging_server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) logging_port = logging_server.add_insecure_port('[::]:0') logging_server.start() logging_servicer = BeamFnLoggingServicer() beam_fn_api_pb2_grpc.add_BeamFnLoggingServicer_to_server( logging_servicer, logging_server) logging_descriptor = text_format.MessageToString( endpoints_pb2.ApiServiceDescriptor(url='localhost:%s' % logging_port)) control_descriptor = text_format.MessageToString( endpoints_pb2.ApiServiceDescriptor(url=self._control_address)) p = subprocess.Popen( self._worker_command_line, shell=True, env=dict( os.environ, CONTROL_API_SERVICE_DESCRIPTOR=control_descriptor, LOGGING_API_SERVICE_DESCRIPTOR=logging_descriptor)) try: p.wait() if p.returncode: raise RuntimeError( 'Worker subprocess exited with return code %s' % p.returncode) finally: if p.poll() is None: p.kill() logging_server.stop(0)
def run(self): logging_server = grpc.server(UnboundedThreadPoolExecutor()) logging_port = logging_server.add_insecure_port('[::]:0') logging_server.start() logging_servicer = BeamFnLoggingServicer() beam_fn_api_pb2_grpc.add_BeamFnLoggingServicer_to_server( logging_servicer, logging_server) logging_descriptor = text_format.MessageToString( endpoints_pb2.ApiServiceDescriptor(url='localhost:%s' % logging_port)) control_descriptor = text_format.MessageToString( endpoints_pb2.ApiServiceDescriptor(url=self._control_address)) env_dict = dict(os.environ, CONTROL_API_SERVICE_DESCRIPTOR=control_descriptor, LOGGING_API_SERVICE_DESCRIPTOR=logging_descriptor) # only add worker_id when it is set. if self._worker_id: env_dict['WORKER_ID'] = self._worker_id with fn_api_runner.SUBPROCESS_LOCK: p = subprocess.Popen(self._worker_command_line, shell=True, env=env_dict) try: p.wait() if p.returncode: raise RuntimeError( 'Worker subprocess exited with return code %s' % p.returncode) finally: if p.poll() is None: p.kill() logging_server.stop(0)
def setUp(self): self.test_logging_service = BeamFnLoggingServicer() self.server = grpc.server(UnboundedThreadPoolExecutor()) beam_fn_api_pb2_grpc.add_BeamFnLoggingServicer_to_server( self.test_logging_service, self.server) self.test_port = self.server.add_insecure_port('[::]:0') self.server.start() self.logging_service_descriptor = endpoints_pb2.ApiServiceDescriptor() self.logging_service_descriptor.url = 'localhost:%s' % self.test_port self.fn_log_handler = log_handler.FnApiLogRecordHandler( self.logging_service_descriptor) logging.getLogger().setLevel(logging.INFO) logging.getLogger().addHandler(self.fn_log_handler)
def setUp(self): self.test_logging_service = BeamFnLoggingServicer() self.server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) beam_fn_api_pb2_grpc.add_BeamFnLoggingServicer_to_server( self.test_logging_service, self.server) self.test_port = self.server.add_insecure_port('[::]:0') self.server.start() self.logging_service_descriptor = endpoints_pb2.ApiServiceDescriptor() self.logging_service_descriptor.url = 'localhost:%s' % self.test_port self.fn_log_handler = log_handler.FnApiLogRecordHandler( self.logging_service_descriptor) logging.getLogger().setLevel(logging.INFO) logging.getLogger().addHandler(self.fn_log_handler)
def run(self): options = [("grpc.http2.max_pings_without_data", 0), ("grpc.http2.max_ping_strikes", 0)] logging_server = grpc.server( thread_pool_executor.shared_unbounded_instance(), options=options) logging_port = logging_server.add_insecure_port('[::]:0') logging_server.start() logging_servicer = BeamFnLoggingServicer() beam_fn_api_pb2_grpc.add_BeamFnLoggingServicer_to_server( logging_servicer, logging_server) logging_descriptor = text_format.MessageToString( endpoints_pb2.ApiServiceDescriptor(url='localhost:%s' % logging_port)) control_descriptor = text_format.MessageToString( endpoints_pb2.ApiServiceDescriptor(url=self._control_address)) pipeline_options = json_format.MessageToJson( self._provision_info.provision_info.pipeline_options) env_dict = dict(os.environ, CONTROL_API_SERVICE_DESCRIPTOR=control_descriptor, LOGGING_API_SERVICE_DESCRIPTOR=logging_descriptor, PIPELINE_OPTIONS=pipeline_options) # only add worker_id when it is set. if self._worker_id: env_dict['WORKER_ID'] = self._worker_id with worker_handlers.SUBPROCESS_LOCK: p = subprocess.Popen(self._worker_command_line, shell=True, env=env_dict) try: p.wait() if p.returncode: raise RuntimeError( 'Worker subprocess exited with return code %s' % p.returncode) finally: if p.poll() is None: p.kill() logging_server.stop(0)
def __init__( self, state, # type: StateServicer provision_info, # type: Optional[ExtendedProvisionInfo] worker_manager, # type: WorkerHandlerManager ): # type: (...) -> None self.state = state self.provision_info = provision_info self.control_server = grpc.server( thread_pool_executor.shared_unbounded_instance()) self.control_port = self.control_server.add_insecure_port('[::]:0') self.control_address = 'localhost:%s' % self.control_port # Options to have no limits (-1) on the size of the messages # received or sent over the data plane. The actual buffer size # is controlled in a layer above. no_max_message_sizes = [("grpc.max_receive_message_length", -1), ("grpc.max_send_message_length", -1)] self.data_server = grpc.server( thread_pool_executor.shared_unbounded_instance(), options=no_max_message_sizes) self.data_port = self.data_server.add_insecure_port('[::]:0') self.state_server = grpc.server( thread_pool_executor.shared_unbounded_instance(), options=no_max_message_sizes) self.state_port = self.state_server.add_insecure_port('[::]:0') self.control_handler = BeamFnControlServicer(worker_manager) beam_fn_api_pb2_grpc.add_BeamFnControlServicer_to_server( self.control_handler, self.control_server) # If we have provision info, serve these off the control port as well. if self.provision_info: if self.provision_info.provision_info: beam_provision_api_pb2_grpc.add_ProvisionServiceServicer_to_server( BasicProvisionService(self.provision_info.provision_info, worker_manager), self.control_server) def open_uncompressed(f): # type: (str) -> BinaryIO return filesystems.FileSystems.open( f, compression_type=CompressionTypes.UNCOMPRESSED) beam_artifact_api_pb2_grpc.add_ArtifactRetrievalServiceServicer_to_server( artifact_service.ArtifactRetrievalService( file_reader=open_uncompressed), self.control_server) self.data_plane_handler = data_plane.BeamFnDataServicer( DATA_BUFFER_TIME_LIMIT_MS) beam_fn_api_pb2_grpc.add_BeamFnDataServicer_to_server( self.data_plane_handler, self.data_server) beam_fn_api_pb2_grpc.add_BeamFnStateServicer_to_server( GrpcStateServicer(state), self.state_server) self.logging_server = grpc.server( thread_pool_executor.shared_unbounded_instance(), options=no_max_message_sizes) self.logging_port = self.logging_server.add_insecure_port('[::]:0') beam_fn_api_pb2_grpc.add_BeamFnLoggingServicer_to_server( BasicLoggingService(), self.logging_server) _LOGGER.info('starting control server on port %s', self.control_port) _LOGGER.info('starting data server on port %s', self.data_port) _LOGGER.info('starting state server on port %s', self.state_port) _LOGGER.info('starting logging server on port %s', self.logging_port) self.logging_server.start() self.state_server.start() self.data_server.start() self.control_server.start()
def __init__( self, state, # type: StateServicer provision_info, # type: Optional[ExtendedProvisionInfo] worker_manager, # type: WorkerHandlerManager ): # type: (...) -> None self.state = state self.provision_info = provision_info self.control_server = grpc.server(UnboundedThreadPoolExecutor()) self.control_port = self.control_server.add_insecure_port('[::]:0') self.control_address = 'localhost:%s' % self.control_port # Options to have no limits (-1) on the size of the messages # received or sent over the data plane. The actual buffer size # is controlled in a layer above. no_max_message_sizes = [("grpc.max_receive_message_length", -1), ("grpc.max_send_message_length", -1)] self.data_server = grpc.server(UnboundedThreadPoolExecutor(), options=no_max_message_sizes) self.data_port = self.data_server.add_insecure_port('[::]:0') self.state_server = grpc.server(UnboundedThreadPoolExecutor(), options=no_max_message_sizes) self.state_port = self.state_server.add_insecure_port('[::]:0') self.control_handler = BeamFnControlServicer(worker_manager) beam_fn_api_pb2_grpc.add_BeamFnControlServicer_to_server( self.control_handler, self.control_server) # If we have provision info, serve these off the control port as well. if self.provision_info: if self.provision_info.provision_info: beam_provision_api_pb2_grpc.add_ProvisionServiceServicer_to_server( BasicProvisionService(self.provision_info.provision_info, worker_manager), self.control_server) if self.provision_info.artifact_staging_dir: service = artifact_service.BeamFilesystemArtifactService( self.provision_info.artifact_staging_dir ) # type: beam_artifact_api_pb2_grpc.LegacyArtifactRetrievalServiceServicer else: service = EmptyArtifactRetrievalService() beam_artifact_api_pb2_grpc.add_LegacyArtifactRetrievalServiceServicer_to_server( service, self.control_server) self.data_plane_handler = data_plane.BeamFnDataServicer( DATA_BUFFER_TIME_LIMIT_MS) beam_fn_api_pb2_grpc.add_BeamFnDataServicer_to_server( self.data_plane_handler, self.data_server) beam_fn_api_pb2_grpc.add_BeamFnStateServicer_to_server( GrpcStateServicer(state), self.state_server) self.logging_server = grpc.server(UnboundedThreadPoolExecutor(), options=no_max_message_sizes) self.logging_port = self.logging_server.add_insecure_port('[::]:0') beam_fn_api_pb2_grpc.add_BeamFnLoggingServicer_to_server( BasicLoggingService(), self.logging_server) _LOGGER.info('starting control server on port %s', self.control_port) _LOGGER.info('starting data server on port %s', self.data_port) _LOGGER.info('starting state server on port %s', self.state_port) _LOGGER.info('starting logging server on port %s', self.logging_port) self.logging_server.start() self.state_server.start() self.data_server.start() self.control_server.start()
def __init__(self, state, provision_info): self.state = state self.provision_info = provision_info self.control_server = grpc.server( futures.ThreadPoolExecutor(max_workers=10)) self.control_port = self.control_server.add_insecure_port('[::]:0') self.control_address = 'localhost:%s' % self.control_port # Options to have no limits (-1) on the size of the messages # received or sent over the data plane. The actual buffer size # is controlled in a layer above. no_max_message_sizes = [("grpc.max_receive_message_length", -1), ("grpc.max_send_message_length", -1)] self.data_server = grpc.server( futures.ThreadPoolExecutor(max_workers=10), options=no_max_message_sizes) self.data_port = self.data_server.add_insecure_port('[::]:0') self.state_server = grpc.server( futures.ThreadPoolExecutor(max_workers=10), options=no_max_message_sizes) self.state_port = self.state_server.add_insecure_port('[::]:0') self.control_handler = BeamFnControlServicer() beam_fn_api_pb2_grpc.add_BeamFnControlServicer_to_server( self.control_handler, self.control_server) # If we have provision info, serve these off the control port as well. if self.provision_info: if self.provision_info.provision_info: provision_info = self.provision_info.provision_info if not provision_info.worker_id: provision_info = copy.copy(provision_info) provision_info.worker_id = str(uuid.uuid4()) beam_provision_api_pb2_grpc.add_ProvisionServiceServicer_to_server( BasicProvisionService(self.provision_info.provision_info), self.control_server) if self.provision_info.artifact_staging_dir: m = beam_artifact_api_pb2_grpc m.add_ArtifactRetrievalServiceServicer_to_server( artifact_service.BeamFilesystemArtifactService( self.provision_info.artifact_staging_dir), self.control_server) self.data_plane_handler = data_plane.GrpcServerDataChannel() beam_fn_api_pb2_grpc.add_BeamFnDataServicer_to_server( self.data_plane_handler, self.data_server) beam_fn_api_pb2_grpc.add_BeamFnStateServicer_to_server( FnApiRunner.GrpcStateServicer(state), self.state_server) self.logging_server = grpc.server( futures.ThreadPoolExecutor(max_workers=2), options=no_max_message_sizes) self.logging_port = self.logging_server.add_insecure_port('[::]:0') beam_fn_api_pb2_grpc.add_BeamFnLoggingServicer_to_server( BasicLoggingService(), self.logging_server) logging.info('starting control server on port %s', self.control_port) logging.info('starting data server on port %s', self.data_port) logging.info('starting state server on port %s', self.state_port) logging.info('starting logging server on port %s', self.logging_port) self.logging_server.start() self.state_server.start() self.data_server.start() self.control_server.start()