def __init__(self, staging_dir=None): self._jobs = {} self._cleanup_staging_dir = staging_dir is None self._staging_dir = staging_dir or tempfile.mkdtemp() self._artifact_service = artifact_service.BeamFilesystemArtifactService( self._staging_dir) self._artifact_staging_endpoint = None
def __init__(self, staging_dir=None): super(LocalJobServicer, self).__init__() self._cleanup_staging_dir = staging_dir is None self._staging_dir = staging_dir or tempfile.mkdtemp() self._artifact_service = artifact_service.BeamFilesystemArtifactService( self._staging_dir) self._artifact_staging_endpoint = None
def __init__(self, staging_dir=None): super(LocalJobServicer, self).__init__() self._cleanup_staging_dir = staging_dir is None self._staging_dir = staging_dir or tempfile.mkdtemp() self._artifact_service = artifact_service.BeamFilesystemArtifactService( self._staging_dir) self._artifact_staging_endpoint = None # type: Optional[endpoints_pb2.ApiServiceDescriptor]
def __init__( self, state, # type: StateServicer provision_info, # type: Optional[ExtendedProvisionInfo] worker_manager, # type: WorkerHandlerManager ): # type: (...) -> None self.state = state self.provision_info = provision_info self.control_server = grpc.server(UnboundedThreadPoolExecutor()) self.control_port = self.control_server.add_insecure_port('[::]:0') self.control_address = 'localhost:%s' % self.control_port # Options to have no limits (-1) on the size of the messages # received or sent over the data plane. The actual buffer size # is controlled in a layer above. no_max_message_sizes = [("grpc.max_receive_message_length", -1), ("grpc.max_send_message_length", -1)] self.data_server = grpc.server(UnboundedThreadPoolExecutor(), options=no_max_message_sizes) self.data_port = self.data_server.add_insecure_port('[::]:0') self.state_server = grpc.server(UnboundedThreadPoolExecutor(), options=no_max_message_sizes) self.state_port = self.state_server.add_insecure_port('[::]:0') self.control_handler = BeamFnControlServicer(worker_manager) beam_fn_api_pb2_grpc.add_BeamFnControlServicer_to_server( self.control_handler, self.control_server) # If we have provision info, serve these off the control port as well. if self.provision_info: if self.provision_info.provision_info: beam_provision_api_pb2_grpc.add_ProvisionServiceServicer_to_server( BasicProvisionService(self.provision_info.provision_info, worker_manager), self.control_server) if self.provision_info.artifact_staging_dir: service = artifact_service.BeamFilesystemArtifactService( self.provision_info.artifact_staging_dir ) # type: beam_artifact_api_pb2_grpc.LegacyArtifactRetrievalServiceServicer else: service = EmptyArtifactRetrievalService() beam_artifact_api_pb2_grpc.add_LegacyArtifactRetrievalServiceServicer_to_server( service, self.control_server) self.data_plane_handler = data_plane.BeamFnDataServicer( DATA_BUFFER_TIME_LIMIT_MS) beam_fn_api_pb2_grpc.add_BeamFnDataServicer_to_server( self.data_plane_handler, self.data_server) beam_fn_api_pb2_grpc.add_BeamFnStateServicer_to_server( GrpcStateServicer(state), self.state_server) self.logging_server = grpc.server(UnboundedThreadPoolExecutor(), options=no_max_message_sizes) self.logging_port = self.logging_server.add_insecure_port('[::]:0') beam_fn_api_pb2_grpc.add_BeamFnLoggingServicer_to_server( BasicLoggingService(), self.logging_server) _LOGGER.info('starting control server on port %s', self.control_port) _LOGGER.info('starting data server on port %s', self.data_port) _LOGGER.info('starting state server on port %s', self.state_port) _LOGGER.info('starting logging server on port %s', self.logging_port) self.logging_server.start() self.state_server.start() self.data_server.start() self.control_server.start()
def create_service(self, staging_dir): return artifact_service.BeamFilesystemArtifactService(staging_dir, chunk_size=10)
def setUp(self): self._staging_dir = tempfile.mkdtemp() self._service = ( artifact_service.BeamFilesystemArtifactService( self._staging_dir, chunk_size=10))