def _stage_files(self, files):
"""Utility method to stage files.
Args:
files: a list of tuples of the form [(local_name, remote_name),...]
describing the name of the artifacts in local temp folder and desired
name in staging location.
"""
server = grpc.server(UnboundedThreadPoolExecutor())
staging_service = TestLocalFileSystemArtifactStagingServiceServicer(
self._remote_dir)
beam_artifact_api_pb2_grpc.add_ArtifactStagingServiceServicer_to_server(
staging_service, server)
test_port = server.add_insecure_port('[::]:0')
server.start()
stager = portable_stager.PortableStager(
artifact_service_channel=grpc.insecure_channel('localhost:%s' %
test_port),
staging_session_token='token')
for from_file, to_file in files:
stager.stage_artifact(local_path_to_artifact=os.path.join(
self._temp_dir, from_file),
artifact_name=to_file)
stager.commit_manifest()
return staging_service.manifest.artifact, staging_service.retrieval_tokens
def _stage_files(self, files):
"""Utility method to stage files.
Args:
files: a list of tuples of the form [(local_name, remote_name),...]
describing the name of the artifacts in local temp folder and desired
name in staging location.
"""
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
staging_service = TestLocalFileSystemArtifactStagingServiceServicer(
self._remote_dir)
beam_artifact_api_pb2_grpc.add_ArtifactStagingServiceServicer_to_server(
staging_service, server)
test_port = server.add_insecure_port('[::]:0')
server.start()
stager = portable_stager.PortableStager(
artifact_service_channel=grpc.insecure_channel(
'localhost:%s' % test_port),
staging_session_token='token')
for from_file, to_file in files:
stager.stage_artifact(
local_path_to_artifact=os.path.join(self._temp_dir, from_file),
artifact_name=to_file)
stager.commit_manifest()
return staging_service.manifest.artifact, staging_service.retrieval_tokens
def start_grpc_server(self, port=0):
self._server = grpc.server(futures.ThreadPoolExecutor(max_workers=3))
port = self._server.add_insecure_port('localhost:%d' % port)
beam_job_api_pb2_grpc.add_JobServiceServicer_to_server(self, self._server)
beam_artifact_api_pb2_grpc.add_ArtifactStagingServiceServicer_to_server(
self._artifact_service, self._server)
self._artifact_staging_endpoint = endpoints_pb2.ApiServiceDescriptor(
url='localhost:%d' % port)
self._server.start()
logging.info('Grpc server started on port %s', port)
return port
def _start_artifact_service(self, jar):
self._artifact_staging_service = artifact_service.ZipFileArtifactService(
jar)
self._artifact_staging_server = grpc.server(futures.ThreadPoolExecutor())
port = self._artifact_staging_server.add_insecure_port('[::]:0')
beam_artifact_api_pb2_grpc.add_ArtifactStagingServiceServicer_to_server(
self._artifact_staging_service, self._artifact_staging_server)
self._artifact_staging_endpoint = endpoints_pb2.ApiServiceDescriptor(
url='localhost:%d' % port)
self._artifact_staging_server.start()
_LOGGER.info('Artifact server started on port %s', port)
return port
def start_grpc_server(self, port=0):
self._server = grpc.server(UnboundedThreadPoolExecutor())
port = self._server.add_insecure_port(
'%s:%d' % (self.get_bind_address(), port))
beam_job_api_pb2_grpc.add_JobServiceServicer_to_server(self, self._server)
beam_artifact_api_pb2_grpc.add_ArtifactStagingServiceServicer_to_server(
self._artifact_service, self._server)
hostname = self.get_service_address()
self._artifact_staging_endpoint = endpoints_pb2.ApiServiceDescriptor(
url='%s:%d' % (hostname, port))
self._server.start()
_LOGGER.info('Grpc server started at %s on port %d' % (hostname, port))
return port
def test_with_grpc(self):
server = grpc.server(UnboundedThreadPoolExecutor())
try:
beam_artifact_api_pb2_grpc.add_ArtifactStagingServiceServicer_to_server(
self._service, server)
beam_artifact_api_pb2_grpc.add_ArtifactRetrievalServiceServicer_to_server(
self._service, server)
port = server.add_insecure_port('[::]:0')
server.start()
channel = grpc.insecure_channel('localhost:%d' % port)
self._run_staging(
beam_artifact_api_pb2_grpc.ArtifactStagingServiceStub(channel),
beam_artifact_api_pb2_grpc.ArtifactRetrievalServiceStub(
channel))
channel.close()
finally:
server.stop(1)
def start_grpc_server(self, port=0):
no_max_message_sizes = [("grpc.max_receive_message_length", -1),
("grpc.max_send_message_length", -1)]
self._server = grpc.server(
thread_pool_executor.shared_unbounded_instance(),
options=no_max_message_sizes)
port = self._server.add_insecure_port(
'%s:%d' % (self.get_bind_address(), port))
beam_job_api_pb2_grpc.add_JobServiceServicer_to_server(self, self._server)
beam_artifact_api_pb2_grpc.add_ArtifactStagingServiceServicer_to_server(
self._artifact_service, self._server)
hostname = self.get_service_address()
self._artifact_staging_endpoint = endpoints_pb2.ApiServiceDescriptor(
url='%s:%d' % (hostname, port))
self._server.start()
_LOGGER.info('Grpc server started at %s on port %d' % (hostname, port))
return port
def _start_artifact_service(self, jar, requested_port):
self._artifact_manager = JarArtifactManager(self._jar,
self.ARTIFACT_FOLDER)
self._artifact_staging_service = artifact_service.ArtifactStagingService(
self._artifact_manager.file_writer)
self._artifact_staging_service.register_job(
self._job_id, {
env_id: env.dependencies
for (env_id, env
) in self._pipeline_proto.components.environments.items()
})
self._artifact_staging_server = grpc.server(
futures.ThreadPoolExecutor())
port = self._artifact_staging_server.add_insecure_port('[::]:%s' %
requested_port)
beam_artifact_api_pb2_grpc.add_ArtifactStagingServiceServicer_to_server(
self._artifact_staging_service, self._artifact_staging_server)
self._artifact_staging_endpoint = endpoints_pb2.ApiServiceDescriptor(
url='localhost:%d' % port)
self._artifact_staging_server.start()
_LOGGER.info('Artifact server started on port %s', port)
return port
