def __init__(
self,
db_url=None,
repo_base_url=None,
s3_endpoint_url=None,
default_namespace=None,
):
cfg = config('yatai_service')
repo_base_url = repo_base_url or cfg.get('repository_base_url')
db_url = db_url or cfg.get('db_url')
s3_endpoint_url = s3_endpoint_url or cfg.get('s3_endpoint_url') or None
default_namespace = default_namespace or cfg.get('default_namespace')
self.default_namespace = default_namespace
self.repo = Repository(repo_base_url, s3_endpoint_url)
self.db = DB(db_url)
def start_yatai_service_grpc_server(
db_url,
grpc_port,
ui_port,
with_ui,
base_url,
repository_type,
file_system_directory,
s3_url,
s3_endpoint_url,
gcs_url,
web_ui_log_path: str = Provide[BentoMLContainer.yatai_logging_path],
):
# Lazily import grpcio for YataiSerivce gRPC related actions
import grpc
from bentoml.yatai.db import DB
from bentoml.yatai.repository import create_repository
from bentoml.yatai.yatai_service_impl import get_yatai_service_impl
from bentoml.yatai.proto.yatai_service_pb2_grpc import add_YataiServicer_to_server
from bentoml.yatai.proto.yatai_service_pb2_grpc import YataiServicer
YataiServicerImpl = get_yatai_service_impl(YataiServicer)
yatai_service = YataiServicerImpl(
repository=create_repository(repository_type, file_system_directory,
s3_url, s3_endpoint_url, gcs_url),
database=DB(db_url),
)
# Define interceptors here
grpc_interceptors = [PromServerInterceptor(), ServiceLatencyInterceptor()]
server = grpc.server(
futures.ThreadPoolExecutor(max_workers=10),
interceptors=grpc_interceptors,
)
add_YataiServicer_to_server(yatai_service, server)
debug_mode = get_debug_mode()
if debug_mode:
try:
logger.debug("Enabling gRPC server reflection for debugging")
from bentoml.yatai.proto import yatai_service_pb2
from grpc_reflection.v1alpha import reflection
SERVICE_NAMES = (
yatai_service_pb2.DESCRIPTOR.services_by_name["Yatai"].
full_name,
reflection.SERVICE_NAME,
)
reflection.enable_server_reflection(SERVICE_NAMES, server)
except ImportError:
logger.debug(
"Failed to enable gRPC server reflection, missing required package: "
'"pip install grpcio-reflection"')
server.add_insecure_port(f"[::]:{grpc_port}")
# NOTE: the current implementation sets prometheus_port to
# 50052 to accomodate with Makefile setups. Currently there
# isn't a way to find the reserve_free_port dynamically inside
# Makefile to find the free ports for prometheus_port without
# the help of a shell scripts.
prometheus_port = 50052
with reserve_free_port() as port:
prometheus_port = port
# prevents wsgi to see prometheus_port as used
start_http_server(prometheus_port)
server.start()
if with_ui:
ensure_node_available_or_raise()
yatai_grpc_server_address = f"localhost:{grpc_port}"
prometheus_address = f"http://localhost:{prometheus_port}"
async_start_yatai_service_web_ui(
yatai_grpc_server_address,
prometheus_address,
ui_port,
web_ui_log_path,
debug_mode,
base_url,
)
# We don't import _echo function from click_utils because of circular dep
if with_ui:
if debug_mode is True:
ui_port = 8080
web_ui_link = f"http://127.0.0.1:{ui_port}"
if base_url != ".":
web_ui_link += f"/{base_url}"
web_ui_message = f"running on {web_ui_link}"
else:
web_ui_message = "off"
if debug_mode:
prom_ui_message = "off"
else:
prom_ui_message = f"running on http://127.0.0.1:{ui_port}/metrics\n"
click.echo(
f"* Starting BentoML YataiService gRPC Server\n"
f'* Debug mode: { "on" if debug_mode else "off"}\n'
f"* Web UI: {web_ui_message}\n"
f"* Running on 127.0.0.1:{grpc_port} (Press CTRL+C to quit)\n"
f"* Prometheus: {prom_ui_message}\n"
f"* Help and instructions: "
f"https://docs.bentoml.org/en/latest/guides/yatai_service.html\n"
f'{f"* Web server log can be found here: {web_ui_log_path}" if with_ui else ""}'
f"\n-----\n"
f"* Usage in Python:\n"
f'* bento_svc.save(yatai_url="127.0.0.1:{grpc_port}")\n'
f"* from bentoml.yatai.client import get_yatai_client\n"
f'* get_yatai_client("127.0.0.1:{grpc_port}").repository.list()\n'
f"* Usage in CLI:\n"
f"* bentoml list --yatai-url=127.0.0.1:{grpc_port}\n"
f"* bentoml containerize IrisClassifier:latest --yatai-url=127.0.0.1:"
f"{grpc_port}\n"
f"* bentoml push IrisClassifier:20200918001645_CD2886 --yatai-url=127.0.0.1:"
f"{grpc_port}\n"
f"* bentoml pull IrisClassifier:20200918001645_CD2886 --yatai-url=127.0.0.1:"
f"{grpc_port}\n"
f"* bentoml retrieve IrisClassifier:20200918001645_CD2886 "
f'--yatai-url=127.0.0.1:{grpc_port} --target_dir="/tmp/foo/bar"\n'
f"* bentoml delete IrisClassifier:20200918001645_CD2886 "
f"--yatai-url=127.0.0.1:{grpc_port}\n"
# TODO: simplify the example usage here once related documentation is ready
)
try:
while True:
time.sleep(_ONE_DAY_IN_SECONDS)
except KeyboardInterrupt:
logger.info("Terminating YataiService gRPC server..")
server.stop(grace=None)
def get_yatai_service(
channel_address: str = Provide[BentoMLContainer.config.yatai.remote.url],
access_token: str = Provide[
BentoMLContainer.config.yatai.remote.access_token],
access_token_header: str = Provide[
BentoMLContainer.config.yatai.remote.access_token_header],
tls_root_ca_cert: str = Provide[BentoMLContainer.yatai_tls_root_ca_cert],
tls_client_key: str = Provide[
BentoMLContainer.config.yatai.remote.tls.client_key],
tls_client_cert: str = Provide[
BentoMLContainer.config.yatai.remote.tls.client_cert],
db_url: str = Provide[BentoMLContainer.yatai_database_url],
default_namespace: str = Provide[BentoMLContainer.config.yatai.namespace],
repository_type: str = Provide[
BentoMLContainer.config.yatai.repository.type],
file_system_directory: str = Provide[
BentoMLContainer.yatai_file_system_directory],
s3_url: str = Provide[BentoMLContainer.config.yatai.repository.s3.url],
gcs_url: str = Provide[BentoMLContainer.config.yatai.repository.gcs.url],
):
if channel_address:
# Lazily import grpcio for YataiSerivce gRPC related actions
import grpc
from bentoml.yatai.client.interceptor import header_client_interceptor
from bentoml.yatai.proto.yatai_service_pb2_grpc import YataiStub
channel_address = channel_address.strip()
logger.debug("Connecting YataiService gRPC server at: %s",
channel_address)
scheme, addr = parse_grpc_url(channel_address)
header_adder_interceptor = header_client_interceptor.header_adder_interceptor(
access_token_header, access_token)
if scheme in ("grpcs", "https"):
tls_root_ca_cert = (tls_root_ca_cert
or certifi.where()) # default: Mozilla ca cert
with open(tls_root_ca_cert, "rb") as fb:
ca_cert = fb.read()
if tls_client_key:
with open(tls_client_key, "rb") as fb:
tls_client_key = fb.read()
if tls_client_cert:
with open(tls_client_cert, "rb") as fb:
tls_client_cert = fb.read()
credentials = grpc.ssl_channel_credentials(ca_cert, tls_client_key,
tls_client_cert)
channel = grpc.intercept_channel(
grpc.secure_channel(addr, credentials),
header_adder_interceptor)
else:
channel = grpc.intercept_channel(grpc.insecure_channel(addr),
header_adder_interceptor)
return YataiStub(channel)
else:
from bentoml.yatai.db import DB
from bentoml.yatai.repository import create_repository
from bentoml.yatai.yatai_service_impl import get_yatai_service_impl
LocalYataiService = get_yatai_service_impl()
logger.debug("Creating local YataiService instance")
return LocalYataiService(
repository=create_repository(repository_type,
file_system_directory, s3_url,
gcs_url),
database=DB(db_url),
default_namespace=default_namespace,
)
def fixture_db():
db_url = 'sqlite:///bentoml/storage.db'
return DB(db_url)
class YataiServiceImpl(base):
# pylint: disable=unused-argument
# pylint: disable=broad-except
def __init__(
self,
db_url=None,
repo_base_url=None,
s3_endpoint_url=None,
default_namespace=None,
):
cfg = config('yatai_service')
repo_base_url = repo_base_url or cfg.get('repository_base_url')
db_url = db_url or cfg.get('db_url')
s3_endpoint_url = s3_endpoint_url or cfg.get('s3_endpoint_url') or None
default_namespace = default_namespace or cfg.get('default_namespace')
self.default_namespace = default_namespace
self.repo = Repository(repo_base_url, s3_endpoint_url)
self.db = DB(db_url)
def HealthCheck(self, request, context=None):
return HealthCheckResponse(status=Status.OK())
def GetYataiServiceVersion(self, request, context=None):
return GetYataiServiceVersionResponse(
status=Status.OK, version=BENTOML_VERSION
)
def ApplyDeployment(self, request, context=None):
with self.db.create_session() as sess:
try:
# apply default namespace if not set
request.deployment.namespace = (
request.deployment.namespace or self.default_namespace
)
validation_errors = validate_deployment_pb(request.deployment)
if validation_errors:
raise InvalidArgument(
'Failed to validate deployment. {errors}'.format(
errors=validation_errors
)
)
previous_deployment = self.db.deployment_store.get(
sess, request.deployment.name, request.deployment.namespace
)
if not previous_deployment:
request.deployment.created_at.GetCurrentTime()
request.deployment.last_updated_at.GetCurrentTime()
self.db.deployment_store.insert_or_update(sess, request.deployment)
# find deployment operator based on deployment spec
operator = get_deployment_operator(self, request.deployment)
# deploying to target platform
if previous_deployment:
response = operator.update(
request.deployment, previous_deployment
)
else:
response = operator.add(request.deployment)
if response.status.status_code == status_pb2.Status.OK:
# update deployment state
if response and response.deployment:
self.db.deployment_store.insert_or_update(
sess, response.deployment
)
else:
raise BentoMLException(
"DeploymentOperator Internal Error: failed to add or "
"update deployment metadata to database"
)
logger.info(
"ApplyDeployment (%s, namespace %s) succeeded",
request.deployment.name,
request.deployment.namespace,
)
else:
if not previous_deployment:
# When failed to create the deployment,
# delete it from active deployments records
self.db.deployment_store.delete(
sess,
request.deployment.name,
request.deployment.namespace,
)
logger.debug(
"ApplyDeployment (%s, namespace %s) failed: %s",
request.deployment.name,
request.deployment.namespace,
response.status.error_message,
)
return response
except BentoMLException as e:
logger.error("RPC ERROR ApplyDeployment: %s", e)
return ApplyDeploymentResponse(status=e.status_proto)
except Exception as e:
logger.error("URPC ERROR ApplyDeployment: %s", e)
return ApplyDeploymentResponse(status=Status.INTERNAL(str(e)))
def DeleteDeployment(self, request, context=None):
with self.db.create_session() as sess:
try:
request.namespace = request.namespace or self.default_namespace
deployment_pb = self.db.deployment_store.get(
sess, request.deployment_name, request.namespace
)
if deployment_pb:
# find deployment operator based on deployment spec
operator = get_deployment_operator(self, deployment_pb)
# executing deployment deletion
response = operator.delete(deployment_pb)
# if delete successful, remove it from active deployment records
# table
if response.status.status_code == status_pb2.Status.OK:
track_deployment_delete(
deployment_pb.spec.operator, deployment_pb.created_at
)
self.db.deployment_store.delete(
sess, request.deployment_name, request.namespace
)
return response
# If force delete flag is True, we will remove the record
# from yatai database.
if request.force_delete:
# Track deployment delete before it
# vanishes from deployment store
track_deployment_delete(
deployment_pb.spec.operator,
deployment_pb.created_at,
True,
)
self.db.deployment_store.delete(
sess, request.deployment_name, request.namespace
)
return DeleteDeploymentResponse(status=Status.OK())
if response.status.status_code == status_pb2.Status.NOT_FOUND:
modified_message = (
'Cloud resources not found, error: {} - it '
'may have been deleted manually. Try delete '
'deployment with "--force" option to ignore this '
'error and force deleting the deployment record'.format(
response.status.error_message
)
)
response.status.error_message = modified_message
return response
else:
return DeleteDeploymentResponse(
status=Status.NOT_FOUND(
'Deployment "{}" in namespace "{}" not found'.format(
request.deployment_name, request.namespace
)
)
)
except BentoMLException as e:
logger.error("RPC ERROR DeleteDeployment: %s", e)
return DeleteDeploymentResponse(status=e.status_proto)
except Exception as e: # pylint: disable=broad-except
logger.error("RPC ERROR DeleteDeployment: %s", e)
return DeleteDeploymentResponse(status=Status.INTERNAL(str(e)))
def GetDeployment(self, request, context=None):
with self.db.create_session() as sess:
try:
request.namespace = request.namespace or self.default_namespace
deployment_pb = self.db.deployment_store.get(
sess, request.deployment_name, request.namespace
)
if deployment_pb:
return GetDeploymentResponse(
status=Status.OK(), deployment=deployment_pb
)
else:
return GetDeploymentResponse(
status=Status.NOT_FOUND(
'Deployment "{}" in namespace "{}" not found'.format(
request.deployment_name, request.namespace
)
)
)
except BentoMLException as e:
logger.error("RPC ERROR GetDeployment: %s", e)
return GetDeploymentResponse(status=e.status_proto)
except Exception as e: # pylint: disable=broad-except
logger.error("RPC ERROR GetDeployment: %s", e)
return GetDeploymentResponse(status=Status.INTERNAL())
def DescribeDeployment(self, request, context=None):
with self.db.create_session() as sess:
try:
request.namespace = request.namespace or self.default_namespace
deployment_pb = self.db.deployment_store.get(
sess, request.deployment_name, request.namespace
)
if deployment_pb:
operator = get_deployment_operator(self, deployment_pb)
response = operator.describe(deployment_pb)
if response.status.status_code == status_pb2.Status.OK:
with self.db.deployment_store.update_deployment(
sess, request.deployment_name, request.namespace
) as deployment:
deployment.state = ProtoMessageToDict(response.state)
return response
else:
return DescribeDeploymentResponse(
status=Status.NOT_FOUND(
'Deployment "{}" in namespace "{}" not found'.format(
request.deployment_name, request.namespace
)
)
)
except BentoMLException as e:
logger.error("RPC ERROR DescribeDeployment: %s", e)
return DeleteDeploymentResponse(status=e.status_proto)
except Exception as e: # pylint: disable=broad-except
logger.error("RPC ERROR DescribeDeployment: %s", e)
return DeleteDeploymentResponse(status=Status.INTERNAL())
def ListDeployments(self, request, context=None):
with self.db.create_session() as sess:
try:
namespace = request.namespace or self.default_namespace
deployment_pb_list = self.db.deployment_store.list(
sess,
namespace=namespace,
label_selectors=request.label_selectors,
offset=request.offset,
limit=request.limit,
operator=request.operator,
order_by=request.order_by,
ascending_order=request.ascending_order,
)
return ListDeploymentsResponse(
status=Status.OK(), deployments=deployment_pb_list
)
except BentoMLException as e:
logger.error("RPC ERROR ListDeployments: %s", e)
return ListDeploymentsResponse(status=e.status_proto)
except Exception as e: # pylint: disable=broad-except
logger.error("RPC ERROR ListDeployments: %s", e)
return ListDeploymentsResponse(status=Status.INTERNAL())
def AddBento(self, request, context=None):
with self.db.create_session() as sess:
try:
# TODO: validate request
bento_pb = self.db.metadata_store.get(
sess, request.bento_name, request.bento_version
)
if bento_pb:
error_message = (
"BentoService bundle: "
"{}:{} already exists".format(
request.bento_name, request.bento_version
)
)
logger.error(error_message)
return AddBentoResponse(status=Status.ABORTED(error_message))
new_bento_uri = self.repo.add(
request.bento_name, request.bento_version
)
self.db.metadata_store.add(
sess,
bento_name=request.bento_name,
bento_version=request.bento_version,
uri=new_bento_uri.uri,
uri_type=new_bento_uri.type,
)
return AddBentoResponse(status=Status.OK(), uri=new_bento_uri)
except BentoMLException as e:
logger.error("RPC ERROR AddBento: %s", e)
return DeleteDeploymentResponse(status=e.status_proto)
except Exception as e: # pylint: disable=broad-except
logger.error("URPC ERROR AddBento: %s", e)
return DeleteDeploymentResponse(status=Status.INTERNAL())
def UpdateBento(self, request, context=None):
with self.db.create_session() as sess:
try:
# TODO: validate request
if request.upload_status:
self.db.metadata_store.update_upload_status(
sess,
request.bento_name,
request.bento_version,
request.upload_status,
)
if request.service_metadata:
self.db.metadata_store.update(
sess,
request.bento_name,
request.bento_version,
request.service_metadata,
)
return UpdateBentoResponse(status=Status.OK())
except BentoMLException as e:
logger.error("RPC ERROR UpdateBento: %s", e)
return UpdateBentoResponse(status=e.status_proto)
except Exception as e: # pylint: disable=broad-except
logger.error("RPC ERROR UpdateBento: %s", e)
return UpdateBentoResponse(status=Status.INTERNAL())
def DangerouslyDeleteBento(self, request, context=None):
with self.db.create_session() as sess:
try:
# TODO: validate request
bento_pb = self.db.metadata_store.get(
sess, request.bento_name, request.bento_version
)
if not bento_pb:
msg = (
f"BentoService "
f"{request.bento_name}:{request.bento_version} "
f"has already been deleted"
)
return DangerouslyDeleteBentoResponse(
status=Status.ABORTED(msg)
)
logger.debug(
'Deleting BentoService %s:%s',
request.bento_name,
request.bento_version,
)
self.db.metadata_store.dangerously_delete(
sess, request.bento_name, request.bento_version
)
self.repo.dangerously_delete(
request.bento_name, request.bento_version
)
return DangerouslyDeleteBentoResponse(status=Status.OK())
except BentoMLException as e:
logger.error("RPC ERROR DangerouslyDeleteBento: %s", e)
return DangerouslyDeleteBentoResponse(status=e.status_proto)
except Exception as e: # pylint: disable=broad-except
logger.error("RPC ERROR DangerouslyDeleteBento: %s", e)
return DangerouslyDeleteBentoResponse(status=Status.INTERNAL())
def GetBento(self, request, context=None):
with self.db.create_session() as sess:
try:
# TODO: validate request
bento_pb = self.db.metadata_store.get(
sess, request.bento_name, request.bento_version
)
if bento_pb:
if request.bento_version.lower() == 'latest':
logger.info(
'Getting latest version %s:%s',
request.bento_name,
bento_pb.version,
)
if bento_pb.uri.type == BentoUri.S3:
bento_pb.uri.s3_presigned_url = self.repo.get(
bento_pb.name, bento_pb.version
)
elif bento_pb.uri.type == BentoUri.GCS:
bento_pb.uri.gcs_presigned_url = self.repo.get(
bento_pb.name, bento_pb.version
)
return GetBentoResponse(status=Status.OK(), bento=bento_pb)
else:
return GetBentoResponse(
status=Status.NOT_FOUND(
"BentoService `{}:{}` is not found".format(
request.bento_name, request.bento_version
)
)
)
except BentoMLException as e:
logger.error("RPC ERROR GetBento: %s", e)
return GetBentoResponse(status=e.status_proto)
except Exception as e: # pylint: disable=broad-except
logger.error("RPC ERROR GetBento: %s", e)
return GetBentoResponse(status=Status.INTERNAL())
def ListBento(self, request, context=None):
with self.db.create_session() as sess:
try:
# TODO: validate request
bento_metadata_pb_list = self.db.metadata_store.list(
sess,
bento_name=request.bento_name,
offset=request.offset,
limit=request.limit,
order_by=request.order_by,
label_selectors=request.label_selectors,
ascending_order=request.ascending_order,
)
return ListBentoResponse(
status=Status.OK(), bentos=bento_metadata_pb_list
)
except BentoMLException as e:
logger.error("RPC ERROR ListBento: %s", e)
return ListBentoResponse(status=e.status_proto)
except Exception as e: # pylint: disable=broad-except
logger.error("RPC ERROR ListBento: %s", e)
return ListBentoResponse(status=Status.INTERNAL())
def ContainerizeBento(self, request, context=None):
with self.db.create_session() as sess:
try:
ensure_docker_available_or_raise()
tag = request.tag
if tag is None or len(tag) == 0:
name = to_valid_docker_image_name(request.bento_name)
version = to_valid_docker_image_version(request.bento_version)
tag = f"{name}:{version}"
if ":" not in tag:
version = to_valid_docker_image_version(request.bento_version)
tag = f"{tag}:{version}"
import docker
docker_client = docker.from_env()
bento_pb = self.db.metadata_store.get(
sess, request.bento_name, request.bento_version
)
if not bento_pb:
raise YataiRepositoryException(
f'BentoService '
f'{request.bento_name}:{request.bento_version} '
f'does not exist'
)
with TempDirectory() as temp_dir:
temp_bundle_path = f'{temp_dir}/{bento_pb.name}'
bento_service_bundle_path = bento_pb.uri.uri
if bento_pb.uri.type == BentoUri.S3:
bento_service_bundle_path = self.repo.get(
bento_pb.name, bento_pb.version
)
elif bento_pb.uri.type == BentoUri.GCS:
bento_service_bundle_path = self.repo.get(
bento_pb.name, bento_pb.version
)
safe_retrieve(bento_service_bundle_path, temp_bundle_path)
try:
docker_client.images.build(
path=temp_bundle_path,
tag=tag,
buildargs=dict(request.build_args),
)
except (
docker.errors.APIError,
docker.errors.BuildError,
) as error:
logger.error(f'Encounter container building issue: {error}')
raise YataiRepositoryException(error)
if request.push is True:
try:
docker_client.images.push(
repository=request.repository, tag=tag
)
except docker.errors.APIError as error:
raise YataiRepositoryException(error)
return ContainerizeBentoResponse(status=Status.OK(), tag=tag)
except BentoMLException as e:
logger.error(f"RPC ERROR ContainerizeBento: {e}")
return ContainerizeBentoResponse(status=e.status_proto)
except Exception as e: # pylint: disable=broad-except
logger.error(f"RPC ERROR ContainerizeBento: {e}")
return ContainerizeBentoResponse(status=Status.INTERNAL(e))