def DescribeDeployment(self, request, context=None):
try:
request.namespace = request.namespace or self.default_namespace
deployment_pb = self.deployment_store.get(
request.deployment_name, request.namespace
)
if deployment_pb:
operator = get_deployment_operator(deployment_pb)
response = operator.describe(deployment_pb, self)
if response.status.status_code == status_pb2.Status.OK:
with self.deployment_store.update_deployment(
request.deployment_name, request.namespace
) as deployment:
deployment.state = ProtoMessageToDict(response.state)
return response
else:
return DescribeDeploymentResponse(
status=Status.NOT_FOUND(
'Deployment "{}" in namespace "{}" not found'.format(
request.deployment_name, request.namespace
)
)
)
except BentoMLException as e:
logger.error("INTERNAL ERROR: %s", e)
return DescribeDeploymentResponse(Status.INTERNAL(str(e)))
def _parse_aws_client_exception_or_raise(e):
"""parse botocore.exceptions.ClientError into Bento StatusProto
We handle two most common errors when deploying to Sagemaker.
1. Authenication issue/invalid access(InvalidSignatureException)
2. resources not found (ValidationException)
It will return correlated StatusProto(NOT_FOUND, UNAUTHENTICATED)
Args:
e: ClientError from botocore.exceptions
Returns:
StatusProto
"""
error_response = e.response.get('Error', {})
error_code = error_response.get('Code')
error_message = error_response.get('Message', 'Unknown')
error_log_message = 'AWS ClientError for {operation}: {code} - {message}'.format(
operation=e.operation_name, code=error_code, message=error_message
)
if error_code == 'ValidationException':
logger.error(error_log_message)
return Status.NOT_FOUND(error_response.get('Message', 'Unknown'))
elif error_code == 'InvalidSignatureException':
logger.error(error_log_message)
return Status.UNAUTHENTICATED(error_response.get('Message', 'Unknown'))
else:
logger.error(error_log_message)
raise e
def DescribeDeployment(self, request, context=None):
with self.db.create_session() as sess:
try:
request.namespace = request.namespace or self.default_namespace
deployment_pb = self.db.deployment_store.get(
sess, request.deployment_name, request.namespace
)
if deployment_pb:
operator = get_deployment_operator(self, deployment_pb)
response = operator.describe(deployment_pb)
if response.status.status_code == status_pb2.Status.OK:
with self.db.deployment_store.update_deployment(
sess, request.deployment_name, request.namespace
) as deployment:
deployment.state = ProtoMessageToDict(response.state)
return response
else:
return DescribeDeploymentResponse(
status=Status.NOT_FOUND(
'Deployment "{}" in namespace "{}" not found'.format(
request.deployment_name, request.namespace
)
)
)
except BentoMLException as e:
logger.error("RPC ERROR DescribeDeployment: %s", e)
return DeleteDeploymentResponse(status=e.status_proto)
except Exception as e: # pylint: disable=broad-except
logger.error("RPC ERROR DescribeDeployment: %s", e)
return DeleteDeploymentResponse(status=Status.INTERNAL())
def GetBento(self, request, context=None):
try:
# TODO: validate request
bento_pb = self.bento_metadata_store.get(request.bento_name,
request.bento_version)
if request.bento_version.lower() == 'latest':
logger.info(
'Getting latest version %s:%s',
request.bento_name,
bento_pb.version,
)
if bento_pb:
if bento_pb.uri.type == BentoUri.S3:
bento_pb.uri.s3_presigned_url = self.repo.get(
bento_pb.name, bento_pb.version)
return GetBentoResponse(status=Status.OK(), bento=bento_pb)
else:
return GetBentoResponse(status=Status.NOT_FOUND(
"BentoService `{}:{}` is not found".format(
request.bento_name, request.bento_version)))
except BentoMLException as e:
logger.error("RPC ERROR GetBento: %s", e)
return GetBentoResponse(status=e.status_proto)
except Exception as e: # pylint: disable=broad-except
logger.error("RPC ERROR GetBento: %s", e)
return GetBentoResponse(status=Status.INTERNAL())
def GetBento(self, request, context=None):
try:
# TODO: validate request
bento_metadata_pb = self.bento_metadata_store.get(
request.bento_name, request.bento_version
)
if request.bento_version.lower() == 'latest':
logger.info(
'Getting latest version %s:%s',
request.bento_name,
bento_metadata_pb.version,
)
if bento_metadata_pb:
return GetBentoResponse(status=Status.OK(), bento=bento_metadata_pb)
else:
return GetBentoResponse(
status=Status.NOT_FOUND(
"BentoService `{}:{}` is not found".format(
request.bento_name, request.bento_version
)
)
)
except BentoMLException as e:
logger.error("RPC ERROR GetBento: %s", e)
return GetBentoResponse(status=e.status_proto)
def GetDeployment(self, request, context=None):
with self.db.create_session() as sess:
try:
request.namespace = request.namespace or self.default_namespace
deployment_pb = self.db.deployment_store.get(
sess, request.deployment_name, request.namespace
)
if deployment_pb:
return GetDeploymentResponse(
status=Status.OK(), deployment=deployment_pb
)
else:
return GetDeploymentResponse(
status=Status.NOT_FOUND(
'Deployment "{}" in namespace "{}" not found'.format(
request.deployment_name, request.namespace
)
)
)
except BentoMLException as e:
logger.error("RPC ERROR GetDeployment: %s", e)
return GetDeploymentResponse(status=e.status_proto)
except Exception as e: # pylint: disable=broad-except
logger.error("RPC ERROR GetDeployment: %s", e)
return GetDeploymentResponse(status=Status.INTERNAL())
def DeleteDeployment(self, request, context=None):
try:
request.namespace = request.namespace or self.default_namespace
deployment_pb = self.deployment_store.get(request.deployment_name,
request.namespace)
if deployment_pb:
# find deployment operator based on deployment spec
operator = get_deployment_operator(deployment_pb)
# executing deployment deletion
response = operator.delete(deployment_pb, self.repo)
# if delete successful, remove it from active deployment records table
if response.status.status_code == status_pb2.Status.OK:
self.deployment_store.delete(request.deployment_name,
request.namespace)
return response
else:
return DeleteDeploymentResponse(status=Status.NOT_FOUND(
'Deployment "{}" in namespace "{}" not found'.format(
request.deployment_name, request.namespace)))
except BentoMLException as e:
logger.error("INTERNAL ERROR: %s", e)
return DeleteDeploymentResponse(status=Status.INTERNAL(e))
def delete(self, deployment_pb, yatai_service=None):
try:
state = self.describe(deployment_pb, yatai_service).state
if state.state != DeploymentState.RUNNING:
message = (
'Failed to delete, no active deployment {name}. '
'The current state is {state}'.format(
name=deployment_pb.name,
state=DeploymentState.State.Name(state.state),
)
)
return DeleteDeploymentResponse(status=Status.ABORTED(message))
deployment_spec = deployment_pb.spec
aws_config = deployment_spec.aws_lambda_operator_config
bento_pb = yatai_service.GetBento(
GetBentoRequest(
bento_name=deployment_spec.bento_name,
bento_version=deployment_spec.bento_version,
)
)
bento_service_metadata = bento_pb.bento.bento_service_metadata
# We are not validating api_name, because for delete, you don't
# need them.
api_names = (
[aws_config.api_name]
if aws_config.api_name
else [api.name for api in bento_service_metadata.apis]
)
with TempDirectory() as serverless_project_dir:
generate_aws_lambda_serverless_config(
bento_service_metadata.env.python_version,
deployment_pb.name,
api_names,
serverless_project_dir,
aws_config.region,
# BentoML deployment namespace is mapping to serverless `stage`
# concept
stage=deployment_pb.namespace,
)
response = call_serverless_command(['remove'], serverless_project_dir)
stack_name = '{name}-{namespace}'.format(
name=deployment_pb.name, namespace=deployment_pb.namespace
)
if "Serverless: Stack removal finished..." in response:
status = Status.OK()
elif "Stack '{}' does not exist".format(stack_name) in response:
status = Status.NOT_FOUND(
'Deployment {} not found'.format(stack_name)
)
else:
status = Status.ABORTED()
return DeleteDeploymentResponse(status=status)
except BentoMLException as error:
return DeleteDeploymentResponse(status=exception_to_return_status(error))
def DeleteDeployment(self, request, context=None):
try:
request.namespace = request.namespace or self.default_namespace
deployment_pb = self.deployment_store.get(
request.deployment_name, request.namespace
)
if deployment_pb:
# find deployment operator based on deployment spec
operator = get_deployment_operator(self, deployment_pb)
# executing deployment deletion
response = operator.delete(deployment_pb)
# if delete successful, remove it from active deployment records table
if response.status.status_code == status_pb2.Status.OK:
self.deployment_store.delete(
request.deployment_name, request.namespace
)
return response
# If force delete flag is True, we will remove the record
# from yatai database.
if request.force_delete:
self.deployment_store.delete(
request.deployment_name, request.namespace
)
return DeleteDeploymentResponse(status=Status.OK())
if response.status.status_code == status_pb2.Status.NOT_FOUND:
modified_message = (
'Cloud resources not found, error: {} - it may have been '
'deleted manually. Try delete deployment '
'with "--force" option to ignore this error '
'and force deleting the deployment record'.format(
response.status.error_message
)
)
response.status.error_message = modified_message
return response
else:
return DeleteDeploymentResponse(
status=Status.NOT_FOUND(
'Deployment "{}" in namespace "{}" not found'.format(
request.deployment_name, request.namespace
)
)
)
except BentoMLException as e:
logger.error("RPC ERROR DeleteDeployment: %s", e)
return DeleteDeploymentResponse(status=e.status_proto)
except Exception as e: # pylint: disable=broad-except
logger.error("RPC ERROR DeleteDeployment: %s", e)
return DeleteDeploymentResponse(status=Status.INTERNAL(str(e)))
def create_yatai_service_mock():
yatai_service_mock = Mock()
yatai_service_mock.ApplyDeployment.return_value = ApplyDeploymentResponse()
yatai_service_mock.DeleteDeployment.return_value = DeleteDeploymentResponse()
yatai_service_mock.DescribeDeployment.return_value = DescribeDeploymentResponse()
yatai_service_mock.GetDeployment.return_value = GetDeploymentResponse(
status=Status.NOT_FOUND()
)
yatai_service_mock.ListDeployments.return_value = ListDeploymentsResponse()
return yatai_service_mock
def GetBento(self, request, context=None):
try:
# TODO: validate request
bento_metadata_pb = self.bento_metadata_store.get(
request.bento_name, request.bento_version)
if bento_metadata_pb:
return GetBentoResponse(status=Status.OK(),
bento=bento_metadata_pb)
else:
return GetBentoResponse(status=Status.NOT_FOUND(
"Bento `{}:{}` is not found".format(
request.bento_name, request.bento_version)))
except BentoMLException as e:
logger.error("INTERNAL ERROR: %s", e)
return GetBentoResponse(status=Status.INTERNAL(str(e)))
def GetDeployment(self, request, context=None):
try:
request.namespace = request.namespace or self.default_namespace
deployment_pb = self.deployment_store.get(request.deployment_name,
request.namespace)
if deployment_pb:
return GetDeploymentResponse(status=Status.OK(),
deployment=deployment_pb)
else:
return GetDeploymentResponse(status=Status.NOT_FOUND(
'Deployment "{}" in namespace "{}" not found'.format(
request.deployment_name, request.namespace)))
except BentoMLException as e:
logger.error("INTERNAL ERROR: %s", e)
return GetDeploymentResponse(status=Status.INTERNAL(str(e)))
def delete(self, deployment_pb, repo=None):
state = self.describe(deployment_pb, repo).state
if state.state != DeploymentState.RUNNING:
message = ('Failed to delete, no active deployment {name}. '
'The current state is {state}'.format(
name=deployment_pb.name,
state=DeploymentState.State.Name(state.state),
))
return DeleteDeploymentResponse(status=Status.ABORTED(message))
deployment_spec = deployment_pb.spec
aws_config = deployment_spec.aws_lambda_operator_config
bento_path = repo.get(deployment_spec.bento_name,
deployment_spec.bento_version)
bento_config = load_bentoml_config(bento_path)
with TemporaryServerlessConfig(
archive_path=bento_path,
deployment_name=deployment_pb.name,
region=aws_config.region,
stage=deployment_pb.namespace,
provider_name='aws',
functions=generate_aws_handler_functions_config(
bento_config['apis']),
) as tempdir:
response = call_serverless_command(['remove'], tempdir)
stack_name = '{name}-{namespace}'.format(
name=deployment_pb.name, namespace=deployment_pb.namespace)
if "Serverless: Stack removal finished..." in response:
status = Status.OK()
elif "Stack '{}' does not exist".format(stack_name) in response:
status = Status.NOT_FOUND('Resource not found')
else:
status = Status.ABORTED()
return DeleteDeploymentResponse(status=status)
def DeleteDeployment(self, request, context=None):
deployment_id = f"{request.deployment_name}_{request.namespace}"
with lock(self.db, [(deployment_id, LockType.WRITE)]) as (
sess,
_,
):
try:
request.namespace = request.namespace or self.default_namespace
deployment_pb = self.db.deployment_store.get(
sess, request.deployment_name, request.namespace)
if deployment_pb:
# find deployment operator based on deployment spec
operator = get_deployment_operator(self, deployment_pb)
# executing deployment deletion
response = operator.delete(deployment_pb)
# if delete successful, remove it from active deployment records
# table
if response.status.status_code == status_pb2.Status.OK:
track_deployment_delete(
deployment_pb.spec.operator,
deployment_pb.created_at)
self.db.deployment_store.delete(
sess, request.deployment_name,
request.namespace)
return response
# If force delete flag is True, we will remove the record
# from yatai database.
if request.force_delete:
# Track deployment delete before it
# vanishes from deployment store
track_deployment_delete(
deployment_pb.spec.operator,
deployment_pb.created_at,
True,
)
self.db.deployment_store.delete(
sess, request.deployment_name,
request.namespace)
return DeleteDeploymentResponse(status=Status.OK())
if response.status.status_code == status_pb2.Status.NOT_FOUND:
modified_message = (
'Cloud resources not found, error: {} - it '
'may have been deleted manually. Try delete '
'deployment with "--force" option to ignore this '
'error and force deleting the deployment record'
.format(response.status.error_message))
response.status.error_message = modified_message
return response
else:
return DeleteDeploymentResponse(
status=Status.NOT_FOUND(
'Deployment "{}" in namespace "{}" not found'.
format(request.deployment_name,
request.namespace)))
except BentoMLException as e:
logger.error("RPC ERROR DeleteDeployment: %s", e)
return DeleteDeploymentResponse(status=e.status_proto)
except Exception as e: # pylint: disable=broad-except
logger.error("RPC ERROR DeleteDeployment: %s", e)
return DeleteDeploymentResponse(
status=Status.INTERNAL(str(e)))
def UploadBento(self, request_iterator, context=None):
if not is_file_system_repo(self.repo):
logger.error(
"UploadBento RPC only works with File System based repository, "
"for other types of repositories(s3, gcs, minio), "
"use pre-signed URL for upload")
return UploadBentoResponse(status=Status.INTERNAL(''))
try:
with self.db.create_session() as sess:
lock_obj = None
bento_pb = None
with TempDirectory() as temp_dir:
temp_tar_path = os.path.join(
temp_dir, f'{uuid.uuid4().hex[:12]}.tar')
file = open(temp_tar_path, 'wb+')
for request in request_iterator:
# Initial request is without bundle
if not request.bento_bundle:
bento_name = request.bento_name
bento_version = request.bento_version
bento_pb = self.db.metadata_store.get(
sess, bento_name, bento_version)
if not bento_pb:
result_status = Status.NOT_FOUND(
"BentoService `{}:{}` is not found".
format(bento_name, bento_version))
return UploadBentoResponse(
status=result_status)
if bento_pb.status:
if bento_pb.status.status == UploadStatus.DONE:
return UploadBentoResponse(
status=Status.CANCELLED(
f"Bento bundle `{bento_name}:"
f"{bento_version}` is uploaded"
))
if bento_pb.status.status == UploadStatus.UPLOADING:
return UploadBentoResponse(
status=Status.CANCELLED(
f"Bento bundle `{bento_name}:"
f"{bento_version}` is currently "
f"uploading"))
if lock_obj is None:
lock_obj = LockStore.acquire(
sess=sess,
lock_type=LockType.WRITE,
resource_id=
f'{bento_name}_{bento_version}',
ttl_min=DEFAULT_TTL_MIN,
)
else:
if (bento_name == request.bento_name
and bento_version
== request.bento_version):
file.write(request.bento_bundle)
else:
lock_obj.release(sess)
raise BadInput(
f"Incoming stream request doesn't match "
f"with initial request info "
f"{bento_name}:{bento_version} - "
f"{request.bento_name}:"
f"{request.bento_version}")
file.seek(0)
with tarfile.open(fileobj=file, mode='r') as tar:
tar.extractall(path=bento_pb.uri.uri)
upload_status = UploadStatus(status=UploadStatus.DONE)
upload_status.updated_at.GetCurrentTime()
self.db.metadata_store.update_upload_status(
sess, bento_name, bento_version, upload_status)
lock_obj.release(sess)
return UploadBentoResponse(status=Status.OK())
except BentoMLException as e:
logger.error("RPC ERROR UploadBento: %s", e)
return UploadBentoResponse(status=e.status_proto)
except Exception as e: # pylint: disable=broad-except
logger.error("RPC ERROR UploadBento: %s", e)
return UploadBentoResponse(status=Status.INTERNAL())
finally:
if file is not None:
file.close()
