def delete(self, deployment_pb):
try:
logger.debug('Deleting AWS Lambda deployment')
deployment_spec = deployment_pb.spec
lambda_deployment_config = deployment_spec.aws_lambda_operator_config
lambda_deployment_config.region = (lambda_deployment_config.region
or get_default_aws_region())
if not lambda_deployment_config.region:
raise InvalidArgument('AWS region is missing')
cf_client = boto3.client('cloudformation',
lambda_deployment_config.region)
stack_name = generate_aws_compatible_string(
deployment_pb.namespace, deployment_pb.name)
if deployment_pb.state.info_json:
deployment_info_json = json.loads(
deployment_pb.state.info_json)
bucket_name = deployment_info_json.get('s3_bucket')
if bucket_name:
cleanup_s3_bucket_if_exist(bucket_name,
lambda_deployment_config.region)
logger.debug(
'Deleting AWS CloudFormation: %s that includes Lambda function '
'and related resources',
stack_name,
)
cf_client.delete_stack(StackName=stack_name)
return DeleteDeploymentResponse(status=Status.OK())
except BentoMLException as error:
return DeleteDeploymentResponse(status=error.status_proto)
def AddBento(self, request, context=None):
try:
# TODO: validate request
bento_pb = self.bento_metadata_store.get(request.bento_name,
request.bento_version)
if bento_pb:
error_message = "BentoService bundle: {}:{} already exist".format(
request.bento_name, request.bento_version)
logger.error(error_message)
return AddBentoResponse(status=Status.ABORTED(error_message))
new_bento_uri = self.repo.add(request.bento_name,
request.bento_version)
self.bento_metadata_store.add(
bento_name=request.bento_name,
bento_version=request.bento_version,
uri=new_bento_uri.uri,
uri_type=new_bento_uri.type,
)
return AddBentoResponse(status=Status.OK(), uri=new_bento_uri)
except BentoMLException as e:
logger.error("RPC ERROR AddBento: %s", e)
return DeleteDeploymentResponse(status=e.status_proto)
except Exception as e: # pylint: disable=broad-except
logger.error("URPC ERROR AddBento: %s", e)
return DeleteDeploymentResponse(status=Status.INTERNAL())
def DescribeDeployment(self, request, context=None):
try:
request.namespace = request.namespace or self.default_namespace
deployment_pb = self.deployment_store.get(request.deployment_name,
request.namespace)
if deployment_pb:
operator = get_deployment_operator(self, deployment_pb)
response = operator.describe(deployment_pb)
if response.status.status_code == status_pb2.Status.OK:
with self.deployment_store.update_deployment(
request.deployment_name,
request.namespace) as deployment:
deployment.state = ProtoMessageToDict(response.state)
return response
else:
return DescribeDeploymentResponse(status=Status.NOT_FOUND(
'Deployment "{}" in namespace "{}" not found'.format(
request.deployment_name, request.namespace)))
except BentoMLException as e:
logger.error("RPC ERROR DescribeDeployment: %s", e)
return DeleteDeploymentResponse(status=e.status_proto)
except Exception as e: # pylint: disable=broad-except
logger.error("RPC ERROR DescribeDeployment: %s", e)
return DeleteDeploymentResponse(status=Status.INTERNAL())
def delete(self, deployment_pb):
try:
deployment_spec = deployment_pb.spec
ec2_deployment_config = deployment_spec.aws_ec2_operator_config
ec2_deployment_config.region = (ec2_deployment_config.region
or get_default_aws_region())
if not ec2_deployment_config.region:
raise InvalidArgument("AWS region is missing")
_, deployment_stack_name, repository_name, _ = generate_ec2_resource_names(
deployment_pb.namespace, deployment_pb.name)
# delete stack
delete_cloudformation_stack(deployment_stack_name,
ec2_deployment_config.region)
# delete repo from ecr
delete_ecr_repository(repository_name,
ec2_deployment_config.region)
# remove bucket
if deployment_pb.state.info_json:
deployment_info_json = json.loads(
deployment_pb.state.info_json)
bucket_name = deployment_info_json.get('S3Bucket')
if bucket_name:
cleanup_s3_bucket_if_exist(bucket_name,
ec2_deployment_config.region)
return DeleteDeploymentResponse(status=Status.OK())
except BentoMLException as error:
return DeleteDeploymentResponse(status=error.status_proto)
def DeleteDeployment(self, request, context=None):
try:
request.namespace = request.namespace or self.default_namespace
deployment_pb = self.deployment_store.get(
request.deployment_name, request.namespace
)
if deployment_pb:
# find deployment operator based on deployment spec
operator = get_deployment_operator(self, deployment_pb)
# executing deployment deletion
response = operator.delete(deployment_pb)
# if delete successful, remove it from active deployment records table
if response.status.status_code == status_pb2.Status.OK:
self.deployment_store.delete(
request.deployment_name, request.namespace
)
return response
# If force delete flag is True, we will remove the record
# from yatai database.
if request.force_delete:
self.deployment_store.delete(
request.deployment_name, request.namespace
)
return DeleteDeploymentResponse(status=Status.OK())
if response.status.status_code == status_pb2.Status.NOT_FOUND:
modified_message = (
'Cloud resources not found, error: {} - it may have been '
'deleted manually. Try delete deployment '
'with "--force" option to ignore this error '
'and force deleting the deployment record'.format(
response.status.error_message
)
)
response.status.error_message = modified_message
return response
else:
return DeleteDeploymentResponse(
status=Status.NOT_FOUND(
'Deployment "{}" in namespace "{}" not found'.format(
request.deployment_name, request.namespace
)
)
)
except BentoMLException as e:
logger.error("RPC ERROR DeleteDeployment: %s", e)
return DeleteDeploymentResponse(status=e.status_proto)
except Exception as e: # pylint: disable=broad-except
logger.error("RPC ERROR DeleteDeployment: %s", e)
return DeleteDeploymentResponse(status=Status.INTERNAL(str(e)))
def delete(self, deployment_pb):
try:
resource_group_name, _, _, _, _ = _generate_azure_resource_names(
namespace=deployment_pb.namespace, deployment_name=deployment_pb.name
)
_call_az_cli(
command=['az', 'group', 'delete', '-y', '--name', resource_group_name],
message='delete Azure resource group',
parse_json=False,
)
return DeleteDeploymentResponse(status=Status.OK())
except BentoMLException as error:
return DeleteDeploymentResponse(status=error.status_proto)
def delete(self, deployment_pb):
try:
deployment_spec = deployment_pb.spec
sagemaker_config = deployment_spec.sagemaker_operator_config
sagemaker_config.region = (sagemaker_config.region
or get_default_aws_region())
if not sagemaker_config.region:
raise InvalidArgument("AWS region is missing")
delete_sagemaker_deployment_resources_if_exist(deployment_pb)
return DeleteDeploymentResponse(status=Status.OK())
except BentoMLException as error:
return DeleteDeploymentResponse(status=error.status_proto)
def ListDeployments(self, request, context=None):
try:
namespace = request.namespace or self.default_namespace
deployment_pb_list = self.deployment_store.list(
namespace=namespace,
labels_query=request.labels_query,
offset=request.offset,
limit=request.limit,
operator=request.operator,
order_by=request.order_by,
ascending_order=request.ascending_order,
)
return ListDeploymentsResponse(status=Status.OK(),
deployments=deployment_pb_list)
except BentoMLException as e:
logger.error("RPC ERROR ListDeployments: %s", e)
return DeleteDeploymentResponse(status=e.status_proto)
except Exception as e: # pylint: disable=broad-except
logger.error("RPC ERROR ListDeployments: %s", e)
return DeleteDeploymentResponse(status=Status.INTERNAL())
def mock_delete_deployment(deployment_pb):
if deployment_pb.name == MOCK_FAILED_DEPLOYMENT_NAME:
return DeleteDeploymentResponse(status=Status.ABORTED())
else:
return DeleteDeploymentResponse(status=Status.OK())
def DeleteDeployment(self, request, context=None):
deployment_id = f"{request.deployment_name}_{request.namespace}"
with lock(self.db, [(deployment_id, LockType.WRITE)]) as (
sess,
_,
):
try:
request.namespace = request.namespace or self.default_namespace
deployment_pb = self.db.deployment_store.get(
sess, request.deployment_name, request.namespace)
if deployment_pb:
# find deployment operator based on deployment spec
operator = get_deployment_operator(self, deployment_pb)
# executing deployment deletion
response = operator.delete(deployment_pb)
# if delete successful, remove it from active deployment records
# table
if response.status.status_code == status_pb2.Status.OK:
track_deployment_delete(
deployment_pb.spec.operator,
deployment_pb.created_at)
self.db.deployment_store.delete(
sess, request.deployment_name,
request.namespace)
return response
# If force delete flag is True, we will remove the record
# from yatai database.
if request.force_delete:
# Track deployment delete before it
# vanishes from deployment store
track_deployment_delete(
deployment_pb.spec.operator,
deployment_pb.created_at,
True,
)
self.db.deployment_store.delete(
sess, request.deployment_name,
request.namespace)
return DeleteDeploymentResponse(status=Status.OK())
if response.status.status_code == status_pb2.Status.NOT_FOUND:
modified_message = (
'Cloud resources not found, error: {} - it '
'may have been deleted manually. Try delete '
'deployment with "--force" option to ignore this '
'error and force deleting the deployment record'
.format(response.status.error_message))
response.status.error_message = modified_message
return response
else:
return DeleteDeploymentResponse(
status=Status.NOT_FOUND(
'Deployment "{}" in namespace "{}" not found'.
format(request.deployment_name,
request.namespace)))
except BentoMLException as e:
logger.error("RPC ERROR DeleteDeployment: %s", e)
return DeleteDeploymentResponse(status=e.status_proto)
except Exception as e: # pylint: disable=broad-except
logger.error("RPC ERROR DeleteDeployment: %s", e)
return DeleteDeploymentResponse(
status=Status.INTERNAL(str(e)))
