def delete(self, deployment_pb, repo=None):
state = self.describe(deployment_pb, repo).state
if state.state != DeploymentState.RUNNING:
message = ('Failed to delete, no active deployment {name}. '
'The current state is {state}'.format(
name=deployment_pb.name,
state=DeploymentState.State.Name(state.state),
))
return DeleteDeploymentResponse(status=Status.ABORTED(message))
deployment_spec = deployment_pb.spec
aws_config = deployment_spec.aws_lambda_operator_config
bento_path = repo.get(deployment_spec.bento_name,
deployment_spec.bento_version)
bento_config = load_bentoml_config(bento_path)
with TemporaryServerlessConfig(
archive_path=bento_path,
deployment_name=deployment_pb.name,
region=aws_config.region,
stage=deployment_pb.namespace,
provider_name='aws',
functions=generate_aws_handler_functions_config(
bento_config['apis']),
) as tempdir:
response = call_serverless_command(['serverless', 'remove'],
tempdir)
if "Serverless: Stack removal finished..." in response:
status = Status.OK()
else:
status = Status.ABORTED()
return DeleteDeploymentResponse(status=status)
def delete(self, deployment_pb, yatai_service=None):
try:
state = self.describe(deployment_pb, yatai_service).state
if state.state != DeploymentState.RUNNING:
message = (
'Failed to delete, no active deployment {name}. '
'The current state is {state}'.format(
name=deployment_pb.name,
state=DeploymentState.State.Name(state.state),
)
)
return DeleteDeploymentResponse(status=Status.ABORTED(message))
deployment_spec = deployment_pb.spec
aws_config = deployment_spec.aws_lambda_operator_config
bento_pb = yatai_service.GetBento(
GetBentoRequest(
bento_name=deployment_spec.bento_name,
bento_version=deployment_spec.bento_version,
)
)
bento_service_metadata = bento_pb.bento.bento_service_metadata
# We are not validating api_name, because for delete, you don't
# need them.
api_names = (
[aws_config.api_name]
if aws_config.api_name
else [api.name for api in bento_service_metadata.apis]
)
with TempDirectory() as serverless_project_dir:
generate_aws_lambda_serverless_config(
bento_service_metadata.env.python_version,
deployment_pb.name,
api_names,
serverless_project_dir,
aws_config.region,
# BentoML deployment namespace is mapping to serverless `stage`
# concept
stage=deployment_pb.namespace,
)
response = call_serverless_command(['remove'], serverless_project_dir)
stack_name = '{name}-{namespace}'.format(
name=deployment_pb.name, namespace=deployment_pb.namespace
)
if "Serverless: Stack removal finished..." in response:
status = Status.OK()
elif "Stack '{}' does not exist".format(stack_name) in response:
status = Status.NOT_FOUND(
'Deployment {} not found'.format(stack_name)
)
else:
status = Status.ABORTED()
return DeleteDeploymentResponse(status=status)
except BentoMLException as error:
return DeleteDeploymentResponse(status=exception_to_return_status(error))
def ApplyDeployment(self, request, context=None):
try:
# apply default namespace if not set
request.deployment.namespace = (
request.deployment.namespace or self.default_namespace
)
validation_errors = validate_deployment_pb_schema(request.deployment)
if validation_errors:
return ApplyDeploymentResponse(
status=Status.ABORTED(
'Failed to validate deployment. {errors}'.format(
errors=validation_errors
)
)
)
previous_deployment = self.deployment_store.get(
request.deployment.name, request.deployment.namespace
)
if previous_deployment:
# check deployment platform
if (
previous_deployment.spec.operator
!= request.deployment.spec.operator
):
return ApplyDeploymentResponse(
status=Status.ABORTED(
'Can not change the target deploy platform of existing '
'active deployment. Try delete existing deployment and '
'deploy to new target platform again'
)
)
request.deployment.state = DeploymentState(
state=DeploymentState.PENDING
)
self.deployment_store.insert_or_update(request.deployment)
# find deployment operator based on deployment spec
operator = get_deployment_operator(request.deployment)
# deploying to target platform
response = operator.apply(
request.deployment, self.repo, previous_deployment
)
# update deployment state
self.deployment_store.insert_or_update(response.deployment)
return response
except BentoMLException as e:
logger.error("INTERNAL ERROR: %s", e)
return ApplyDeploymentResponse(status=Status.INTERNAL(str(e)))
def AddBento(self, request, context=None):
try:
# TODO: validate request
bento_metadata_pb = self.bento_metadata_store.get(
request.bento_name, request.bento_version
)
if bento_metadata_pb:
error_message = "BentoService bundle: {}:{} already exist".format(
request.bento_name, request.bento_version
)
logger.error(error_message)
return AddBentoResponse(status=Status.ABORTED(error_message))
new_bento_uri = self.repo.add(request.bento_name, request.bento_version)
self.bento_metadata_store.add(
bento_name=request.bento_name,
bento_version=request.bento_version,
uri=new_bento_uri.uri,
uri_type=new_bento_uri.type,
)
return AddBentoResponse(status=Status.OK(), uri=new_bento_uri)
except BentoMLException as e:
logger.error("INTERNAL ERROR: %s", e)
return AddBentoResponse(status=Status.INTERNAL(str(e)))
def DangerouslyDeleteBento(self, request, context=None):
try:
# TODO: validate request
bento_pb = self.bento_metadata_store.get(request.bento_name,
request.bento_version)
if not bento_pb:
msg = (
f"BentoService {request.bento_name}:{request.bento_version} "
f"has already been deleted")
return DangerouslyDeleteBentoResponse(
status=Status.ABORTED(msg))
logger.debug('Deleting BentoService %s:%s', request.bento_name,
request.bento_version)
self.bento_metadata_store.dangerously_delete(
request.bento_name, request.bento_version)
self.repo.dangerously_delete(request.bento_name,
request.bento_version)
return DangerouslyDeleteBentoResponse(status=Status.OK())
except BentoMLException as e:
logger.error("RPC ERROR DangerouslyDeleteBento: %s", e)
return DangerouslyDeleteBentoResponse(status=e.status_proto)
except Exception as e: # pylint: disable=broad-except
logger.error("RPC ERROR DangerouslyDeleteBento: %s", e)
return DangerouslyDeleteBentoResponse(status=Status.INTERNAL())
def AddBento(self, request, context=None):
try:
# TODO: validate request
bento_pb = self.bento_metadata_store.get(request.bento_name,
request.bento_version)
if bento_pb:
error_message = "BentoService bundle: {}:{} already exist".format(
request.bento_name, request.bento_version)
logger.error(error_message)
return AddBentoResponse(status=Status.ABORTED(error_message))
new_bento_uri = self.repo.add(request.bento_name,
request.bento_version)
self.bento_metadata_store.add(
bento_name=request.bento_name,
bento_version=request.bento_version,
uri=new_bento_uri.uri,
uri_type=new_bento_uri.type,
)
return AddBentoResponse(status=Status.OK(), uri=new_bento_uri)
except BentoMLException as e:
logger.error("RPC ERROR AddBento: %s", e)
return DeleteDeploymentResponse(status=e.status_proto)
except Exception as e: # pylint: disable=broad-except
logger.error("URPC ERROR AddBento: %s", e)
return DeleteDeploymentResponse(status=Status.INTERNAL())
def delete(self, deployment_pb, yatai_service=None):
try:
state = self.describe(deployment_pb, yatai_service).state
if state.state != DeploymentState.RUNNING:
message = ('Failed to delete, no active deployment {name}. '
'The current state is {state}'.format(
name=deployment_pb.name,
state=DeploymentState.State.Name(state.state),
))
return DeleteDeploymentResponse(status=Status.ABORTED(message))
deployment_spec = deployment_pb.spec
gcp_config = deployment_spec.gcp_function_operator_config
bento_pb = yatai_service.GetBento(
GetBentoRequest(
bento_name=deployment_spec.bento_name,
bento_version=deployment_spec.bento_version,
))
bento_service_metadata = bento_pb.bento.bento_service_metadata
api_names = ([gcp_config.api_name] if gcp_config.api_name else
[api.name for api in bento_service_metadata.apis])
with TempDirectory() as serverless_project_dir:
generate_gcp_function_serverless_config(
deployment_pb.name,
api_names,
serverless_project_dir,
gcp_config.region,
# BentoML namespace is mapping to serverless stage.
stage=deployment_pb.namespace,
)
try:
response = call_serverless_command(['remove'],
serverless_project_dir)
if "Serverless: Stack removal finished..." in response:
status = Status.OK()
else:
status = Status.ABORTED()
except BentoMLException as e:
status = Status.INTERNAL(str(e))
return DeleteDeploymentResponse(status=status)
except BentoMLException as error:
return DeleteDeploymentResponse(
status=exception_to_return_status(error))
def mock_delete_deployment(deployment_pb):
if deployment_pb.name == MOCK_FAILED_DEPLOYMENT_NAME:
return DeleteDeploymentResponse(status=Status.ABORTED())
else:
return DeleteDeploymentResponse(status=Status.OK())
def describe(self, deployment_pb):
try:
deployment_spec = deployment_pb.spec
lambda_deployment_config = deployment_spec.aws_lambda_operator_config
lambda_deployment_config.region = (lambda_deployment_config.region
or get_default_aws_region())
if not lambda_deployment_config.region:
raise InvalidArgument('AWS region is missing')
bento_pb = self.yatai_service.GetBento(
GetBentoRequest(
bento_name=deployment_spec.bento_name,
bento_version=deployment_spec.bento_version,
))
bento_service_metadata = bento_pb.bento.bento_service_metadata
api_names = ([lambda_deployment_config.api_name]
if lambda_deployment_config.api_name else
[api.name for api in bento_service_metadata.apis])
try:
cf_client = boto3.client('cloudformation',
lambda_deployment_config.region)
stack_name = generate_aws_compatible_string(
'{ns}-{name}'.format(ns=deployment_pb.namespace,
name=deployment_pb.name))
cloud_formation_stack_result = cf_client.describe_stacks(
StackName=stack_name)
stack_result = cloud_formation_stack_result.get('Stacks')[0]
# https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/\
# using-cfn-describing-stacks.html
success_status = ['CREATE_COMPLETE', 'UPDATE_COMPLETE']
if stack_result['StackStatus'] in success_status:
if stack_result.get('Outputs'):
outputs = stack_result['Outputs']
else:
return DescribeDeploymentResponse(
status=Status.ABORTED(
'"Outputs" field is not present'),
state=DeploymentState(
state=DeploymentState.ERROR,
error_message='"Outputs" field is not present',
),
)
elif stack_result[
'StackStatus'] in FAILED_CLOUDFORMATION_STACK_STATUS:
state = DeploymentState(state=DeploymentState.FAILED)
state.timestamp.GetCurrentTime()
return DescribeDeploymentResponse(status=Status.OK(),
state=state)
else:
state = DeploymentState(state=DeploymentState.PENDING)
state.timestamp.GetCurrentTime()
return DescribeDeploymentResponse(status=Status.OK(),
state=state)
except Exception as error: # pylint: disable=broad-except
state = DeploymentState(state=DeploymentState.ERROR,
error_message=str(error))
state.timestamp.GetCurrentTime()
return DescribeDeploymentResponse(status=Status.INTERNAL(
str(error)),
state=state)
outputs = {o['OutputKey']: o['OutputValue'] for o in outputs}
info_json = {}
if 'EndpointUrl' in outputs:
info_json['endpoints'] = [
outputs['EndpointUrl'] + '/' + api_name
for api_name in api_names
]
if 'S3Bucket' in outputs:
info_json['s3_bucket'] = outputs['S3Bucket']
state = DeploymentState(state=DeploymentState.RUNNING,
info_json=json.dumps(info_json))
state.timestamp.GetCurrentTime()
return DescribeDeploymentResponse(status=Status.OK(), state=state)
except BentoMLException as error:
return DescribeDeploymentResponse(status=error.status_proto)
def describe(self, deployment_pb):
try:
deployment_spec = deployment_pb.spec
ec2_deployment_config = deployment_spec.aws_ec2_operator_config
ec2_deployment_config.region = (ec2_deployment_config.region
or get_default_aws_region())
if not ec2_deployment_config.region:
raise InvalidArgument("AWS region is missing")
bento_pb = self.yatai_service.GetBento(
GetBentoRequest(
bento_name=deployment_spec.bento_name,
bento_version=deployment_spec.bento_version,
))
bento_service_metadata = bento_pb.bento.bento_service_metadata
api_names = [api.name for api in bento_service_metadata.apis]
deployment_stack_name = generate_aws_compatible_string(
"btml-stack-{namespace}-{name}".format(
namespace=deployment_pb.namespace,
name=deployment_pb.name))
try:
cf_client = boto3.client("cloudformation",
ec2_deployment_config.region)
cloudformation_stack_result = cf_client.describe_stacks(
StackName=deployment_stack_name)
stack_result = cloudformation_stack_result.get("Stacks")[0]
if stack_result.get("Outputs"):
outputs = stack_result.get("Outputs")
else:
return DescribeDeploymentResponse(
status=Status.ABORTED(
'"Outputs" field is not present'),
state=DeploymentState(
state=DeploymentState.ERROR,
error_message='"Outputs" field is not present',
),
)
if stack_result[
"StackStatus"] in FAILED_CLOUDFORMATION_STACK_STATUS:
state = DeploymentState(state=DeploymentState.FAILED)
return DescribeDeploymentResponse(status=Status.OK(),
state=state)
except Exception as error: # pylint: disable=broad-except
state = DeploymentState(state=DeploymentState.ERROR,
error_message=str(error))
return DescribeDeploymentResponse(status=Status.INTERNAL(
str(error)),
state=state)
info_json = {}
outputs = {o["OutputKey"]: o["OutputValue"] for o in outputs}
if "AutoScalingGroup" in outputs:
info_json[
"InstanceDetails"] = get_instance_ip_from_scaling_group(
[outputs["AutoScalingGroup"]],
ec2_deployment_config.region)
info_json["Endpoints"] = get_endpoints_from_instance_address(
info_json["InstanceDetails"], api_names)
if "S3Bucket" in outputs:
info_json["S3Bucket"] = outputs["S3Bucket"]
if "TargetGroup" in outputs:
info_json["TargetGroup"] = outputs["TargetGroup"]
if "Url" in outputs:
info_json["Url"] = outputs["Url"]
healthy_target = get_healthy_target(outputs["TargetGroup"],
ec2_deployment_config.region)
if healthy_target:
deployment_state = DeploymentState.RUNNING
else:
deployment_state = DeploymentState.PENDING
state = DeploymentState(state=deployment_state,
info_json=json.dumps(info_json))
return DescribeDeploymentResponse(status=Status.OK(), state=state)
except BentoMLException as error:
return DescribeDeploymentResponse(status=error.status_proto)
