def describe(self, deployment_pb):
try:
(
resource_group_name,
_,
_,
function_name,
_,
) = _generate_azure_resource_names(deployment_pb.namespace,
deployment_pb.name)
show_function_result = _call_az_cli(
command=[
'az',
'functionapp',
'show',
'--name',
function_name,
'--resource-group',
resource_group_name,
],
message='show Azure functionapp detail',
)
keys = [
'defaultHostName',
'enabledHostNames',
'hostNames',
'id',
'kind',
'lastModifiedTimeUtc',
'location',
'name',
'repositorySiteName',
'reserved',
'resourceGroup',
'state',
'type',
'usageState',
]
# Need find more documentation on the status of functionapp. For now, any
# other status is error.
if show_function_result['state'] == 'Running':
state = DeploymentState.RUNNING
else:
state = DeploymentState.ERROR
info_json = {
k: v
for k, v in show_function_result.items() if k in keys
}
deployment_state = DeploymentState(
info_json=json.dumps(info_json, default=str),
state=state,
)
deployment_state.timestamp.GetCurrentTime()
return DescribeDeploymentResponse(state=deployment_state,
status=Status.OK())
except BentoMLException as error:
return DescribeDeploymentResponse(status=error.status_proto)
def DescribeDeployment(self, request, context=None):
try:
request.namespace = request.namespace or self.default_namespace
deployment_pb = self.deployment_store.get(request.deployment_name,
request.namespace)
if deployment_pb:
operator = get_deployment_operator(self, deployment_pb)
response = operator.describe(deployment_pb)
if response.status.status_code == status_pb2.Status.OK:
with self.deployment_store.update_deployment(
request.deployment_name,
request.namespace) as deployment:
deployment.state = ProtoMessageToDict(response.state)
return response
else:
return DescribeDeploymentResponse(status=Status.NOT_FOUND(
'Deployment "{}" in namespace "{}" not found'.format(
request.deployment_name, request.namespace)))
except BentoMLException as e:
logger.error("RPC ERROR DescribeDeployment: %s", e)
return DeleteDeploymentResponse(status=e.status_proto)
except Exception as e: # pylint: disable=broad-except
logger.error("RPC ERROR DescribeDeployment: %s", e)
return DeleteDeploymentResponse(status=Status.INTERNAL())
def describe(self, deployment_pb):
try:
deployment_spec = deployment_pb.spec
sagemaker_config = deployment_spec.sagemaker_operator_config
sagemaker_config.region = (sagemaker_config.region
or get_default_aws_region())
if not sagemaker_config.region:
raise InvalidArgument("AWS region is missing")
sagemaker_client = boto3.client("sagemaker",
sagemaker_config.region)
_, _, sagemaker_endpoint_name = _get_sagemaker_resource_names(
deployment_pb)
try:
endpoint_status_response = sagemaker_client.describe_endpoint(
EndpointName=sagemaker_endpoint_name)
except ClientError as e:
raise _aws_client_error_to_bentoml_exception(
e,
f"Failed to fetch current status of sagemaker endpoint "
f"'{sagemaker_endpoint_name}'",
)
logger.debug("AWS describe endpoint response: %s",
endpoint_status_response)
endpoint_status = endpoint_status_response["EndpointStatus"]
service_state = ENDPOINT_STATUS_TO_STATE[endpoint_status]
deployment_state = DeploymentState(
state=service_state,
info_json=json.dumps(endpoint_status_response, default=str),
)
deployment_state.timestamp.GetCurrentTime()
return DescribeDeploymentResponse(state=deployment_state,
status=Status.OK())
except BentoMLException as error:
return DescribeDeploymentResponse(status=error.status_proto)
def describe(self, deployment_pb):
try:
deployment_spec = deployment_pb.spec
lambda_deployment_config = deployment_spec.aws_lambda_operator_config
lambda_deployment_config.region = (lambda_deployment_config.region
or get_default_aws_region())
if not lambda_deployment_config.region:
raise InvalidArgument('AWS region is missing')
bento_pb = self.yatai_service.GetBento(
GetBentoRequest(
bento_name=deployment_spec.bento_name,
bento_version=deployment_spec.bento_version,
))
bento_service_metadata = bento_pb.bento.bento_service_metadata
api_names = ([lambda_deployment_config.api_name]
if lambda_deployment_config.api_name else
[api.name for api in bento_service_metadata.apis])
try:
cf_client = boto3.client('cloudformation',
lambda_deployment_config.region)
stack_name = generate_aws_compatible_string(
'{ns}-{name}'.format(ns=deployment_pb.namespace,
name=deployment_pb.name))
cloud_formation_stack_result = cf_client.describe_stacks(
StackName=stack_name)
stack_result = cloud_formation_stack_result.get('Stacks')[0]
# https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/\
# using-cfn-describing-stacks.html
success_status = ['CREATE_COMPLETE', 'UPDATE_COMPLETE']
if stack_result['StackStatus'] in success_status:
if stack_result.get('Outputs'):
outputs = stack_result['Outputs']
else:
return DescribeDeploymentResponse(
status=Status.ABORTED(
'"Outputs" field is not present'),
state=DeploymentState(
state=DeploymentState.ERROR,
error_message='"Outputs" field is not present',
),
)
elif stack_result[
'StackStatus'] in FAILED_CLOUDFORMATION_STACK_STATUS:
state = DeploymentState(state=DeploymentState.FAILED)
state.timestamp.GetCurrentTime()
return DescribeDeploymentResponse(status=Status.OK(),
state=state)
else:
state = DeploymentState(state=DeploymentState.PENDING)
state.timestamp.GetCurrentTime()
return DescribeDeploymentResponse(status=Status.OK(),
state=state)
except Exception as error: # pylint: disable=broad-except
state = DeploymentState(state=DeploymentState.ERROR,
error_message=str(error))
state.timestamp.GetCurrentTime()
return DescribeDeploymentResponse(status=Status.INTERNAL(
str(error)),
state=state)
outputs = {o['OutputKey']: o['OutputValue'] for o in outputs}
info_json = {}
if 'EndpointUrl' in outputs:
info_json['endpoints'] = [
outputs['EndpointUrl'] + '/' + api_name
for api_name in api_names
]
if 'S3Bucket' in outputs:
info_json['s3_bucket'] = outputs['S3Bucket']
state = DeploymentState(state=DeploymentState.RUNNING,
info_json=json.dumps(info_json))
state.timestamp.GetCurrentTime()
return DescribeDeploymentResponse(status=Status.OK(), state=state)
except BentoMLException as error:
return DescribeDeploymentResponse(status=error.status_proto)
def describe(self, deployment_pb):
try:
deployment_spec = deployment_pb.spec
ec2_deployment_config = deployment_spec.aws_ec2_operator_config
ec2_deployment_config.region = (ec2_deployment_config.region
or get_default_aws_region())
if not ec2_deployment_config.region:
raise InvalidArgument("AWS region is missing")
bento_pb = self.yatai_service.GetBento(
GetBentoRequest(
bento_name=deployment_spec.bento_name,
bento_version=deployment_spec.bento_version,
))
bento_service_metadata = bento_pb.bento.bento_service_metadata
api_names = [api.name for api in bento_service_metadata.apis]
deployment_stack_name = generate_aws_compatible_string(
"btml-stack-{namespace}-{name}".format(
namespace=deployment_pb.namespace,
name=deployment_pb.name))
try:
cf_client = boto3.client("cloudformation",
ec2_deployment_config.region)
cloudformation_stack_result = cf_client.describe_stacks(
StackName=deployment_stack_name)
stack_result = cloudformation_stack_result.get("Stacks")[0]
if stack_result.get("Outputs"):
outputs = stack_result.get("Outputs")
else:
return DescribeDeploymentResponse(
status=Status.ABORTED(
'"Outputs" field is not present'),
state=DeploymentState(
state=DeploymentState.ERROR,
error_message='"Outputs" field is not present',
),
)
if stack_result[
"StackStatus"] in FAILED_CLOUDFORMATION_STACK_STATUS:
state = DeploymentState(state=DeploymentState.FAILED)
return DescribeDeploymentResponse(status=Status.OK(),
state=state)
except Exception as error: # pylint: disable=broad-except
state = DeploymentState(state=DeploymentState.ERROR,
error_message=str(error))
return DescribeDeploymentResponse(status=Status.INTERNAL(
str(error)),
state=state)
info_json = {}
outputs = {o["OutputKey"]: o["OutputValue"] for o in outputs}
if "AutoScalingGroup" in outputs:
info_json[
"InstanceDetails"] = get_instance_ip_from_scaling_group(
[outputs["AutoScalingGroup"]],
ec2_deployment_config.region)
info_json["Endpoints"] = get_endpoints_from_instance_address(
info_json["InstanceDetails"], api_names)
if "S3Bucket" in outputs:
info_json["S3Bucket"] = outputs["S3Bucket"]
if "TargetGroup" in outputs:
info_json["TargetGroup"] = outputs["TargetGroup"]
if "Url" in outputs:
info_json["Url"] = outputs["Url"]
healthy_target = get_healthy_target(outputs["TargetGroup"],
ec2_deployment_config.region)
if healthy_target:
deployment_state = DeploymentState.RUNNING
else:
deployment_state = DeploymentState.PENDING
state = DeploymentState(state=deployment_state,
info_json=json.dumps(info_json))
return DescribeDeploymentResponse(status=Status.OK(), state=state)
except BentoMLException as error:
return DescribeDeploymentResponse(status=error.status_proto)
@patch(
"bentoml.yatai.deployment.aws_ec2.operator.ensure_sam_available_or_raise",
MagicMock(),
)
@patch(
"bentoml.yatai.deployment.aws_ec2.operator.ensure_docker_available_or_raise",
MagicMock(),
)
@patch(
"bentoml.yatai.deployment.aws_ec2.operator.AwsEc2DeploymentOperator.deploy_service",
MagicMock(),
)
@patch(
"bentoml.yatai.deployment.aws_ec2.operator.AwsEc2DeploymentOperator.describe",
MagicMock(return_value=DescribeDeploymentResponse(status=status_pb2.Status(
status_code=status_pb2.Status.INTERNAL, error_message="failed"))),
)
def test_ec2_update_describe_failure():
yatai_service_mock = create_yatai_service_mock()
test_deployment_pb = generate_ec2_deployment_pb()
operator = AwsEc2DeploymentOperator(yatai_service_mock)
result_pb = operator.update(test_deployment_pb, test_deployment_pb)
assert result_pb.status.status_code == status_pb2.Status.INTERNAL
assert result_pb.deployment.state.state == DeploymentState.ERROR
@patch(
"bentoml.yatai.deployment.aws_ec2.operator.ensure_sam_available_or_raise",
MagicMock(),
)