Пример #1
0
    def _update(self, deployment_pb, current_deployment, bento_pb, bento_path):
        if loader._is_remote_path(bento_path):
            with loader._resolve_remote_bundle_path(bento_path) as local_path:
                return self._update(deployment_pb, current_deployment,
                                    bento_pb, local_path)
        updated_deployment_spec = deployment_pb.spec
        updated_sagemaker_config = updated_deployment_spec.sagemaker_operator_config
        sagemaker_client = boto3.client(
            'sagemaker', updated_sagemaker_config.region
            or get_default_aws_region())

        try:
            raise_if_api_names_not_found_in_bento_service_metadata(
                bento_pb.bento.bento_service_metadata,
                [updated_sagemaker_config.api_name],
            )
            describe_latest_deployment_state = self.describe(deployment_pb)
            current_deployment_spec = current_deployment.spec
            current_sagemaker_config = current_deployment_spec.sagemaker_operator_config
            latest_deployment_state = json.loads(
                describe_latest_deployment_state.state.info_json)

            current_ecr_image_tag = latest_deployment_state[
                'ProductionVariants'][0]['DeployedImages'][0]['SpecifiedImage']
            if (updated_deployment_spec.bento_name !=
                    current_deployment_spec.bento_name
                    or updated_deployment_spec.bento_version !=
                    current_deployment_spec.bento_version):
                logger.debug(
                    'BentoService tag is different from current deployment, '
                    'creating new docker image and push to ECR')
                with TempDirectory() as temp_dir:
                    sagemaker_project_dir = os.path.join(
                        temp_dir, updated_deployment_spec.bento_name)
                    _init_sagemaker_project(
                        sagemaker_project_dir,
                        bento_path,
                        bento_pb.bento.bento_service_metadata.env.
                        docker_base_image,
                    )
                    ecr_image_path = create_and_push_docker_image_to_ecr(
                        updated_sagemaker_config.region,
                        updated_deployment_spec.bento_name,
                        updated_deployment_spec.bento_version,
                        sagemaker_project_dir,
                    )
            else:
                logger.debug('Using existing ECR image for Sagemaker model')
                ecr_image_path = current_ecr_image_tag

            (
                updated_sagemaker_model_name,
                updated_sagemaker_endpoint_config_name,
                sagemaker_endpoint_name,
            ) = _get_sagemaker_resource_names(deployment_pb)
            (
                current_sagemaker_model_name,
                current_sagemaker_endpoint_config_name,
                _,
            ) = _get_sagemaker_resource_names(current_deployment)

            if (updated_sagemaker_config.api_name !=
                    current_sagemaker_config.api_name
                    or updated_sagemaker_config.
                    num_of_gunicorn_workers_per_instance !=
                    current_sagemaker_config.
                    num_of_gunicorn_workers_per_instance
                    or ecr_image_path != current_ecr_image_tag):
                logger.debug(
                    'Sagemaker model requires update. Delete current sagemaker model %s'
                    'and creating new model %s',
                    current_sagemaker_model_name,
                    updated_sagemaker_model_name,
                )
                _delete_sagemaker_model_if_exist(sagemaker_client,
                                                 current_sagemaker_model_name)
                _create_sagemaker_model(
                    sagemaker_client,
                    updated_sagemaker_model_name,
                    ecr_image_path,
                    updated_sagemaker_config,
                )
            # When bento service tag is not changed, we need to delete the current
            # endpoint configuration in order to create new one to avoid name collation
            if (current_sagemaker_endpoint_config_name ==
                    updated_sagemaker_endpoint_config_name):
                logger.debug(
                    'Current sagemaker config name %s is same as updated one, '
                    'delete it before create new endpoint config',
                    current_sagemaker_endpoint_config_name,
                )
                _delete_sagemaker_endpoint_config_if_exist(
                    sagemaker_client, current_sagemaker_endpoint_config_name)
            logger.debug(
                'Create new endpoint configuration %s',
                updated_sagemaker_endpoint_config_name,
            )
            _create_sagemaker_endpoint_config(
                sagemaker_client,
                updated_sagemaker_model_name,
                updated_sagemaker_endpoint_config_name,
                updated_sagemaker_config,
            )
            logger.debug(
                'Updating endpoint to new endpoint configuration %s',
                updated_sagemaker_endpoint_config_name,
            )
            _update_sagemaker_endpoint(
                sagemaker_client,
                sagemaker_endpoint_name,
                updated_sagemaker_endpoint_config_name,
            )
            logger.debug(
                'Delete old sagemaker endpoint config %s',
                current_sagemaker_endpoint_config_name,
            )
            _delete_sagemaker_endpoint_config_if_exist(
                sagemaker_client, current_sagemaker_endpoint_config_name)
        except AWSServiceError as e:
            delete_sagemaker_deployment_resources_if_exist(deployment_pb)
            raise e

        return ApplyDeploymentResponse(status=Status.OK(),
                                       deployment=deployment_pb)
Пример #2
0
 def HealthCheck(self, request, context=None):
     return HealthCheckResponse(status=Status.OK())
Пример #3
0
    def describe(self, deployment_pb):
        try:
            deployment_spec = deployment_pb.spec
            lambda_deployment_config = deployment_spec.aws_lambda_operator_config
            lambda_deployment_config.region = (lambda_deployment_config.region
                                               or get_default_aws_region())
            if not lambda_deployment_config.region:
                raise InvalidArgument('AWS region is missing')

            bento_pb = self.yatai_service.GetBento(
                GetBentoRequest(
                    bento_name=deployment_spec.bento_name,
                    bento_version=deployment_spec.bento_version,
                ))
            bento_service_metadata = bento_pb.bento.bento_service_metadata
            api_names = ([lambda_deployment_config.api_name]
                         if lambda_deployment_config.api_name else
                         [api.name for api in bento_service_metadata.apis])

            try:
                cf_client = boto3.client('cloudformation',
                                         lambda_deployment_config.region)
                cloud_formation_stack_result = cf_client.describe_stacks(
                    StackName='{ns}-{name}'.format(ns=deployment_pb.namespace,
                                                   name=deployment_pb.name))
                stack_result = cloud_formation_stack_result.get('Stacks')[0]
                # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/\
                # using-cfn-describing-stacks.html
                success_status = ['CREATE_COMPLETE', 'UPDATE_COMPLETE']
                if stack_result['StackStatus'] in success_status:
                    if stack_result.get('Outputs'):
                        outputs = stack_result['Outputs']
                    else:
                        return DescribeDeploymentResponse(
                            status=Status.ABORTED(
                                '"Outputs" field is not present'),
                            state=DeploymentState(
                                state=DeploymentState.ERROR,
                                error_message='"Outputs" field is not present',
                            ),
                        )
                elif stack_result[
                        'StackStatus'] in FAILED_CLOUDFORMATION_STACK_STATUS:
                    state = DeploymentState(state=DeploymentState.FAILED)
                    state.timestamp.GetCurrentTime()
                    return DescribeDeploymentResponse(status=Status.OK(),
                                                      state=state)
                else:
                    state = DeploymentState(state=DeploymentState.PENDING)
                    state.timestamp.GetCurrentTime()
                    return DescribeDeploymentResponse(status=Status.OK(),
                                                      state=state)
            except Exception as error:  # pylint: disable=broad-except
                state = DeploymentState(state=DeploymentState.ERROR,
                                        error_message=str(error))
                state.timestamp.GetCurrentTime()
                return DescribeDeploymentResponse(status=Status.INTERNAL(
                    str(error)),
                                                  state=state)
            outputs = {o['OutputKey']: o['OutputValue'] for o in outputs}
            info_json = {}

            if 'EndpointUrl' in outputs:
                info_json['endpoints'] = [
                    outputs['EndpointUrl'] + '/' + api_name
                    for api_name in api_names
                ]
            if 'S3Bucket' in outputs:
                info_json['s3_bucket'] = outputs['S3Bucket']

            state = DeploymentState(state=DeploymentState.RUNNING,
                                    info_json=json.dumps(info_json))
            state.timestamp.GetCurrentTime()
            return DescribeDeploymentResponse(status=Status.OK(), state=state)
        except BentoMLException as error:
            return DescribeDeploymentResponse(status=error.status_proto)
Пример #4
0
    def apply(self, deployment_pb, yatai_service, prev_deployment=None):
        try:
            ensure_docker_available_or_raise()
            deployment_spec = deployment_pb.spec
            aws_config = deployment_spec.aws_lambda_operator_config

            bento_pb = yatai_service.GetBento(
                GetBentoRequest(
                    bento_name=deployment_spec.bento_name,
                    bento_version=deployment_spec.bento_version,
                )
            )
            if bento_pb.bento.uri.type != BentoUri.LOCAL:
                raise BentoMLException(
                    'BentoML currently only support local repository'
                )
            else:
                bento_path = bento_pb.bento.uri.uri
            bento_service_metadata = bento_pb.bento.bento_service_metadata

            template = 'aws-python3'
            if version.parse(bento_service_metadata.env.python_version) < version.parse(
                '3.0.0'
            ):
                template = 'aws-python'

            api_names = (
                [aws_config.api_name]
                if aws_config.api_name
                else [api.name for api in bento_service_metadata.apis]
            )
            ensure_deploy_api_name_exists_in_bento(
                [api.name for api in bento_service_metadata.apis], api_names
            )

            with TempDirectory() as serverless_project_dir:
                init_serverless_project_dir(
                    serverless_project_dir,
                    bento_path,
                    deployment_pb.name,
                    deployment_spec.bento_name,
                    template,
                )
                generate_aws_lambda_handler_py(
                    deployment_spec.bento_name, api_names, serverless_project_dir
                )
                generate_aws_lambda_serverless_config(
                    bento_service_metadata.env.python_version,
                    deployment_pb.name,
                    api_names,
                    serverless_project_dir,
                    aws_config.region,
                    # BentoML deployment namespace is mapping to serverless `stage`
                    # concept
                    stage=deployment_pb.namespace,
                )
                logger.info(
                    'Installing additional packages: serverless-python-requirements, '
                    'serverless-apigw-binary'
                )
                install_serverless_plugin(
                    "serverless-python-requirements", serverless_project_dir
                )
                install_serverless_plugin(
                    "serverless-apigw-binary", serverless_project_dir
                )
                logger.info('Deploying to AWS Lambda')
                call_serverless_command(["deploy"], serverless_project_dir)

            res_deployment_pb = Deployment(state=DeploymentState())
            res_deployment_pb.CopyFrom(deployment_pb)
            state = self.describe(res_deployment_pb, yatai_service).state
            res_deployment_pb.state.CopyFrom(state)
            return ApplyDeploymentResponse(
                status=Status.OK(), deployment=res_deployment_pb
            )
        except BentoMLException as error:
            return ApplyDeploymentResponse(status=exception_to_return_status(error))
Пример #5
0
def mock_delete_deployment(deployment_pb):
    if deployment_pb.name == MOCK_FAILED_DEPLOYMENT_NAME:
        return DeleteDeploymentResponse(status=Status.ABORTED())
    else:
        return DeleteDeploymentResponse(status=Status.OK())
Пример #6
0
    def describe(self, deployment_pb):
        try:
            deployment_spec = deployment_pb.spec
            ec2_deployment_config = deployment_spec.aws_ec2_operator_config
            ec2_deployment_config.region = (ec2_deployment_config.region
                                            or get_default_aws_region())
            if not ec2_deployment_config.region:
                raise InvalidArgument("AWS region is missing")

            bento_pb = self.yatai_service.GetBento(
                GetBentoRequest(
                    bento_name=deployment_spec.bento_name,
                    bento_version=deployment_spec.bento_version,
                ))
            bento_service_metadata = bento_pb.bento.bento_service_metadata
            api_names = [api.name for api in bento_service_metadata.apis]

            _, deployment_stack_name, _, _ = generate_ec2_resource_names(
                deployment_pb.namespace, deployment_pb.name)
            try:
                stack_result = describe_cloudformation_stack(
                    ec2_deployment_config.region, deployment_stack_name)
                if stack_result.get("Outputs"):
                    outputs = stack_result.get("Outputs")
                else:
                    return DescribeDeploymentResponse(
                        status=Status.ABORTED(
                            '"Outputs" field is not present'),
                        state=DeploymentState(
                            state=DeploymentState.ERROR,
                            error_message='"Outputs" field is not present',
                        ),
                    )

                if stack_result[
                        "StackStatus"] in FAILED_CLOUDFORMATION_STACK_STATUS:
                    state = DeploymentState(state=DeploymentState.FAILED)
                    return DescribeDeploymentResponse(status=Status.OK(),
                                                      state=state)

            except Exception as error:  # pylint: disable=broad-except
                state = DeploymentState(state=DeploymentState.ERROR,
                                        error_message=str(error))
                return DescribeDeploymentResponse(status=Status.INTERNAL(
                    str(error)),
                                                  state=state)

            info_json = {}
            outputs = {o["OutputKey"]: o["OutputValue"] for o in outputs}
            if "AutoScalingGroup" in outputs:
                info_json[
                    "InstanceDetails"] = get_instance_ip_from_scaling_group(
                        [outputs["AutoScalingGroup"]],
                        ec2_deployment_config.region)
                info_json["Endpoints"] = get_endpoints_from_instance_address(
                    info_json["InstanceDetails"], api_names)
            if "S3Bucket" in outputs:
                info_json["S3Bucket"] = outputs["S3Bucket"]
            if "TargetGroup" in outputs:
                info_json["TargetGroup"] = outputs["TargetGroup"]
            if "Url" in outputs:
                info_json["Url"] = outputs["Url"]

            healthy_target = get_healthy_target(outputs["TargetGroup"],
                                                ec2_deployment_config.region)
            if healthy_target:
                deployment_state = DeploymentState.RUNNING
            else:
                deployment_state = DeploymentState.PENDING
            state = DeploymentState(state=deployment_state,
                                    info_json=json.dumps(info_json))
            return DescribeDeploymentResponse(status=Status.OK(), state=state)

        except BentoMLException as error:
            return DescribeDeploymentResponse(status=error.status_proto)
Пример #7
0
def test_deployment_labels():
    runner = CliRunner()
    cli = create_bentoml_cli()

    failed_result = runner.invoke(
        cli.commands['lambda'],
        [
            'deploy',
            'failed-name',
            '-b',
            'ExampleBentoService:version',
            '--labels',
            'test=abc',
        ],
    )
    assert failed_result.exit_code == 2

    with mock.patch(
            'bentoml.yatai.deployment.aws_lambda.operator.AwsLambdaDeploymentOperator.add'
    ) as mock_operator_add:
        bento_name = 'MockService'
        bento_version = 'MockVersion'
        deployment_name = f'test-label-{uuid.uuid4().hex[:8]}'
        deployment_namespace = 'test-namespace'
        mocked_deployment_pb = Deployment(name=deployment_name,
                                          namespace=deployment_namespace)
        mocked_deployment_pb.spec.bento_name = bento_name
        mocked_deployment_pb.spec.bento_version = bento_version
        mocked_deployment_pb.spec.operator = DeploymentSpec.AWS_LAMBDA
        mocked_deployment_pb.spec.aws_lambda_operator_config.memory_size = 1000
        mocked_deployment_pb.spec.aws_lambda_operator_config.timeout = 60
        mocked_deployment_pb.spec.aws_lambda_operator_config.region = 'us-west-2'
        mock_operator_add.return_value = ApplyDeploymentResponse(
            status=Status.OK(), deployment=mocked_deployment_pb)

        success_result = runner.invoke(
            cli.commands['lambda'],
            [
                'deploy',
                deployment_name,
                '-b',
                f'{bento_name}:{bento_version}',
                '--namespace',
                deployment_namespace,
                '--labels',
                'created_by:admin,cicd:passed',
                '--region',
                'us-west-2',
            ],
        )
        assert success_result.exit_code == 0

        list_result = runner.invoke(
            cli.commands['deployment'],
            [
                'list',
                '--labels',
                'created_by=admin,cicd NotIn (failed, unsuccessful)',
                '--output',
                'wide',
            ],
        )
        assert list_result.exit_code == 0
        assert deployment_name in list_result.output.strip()
        assert 'created_by:admin' in list_result.output.strip()