def delete(self, deployment_pb, repo=None): state = self.describe(deployment_pb, repo).state if state.state != DeploymentState.RUNNING: message = ('Failed to delete, no active deployment {name}. ' 'The current state is {state}'.format( name=deployment_pb.name, state=DeploymentState.State.Name(state.state), )) return DeleteDeploymentResponse(status=Status.ABORTED(message)) deployment_spec = deployment_pb.spec aws_config = deployment_spec.aws_lambda_operator_config bento_path = repo.get(deployment_spec.bento_name, deployment_spec.bento_version) bento_config = load_bentoml_config(bento_path) with TemporaryServerlessConfig( archive_path=bento_path, deployment_name=deployment_pb.name, region=aws_config.region, stage=deployment_pb.namespace, provider_name='aws', functions=generate_aws_handler_functions_config( bento_config['apis']), ) as tempdir: response = call_serverless_command(['serverless', 'remove'], tempdir) if "Serverless: Stack removal finished..." in response: status = Status.OK() else: status = Status.ABORTED() return DeleteDeploymentResponse(status=status)
def delete(self, deployment_pb, yatai_service=None): try: state = self.describe(deployment_pb, yatai_service).state if state.state != DeploymentState.RUNNING: message = ( 'Failed to delete, no active deployment {name}. ' 'The current state is {state}'.format( name=deployment_pb.name, state=DeploymentState.State.Name(state.state), ) ) return DeleteDeploymentResponse(status=Status.ABORTED(message)) deployment_spec = deployment_pb.spec aws_config = deployment_spec.aws_lambda_operator_config bento_pb = yatai_service.GetBento( GetBentoRequest( bento_name=deployment_spec.bento_name, bento_version=deployment_spec.bento_version, ) ) bento_service_metadata = bento_pb.bento.bento_service_metadata # We are not validating api_name, because for delete, you don't # need them. api_names = ( [aws_config.api_name] if aws_config.api_name else [api.name for api in bento_service_metadata.apis] ) with TempDirectory() as serverless_project_dir: generate_aws_lambda_serverless_config( bento_service_metadata.env.python_version, deployment_pb.name, api_names, serverless_project_dir, aws_config.region, # BentoML deployment namespace is mapping to serverless `stage` # concept stage=deployment_pb.namespace, ) response = call_serverless_command(['remove'], serverless_project_dir) stack_name = '{name}-{namespace}'.format( name=deployment_pb.name, namespace=deployment_pb.namespace ) if "Serverless: Stack removal finished..." in response: status = Status.OK() elif "Stack '{}' does not exist".format(stack_name) in response: status = Status.NOT_FOUND( 'Deployment {} not found'.format(stack_name) ) else: status = Status.ABORTED() return DeleteDeploymentResponse(status=status) except BentoMLException as error: return DeleteDeploymentResponse(status=exception_to_return_status(error))
def ApplyDeployment(self, request, context=None): try: # apply default namespace if not set request.deployment.namespace = ( request.deployment.namespace or self.default_namespace ) validation_errors = validate_deployment_pb_schema(request.deployment) if validation_errors: return ApplyDeploymentResponse( status=Status.ABORTED( 'Failed to validate deployment. {errors}'.format( errors=validation_errors ) ) ) previous_deployment = self.deployment_store.get( request.deployment.name, request.deployment.namespace ) if previous_deployment: # check deployment platform if ( previous_deployment.spec.operator != request.deployment.spec.operator ): return ApplyDeploymentResponse( status=Status.ABORTED( 'Can not change the target deploy platform of existing ' 'active deployment. Try delete existing deployment and ' 'deploy to new target platform again' ) ) request.deployment.state = DeploymentState( state=DeploymentState.PENDING ) self.deployment_store.insert_or_update(request.deployment) # find deployment operator based on deployment spec operator = get_deployment_operator(request.deployment) # deploying to target platform response = operator.apply( request.deployment, self.repo, previous_deployment ) # update deployment state self.deployment_store.insert_or_update(response.deployment) return response except BentoMLException as e: logger.error("INTERNAL ERROR: %s", e) return ApplyDeploymentResponse(status=Status.INTERNAL(str(e)))
def AddBento(self, request, context=None): try: # TODO: validate request bento_metadata_pb = self.bento_metadata_store.get( request.bento_name, request.bento_version ) if bento_metadata_pb: error_message = "BentoService bundle: {}:{} already exist".format( request.bento_name, request.bento_version ) logger.error(error_message) return AddBentoResponse(status=Status.ABORTED(error_message)) new_bento_uri = self.repo.add(request.bento_name, request.bento_version) self.bento_metadata_store.add( bento_name=request.bento_name, bento_version=request.bento_version, uri=new_bento_uri.uri, uri_type=new_bento_uri.type, ) return AddBentoResponse(status=Status.OK(), uri=new_bento_uri) except BentoMLException as e: logger.error("INTERNAL ERROR: %s", e) return AddBentoResponse(status=Status.INTERNAL(str(e)))
def DangerouslyDeleteBento(self, request, context=None): try: # TODO: validate request bento_pb = self.bento_metadata_store.get(request.bento_name, request.bento_version) if not bento_pb: msg = ( f"BentoService {request.bento_name}:{request.bento_version} " f"has already been deleted") return DangerouslyDeleteBentoResponse( status=Status.ABORTED(msg)) logger.debug('Deleting BentoService %s:%s', request.bento_name, request.bento_version) self.bento_metadata_store.dangerously_delete( request.bento_name, request.bento_version) self.repo.dangerously_delete(request.bento_name, request.bento_version) return DangerouslyDeleteBentoResponse(status=Status.OK()) except BentoMLException as e: logger.error("RPC ERROR DangerouslyDeleteBento: %s", e) return DangerouslyDeleteBentoResponse(status=e.status_proto) except Exception as e: # pylint: disable=broad-except logger.error("RPC ERROR DangerouslyDeleteBento: %s", e) return DangerouslyDeleteBentoResponse(status=Status.INTERNAL())
def AddBento(self, request, context=None): try: # TODO: validate request bento_pb = self.bento_metadata_store.get(request.bento_name, request.bento_version) if bento_pb: error_message = "BentoService bundle: {}:{} already exist".format( request.bento_name, request.bento_version) logger.error(error_message) return AddBentoResponse(status=Status.ABORTED(error_message)) new_bento_uri = self.repo.add(request.bento_name, request.bento_version) self.bento_metadata_store.add( bento_name=request.bento_name, bento_version=request.bento_version, uri=new_bento_uri.uri, uri_type=new_bento_uri.type, ) return AddBentoResponse(status=Status.OK(), uri=new_bento_uri) except BentoMLException as e: logger.error("RPC ERROR AddBento: %s", e) return DeleteDeploymentResponse(status=e.status_proto) except Exception as e: # pylint: disable=broad-except logger.error("URPC ERROR AddBento: %s", e) return DeleteDeploymentResponse(status=Status.INTERNAL())
def delete(self, deployment_pb, yatai_service=None): try: state = self.describe(deployment_pb, yatai_service).state if state.state != DeploymentState.RUNNING: message = ('Failed to delete, no active deployment {name}. ' 'The current state is {state}'.format( name=deployment_pb.name, state=DeploymentState.State.Name(state.state), )) return DeleteDeploymentResponse(status=Status.ABORTED(message)) deployment_spec = deployment_pb.spec gcp_config = deployment_spec.gcp_function_operator_config bento_pb = yatai_service.GetBento( GetBentoRequest( bento_name=deployment_spec.bento_name, bento_version=deployment_spec.bento_version, )) bento_service_metadata = bento_pb.bento.bento_service_metadata api_names = ([gcp_config.api_name] if gcp_config.api_name else [api.name for api in bento_service_metadata.apis]) with TempDirectory() as serverless_project_dir: generate_gcp_function_serverless_config( deployment_pb.name, api_names, serverless_project_dir, gcp_config.region, # BentoML namespace is mapping to serverless stage. stage=deployment_pb.namespace, ) try: response = call_serverless_command(['remove'], serverless_project_dir) if "Serverless: Stack removal finished..." in response: status = Status.OK() else: status = Status.ABORTED() except BentoMLException as e: status = Status.INTERNAL(str(e)) return DeleteDeploymentResponse(status=status) except BentoMLException as error: return DeleteDeploymentResponse( status=exception_to_return_status(error))
def mock_delete_deployment(deployment_pb): if deployment_pb.name == MOCK_FAILED_DEPLOYMENT_NAME: return DeleteDeploymentResponse(status=Status.ABORTED()) else: return DeleteDeploymentResponse(status=Status.OK())
def describe(self, deployment_pb): try: deployment_spec = deployment_pb.spec lambda_deployment_config = deployment_spec.aws_lambda_operator_config lambda_deployment_config.region = (lambda_deployment_config.region or get_default_aws_region()) if not lambda_deployment_config.region: raise InvalidArgument('AWS region is missing') bento_pb = self.yatai_service.GetBento( GetBentoRequest( bento_name=deployment_spec.bento_name, bento_version=deployment_spec.bento_version, )) bento_service_metadata = bento_pb.bento.bento_service_metadata api_names = ([lambda_deployment_config.api_name] if lambda_deployment_config.api_name else [api.name for api in bento_service_metadata.apis]) try: cf_client = boto3.client('cloudformation', lambda_deployment_config.region) stack_name = generate_aws_compatible_string( '{ns}-{name}'.format(ns=deployment_pb.namespace, name=deployment_pb.name)) cloud_formation_stack_result = cf_client.describe_stacks( StackName=stack_name) stack_result = cloud_formation_stack_result.get('Stacks')[0] # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/\ # using-cfn-describing-stacks.html success_status = ['CREATE_COMPLETE', 'UPDATE_COMPLETE'] if stack_result['StackStatus'] in success_status: if stack_result.get('Outputs'): outputs = stack_result['Outputs'] else: return DescribeDeploymentResponse( status=Status.ABORTED( '"Outputs" field is not present'), state=DeploymentState( state=DeploymentState.ERROR, error_message='"Outputs" field is not present', ), ) elif stack_result[ 'StackStatus'] in FAILED_CLOUDFORMATION_STACK_STATUS: state = DeploymentState(state=DeploymentState.FAILED) state.timestamp.GetCurrentTime() return DescribeDeploymentResponse(status=Status.OK(), state=state) else: state = DeploymentState(state=DeploymentState.PENDING) state.timestamp.GetCurrentTime() return DescribeDeploymentResponse(status=Status.OK(), state=state) except Exception as error: # pylint: disable=broad-except state = DeploymentState(state=DeploymentState.ERROR, error_message=str(error)) state.timestamp.GetCurrentTime() return DescribeDeploymentResponse(status=Status.INTERNAL( str(error)), state=state) outputs = {o['OutputKey']: o['OutputValue'] for o in outputs} info_json = {} if 'EndpointUrl' in outputs: info_json['endpoints'] = [ outputs['EndpointUrl'] + '/' + api_name for api_name in api_names ] if 'S3Bucket' in outputs: info_json['s3_bucket'] = outputs['S3Bucket'] state = DeploymentState(state=DeploymentState.RUNNING, info_json=json.dumps(info_json)) state.timestamp.GetCurrentTime() return DescribeDeploymentResponse(status=Status.OK(), state=state) except BentoMLException as error: return DescribeDeploymentResponse(status=error.status_proto)
def describe(self, deployment_pb): try: deployment_spec = deployment_pb.spec ec2_deployment_config = deployment_spec.aws_ec2_operator_config ec2_deployment_config.region = (ec2_deployment_config.region or get_default_aws_region()) if not ec2_deployment_config.region: raise InvalidArgument("AWS region is missing") bento_pb = self.yatai_service.GetBento( GetBentoRequest( bento_name=deployment_spec.bento_name, bento_version=deployment_spec.bento_version, )) bento_service_metadata = bento_pb.bento.bento_service_metadata api_names = [api.name for api in bento_service_metadata.apis] deployment_stack_name = generate_aws_compatible_string( "btml-stack-{namespace}-{name}".format( namespace=deployment_pb.namespace, name=deployment_pb.name)) try: cf_client = boto3.client("cloudformation", ec2_deployment_config.region) cloudformation_stack_result = cf_client.describe_stacks( StackName=deployment_stack_name) stack_result = cloudformation_stack_result.get("Stacks")[0] if stack_result.get("Outputs"): outputs = stack_result.get("Outputs") else: return DescribeDeploymentResponse( status=Status.ABORTED( '"Outputs" field is not present'), state=DeploymentState( state=DeploymentState.ERROR, error_message='"Outputs" field is not present', ), ) if stack_result[ "StackStatus"] in FAILED_CLOUDFORMATION_STACK_STATUS: state = DeploymentState(state=DeploymentState.FAILED) return DescribeDeploymentResponse(status=Status.OK(), state=state) except Exception as error: # pylint: disable=broad-except state = DeploymentState(state=DeploymentState.ERROR, error_message=str(error)) return DescribeDeploymentResponse(status=Status.INTERNAL( str(error)), state=state) info_json = {} outputs = {o["OutputKey"]: o["OutputValue"] for o in outputs} if "AutoScalingGroup" in outputs: info_json[ "InstanceDetails"] = get_instance_ip_from_scaling_group( [outputs["AutoScalingGroup"]], ec2_deployment_config.region) info_json["Endpoints"] = get_endpoints_from_instance_address( info_json["InstanceDetails"], api_names) if "S3Bucket" in outputs: info_json["S3Bucket"] = outputs["S3Bucket"] if "TargetGroup" in outputs: info_json["TargetGroup"] = outputs["TargetGroup"] if "Url" in outputs: info_json["Url"] = outputs["Url"] healthy_target = get_healthy_target(outputs["TargetGroup"], ec2_deployment_config.region) if healthy_target: deployment_state = DeploymentState.RUNNING else: deployment_state = DeploymentState.PENDING state = DeploymentState(state=deployment_state, info_json=json.dumps(info_json)) return DescribeDeploymentResponse(status=Status.OK(), state=state) except BentoMLException as error: return DescribeDeploymentResponse(status=error.status_proto)