def add(self, deployment_pb):
try:
deployment_spec = deployment_pb.spec
deployment_spec.aws_lambda_operator_config.region = (
deployment_spec.aws_lambda_operator_config.region
or get_default_aws_region())
if not deployment_spec.aws_lambda_operator_config.region:
raise InvalidArgument('AWS region is missing')
ensure_sam_available_or_raise()
ensure_docker_available_or_raise()
bento_pb = self.yatai_service.GetBento(
GetBentoRequest(
bento_name=deployment_spec.bento_name,
bento_version=deployment_spec.bento_version,
))
if bento_pb.bento.uri.type not in (BentoUri.LOCAL, BentoUri.S3):
raise BentoMLException(
'BentoML currently not support {} repository'.format(
BentoUri.StorageType.Name(bento_pb.bento.uri.type)))
return self._add(deployment_pb, bento_pb, bento_pb.bento.uri.uri)
except BentoMLException as error:
deployment_pb.state.state = DeploymentState.ERROR
deployment_pb.state.error_message = f'Error: {str(error)}'
return ApplyDeploymentResponse(status=error.status_proto,
deployment=deployment_pb)
def temporary_docker_postgres_url():
ensure_docker_available_or_raise()
container_name = f'e2e-test-yatai-service-postgres-db-{uuid.uuid4().hex[:6]}'
db_url = 'postgresql://*****:*****@localhost:5432/bentoml'
command = [
'docker',
'run',
'--rm',
'--name',
container_name,
'-e',
'POSTGRES_PASSWORD=postgres',
'-p',
'5432:5432',
'postgres',
]
docker_proc = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
wait_for_docker_container_ready(
container_name, b'database system is ready to accept connections')
from sqlalchemy_utils import create_database
create_database(db_url)
yield db_url
docker_proc.terminate()
def add(self, deployment_pb):
try:
ensure_docker_available_or_raise()
deployment_spec = deployment_pb.spec
sagemaker_config = deployment_spec.sagemaker_operator_config
if sagemaker_config is None:
raise YataiDeploymentException('Sagemaker configuration is missing.')
bento_pb = self.yatai_service.GetBento(
GetBentoRequest(
bento_name=deployment_spec.bento_name,
bento_version=deployment_spec.bento_version,
)
)
if bento_pb.bento.uri.type not in (BentoUri.LOCAL, BentoUri.S3):
raise BentoMLException(
'BentoML currently not support {} repository'.format(
BentoUri.StorageType.Name(bento_pb.bento.uri.type)
)
)
return self._add(deployment_pb, bento_pb, bento_pb.bento.uri.uri)
except BentoMLException as error:
deployment_pb.state.state = DeploymentState.ERROR
deployment_pb.state.error_message = (
f'Error creating SageMaker deployment: {str(error)}'
)
return ApplyDeploymentResponse(
status=error.status_proto, deployment=deployment_pb
)
def apply(self, deployment_pb, yatai_service, prev_deployment=None):
try:
ensure_docker_available_or_raise()
deployment_spec = deployment_pb.spec
sagemaker_config = deployment_spec.sagemaker_operator_config
if sagemaker_config is None:
raise BentoMLDeploymentException('Sagemaker configuration is missing.')
bento_pb = yatai_service.GetBento(
GetBentoRequest(
bento_name=deployment_spec.bento_name,
bento_version=deployment_spec.bento_version,
)
)
if bento_pb.bento.uri.type not in (BentoUri.LOCAL, BentoUri.S3):
raise BentoMLException(
'BentoML currently not support {} repository'.format(
bento_pb.bento.uri.type
)
)
return self._apply(
deployment_pb,
bento_pb,
yatai_service,
bento_pb.bento.uri.uri,
prev_deployment,
)
except BentoMLException as error:
return ApplyDeploymentResponse(status=exception_to_return_status(error))
def update(self, deployment_pb, previous_deployment):
try:
ensure_sam_available_or_raise()
ensure_docker_available_or_raise()
deployment_spec = deployment_pb.spec
bento_pb = self.yatai_service.GetBento(
GetBentoRequest(
bento_name=deployment_spec.bento_name,
bento_version=deployment_spec.bento_version,
)
)
if bento_pb.bento.uri.type not in (BentoUri.LOCAL, BentoUri.S3):
raise BentoMLException(
'BentoML currently not support {} repository'.format(
BentoUri.StorageType.Name(bento_pb.bento.uri.type)
)
)
return self._update(
deployment_pb, previous_deployment, bento_pb, bento_pb.bento.uri.uri
)
except BentoMLException as error:
deployment_pb.state.state = DeploymentState.ERROR
deployment_pb.state.error_message = f'Error: {str(error)}'
return ApplyDeploymentResponse(
status=error.status_code, deployment_pb=deployment_pb
)
def temporary_yatai_service_url():
ensure_docker_available_or_raise()
docker_client = docker.from_env()
local_bentoml_repo_path = os.path.abspath(
os.path.join(__file__, '..', '..'))
docker_tag = f'bentoml/yatai-service:e2e-test-{uuid.uuid4().hex[:6]}'
# Note: When set both `custom_context` and `fileobj`, docker api will not use the
# `path` provide... docker/api/build.py L138. The solution is create an actual
# Dockerfile along with path, instead of fileobj and custom_context.
with TempDirectory() as temp_dir:
temp_docker_file_path = os.path.join(temp_dir, 'Dockerfile')
with open(temp_docker_file_path, 'w') as f:
f.write(f"""\
FROM bentoml/yatai-service:{PREV_PYPI_RELEASE_VERSION}
ADD . /bentoml-local-repo
RUN pip install /bentoml-local-repo
""")
logger.info('building docker image')
docker_client.images.build(
path=local_bentoml_repo_path,
dockerfile=temp_docker_file_path,
tag=docker_tag,
)
logger.info('complete build docker image')
container_name = f'e2e-test-yatai-service-container-{uuid.uuid4().hex[:6]}'
yatai_service_url = 'localhost:50051'
command = [
'docker',
'run',
'--rm',
'--name',
container_name,
'-p',
'50051:50051',
'-p',
'3000:3000',
docker_tag,
'--repo-base-url',
'/tmp',
]
logger.info(f'Running docker command {" ".join(command)}')
docker_proc = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
wait_for_docker_container_ready(
container_name, b'* Starting BentoML YataiService gRPC Server')
yield yatai_service_url
docker_proc.terminate()
def test_ensure_docker_available_or_raise():
if version_info.major < 3:
not_found_error = OSError
else:
not_found_error = FileNotFoundError
with patch('subprocess.check_output',
new=lambda x: raise_(not_found_error())):
with pytest.raises(BentoMLMissingDependencyException) as error:
ensure_docker_available_or_raise()
assert str(error.value).startswith('Docker is required')
def temporary_docker_postgres_url():
ensure_docker_available_or_raise()
container_name = f'e2e-test-yatai-service-postgres-db-{uuid.uuid4().hex[:6]}'
db_url = 'postgresql://*****:*****@localhost:5432/bentoml'
command = [
'docker',
'run',
'--rm',
'--name',
container_name,
'-e',
'POSTGRES_PASSWORD=postgres',
'-p',
'5432:5432',
'postgres',
]
docker_proc = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
# wait until postgres db is ready
docker_client = docker.from_env()
while True:
time.sleep(1)
container_list = docker_client.containers.list(filters={
'name': container_name,
'status': 'running'
})
logger.info("container_list: " + str(container_list))
if not container_list:
continue
assert len(
container_list) == 1, "should be only one container with name"
postgres_container = container_list[0]
logger.info("container_log:" + str(postgres_container.logs()))
if (b'database system is ready to accept connections'
in postgres_container.logs()):
break
from sqlalchemy_utils import create_database
create_database(db_url)
yield db_url
docker_proc.terminate()
def apply(self, deployment_pb, yatai_service, prev_deployment=None):
try:
ensure_docker_available_or_raise()
deployment_spec = deployment_pb.spec
sagemaker_config = deployment_spec.sagemaker_operator_config
if sagemaker_config is None:
raise BentoMLDeploymentException('Sagemaker configuration is missing.')
bento_pb = yatai_service.GetBento(
GetBentoRequest(
bento_name=deployment_spec.bento_name,
bento_version=deployment_spec.bento_version,
)
)
if bento_pb.bento.uri.type != BentoUri.LOCAL:
raise BentoMLException(
'BentoML currently only support local repository'
)
else:
bento_path = bento_pb.bento.uri.uri
ensure_deploy_api_name_exists_in_bento(
[api.name for api in bento_pb.bento.bento_service_metadata.apis],
[sagemaker_config.api_name],
)
sagemaker_client = boto3.client('sagemaker', sagemaker_config.region)
with TempDirectory() as temp_dir:
sagemaker_project_dir = os.path.jon(
temp_dir, deployment_spec.bento_name
)
init_sagemaker_project(sagemaker_project_dir, bento_path)
ecr_image_path = create_push_docker_image_to_ecr(
deployment_spec.bento_name,
deployment_spec.bento_version,
sagemaker_project_dir,
)
execution_role_arn = get_arn_role_from_current_aws_user()
model_name = create_sagemaker_model_name(
deployment_spec.bento_name, deployment_spec.bento_version
)
sagemaker_model_info = {
"ModelName": model_name,
"PrimaryContainer": {
"ContainerHostname": model_name,
"Image": ecr_image_path,
"Environment": {
"API_NAME": sagemaker_config.api_name,
"BENTO_SERVER_TIMEOUT": config().get(
'apiserver', 'default_timeout'
),
"BENTO_SERVER_WORKERS": config().get(
'apiserver', 'default_gunicorn_workers_count'
),
},
},
"ExecutionRoleArn": execution_role_arn,
}
logger.info("Creating sagemaker model %s", model_name)
try:
create_model_response = sagemaker_client.create_model(
**sagemaker_model_info
)
logger.debug("AWS create model response: %s", create_model_response)
except ClientError as e:
status = _parse_aws_client_exception_or_raise(e)
status.error_message = (
'Failed to create model for SageMaker Deployment: %s',
status.error_message,
)
return ApplyDeploymentResponse(status=status, deployment=deployment_pb)
production_variants = [
{
"VariantName": generate_aws_compatible_string(
deployment_spec.bento_name
),
"ModelName": model_name,
"InitialInstanceCount": sagemaker_config.instance_count,
"InstanceType": sagemaker_config.instance_type,
}
]
endpoint_config_name = create_sagemaker_endpoint_config_name(
deployment_spec.bento_name, deployment_spec.bento_version
)
logger.info(
"Creating Sagemaker endpoint %s configuration", endpoint_config_name
)
try:
create_config_response = sagemaker_client.create_endpoint_config(
EndpointConfigName=endpoint_config_name,
ProductionVariants=production_variants,
)
logger.debug(
"AWS create endpoint config response: %s", create_config_response
)
except ClientError as e:
# create endpoint failed, will remove previously created model
cleanup_model_error = _cleanup_sagemaker_model(
sagemaker_client,
deployment_spec.bento_name,
deployment_spec.bento_version,
)
if cleanup_model_error:
cleanup_model_error.error_message = (
'Failed to clean up model after unsuccessfully '
'create endpoint config: %s',
cleanup_model_error.error_message,
)
return ApplyDeploymentResponse(
status=cleanup_model_error, deployment=deployment_pb
)
status = _parse_aws_client_exception_or_raise(e)
status.error_message = (
'Failed to create endpoint config for SageMaker deployment: %s',
status.error_message,
)
return ApplyDeploymentResponse(status=status, deployment=deployment_pb)
endpoint_name = generate_aws_compatible_string(
deployment_pb.namespace + '-' + deployment_spec.bento_name
)
try:
if prev_deployment:
logger.debug("Updating sagemaker endpoint %s", endpoint_name)
update_endpoint_response = sagemaker_client.update_endpoint(
EndpointName=endpoint_name,
EndpointConfigName=endpoint_config_name,
)
logger.debug(
"AWS update endpoint response: %s", update_endpoint_response
)
else:
logger.debug("Creating sagemaker endpoint %s", endpoint_name)
create_endpoint_response = sagemaker_client.create_endpoint(
EndpointName=endpoint_name,
EndpointConfigName=endpoint_config_name,
)
logger.debug(
"AWS create endpoint response: %s", create_endpoint_response
)
except ClientError as e:
# create/update endpoint failed, will remove previously created config
# and then remove the model
cleanup_endpoint_config_error = _cleanup_sagemaker_endpoint_config(
client=sagemaker_client,
name=deployment_spec.bento_name,
version=deployment_spec.bento_version,
)
if cleanup_endpoint_config_error:
cleanup_endpoint_config_error.error_message = (
'Failed to clean up endpoint config after unsuccessfully '
'apply SageMaker deployment: %s',
cleanup_endpoint_config_error.error_message,
)
return ApplyDeploymentResponse(
status=cleanup_endpoint_config_error, deployment=deployment_pb
)
cleanup_model_error = _cleanup_sagemaker_model(
client=sagemaker_client,
name=deployment_spec.bento_name,
version=deployment_spec.bento_version,
)
if cleanup_model_error:
cleanup_model_error.error_message = (
'Failed to clean up model after unsuccessfully '
'apply SageMaker deployment: %s',
cleanup_model_error.error_message,
)
return ApplyDeploymentResponse(
status=cleanup_model_error, deployment=deployment_pb
)
status = _parse_aws_client_exception_or_raise(e)
status.error_message = (
'Failed to apply SageMaker deployment: %s',
status.error_message,
)
return ApplyDeploymentResponse(status=status, deployment=deployment_pb)
res_deployment_pb = Deployment(state=DeploymentState())
res_deployment_pb.CopyFrom(deployment_pb)
return ApplyDeploymentResponse(
status=Status.OK(), deployment=res_deployment_pb
)
except BentoMLException as error:
return ApplyDeploymentResponse(status=exception_to_return_status(error))
def deploy_bentoml(
clipper_conn, bundle_path, api_name, model_name=None, labels=None, build_envs=None
):
"""Deploy bentoml bundle to clipper cluster
Args:
clipper_conn(clipper_admin.ClipperConnection): Clipper connection instance
bundle_path(str): Path to the saved BentomlService bundle.
api_name(str): name of the api that will be used as prediction function for
clipper cluster
model_name(str): Model's name for clipper cluster
labels(:obj:`list(str)`, optional): labels for clipper model
Returns:
tuple: Model name and model version that deployed to clipper
"""
track("clipper-deploy", {'bento_service_bundle_path': bundle_path})
build_envs = {} if build_envs is None else build_envs
# docker is required to build clipper model image
ensure_docker_available_or_raise()
if not clipper_conn.connected:
raise BentoMLException(
"No connection to Clipper cluster. CallClipperConnection.connect to "
"connect to an existing cluster or ClipperConnnection.start_clipper to "
"create a new one"
)
bento_service_metadata = load_bento_service_metadata(bundle_path)
try:
api_metadata = next(
(api for api in bento_service_metadata.apis if api.name == api_name)
)
except StopIteration:
raise BentoMLException(
"Can't find API '{}' in BentoService bundle {}".format(
api_name, bento_service_metadata.name
)
)
if api_metadata.handler_type not in HANDLER_TYPE_TO_INPUT_TYPE:
raise BentoMLException(
"Only BentoService APIs using ClipperHandler can be deployed to Clipper"
)
input_type = HANDLER_TYPE_TO_INPUT_TYPE[api_metadata.handler_type]
model_name = model_name or get_clipper_compatiable_string(
bento_service_metadata.name + "-" + api_metadata.name
)
model_version = get_clipper_compatiable_string(bento_service_metadata.version)
with TempDirectory() as tempdir:
entry_py_content = CLIPPER_ENTRY.format(api_name=api_name)
model_path = os.path.join(tempdir, "bento")
shutil.copytree(bundle_path, model_path)
with open(os.path.join(tempdir, "clipper_entry.py"), "w") as f:
f.write(entry_py_content)
if bento_service_metadata.env.python_version.startswith("3.6"):
base_image = "clipper/python36-closure-container:0.4.1"
elif bento_service_metadata.env.python_version.startswith("2.7"):
base_image = "clipper/python-closure-container:0.4.1"
else:
raise BentoMLException(
"Python version {} is not supported in Clipper".format(
bento_service_metadata.env.python_version
)
)
docker_content = CLIPPER_DOCKERFILE.format(
model_name=model_name,
model_version=model_version,
base_image=base_image,
pip_index_url=build_envs.get("PIP_INDEX_URL", ""),
pip_trusted_url=build_envs.get("PIP_TRUSTED_HOST", ""),
)
with open(os.path.join(tempdir, "Dockerfile-clipper"), "w") as f:
f.write(docker_content)
docker_api = docker.APIClient()
clipper_model_docker_image_tag = "clipper-model-{}:{}".format(
bento_service_metadata.name.lower(), bento_service_metadata.version
)
for line in docker_api.build(
path=tempdir,
dockerfile="Dockerfile-clipper",
tag=clipper_model_docker_image_tag,
):
process_docker_api_line(line)
logger.info(
"Successfully built docker image %s for Clipper deployment",
clipper_model_docker_image_tag,
)
clipper_conn.deploy_model(
name=model_name,
version=model_version,
input_type=input_type,
image=clipper_model_docker_image_tag,
labels=labels,
)
track("clipper-deploy-success", {'bento_service_bundle_path': bundle_path})
return model_name, model_version
def apply(self, deployment_pb, yatai_service, prev_deployment=None):
try:
ensure_docker_available_or_raise()
deployment_spec = deployment_pb.spec
aws_config = deployment_spec.aws_lambda_operator_config
bento_pb = yatai_service.GetBento(
GetBentoRequest(
bento_name=deployment_spec.bento_name,
bento_version=deployment_spec.bento_version,
)
)
if bento_pb.bento.uri.type != BentoUri.LOCAL:
raise BentoMLException(
'BentoML currently only support local repository'
)
else:
bento_path = bento_pb.bento.uri.uri
bento_service_metadata = bento_pb.bento.bento_service_metadata
template = 'aws-python3'
if version.parse(bento_service_metadata.env.python_version) < version.parse(
'3.0.0'
):
template = 'aws-python'
api_names = (
[aws_config.api_name]
if aws_config.api_name
else [api.name for api in bento_service_metadata.apis]
)
ensure_deploy_api_name_exists_in_bento(
[api.name for api in bento_service_metadata.apis], api_names
)
with TempDirectory() as serverless_project_dir:
init_serverless_project_dir(
serverless_project_dir,
bento_path,
deployment_pb.name,
deployment_spec.bento_name,
template,
)
generate_aws_lambda_handler_py(
deployment_spec.bento_name, api_names, serverless_project_dir
)
generate_aws_lambda_serverless_config(
bento_service_metadata.env.python_version,
deployment_pb.name,
api_names,
serverless_project_dir,
aws_config.region,
# BentoML deployment namespace is mapping to serverless `stage`
# concept
stage=deployment_pb.namespace,
)
logger.info(
'Installing additional packages: serverless-python-requirements'
)
install_serverless_plugin(
"serverless-python-requirements", serverless_project_dir
)
logger.info('Deploying to AWS Lambda')
call_serverless_command(["deploy"], serverless_project_dir)
res_deployment_pb = Deployment(state=DeploymentState())
res_deployment_pb.CopyFrom(deployment_pb)
state = self.describe(res_deployment_pb, yatai_service).state
res_deployment_pb.state.CopyFrom(state)
return ApplyDeploymentResponse(
status=Status.OK(), deployment=res_deployment_pb
)
except BentoMLException as error:
return ApplyDeploymentResponse(status=exception_to_return_status(error))
