def __init__(
self,
bento_bundle_path,
outbound_host="localhost",
outbound_port=None,
outbound_workers=1,
mb_max_batch_size: int = None,
mb_max_latency: int = None,
):
self.outbound_host = outbound_host
self.outbound_port = outbound_port
self.outbound_workers = outbound_workers
self.mb_max_batch_size = mb_max_batch_size
self.mb_max_latency = mb_max_latency
self.batch_handlers = dict()
self._outbound_sema = None # the semaphore to limit outbound connections
self.bento_service_metadata_pb = load_bento_service_metadata(
bento_bundle_path)
self.setup_routes_from_pb(self.bento_service_metadata_pb)
if psutil.POSIX:
import resource
self.CONNECTION_LIMIT = resource.getrlimit(
resource.RLIMIT_NOFILE)[0]
else:
self.CONNECTION_LIMIT = 1024
logger.info(
"Your system nofile limit is %d, which means each instance of microbatch "
"service is able to hold this number of connections at same time. "
"You can increase the number of file descriptors for the server process, "
"or launch more microbatch instances to accept more concurrent connection.",
self.CONNECTION_LIMIT,
)
def test_auto_artifact_dependencies():
clf = svm.SVC(gamma='scale')
iris = datasets.load_iris()
X, y = iris.data, iris.target
clf.fit(X, y)
# Create a iris classifier service
iris_classifier_service = IrisClassifier()
# Pack it with the newly trained model artifact
iris_classifier_service.pack('model', clf)
# Save the prediction service to a BentoService bundle
saved_path = iris_classifier_service.save()
with open(os.path.join(saved_path, 'requirements.txt')) as f:
requirements_txt_content = f.read()
dependencies = requirements_txt_content.split('\n')
dependencies = [dep.split('==')[0] for dep in dependencies]
assert 'scikit-learn' in dependencies
assert 'bentoml' in dependencies
# Test that dependencies also wrote to BentoServiceMetadat config file
bs_matadata = load_bento_service_metadata(saved_path)
dependencies = bs_matadata.env.pip_dependencies.split('\n')
dependencies = [dep.split('==')[0] for dep in dependencies]
assert 'scikit-learn' in dependencies
assert 'bentoml' in dependencies
delete_saved_bento_service(iris_classifier_service.name,
iris_classifier_service.version)
def _get_bento_service_event_properties_from_bundle_path(
bundle_path, properties=None):
from bentoml.saved_bundle import load_bento_service_metadata
bento_service_metadata = load_bento_service_metadata(bundle_path)
return _bento_service_metadata_to_event_properties(bento_service_metadata,
properties)
def test_auto_artifact_dependencies():
clf = _fit_clf()
# Create a iris classifier service
iris_classifier_service = IrisClassifier()
# Pack it with the newly trained model artifact
iris_classifier_service.pack('model', clf)
# Save the prediction service to a BentoService bundle
saved_path = iris_classifier_service.save()
# parse generated requirements.txt
dependencies = _dependencies_to_requirements(
_parse_dependencies(saved_path))
_assert_in_dependencies(['scikit-learn', 'bentoml'], dependencies)
# Test that dependencies also wrote to BentoServiceMetadata config file
bs_metadata = load_bento_service_metadata(saved_path)
dependencies = bs_metadata.env.pip_packages
dependencies = _dependencies_to_requirements(dependencies)
_assert_in_dependencies(['scikit-learn', 'bentoml'], dependencies)
# Clean up
delete_saved_bento_service(iris_classifier_service.name,
iris_classifier_service.version)
def info(bento=None):
"""
List all APIs defined in the BentoService loaded from saved bundle
"""
saved_bundle_path = resolve_bundle_path(bento, pip_installed_bundle_path)
bento_service_metadata_pb = load_bento_service_metadata(saved_bundle_path)
output = json.dumps(ProtoMessageToDict(bento_service_metadata_pb), indent=2)
_echo(output)
def containerize(bento, push, tag, build_arg, yatai_url):
"""Containerize specified BentoService.
BENTO is the target BentoService to be containerized, referenced by its name
and version in format of name:version. For example: "iris_classifier:v1.2.0"
`bentoml containerize` command also supports the use of the `latest` tag
which will automatically use the last built version of your Bento.
You can provide a tag for the image built by Bento using the
`--tag` flag. Additionally, you can provide a `--push` flag,
which will push the built image to the Docker repository specified by the
image tag.
You can also prefixing the tag with a hostname for the repository you wish
to push to.
e.g. `bentoml containerize IrisClassifier:latest --push --tag
repo-address.com:username/iris` would build a Docker image called
`username/iris:latest` and push that to docker repository at repo-address.com.
By default, the `containerize` command will use the current credentials
provided by Docker daemon.
"""
saved_bundle_path = resolve_bundle_path(bento,
pip_installed_bundle_path,
yatai_url)
_echo(f"Found Bento: {saved_bundle_path}")
# fmt: off
bento_metadata: "BentoServiceMetadata" = load_bento_service_metadata(
saved_bundle_path) # noqa: E501
# fmt: on
bento_tag = f'{bento_metadata.name}:{bento_metadata.version}'
yatai_client: "YataiClient" = get_yatai_client(yatai_url)
docker_build_args = {}
if build_arg:
for arg in build_arg:
key, value = arg.split("=", 1)
docker_build_args[key] = value
if yatai_url is not None:
spinner_message = f'Sending containerize RPC to YataiService at {yatai_url}'
else:
spinner_message = (
f'Containerizing {bento_tag} with local YataiService and docker '
f'daemon from local environment')
with Spinner(spinner_message):
tag: str = yatai_client.repository.containerize(
bento=bento_tag,
tag=tag,
build_args=docker_build_args,
push=push,
)
_echo(f'\nBuild container image: {tag}', CLI_COLOR_SUCCESS)
def __init__(
self,
bento_bundle_path,
outbound_host="localhost",
outbound_port=None,
outbound_workers: int = Provide[
BentoMLContainer.config.api_server.workers],
mb_max_batch_size: int = Provide[
BentoMLContainer.config.marshal_server.max_batch_size],
mb_max_latency: int = Provide[
BentoMLContainer.config.marshal_server.max_latency],
request_header_flag: str = Provide[
BentoMLContainer.config.marshal_server.request_header_flag],
max_request_size: int = Provide[
BentoMLContainer.config.api_server.max_request_size],
outbound_unix_socket: str = None,
enable_microbatch: bool = Provide[
BentoMLContainer.config.api_server.enable_microbatch],
):
self._client = None
self.outbound_unix_socket = outbound_unix_socket
self.outbound_host = outbound_host
self.outbound_port = outbound_port
self.outbound_workers = outbound_workers
self.mb_max_batch_size = mb_max_batch_size
self.mb_max_latency = mb_max_latency
self.batch_handlers = dict()
self._outbound_sema = None # the semaphore to limit outbound connections
self.request_header_flag = request_header_flag
self.max_request_size = max_request_size
self.bento_service_metadata_pb = load_bento_service_metadata(
bento_bundle_path)
if enable_microbatch:
self.setup_routes_from_pb(self.bento_service_metadata_pb)
if psutil.POSIX:
import resource
self.CONNECTION_LIMIT = resource.getrlimit(
resource.RLIMIT_NOFILE)[0]
else:
self.CONNECTION_LIMIT = 1024
logger.info(
"Your system nofile limit is %d, which means each instance of microbatch "
"service is able to hold this number of connections at same time. "
"You can increase the number of file descriptors for the server process, "
"or launch more microbatch instances to accept more concurrent connection.",
self.CONNECTION_LIMIT,
)
def test_save_duplicated_bento_exception_raised(example_bento_service_class):
test_model = TestModel()
svc = example_bento_service_class()
svc.pack("model", test_model)
saved_path = svc.save()
svc_metadata = load_bento_service_metadata(saved_path)
assert svc.version == svc_metadata.version
with pytest.raises(BentoMLException):
with patch.object(bentoml.BentoService, 'save_to_dir') as save_to_dir_method:
# attempt to save again
svc.save()
save_to_dir_method.assert_not_called()
# reset svc version
svc.set_version()
saved_path = svc.save()
svc_metadata_new = load_bento_service_metadata(saved_path)
assert svc.version == svc_metadata_new.version
delete_saved_bento_service(svc_metadata.name, svc_metadata.version)
delete_saved_bento_service(svc_metadata_new.name, svc_metadata_new.version)
def test_requirements_txt_file():
clf = _fit_clf()
iris_classifier_service = IrisClassifierPipEnv()
iris_classifier_service.pack('model', clf)
saved_path = iris_classifier_service.save()
dependencies = _dependencies_to_requirements(
_parse_dependencies(saved_path))
_assert_in_dependencies(
['scikit-learn', 'azure-cli', 'psycopg2-binary', 'bentoml'],
dependencies)
bs_metadata = load_bento_service_metadata(saved_path)
requirements_txt = bs_metadata.env.requirements_txt
requirements_content = open("./tests/pipenv_requirements.txt", "r").read()
assert requirements_txt == requirements_content
delete_saved_bento_service(iris_classifier_service.name,
iris_classifier_service.version)
def containerize(bento, push, tag, build_arg, username, password):
"""Containerize specified BentoService.
BENTO is the target BentoService to be containerized, referenced by its name
and version in format of name:version. For example: "iris_classifier:v1.2.0"
`bentoml containerize` command also supports the use of the `latest` tag
which will automatically use the last built version of your Bento.
You can provide a tag for the image built by Bento using the
`--docker-image-tag` flag. Additionally, you can provide a `--push` flag,
which will push the built image to the Docker repository specified by the
image tag.
You can also prefixing the tag with a hostname for the repository you wish
to push to.
e.g. `bentoml containerize IrisClassifier:latest --push --tag username/iris`
would build a Docker image called `username/iris:latest` and push that to
Docker Hub.
By default, the `containerize` command will use the credentials provided by
Docker. You may provide your own through `--username` and `--password`.
"""
saved_bundle_path = resolve_bundle_path(bento,
pip_installed_bundle_path)
_echo(f"Found Bento: {saved_bundle_path}")
bento_metadata = load_bento_service_metadata(saved_bundle_path)
name = to_valid_docker_image_name(bento_metadata.name)
version = to_valid_docker_image_version(bento_metadata.version)
if not tag:
_echo("Tag not specified, using tag parsed from "
f"BentoService: '{name}:{version}'")
tag = f"{name}:{version}"
if ":" not in tag:
_echo(
"Image version not specified, using version parsed "
f"from BentoService: '{version}'",
CLI_COLOR_WARNING,
)
tag = f"{tag}:{version}"
docker_build_args = {}
if build_arg:
for arg in build_arg:
key, value = arg.split("=")
docker_build_args[key] = value
import docker
docker_api = docker.APIClient()
try:
with Spinner(f"Building Docker image {tag} from {bento} \n"):
for line in echo_docker_api_result(
docker_api.build(
path=saved_bundle_path,
tag=tag,
decode=True,
buildargs=docker_build_args,
)):
_echo(line)
except docker.errors.APIError as error:
raise CLIException(f'Could not build Docker image: {error}')
_echo(
f'Finished building {tag} from {bento}',
CLI_COLOR_SUCCESS,
)
if push:
auth_config_payload = ({
"username": username,
"password": password
} if username or password else None)
try:
with Spinner(f"Pushing docker image to {tag}\n"):
for line in echo_docker_api_result(
docker_api.push(
repository=tag,
stream=True,
decode=True,
auth_config=auth_config_payload,
)):
_echo(line)
_echo(
f'Pushed {tag} to {name}',
CLI_COLOR_SUCCESS,
)
except (docker.errors.APIError, BentoMLException) as error:
raise CLIException(f'Could not push Docker image: {error}')
def upload_from_dir(self, saved_bento_path, labels=None):
bento_service_metadata = load_bento_service_metadata(saved_bento_path)
if labels:
_validate_labels(labels)
bento_service_metadata.labels.update(labels)
get_bento_response = self.yatai_service.GetBento(
GetBentoRequest(
bento_name=bento_service_metadata.name,
bento_version=bento_service_metadata.version,
))
if get_bento_response.status.status_code == status_pb2.Status.OK:
raise BentoMLException(
"BentoService bundle {}:{} already registered in repository. Reset "
"BentoService version with BentoService#set_version or bypass BentoML's"
" model registry feature with BentoService#save_to_dir".format(
bento_service_metadata.name,
bento_service_metadata.version))
elif get_bento_response.status.status_code != status_pb2.Status.NOT_FOUND:
raise BentoMLException(
'Failed accessing YataiService. {error_code}:'
'{error_message}'.format(
error_code=Status.Name(
get_bento_response.status.status_code),
error_message=get_bento_response.status.error_message,
))
request = AddBentoRequest(
bento_name=bento_service_metadata.name,
bento_version=bento_service_metadata.version,
)
response = self.yatai_service.AddBento(request)
if response.status.status_code != status_pb2.Status.OK:
raise BentoMLException(
"Error adding BentoService bundle to repository: {}:{}".format(
Status.Name(response.status.status_code),
response.status.error_message,
))
if response.uri.type == BentoUri.LOCAL:
if os.path.exists(response.uri.uri):
# due to copytree dst must not already exist
shutil.rmtree(response.uri.uri)
shutil.copytree(saved_bento_path, response.uri.uri)
self._update_bento_upload_progress(bento_service_metadata)
logger.info(
"BentoService bundle '%s:%s' saved to: %s",
bento_service_metadata.name,
bento_service_metadata.version,
response.uri.uri,
)
# Return URI to saved bento in repository storage
return response.uri.uri
elif response.uri.type == BentoUri.S3 or response.uri.type == BentoUri.GCS:
uri_type = 'S3' if response.uri.type == BentoUri.S3 else 'GCS'
self._update_bento_upload_progress(bento_service_metadata,
UploadStatus.UPLOADING, 0)
fileobj = io.BytesIO()
with tarfile.open(mode="w:gz", fileobj=fileobj) as tar:
tar.add(saved_bento_path, arcname=bento_service_metadata.name)
fileobj.seek(0, 0)
if response.uri.type == BentoUri.S3:
http_response = requests.put(response.uri.s3_presigned_url,
data=fileobj)
elif response.uri.type == BentoUri.GCS:
http_response = requests.put(response.uri.gcs_presigned_url,
data=fileobj)
if http_response.status_code != 200:
self._update_bento_upload_progress(bento_service_metadata,
UploadStatus.ERROR)
raise BentoMLException(
f"Error saving BentoService bundle to {uri_type}."
f"{http_response.status_code}: {http_response.text}")
self._update_bento_upload_progress(bento_service_metadata)
logger.info(
"Successfully saved BentoService bundle '%s:%s' to {uri_type}: %s",
bento_service_metadata.name,
bento_service_metadata.version,
response.uri.uri,
)
return response.uri.uri
else:
raise BentoMLException(
f"Error saving Bento to target repository, URI type {response.uri.type}"
f" at {response.uri.uri} not supported")
def deploy_bentoml(clipper_conn,
bundle_path,
api_name,
model_name=None,
labels=None,
build_envs=None):
"""Deploy bentoml bundle to clipper cluster
Args:
clipper_conn(clipper_admin.ClipperConnection): Clipper connection instance
bundle_path(str): Path to the saved BentomlService bundle.
api_name(str): name of the api that will be used as prediction function for
clipper cluster
model_name(str): Model's name for clipper cluster
labels(:obj:`list(str)`, optional): labels for clipper model
Returns:
tuple: Model name and model version that deployed to clipper
"""
track("clipper-deploy", {'bento_service_bundle_path': bundle_path})
build_envs = {} if build_envs is None else build_envs
# docker is required to build clipper model image
ensure_docker_available_or_raise()
if not clipper_conn.connected:
raise BentoMLException(
"No connection to Clipper cluster. CallClipperConnection.connect to "
"connect to an existing cluster or ClipperConnection.start_clipper to "
"create a new one")
bento_service_metadata = load_bento_service_metadata(bundle_path)
try:
api_metadata = next((api for api in bento_service_metadata.apis
if api.name == api_name))
except StopIteration:
raise BentoMLException(
"Can't find API '{}' in BentoService bundle {}".format(
api_name, bento_service_metadata.name))
if api_metadata.input_type not in ADAPTER_TYPE_TO_INPUT_TYPE:
raise BentoMLException(
"Only BentoService APIs using ClipperInput can be deployed to Clipper"
)
input_type = ADAPTER_TYPE_TO_INPUT_TYPE[api_metadata.input_type]
model_name = model_name or get_clipper_compatible_string(
bento_service_metadata.name + "-" + api_metadata.name)
model_version = get_clipper_compatible_string(
bento_service_metadata.version)
with TempDirectory() as tempdir:
entry_py_content = CLIPPER_ENTRY.format(api_name=api_name)
build_path = os.path.join(tempdir, "bento")
shutil.copytree(bundle_path, build_path)
with open(os.path.join(build_path, "clipper_entry.py"), "w") as f:
f.write(entry_py_content)
if bento_service_metadata.env.python_version.startswith("3.6"):
base_image = "clipper/python36-closure-container:0.4.1"
elif bento_service_metadata.env.python_version.startswith("2.7"):
base_image = "clipper/python-closure-container:0.4.1"
else:
raise BentoMLException(
"Python version {} is not supported in Clipper".format(
bento_service_metadata.env.python_version))
docker_content = CLIPPER_DOCKERFILE.format(
model_name=model_name,
model_version=model_version,
base_image=base_image,
pip_index_url=build_envs.get("PIP_INDEX_URL", ""),
pip_trusted_url=build_envs.get("PIP_TRUSTED_HOST", ""),
)
with open(os.path.join(build_path, "Dockerfile-clipper"), "w") as f:
f.write(docker_content)
docker_api = docker.APIClient()
clipper_model_docker_image_tag = "clipper-model-{}:{}".format(
bento_service_metadata.name.lower(),
bento_service_metadata.version)
for line in docker_api.build(
path=build_path,
dockerfile="Dockerfile-clipper",
tag=clipper_model_docker_image_tag,
):
process_docker_api_line(line)
logger.info(
"Successfully built docker image %s for Clipper deployment",
clipper_model_docker_image_tag,
)
clipper_conn.deploy_model(
name=model_name,
version=model_version,
input_type=input_type,
image=clipper_model_docker_image_tag,
labels=labels,
)
track("clipper-deploy-success", {'bento_service_bundle_path': bundle_path})
return model_name, model_version
def upload_from_dir(self,
saved_bento_path: str,
labels: Dict = None) -> "BentoUri":
from bentoml.yatai.db.stores.label import _validate_labels
bento_service_metadata = load_bento_service_metadata(saved_bento_path)
if labels:
_validate_labels(labels)
bento_service_metadata.labels.update(labels)
get_bento_response = self.yatai_service.GetBento(
GetBentoRequest(
bento_name=bento_service_metadata.name,
bento_version=bento_service_metadata.version,
))
if get_bento_response.status.status_code == status_pb2.Status.OK:
raise BentoMLException(
"BentoService bundle {}:{} already registered in repository. Reset "
"BentoService version with BentoService#set_version or bypass BentoML's"
" model registry feature with BentoService#save_to_dir".format(
bento_service_metadata.name,
bento_service_metadata.version))
elif get_bento_response.status.status_code != status_pb2.Status.NOT_FOUND:
raise BentoMLException(
'Failed accessing YataiService. {error_code}:'
'{error_message}'.format(
error_code=Status.Name(
get_bento_response.status.status_code),
error_message=get_bento_response.status.error_message,
))
request = AddBentoRequest(
bento_name=bento_service_metadata.name,
bento_version=bento_service_metadata.version,
)
response = self.yatai_service.AddBento(request)
if response.status.status_code != status_pb2.Status.OK:
raise BentoMLException(
"Error adding BentoService bundle to repository: {}:{}".format(
Status.Name(response.status.status_code),
response.status.error_message,
))
if response.uri.type == BentoUri.LOCAL:
# When using Yatai backed by File System repository,
# if Yatai is a local instance, copy the files directly.
# Otherwise, use UploadBento RPC to stream files to remote Yatai server
if is_remote_yatai(self.yatai_service):
self._upload_bento(
bento_service_metadata.name,
bento_service_metadata.version,
saved_bento_path,
)
update_bento_service = UpdateBentoRequest(
bento_name=bento_service_metadata.name,
bento_version=bento_service_metadata.version,
service_metadata=bento_service_metadata,
)
self.yatai_service.UpdateBento(update_bento_service)
else:
if os.path.exists(response.uri.uri):
raise BentoMLException(
f'Bento bundle directory {response.uri.uri} already exist'
)
shutil.copytree(saved_bento_path, response.uri.uri)
upload_status = UploadStatus.DONE
self._update_bento_upload_progress(bento_service_metadata,
status=upload_status)
logger.info(
"BentoService bundle '%s:%s' saved to: %s",
bento_service_metadata.name,
bento_service_metadata.version,
response.uri.uri,
)
# Return URI to saved bento in repository storage
return response.uri.uri
elif response.uri.type == BentoUri.S3 or response.uri.type == BentoUri.GCS:
uri_type = 'S3' if response.uri.type == BentoUri.S3 else 'GCS'
self._update_bento_upload_progress(bento_service_metadata,
UploadStatus.UPLOADING, 0)
fileobj = io.BytesIO()
with tarfile.open(mode="w:gz", fileobj=fileobj) as tar:
tar.add(saved_bento_path, arcname=bento_service_metadata.name)
fileobj.seek(0, 0)
if response.uri.type == BentoUri.S3:
http_response = requests.put(response.uri.s3_presigned_url,
data=fileobj)
elif response.uri.type == BentoUri.GCS:
http_response = requests.put(response.uri.gcs_presigned_url,
data=fileobj)
if http_response.status_code != 200:
self._update_bento_upload_progress(bento_service_metadata,
UploadStatus.ERROR)
raise BentoMLException(
f"Error saving BentoService bundle to {uri_type}."
f"{http_response.status_code}: {http_response.text}")
self._update_bento_upload_progress(bento_service_metadata)
logger.info(
"Successfully saved BentoService bundle '%s:%s' to {uri_type}: %s",
bento_service_metadata.name,
bento_service_metadata.version,
response.uri.uri,
)
return response.uri.uri
else:
raise BentoMLException(
f"Error saving Bento to target repository, URI type {response.uri.type}"
f" at {response.uri.uri} not supported")
def __init__(
self,
bento_bundle_path: str = Provide[BentoMLContainer.bundle_path],
outbound_host: str = Provide[BentoMLContainer.forward_host],
outbound_port: int = Provide[BentoMLContainer.forward_port],
outbound_workers: int = Provide[BentoMLContainer.api_server_workers],
mb_max_batch_size: int = Provide[
BentoMLContainer.config.bento_server.microbatch.max_batch_size
],
mb_max_latency: int = Provide[
BentoMLContainer.config.bento_server.microbatch.max_latency
],
max_request_size: int = Provide[
BentoMLContainer.config.bento_server.max_request_size
],
outbound_unix_socket: str = None,
enable_access_control: bool = Provide[
BentoMLContainer.config.bento_server.cors.enabled
],
access_control_allow_origin: Optional[str] = Provide[
BentoMLContainer.config.bento_server.cors.access_control_allow_origin
],
access_control_options: Optional["ResourceOptions"] = Provide[
BentoMLContainer.access_control_options
],
timeout: int = Provide[BentoMLContainer.config.bento_server.timeout],
tracer=Provide[BentoMLContainer.tracer],
):
self._conn: Optional["BaseConnector"] = None
self._client: Optional["ClientSession"] = None
self.outbound_unix_socket = outbound_unix_socket
self.outbound_host = outbound_host
self.outbound_port = outbound_port
self.outbound_workers = outbound_workers
self.mb_max_batch_size = mb_max_batch_size
self.mb_max_latency = mb_max_latency
self.batch_handlers = dict()
self._outbound_sema = None # the semaphore to limit outbound connections
self._cleanup_tasks = None
self.max_request_size = max_request_size
self.tracer = tracer
self.enable_access_control = enable_access_control
self.access_control_allow_origin = access_control_allow_origin
self.access_control_options = access_control_options
self.bento_service_metadata_pb = load_bento_service_metadata(bento_bundle_path)
self.setup_routes_from_pb(self.bento_service_metadata_pb)
self.timeout = timeout
if psutil.POSIX:
import resource
self.CONNECTION_LIMIT = resource.getrlimit(resource.RLIMIT_NOFILE)[0]
else:
self.CONNECTION_LIMIT = 1024
logger.info(
"Your system nofile limit is %d, which means each instance of microbatch "
"service is able to hold this number of connections at same time. "
"You can increase the number of file descriptors for the server process, "
"or launch more microbatch instances to accept more concurrent connection.",
self.CONNECTION_LIMIT,
)
