def pack(cls, *args, **kwargs):
if args and isinstance(args[0], ArtifactCollection):
return cls(args[0])
artifacts = ArtifactCollection()
for artifact_spec in cls._artifacts_spec:
if artifact_spec.name in kwargs:
artifact_instance = artifact_spec.pack(kwargs[artifact_spec.name])
artifacts.add(artifact_instance)
return cls(artifacts)
class BentoService(BentoServiceBase):
"""
BentoService is the base component for building prediction services using BentoML.
BentoService provide an abstraction for describing model artifacts and environment
dependencies required for a prediction service. And allows users to write custom
prediction API handling logic via BentoService API callback function.
Each BentoService can contain multiple models via the BentoML Artifact class, and
can define multiple APIs for accessing this service. Each API should specify a type
of Handler, which defines the expected input data format for this API.
>>> from bentoml import BentoService, env, api, artifacts, ver
>>> from bentoml.handlers import DataframeHandler
>>> from bentoml.artifact import SklearnModelArtifact
>>>
>>> @ver(major=1, minor=4)
>>> @artifacts([SklearnModelArtifact('clf')])
>>> @env(pip_dependencies=["scikit-learn"])
>>> class MyMLService(BentoService):
>>>
>>> @api(DataframeHandler)
>>> def predict(self, df):
>>> return self.artifacts.clf.predict(df)
>>>
>>> bento_service = MyMLService()
>>> bento_service.pack('clf', trained_classifier_model)
>>> bento_service.save_to_dir('/bentoml_bundles')
"""
# User may use @name to override this if they don't want the generated model
# to have the same name as their Python model class name
_bento_service_name = None
# For BentoService loaded from saved bundle, this will be set to the path of bundle.
# When user install BentoService bundle as a PyPI package, this will be set to the
# installed site-package location of current python environment
_bento_service_bundle_path = None
# list of artifacts required by this BentoService
_artifacts = []
# Describe the desired environment for this BentoService using
# `bentoml.service_env.BentoServiceEnv`
_env = None
# When loading BentoService from saved bundle, this will be set to the version of
# the saved BentoService bundle
_bento_service_bundle_version = None
# See `ver_decorator` function above for more information
_version_major = None
_version_minor = None
def __init__(self):
from bentoml.artifact import ArtifactCollection
self._bento_service_version = self.__class__._bento_service_bundle_version
self._packed_artifacts = ArtifactCollection()
if self._bento_service_bundle_path:
# load artifacts from saved BentoService bundle
self._load_artifacts(self._bento_service_bundle_path)
self._config_service_apis()
self._init_env()
def _init_env(self):
self._env = self.__class__._env or BentoServiceEnv(self.name)
for api in self._service_apis:
self._env._add_pip_dependencies_if_missing(api.handler.pip_dependencies)
for artifact in self._artifacts:
self._env._add_pip_dependencies_if_missing(artifact.pip_dependencies)
@property
def artifacts(self):
"""
:return: List of model artifacts
"""
return self._packed_artifacts
@property
def env(self):
return self._env
@hybridmethod
@property
def name(self):
return self.__class__.name() # pylint: disable=no-value-for-parameter
@name.classmethod
def name(cls): # pylint: disable=no-self-argument,invalid-overridden-method
if cls._bento_service_name is not None:
if not isidentifier(cls._bento_service_name):
raise InvalidArgument(
'BentoService#_bento_service_name must be valid python identifier'
'matching regex `(letter|"_")(letter|digit|"_")*`'
)
return cls._bento_service_name
else:
# Use python class name as service name
return cls.__name__
def set_version(self, version_str=None):
"""Manually override the version of this BentoService instance
"""
if version_str is None:
version_str = self.versioneer()
if self._version_major is not None and self._version_minor is not None:
# BentoML uses semantic versioning for BentoService distribution
# when user specified the MAJOR and MINOR version number along with
# the BentoService class definition with '@ver' decorator.
# The parameter version(or auto generated version) here will be used as
# PATCH field in the final version:
version_str = ".".join(
[str(self._version_major), str(self._version_minor), version_str]
)
_validate_version_str(version_str)
if self.__class__._bento_service_bundle_version is not None:
logger.warning(
"Overriding loaded BentoService(%s) version:%s to %s",
self.__class__._bento_service_bundle_path,
self.__class__._bento_service_bundle_version,
version_str,
)
self.__class__._bento_service_bundle_version = None
if (
self._bento_service_version is not None
and self._bento_service_version != version_str
):
logger.warning(
"Resetting BentoService '%s' version from %s to %s",
self.name,
self._bento_service_version,
version_str,
)
self._bento_service_version = version_str
return self._bento_service_version
def versioneer(self):
"""
Function used to generate a new version string when saving a new BentoService
bundle. User can also override this function to get a customized version format
"""
datetime_string = datetime.now().strftime("%Y%m%d%H%M%S")
random_hash = uuid.uuid4().hex[:6].upper()
# Example output: '20191009135240_D246ED'
return datetime_string + "_" + random_hash
@property
def version(self):
if self.__class__._bento_service_bundle_version is not None:
return self.__class__._bento_service_bundle_version
if self._bento_service_version is None:
self.set_version(self.versioneer())
return self._bento_service_version
def save(self, base_path=None, version=None):
"""
Save and register this BentoService via BentoML's built-in model management
system. BentoML by default keeps track of all the SavedBundle's files and
metadata in local file system under the $BENTOML_HOME(~/bentoml) directory.
Users can also configure BentoML to save their BentoService to a shared Database
and cloud object storage such as AWS S3.
:param base_path: optional - override repository base path
:param version: optional - save with version override
:return: saved_path: file path to where the BentoService is saved
"""
return save(self, base_path, version)
def save_to_dir(self, path, version=None):
"""Save this BentoService along with all its artifacts, source code and
dependencies to target file path, assuming path exist and empty. If target path
is not empty, this call may override existing files in the given path.
:param path (str): Destination of where the bento service will be saved
:param version: optional - save with version override
"""
return save_to_dir(self, path, version)
@hybridmethod
def pack(self, name, *args, **kwargs):
"""
BentoService#pack method is used for packing trained model instances with a
BentoService instance and make it ready for BentoService#save.
pack(name, *args, **kwargs):
:param name: name of the declared model artifact
:param args: args passing to the target model artifact to be packed
:param kwargs: kwargs passing to the target model artifact to be packed
:return: this BentoService instance
"""
if name in self.artifacts:
logger.warning(
"BentoService '%s' #pack overriding existing artifact '%s'",
self.name,
name,
)
del self.artifacts[name]
artifact = next(
artifact for artifact in self._artifacts if artifact.name == name
)
packed_artifact = artifact.pack(*args, **kwargs)
self._packed_artifacts.add(packed_artifact)
return self
@pack.classmethod
def pack(cls, *args, **kwargs): # pylint: disable=no-self-argument
"""
**Deprecated**: Legacy `BentoService#pack` class method, which can be used to
initialize a BentoService instance along with trained model artifacts. This will
be deprecated soon:
:param args: args passing to the BentoService class
:param kwargs: kwargs passing to the BentoService class and (artifact_name,
args) pair for creating declared model artifacts
:return: a new BentoService instance
"""
logger.warning(
"BentoService#pack class method is deprecated, use instance method `pack` "
"instead. e.g.: svc = MyBentoService(); svc.pack('model', model_object)"
)
from bentoml.artifact import ArtifactCollection
if args and isinstance(args[0], ArtifactCollection):
bento_svc = cls(*args[1:], **kwargs) # pylint: disable=not-callable
bento_svc._packed_artifacts = args[0]
return bento_svc
packed_artifacts = []
for artifact in cls._artifacts:
if artifact.name in kwargs:
artifact_args = kwargs.pop(artifact.name)
packed_artifacts.append(artifact.pack(artifact_args))
bento_svc = cls(*args, **kwargs) # pylint: disable=not-callable
for packed_artifact in packed_artifacts:
bento_svc.artifacts.add(packed_artifact)
return bento_svc
def _load_artifacts(self, path):
# For pip installed BentoService, artifacts directory is located at
# 'package_path/artifacts/', but for loading from bundle directory, it is
# in 'path/{service_name}/artifacts/'
if not os.path.isdir(os.path.join(path, ARTIFACTS_DIR_NAME)):
artifacts_path = os.path.join(path, self.name, ARTIFACTS_DIR_NAME)
else:
artifacts_path = os.path.join(path, ARTIFACTS_DIR_NAME)
for artifact in self._artifacts:
packed_artifact = artifact.load(artifacts_path)
self._packed_artifacts.add(packed_artifact)
def get_bento_service_metadata_pb(self):
return SavedBundleConfig(self).get_bento_service_metadata_pb()
class BentoService:
"""
BentoService is the base component for building prediction services using BentoML.
BentoService provide an abstraction for describing model artifacts and environment
dependencies required for a prediction service. And allows users to create inference
APIs that defines the inferencing logic and how the underlying model can be served.
Each BentoService can contain multiple models and serve multiple inference APIs.
Usage example:
>>> from bentoml import BentoService, env, api, artifacts
>>> from bentoml.adapters import DataframeInput
>>> from bentoml.artifact import SklearnModelArtifact
>>>
>>> @artifacts([SklearnModelArtifact('clf')])
>>> @env(pip_dependencies=["scikit-learn"])
>>> class MyMLService(BentoService):
>>>
>>> @api(input=DataframeInput())
>>> def predict(self, df):
>>> return self.artifacts.clf.predict(df)
>>>
>>> if __name__ == "__main__":
>>> bento_service = MyMLService()
>>> bento_service.pack('clf', trained_classifier_model)
>>> bento_service.save_to_dir('/bentoml_bundles')
"""
# List of inference APIs that this BentoService provides
_inference_apis = []
# Name of this BentoService. It is default the class name of this BentoService class
_bento_service_name = None
# For BentoService loaded from saved bundle, this will be set to the path of bundle.
# When user install BentoService bundle as a PyPI package, this will be set to the
# installed site-package location of current python environment
_bento_service_bundle_path = None
# A list of artifacts required by this BentoService
_artifacts = []
# A `BentoServiceEnv` instance specifying the required dependencies and all system
# environment setups
_env = None
# When loading BentoService from saved bundle, this will be set to the version of
# the saved BentoService bundle
_bento_service_bundle_version = None
# See `ver_decorator` function above for more information
_version_major = None
_version_minor = None
# See `web_static_content` function above for more
_web_static_content = None
def __init__(self):
from bentoml.artifact import ArtifactCollection
self._bento_service_version = self.__class__._bento_service_bundle_version
self._packed_artifacts = ArtifactCollection()
if self._bento_service_bundle_path:
# load artifacts from saved BentoService bundle
self._load_artifacts(self._bento_service_bundle_path)
self._config_inference_apis()
self._config_environments()
def _config_environments(self):
self._env = self.__class__._env or BentoServiceEnv(self.name)
for api in self._inference_apis:
self._env._add_pip_dependencies_if_missing(
api.handler.pip_dependencies)
self._env._add_pip_dependencies_if_missing(
api.output_adapter.pip_dependencies)
for artifact in self._artifacts:
self._env._add_pip_dependencies_if_missing(
artifact.pip_dependencies)
def _config_inference_apis(self):
self._inference_apis = []
for _, function in inspect.getmembers(
self.__class__,
predicate=lambda x: inspect.isfunction(x) or inspect.ismethod(
x),
):
if hasattr(function, "_is_api"):
api_name = getattr(function, "_api_name")
api_doc = getattr(function, "_api_doc")
handler = getattr(function, "_handler")
mb_max_latency = getattr(function, "_mb_max_latency")
mb_max_batch_size = getattr(function, "_mb_max_batch_size")
# Bind api method call with self(BentoService instance)
func = function.__get__(self)
self._inference_apis.append(
InferenceAPI(
self,
api_name,
api_doc,
handler=handler,
func=func,
mb_max_latency=mb_max_latency,
mb_max_batch_size=mb_max_batch_size,
))
@property
def inference_apis(self):
"""Return a list of user defined API functions
Returns:
list(InferenceAPI): List of Inference API objects
"""
return self._inference_apis
def get_inference_api(self, api_name):
"""Find the inference API in this BentoService with a specific name.
When the api_name is None, this returns the first Inference API found in the
`self.inference_apis` list.
:param api_name: the target Inference API's name
:return:
"""
if api_name:
try:
return next((api for api in self.inference_apis
if api.name == api_name))
except StopIteration:
raise NotFound("Can't find API '{}' in service '{}'".format(
api_name, self.name))
elif len(self.inference_apis) > 0:
return self.inference_apis[0]
else:
raise NotFound(
f"Can't find any inference API in service '{self.name}'")
@property
def artifacts(self):
""" Returns all packed artifacts in an ArtifactCollection object
Returns:
artifacts(ArtifactCollection): A dictionary of packed artifacts from the
artifact name to the loaded artifact model instance in its native form
"""
return self._packed_artifacts
@property
def env(self):
return self._env
@property
def web_static_content(self):
return self._web_static_content
def get_web_static_content_path(self):
if not self.web_static_content:
return None
if self._bento_service_bundle_path:
return os.path.join(
self._bento_service_bundle_path,
self.name,
'web_static_content',
)
else:
return os.path.join(os.getcwd(), self.web_static_content)
@hybridmethod
@property
def name(self):
"""
:return: BentoService name
"""
return self.__class__.name() # pylint: disable=no-value-for-parameter
@name.classmethod
def name(cls): # pylint: disable=no-self-argument,invalid-overridden-method
"""
:return: BentoService name
"""
if cls._bento_service_name is not None:
if not isidentifier(cls._bento_service_name):
raise InvalidArgument(
'BentoService#_bento_service_name must be valid python identifier'
'matching regex `(letter|"_")(letter|digit|"_")*`')
return cls._bento_service_name
else:
# Use python class name as service name
return cls.__name__
def set_version(self, version_str=None):
"""Set the version of this BentoService instance. Once the version is set
explicitly via `set_version`, the `self.versioneer` method will no longer be
invoked when saving this BentoService.
"""
if version_str is None:
version_str = self.versioneer()
if self._version_major is not None and self._version_minor is not None:
# BentoML uses semantic versioning for BentoService distribution
# when user specified the MAJOR and MINOR version number along with
# the BentoService class definition with '@ver' decorator.
# The parameter version(or auto generated version) here will be used as
# PATCH field in the final version:
version_str = ".".join([
str(self._version_major),
str(self._version_minor), version_str
])
_validate_version_str(version_str)
if self.__class__._bento_service_bundle_version is not None:
logger.warning(
"Overriding loaded BentoService(%s) version:%s to %s",
self.__class__._bento_service_bundle_path,
self.__class__._bento_service_bundle_version,
version_str,
)
self.__class__._bento_service_bundle_version = None
if (self._bento_service_version is not None
and self._bento_service_version != version_str):
logger.warning(
"Resetting BentoService '%s' version from %s to %s",
self.name,
self._bento_service_version,
version_str,
)
self._bento_service_version = version_str
return self._bento_service_version
def versioneer(self):
"""
Function used to generate a new version string when saving a new BentoService
bundle. User can also override this function to get a customized version format
"""
datetime_string = datetime.now().strftime("%Y%m%d%H%M%S")
random_hash = uuid.uuid4().hex[:6].upper()
# Example output: '20191009135240_D246ED'
return datetime_string + "_" + random_hash
@property
def version(self):
"""
Return the version of this BentoService. If the version of this BentoService has
not been set explicitly via `self.set_version`, a new version will be generated
with the `self.versioneer` method. User can customize this version str either by
setting the version with `self.set_version` before a `save` call, or override
the `self.versioneer` method to customize the version str generator logic.
For BentoService loaded from a saved bundle, this will simply return the version
information found in the saved bundle.
:return: BentoService version str
"""
if self.__class__._bento_service_bundle_version is not None:
return self.__class__._bento_service_bundle_version
if self._bento_service_version is None:
self.set_version(self.versioneer())
return self._bento_service_version
def save(self, base_path=None, version=None):
"""
Save and register this BentoService via BentoML's built-in model management
system. BentoML by default keeps track of all the SavedBundle's files and
metadata in local file system under the $BENTOML_HOME(~/bentoml) directory.
Users can also configure BentoML to save their BentoService to a shared Database
and cloud object storage such as AWS S3.
:param base_path: optional - override repository base path
:param version: optional - save with version override
:return: saved_path: file path to where the BentoService is saved
"""
return save(self, base_path, version)
def save_to_dir(self, path, version=None):
"""Save this BentoService along with all its artifacts, source code and
dependencies to target file path, assuming path exist and empty. If target path
is not empty, this call may override existing files in the given path.
:param path (str): Destination of where the bento service will be saved
:param version: optional - save with version override
"""
return save_to_dir(self, path, version)
@hybridmethod
def pack(self, name, *args, **kwargs):
"""
BentoService#pack method is used for packing trained model instances with a
BentoService instance and make it ready for BentoService#save.
pack(name, *args, **kwargs):
:param name: name of the declared model artifact
:param args: args passing to the target model artifact to be packed
:param kwargs: kwargs passing to the target model artifact to be packed
:return: this BentoService instance
"""
if name in self.artifacts:
logger.warning(
"BentoService '%s' #pack overriding existing artifact '%s'",
self.name,
name,
)
del self.artifacts[name]
artifact = next(artifact for artifact in self._artifacts
if artifact.name == name)
packed_artifact = artifact.pack(*args, **kwargs)
self._packed_artifacts.add(packed_artifact)
return self
@pack.classmethod
def pack(cls, *args, **kwargs): # pylint: disable=no-self-argument
"""
**Deprecated**: Legacy `BentoService#pack` class method, no longer supported
"""
raise BentoMLException(
"BentoService#pack class method is deprecated, use instance method `pack` "
"instead. e.g.: svc = MyBentoService(); svc.pack('model', model_object)"
)
def _load_artifacts(self, path):
# For pip installed BentoService, artifacts directory is located at
# 'package_path/artifacts/', but for loading from bundle directory, it is
# in 'path/{service_name}/artifacts/'
if not os.path.isdir(os.path.join(path, ARTIFACTS_DIR_NAME)):
artifacts_path = os.path.join(path, self.name, ARTIFACTS_DIR_NAME)
else:
artifacts_path = os.path.join(path, ARTIFACTS_DIR_NAME)
for artifact in self._artifacts:
packed_artifact = artifact.load(artifacts_path)
self._packed_artifacts.add(packed_artifact)
def get_bento_service_metadata_pb(self):
return SavedBundleConfig(self).get_bento_service_metadata_pb()
class BentoService(BentoServiceBase):
"""BentoService packs a list of artifacts and exposes service APIs
for BentoAPIServer and BentoCLI to execute. By subclassing BentoService,
users can customize the artifacts and environments required for
a ML service.
>>> from bentoml import BentoService, env, api, artifacts, ver
>>> from bentoml.handlers import DataframeHandler
>>> from bentoml.artifact import SklearnModelArtifact
>>>
>>> @ver(major=1, minor=4)
>>> @artifacts([SklearnModelArtifact('clf')])
>>> @env(pip_dependencies=["scikit-learn"])
>>> class MyMLService(BentoService):
>>>
>>> @api(DataframeHandler)
>>> def predict(self, df):
>>> return self.artifacts.clf.predict(df)
>>>
>>> bento_service = MyMLService()
>>> bento_service.pack('clf', trained_classifier_model)
>>> bento_service.save_to_dir('/bentoml_bundles')
"""
# User may use @name to override this if they don't want the generated model
# to have the same name as their Python model class name
_bento_service_name = None
# For BentoService loaded from saved bundle, this will be set to the path of bundle.
# When user install BentoService bundle as a PyPI package, this will be set to the
# installed site-package location of current python environment
_bento_service_bundle_path = None
# list of artifacts required by this BentoService
_artifacts = []
# Describe the desired environment for this BentoService using
# `bentoml.service_env.BentoServiceEnv`
_env = None
# When loading BentoService from saved bundle, this will be set to the version of
# the saved BentoService bundle
_bento_service_bundle_version = None
# See `ver_decorator` function above for more information
_version_major = None
_version_minor = None
def __init__(self):
from bentoml.artifact import ArtifactCollection
self._bento_service_version = self.__class__._bento_service_bundle_version
self._packed_artifacts = ArtifactCollection()
if self._bento_service_bundle_path:
# load artifacts from saved BentoService bundle
self._load_artifacts(self._bento_service_bundle_path)
self._config_service_apis()
self._init_env()
def _init_env(self):
self._env = self.__class__._env or BentoServiceEnv(self.name)
for api in self._service_apis:
self._env.add_handler_dependencies(api.handler.pip_dependencies)
@property
def artifacts(self):
return self._packed_artifacts
@property
def env(self):
return self._env
@hybridmethod
@property
def name(self):
return self.__class__.name() # pylint: disable=no-value-for-parameter
@name.classmethod
def name(cls): # pylint: disable=no-self-argument,invalid-overridden-method
if cls._bento_service_name is not None:
if not isidentifier(cls._bento_service_name):
raise InvalidArgument(
'BentoService#_bento_service_name must be valid python identifier'
'matching regex `(letter|"_")(letter|digit|"_")*`'
)
return cls._bento_service_name
else:
# Use python class name as service name
return cls.__name__
def set_version(self, version_str=None):
"""Manually override the version of this BentoService instance
"""
if version_str is None:
version_str = self.versioneer()
if self._version_major is not None and self._version_minor is not None:
# BentoML uses semantic versioning for BentoService distribution
# when user specified the MAJOR and MINOR version number along with
# the BentoService class definition with '@ver' decorator.
# The parameter version(or auto generated version) here will be used as
# PATCH field in the final version:
version_str = ".".join(
[str(self._version_major), str(self._version_minor), version_str]
)
_validate_version_str(version_str)
if self.__class__._bento_service_bundle_version is not None:
logger.warning(
"Overriding loaded BentoService(%s) version:%s to %s",
self.__class__._bento_service_bundle_path,
self.__class__._bento_service_bundle_version,
version_str,
)
self.__class__._bento_service_bundle_version = None
if (
self._bento_service_version is not None
and self._bento_service_version != version_str
):
logger.warning(
"Reseting BentoServive '%s' version from %s to %s",
self.name,
self._bento_service_version,
version_str,
)
self._bento_service_version = version_str
return self._bento_service_version
def versioneer(self):
"""
Function used to generate a new version string when saving a new BentoService
bundle. User can also override this function to get a customized version format
"""
datetime_string = datetime.now().strftime("%Y%m%d%H%M%S")
random_hash = uuid.uuid4().hex[:6].upper()
# Example output: '20191009135240_D246ED'
return datetime_string + "_" + random_hash
@property
def version(self):
if self.__class__._bento_service_bundle_version is not None:
return self.__class__._bento_service_bundle_version
if self._bento_service_version is None:
self.set_version(self.versioneer())
return self._bento_service_version
def save(self, base_path=None, version=None):
return save(self, base_path, version)
def save_to_dir(self, path, version=None):
return save_to_dir(self, path, version)
@hybridmethod
def pack(self, name, *args, **kwargs):
if name in self.artifacts:
logger.warning(
"BentoService '%s' #pack overriding existing artifact '%s'",
self.name,
name,
)
del self.artifacts[name]
artifact = next(
artifact for artifact in self._artifacts if artifact.name == name
)
packed_artifact = artifact.pack(*args, **kwargs)
self._packed_artifacts.add(packed_artifact)
return self
@pack.classmethod
def pack(cls, *args, **kwargs): # pylint: disable=no-self-argument
from bentoml.artifact import ArtifactCollection
if args and isinstance(args[0], ArtifactCollection):
bento_svc = cls(*args[1:], **kwargs) # pylint: disable=not-callable
bento_svc._packed_artifacts = args[0]
return bento_svc
packed_artifacts = []
for artifact in cls._artifacts:
if artifact.name in kwargs:
artifact_args = kwargs.pop(artifact.name)
packed_artifacts.append(artifact.pack(artifact_args))
bento_svc = cls(*args, **kwargs) # pylint: disable=not-callable
for packed_artifact in packed_artifacts:
bento_svc.artifacts.add(packed_artifact)
return bento_svc
def _load_artifacts(self, path):
# For pip installed BentoService, artifacts directory is located at
# 'package_path/artifacts/', but for loading from bundle directory, it is
# in 'path/{service_name}/artifacts/'
if not os.path.isdir(os.path.join(path, ARTIFACTS_DIR_NAME)):
artifacts_path = os.path.join(path, self.name, ARTIFACTS_DIR_NAME)
else:
artifacts_path = os.path.join(path, ARTIFACTS_DIR_NAME)
for artifact in self._artifacts:
packed_artifact = artifact.load(artifacts_path)
self._packed_artifacts.add(packed_artifact)
def get_bento_service_metadata_pb(self):
return SavedBundleConfig(self).get_bento_service_metadata_pb()