def __init__(
self,
service,
name,
doc,
input_adapter: BaseInputAdapter,
user_func: callable,
output_adapter: BaseOutputAdapter,
mb_max_latency=10000,
mb_max_batch_size=1000,
batch=False,
):
"""
:param service: ref to service containing this API
:param name: API name
:param doc: the user facing document of this inference API, default to the
docstring of the inference API function
:param input_adapter: A InputAdapter that transforms HTTP Request and/or
CLI options into parameters for API func
:param user_func: the user-defined API callback function, this is
typically the 'predict' method on a model
:param output_adapter: A OutputAdapter is an layer between result of user
defined API callback function
and final output in a variety of different forms,
such as HTTP response, command line stdout or AWS Lambda event object.
:param mb_max_latency: The latency goal of this inference API in milliseconds.
Default: 10000.
:param mb_max_batch_size: The maximum size of requests batch accepted by this
inference API. This parameter governs the throughput/latency trade off, and
avoids having large batches that exceed some resource constraint (e.g. GPU
memory to hold the entire batch's data). Default: 1000.
:param batch: If true, the user API functool would take a batch of input data
a time.
"""
self._service = service
self._name = name
self._input_adapter = input_adapter
self._user_func = user_func
self._output_adapter = output_adapter
self.mb_max_latency = mb_max_latency
self.mb_max_batch_size = mb_max_batch_size
self.batch = batch
if not self.input_adapter.BATCH_MODE_SUPPORTED and batch:
raise BentoMLConfigException(
f"{input_adapter.__class__.__name__} does not support `batch=True`"
)
if not self.input_adapter.SINGLE_MODE_SUPPORTED and not batch:
raise BentoMLConfigException(
f"{input_adapter.__class__.__name__} does not support `batch=False`, "
"its output passed to API functions could only be a batch of data."
)
if doc is None:
# generate a default doc string for this inference API
doc = (f"BentoService inference API '{self.name}', input: "
f"'{type(input_adapter).__name__}', output: "
f"'{type(output_adapter).__name__}'")
self._doc = doc
def load(cls, filepath):
conf = cls()
with open(filepath, "rb") as config_file:
yml_content = config_file.read()
conf.config = conf._yaml.load(yml_content)
ver = str(conf["version"])
py_ver = conf.config["env"]["python_version"]
if ver != BENTOML_VERSION:
msg = (
"Saved BentoService bundle version mismatch: loading BentoService "
"bundle create with BentoML version {}, but loading from BentoML "
"version {}".format(conf["version"], BENTOML_VERSION))
# If major version is different, then there could be incompatible API
# changes. Raise error in this case.
if ver.split(".")[0] != BENTOML_VERSION.split(".")[0]:
if not BENTOML_VERSION.startswith('0+untagged'):
raise BentoMLConfigException(msg)
else:
logger.warning(msg)
else: # Otherwise just show a warning.
logger.warning(msg)
if py_ver != PYTHON_VERSION:
logger.warning(
f"Saved BentoService Python version mismatch: loading "
f"BentoService bundle created with Python version {py_ver}, "
f"but current environment version is {PYTHON_VERSION}.")
return conf
def load_config():
global BENTOML_HOME # pylint: disable=global-statement
try:
Path(BENTOML_HOME).mkdir(exist_ok=True)
except OSError as err:
raise BentoMLConfigException(
"Error creating bentoml home directory '{}': {}".format(
BENTOML_HOME, err.strerror))
with open(DEFAULT_CONFIG_FILE, "rb") as f:
DEFAULT_CONFIG = f.read().decode(CONFIG_FILE_ENCODING)
loaded_config = BentoMLConfigParser(
default_config=parameterized_config(DEFAULT_CONFIG))
local_config_file = get_local_config_file()
if os.path.isfile(local_config_file):
logger.info("Loading local BentoML config file: %s", local_config_file)
with open(local_config_file, "rb") as f:
loaded_config.read_string(
parameterized_config(f.read().decode(CONFIG_FILE_ENCODING)))
else:
logger.info(
"No local BentoML config file found, using default configurations")
return loaded_config
def __init__(
self,
default_config_file: str = None,
override_config_file: str = None,
validate_schema: bool = True,
):
# Default configuraiton
if default_config_file is None:
default_config_file = os.path.join(
os.path.dirname(__file__), "default_bentoml.yml"
)
with open(default_config_file, "rb") as f:
self.config = YAML().load(f.read())
if validate_schema:
try:
SCHEMA.validate(self.config)
except SchemaError as e:
raise BentoMLConfigException(
"Default configuration 'default_bentoml.yml' does not"
" conform to the required schema."
) from e
# User override configuration
if override_config_file is not None:
LOGGER.info("Applying user config override from %s" % override_config_file)
if not os.path.exists(override_config_file):
raise BentoMLConfigException(
f"Config file {override_config_file} not found"
)
with open(override_config_file, "rb") as f:
override_config = YAML().load(f.read())
always_merger.merge(self.config, override_config)
if validate_schema:
try:
SCHEMA.validate(self.config)
except SchemaError as e:
raise BentoMLConfigException(
"Configuration after user override does not conform to"
" the required schema."
) from e
def override(self, keys: list, value):
if keys is None:
raise BentoMLConfigException("Configuration override key is None.")
if len(keys) == 0:
raise BentoMLConfigException("Configuration override key is empty.")
if value is None:
return
c = self.config
for key in keys[:-1]:
if key not in c:
raise BentoMLConfigException(
"Configuration override key is invalid, %s" % keys
)
c = c[key]
c[keys[-1]] = value
try:
SCHEMA.validate(self.config)
except SchemaError as e:
raise BentoMLConfigException(
"Configuration after applying override does not conform"
" to the required schema, key=%s, value=%s." % (keys, value)
) from e
def get(self, section, key, **kwargs): # pylint:disable=arguments-differ
""" A simple hierachical config access, priority order:
1. environment var
2. user config file
3. bentoml default config file
"""
section = str(section).lower()
key = str(key).lower()
env_var = self._env_var_name(section, key)
if env_var in os.environ:
return os.environ[env_var]
if ConfigParser.has_option(self, section, key):
return ConfigParser.get(self, section, key, **kwargs)
else:
raise BentoMLConfigException(
"section/key '{}/{}' not found in BentoML config".format(
section, key))
def load(cls, filepath):
conf = cls()
with open(filepath, "rb") as config_file:
yml_content = config_file.read()
conf.config = conf._yaml.load(yml_content)
if conf["version"] != BENTOML_VERSION:
msg = (
"Saved BentoService bundle version mismatch: loading BentoServie "
"bundle create with BentoML version {}, but loading from BentoML "
"version {}".format(conf["version"], BENTOML_VERSION))
# If major version is different, then there could be incompatible API
# changes. Raise error in this case.
if conf["version"].split(".")[0] != BENTOML_VERSION.split(".")[0]:
if not BENTOML_VERSION.startswith('0+untagged'):
raise BentoMLConfigException(msg)
else:
logger.warning(msg)
else: # Otherwise just show a warning.
logger.warning(msg)
return conf
def __init__(
self,
default_config_file: str = None,
override_config_file: str = None,
validate_schema: bool = True,
legacy_compatibility: bool = True,
):
# Default configuraiton
if default_config_file is None:
default_config_file = os.path.join(os.path.dirname(__file__),
"default_bentoml.yml")
with open(default_config_file, "rb") as f:
self.config = YAML().load(f.read())
if validate_schema:
try:
SCHEMA.validate(self.config)
except SchemaError as e:
raise BentoMLConfigException(
"Default configuration 'default_bentoml.yml' does not"
" conform to the required schema.") from e
# Legacy configuration compatibility
if legacy_compatibility:
try:
self.config["bento_server"]["port"] = config(
"apiserver").getint("default_port")
self.config["bento_server"]["workers"] = config(
"apiserver").getint("default_gunicorn_workers_count")
self.config["bento_server"]["max_request_size"] = config(
"apiserver").getint("default_max_request_size")
if "default_max_batch_size" in config("marshal_server"):
self.config["bento_server"]["microbatch"][
"max_batch_size"] = config("marshal_server").getint(
"default_max_batch_size")
if "default_max_latency" in config("marshal_server"):
self.config["bento_server"]["microbatch"][
"max_latency"] = config("marshal_server").getint(
"default_max_latency")
self.config["bento_server"]["metrics"]["namespace"] = config(
"instrument").get("default_namespace")
self.config["adapters"]["image_input"][
"default_extensions"] = [
extension.strip()
for extension in config("apiserver").get(
"default_image_input_accept_file_extensions").
split(",")
]
except KeyError as e:
raise BentoMLConfigException(
"Overriding a non-existent configuration key in compatibility mode."
) from e
if validate_schema:
try:
SCHEMA.validate(self.config)
except SchemaError as e:
raise BentoMLConfigException(
"Configuration after applying legacy compatibility"
" does not conform to the required schema.") from e
# User override configuration
if override_config_file is not None:
LOGGER.info("Applying user config override from %s" %
override_config_file)
if not os.path.exists(override_config_file):
raise BentoMLConfigException(
f"Config file {override_config_file} not found")
with open(override_config_file, "rb") as f:
override_config = YAML().load(f.read())
always_merger.merge(self.config, override_config)
if validate_schema:
try:
SCHEMA.validate(self.config)
except SchemaError as e:
raise BentoMLConfigException(
"Configuration after user override does not conform to"
" the required schema.") from e
Args:
:param template: a config content templated with {{variables}}
Returns:
string: config content after templated with locals() and globals()
"""
all_vars = {k: v for d in [globals(), locals()] for k, v in d.items()}
return template.format(**all_vars)
BENTOML_HOME = expand_env_var(os.environ.get("BENTOML_HOME", "~/bentoml"))
try:
Path(BENTOML_HOME).mkdir(exist_ok=True)
except OSError as err:
raise BentoMLConfigException(
"Error creating bentoml home dir '{}': {}".format(
BENTOML_HOME, err.strerror))
# Default bentoml config comes with the library bentoml/config/default_bentoml.cfg
DEFAULT_CONFIG_FILE = os.path.join(os.path.dirname(__file__),
"default_bentoml.cfg")
with open(DEFAULT_CONFIG_FILE, "rb") as f:
DEFAULT_CONFIG = f.read().decode("utf-8")
config = BentoMLConfigParser(
default_config=parameterized_config(DEFAULT_CONFIG))
if "BENTML_CONFIG" in os.environ:
# User local config file for customizing bentoml
LOCAL_CONFIG_FILE = expand_env_var(os.environ.get("BENTML_CONFIG"))
logger.info("Using BentoML config file $BENTML_CONFIG: %s",
def __init__(
self,
default_config_file: str = None,
override_config_file: str = None,
validate_schema: bool = True,
legacy_compatibility: bool = True,
):
# Default configuraiton
if default_config_file is None:
default_config_file = os.path.join(os.path.dirname(__file__),
"default_bentoml.yml")
with open(default_config_file, "rb") as f:
self.config = YAML().load(f.read())
if validate_schema:
try:
SCHEMA.validate(self.config)
except SchemaError as e:
raise BentoMLConfigException(
"Default configuration 'default_bentoml.yml' does not"
" conform to the required schema.") from e
# Legacy configuration compatibility
if legacy_compatibility:
try:
self.config["api_server"]["port"] = config("apiserver").getint(
"default_port")
self.config["api_server"]["max_request_size"] = config(
"apiserver").getint("default_max_request_size")
self.config["marshal_server"]["max_batch_size"] = config(
"marshal_server").getint("default_max_batch_size")
self.config["marshal_server"]["max_latency"] = config(
"marshal_server").getint("default_max_latency")
self.config["marshal_server"]["request_header_flag"] = config(
"marshal_server").get("marshal_request_header_flag")
self.config["yatai"]["url"] = config("yatai_service").get(
"url")
self.config["tracing"]["zipkin_api_url"] = config(
"tracing").get("zipkin_api_url")
self.config["instrument"]["namespace"] = config(
"instrument").get("default_namespace")
except KeyError as e:
raise BentoMLConfigException(
"Overriding a non-existent configuration key in compatibility mode."
) from e
if validate_schema:
try:
SCHEMA.validate(self.config)
except SchemaError as e:
raise BentoMLConfigException(
"Configuration after applying legacy compatibility"
" does not conform to the required schema.") from e
# User override configuration
if override_config_file is not None and os.path.exists(
override_config_file):
LOGGER.info("Applying user config override from %s" %
override_config_file)
with open(override_config_file, "rb") as f:
override_config = YAML().load(f.read())
always_merger.merge(self.config, override_config)
if validate_schema:
try:
SCHEMA.validate(self.config)
except SchemaError as e:
raise BentoMLConfigException(
"Configuration after user override does not conform to"
" the required schema.") from e