def start_dev_server(
saved_bundle_path: str,
port: int = Provide[BentoMLContainer.config.api_server.port],
enable_microbatch: bool = Provide[
BentoMLContainer.config.api_server.enable_microbatch],
mb_max_batch_size: int = Provide[
BentoMLContainer.config.marshal_server.max_batch_size],
mb_max_latency: int = Provide[
BentoMLContainer.config.marshal_server.max_latency],
run_with_ngrok: bool = Provide[
BentoMLContainer.config.api_server.run_with_ngrok],
enable_swagger: bool = Provide[
BentoMLContainer.config.api_server.enable_swagger],
):
logger.info("Starting BentoML API server in development mode..")
import multiprocessing
from bentoml.saved_bundle import load_from_dir
from bentoml.server.api_server import BentoAPIServer
from bentoml.utils import reserve_free_port
if run_with_ngrok:
from threading import Timer
from bentoml.utils.flask_ngrok import start_ngrok
thread = Timer(1, start_ngrok, args=(port, ))
thread.setDaemon(True)
thread.start()
if enable_microbatch:
with reserve_free_port() as api_server_port:
# start server right after port released
# to reduce potential race
marshal_proc = multiprocessing.Process(
target=start_dev_batching_server,
kwargs=dict(
api_server_port=api_server_port,
saved_bundle_path=saved_bundle_path,
port=port,
mb_max_latency=mb_max_latency,
mb_max_batch_size=mb_max_batch_size,
),
daemon=True,
)
marshal_proc.start()
bento_service = load_from_dir(saved_bundle_path)
api_server = BentoAPIServer(bento_service,
port=api_server_port,
enable_swagger=enable_swagger)
api_server.start()
else:
bento_service = load_from_dir(saved_bundle_path)
api_server = BentoAPIServer(bento_service,
port=port,
enable_swagger=enable_swagger)
api_server.start()
def load(self, bento):
"""
Load bento service from bento tag or from a bento bundle path.
Args:
bento: string,
Returns:
BentoService instance
Example:
>>> yatai_client = get_yatai_client()
>>> # Load BentoService bases on bento tag.
>>> bento = yatai_client.repository.load('Service_name:version')
>>> # Load BentoService from bento bundle path
>>> bento = yatai_client.repository.load('/path/to/bento/bundle')
>>> # Load BentoService from s3 storage
>>> bento = yatai_client.repository.load('s3://bucket/path/bundle.tar.gz')
"""
track('py-api-load')
if os.path.isdir(bento) or is_s3_url(bento) or is_gcs_url(bento):
saved_bundle_path = bento
else:
bento_pb = self.get(bento)
saved_bundle_path = resolve_bento_bundle_uri(bento_pb)
svc = load_from_dir(saved_bundle_path)
return svc
def load(self):
bento_service = load_from_dir(self.bento_service_bundle_path)
api_server = GunicornBentoAPIServer(
bento_service,
port=self.port,
enable_swagger=self.enable_swagger)
return api_server.app
def open_api_spec(bento=None, yatai_url=None):
saved_bundle_path = resolve_bundle_path(
bento, pip_installed_bundle_path, yatai_url
)
bento_service = load_from_dir(saved_bundle_path)
_echo(json.dumps(get_open_api_spec_json(bento_service), indent=2))
def _start_dev_server(
saved_bundle_path: str,
api_server_port: int,
enable_swagger: bool,
):
logger.info("Starting BentoML API server in development mode..")
from bentoml.server.api_server import BentoAPIServer
from bentoml.saved_bundle import load_from_dir
bento_service = load_from_dir(saved_bundle_path)
api_server = BentoAPIServer(bento_service, enable_swagger=enable_swagger)
api_server.start(port=api_server_port)
def _start_dev_server(
saved_bundle_path: str,
api_server_port: int,
config: BentoMLConfiguration,
):
logger.info("Starting BentoML API server in development mode..")
from bentoml.saved_bundle import load_from_dir
bento_service = load_from_dir(saved_bundle_path)
from bentoml.server.api_server import BentoAPIServer
container = BentoMLContainer()
container.config.from_dict(config.as_dict())
container.wire(packages=[sys.modules[__name__]])
api_server = BentoAPIServer(bento_service)
api_server.start(port=api_server_port)
def load(self, bento: str) -> "BentoService":
"""
Load :class:`~bentoml.BentoService` from nametag identifier
or from a bento bundle path.
Args:
bento (`str`):
:class:`~bentoml.BentoService` identifier or bundle path. Note
that nametag identifier will have the following format: :code:`NAME:VERSION`
Returns:
:class:`~bentoml.BentoService`
Example::
from bentoml.yatai.client import get_yatai_client
yatai_client = get_yatai_client()
# Load BentoService bases on bento tag.
bento_from_name = yatai_client.repository.load('service_name:version')
# Load BentoService from bento bundle path
bento_from_path = yatai_client.repository.load('/path/to/bento/bundle')
# Load BentoService from s3 storage
bento_from_reg = yatai_client.repository.load('s3://bucket/path/bundle.tar.gz')
""" # noqa: E501
if os.path.isdir(bento) or is_s3_url(bento) or is_gcs_url(bento):
saved_bundle_path = bento
else:
bento_pb = self.get(bento)
if bento_pb.uri.type == BentoUri.LOCAL and is_remote_yatai(
self.yatai_service):
saved_bundle_path = self._download_bento(
bento_pb.name, bento_pb.version)
else:
saved_bundle_path = resolve_bento_bundle_uri(bento_pb)
svc = load_from_dir(saved_bundle_path)
return svc
def load():
return saved_bundle.load_from_dir(__module_path)
# Copyright 2019 Atalaya Tech, Inc.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import azure.functions as func # pylint: disable=import-error
from bentoml.server.api_server import BentoAPIServer
from bentoml.saved_bundle import load_from_dir
bento_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
svc = load_from_dir(bento_path)
bento_server = BentoAPIServer(svc)
def main(req: func.HttpRequest, context: func.Context) -> func.HttpResponse:
return func.WsgiMiddleware(bento_server.app.wsgi_app).handle(req, context)
# Copyright 2019 Atalaya Tech, Inc.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from bentoml.saved_bundle import load_from_dir
from bentoml.yatai.deployment.sagemaker.model_server import BentomlSagemakerServer
api_name = os.environ.get('API_NAME', None)
model_service = load_from_dir('/bento')
server = BentomlSagemakerServer(model_service, api_name)
app = server.app
# Set BENTOML_HOME to /tmp directory due to AWS lambda disk access restrictions
os.environ['BENTOML_HOME'] = '/tmp/bentoml/'
from bentoml.saved_bundle import load_from_dir # noqa
logger = logging.getLogger('bentoml.lambda_app')
bento_name = os.environ['BENTOML_BENTO_SERVICE_NAME']
api_name = os.environ["BENTOML_API_NAME"]
bento_bundle_path = os.path.join('./', bento_name)
if not os.path.exists(bento_bundle_path):
bento_bundle_path = os.path.join('/tmp/requirements', bento_name)
logger.debug('Loading BentoService bundle from path: "%s"', bento_bundle_path)
bento_service = load_from_dir(bento_bundle_path)
logger.debug('BentoService "%s" loaded successfully', bento_service.name)
bento_service_api = bento_service.get_inference_api(api_name)
logger.debug('BentoService API "%s" loaded successfully', {bento_service_api.name})
this_module = sys.modules[__name__]
def api_func(event, context): # pylint: disable=unused-argument
"""
Event – AWS Lambda uses this parameter to pass in event data to the handler. This
parameter is usually of the Python dict type. It can also be list, str, int,
float, or NoneType type. Currently BentoML only handles Lambda event coming from
Application Load Balancer, in which case the parameter `event` must be type `dict`
containing the HTTP request headers and body.
