def start_dev_batching_server(
saved_bundle_path: str,
port: int,
api_server_port: int,
mb_max_batch_size: int,
mb_max_latency: int,
):
from bentoml.marshal.marshal import MarshalService
marshal_server = MarshalService(
saved_bundle_path,
outbound_host="localhost",
outbound_port=api_server_port,
outbound_workers=1,
mb_max_batch_size=mb_max_batch_size,
mb_max_latency=mb_max_latency,
)
logger.info("Running micro batch service on :%d", port)
marshal_server.fork_start_app(port=port)
def _start_dev_proxy(
port: int,
saved_bundle_path: str,
api_server_port: int,
enable_microbatch: bool,
mb_max_batch_size: int,
mb_max_latency: int,
):
logger.info("Starting BentoML API proxy in development mode..")
from bentoml.marshal.marshal import MarshalService
marshal_server = MarshalService(
saved_bundle_path,
outbound_host="localhost",
outbound_port=api_server_port,
enable_microbatch=enable_microbatch,
mb_max_batch_size=mb_max_batch_size,
mb_max_latency=mb_max_latency,
)
marshal_server.fork_start_app(port=port)
def _start_dev_proxy(
saved_bundle_path: str,
api_server_port: int,
config: BentoMLConfiguration,
):
logger.info("Starting BentoML API proxy in development mode..")
from bentoml import marshal
container = BentoMLContainer()
container.config.from_dict(config.as_dict())
container.wire(packages=[marshal])
from bentoml.marshal.marshal import MarshalService
marshal_server = MarshalService(
saved_bundle_path,
outbound_host="localhost",
outbound_port=api_server_port,
)
marshal_server.fork_start_app()