def __init__(
self,
bento_bundle_path,
outbound_host="localhost",
outbound_port=None,
outbound_workers=1,
):
self.outbound_host = outbound_host
self.outbound_port = outbound_port
self.outbound_workers = outbound_workers
self.batch_handlers = dict()
self._outbound_sema = None # the semaphore to limit outbound connections
self.bento_service_metadata_pb = load_bento_service_metadata(
bento_bundle_path)
self.setup_routes_from_pb(self.bento_service_metadata_pb)
self.CONNECTION_LIMIT = resource.getrlimit(resource.RLIMIT_NOFILE)[0]
logger.info(
"Your system nofile limit is %d, which means each instance of microbatch "
"service is able to hold this number of connections at same time. "
"You can increase the number of file descriptors for the server process, "
"or launch more microbatch instances to accept more concurrent connection.",
self.CONNECTION_LIMIT,
)
def _get_bento_service_event_properties_from_bundle_path(
bundle_path, properties=None):
from bentoml.bundler import load_bento_service_metadata
bento_service_metadata = load_bento_service_metadata(bundle_path)
return _bento_service_metadata_to_event_properties(bento_service_metadata,
properties)
def serve(port, bento=None, with_conda=False, enable_microbatch=False):
track_cli('serve')
bento_service_bundle_path = resolve_bundle_path(
bento, pip_installed_bundle_path)
bento_service = load(bento_service_bundle_path)
if with_conda:
run_with_conda_env(
bento_service_bundle_path,
'bentoml serve {bento} --port {port} {flags}'.format(
bento=bento_service_bundle_path,
port=port,
flags="--enable-microbatch" if enable_microbatch else "",
),
)
return
if enable_microbatch:
with reserve_free_port() as api_server_port:
# start server right after port released
# to reduce potential race
bento_service_metadata_pb = load_bento_service_metadata(
bento_service_bundle_path)
marshal_server = MarshalServer(port=port,
target_host="localhost",
target_port=api_server_port)
marshal_server.setup_routes_from_pb(bento_service_metadata_pb)
api_server = BentoAPIServer(bento_service,
port=api_server_port)
marshal_server.async_start()
api_server.start()
else:
api_server = BentoAPIServer(bento_service, port=port)
api_server.start()
def test_auto_artifact_dependencies():
clf = svm.SVC(gamma='scale')
iris = datasets.load_iris()
X, y = iris.data, iris.target
clf.fit(X, y)
# Create a iris classifier service
iris_classifier_service = IrisClassifier()
# Pack it with the newly trained model artifact
iris_classifier_service.pack('model', clf)
# Save the prediction service to a BentoService bundle
saved_path = iris_classifier_service.save()
with open(os.path.join(saved_path, 'requirements.txt')) as f:
requirements_txt_content = f.read()
dependencies = requirements_txt_content.split('\n')
dependencies = [dep.split('==')[0] for dep in dependencies]
assert 'scikit-learn' in dependencies
assert 'bentoml' in dependencies
# Test that dependencies also wrote to BentoServiceMetadat config file
bs_matadata = load_bento_service_metadata(saved_path)
dependencies = bs_matadata.env.pip_dependencies.split('\n')
dependencies = [dep.split('==')[0] for dep in dependencies]
assert 'scikit-learn' in dependencies
assert 'bentoml' in dependencies
delete_saved_bento_service(iris_classifier_service.name,
iris_classifier_service.version)
def info(bundle_path=bundle_path):
"""
List all APIs defined in the BentoService loaded from saved bundle
"""
track_cli('info')
bento_service_metadata_pb = load_bento_service_metadata(bundle_path)
output = json.dumps(ProtoMessageToDict(bento_service_metadata_pb), indent=2)
_echo(output)
def load(self):
server = MarshalServer(self.target_host,
self.target_port,
port=self.port)
bento_service_metadata_pb = load_bento_service_metadata(
self.bento_service_bundle_path)
server.setup_routes_from_pb(bento_service_metadata_pb)
return server.marshal_app.make_app()
def test_save_duplicated_bento_exception_raised(tmpdir,
test_bento_service_class):
test_model = TestModel()
svc = test_bento_service_class()
svc.pack("model", test_model)
saved_path = svc.save(str(tmpdir))
svc_metadata = load_bento_service_metadata(saved_path)
assert svc.version == svc_metadata.version
with pytest.raises(BentoMLException):
with patch.object(bentoml.BentoService,
'save_to_dir') as save_to_dir_method:
# attempt to save again
svc.save(str(tmpdir))
save_to_dir_method.assert_not_called()
# reset svc version
svc.set_version()
saved_path = svc.save(str(tmpdir))
svc_metadata = load_bento_service_metadata(saved_path)
assert svc.version == svc_metadata.version
def _upload_bento_service(self, saved_bento_path):
bento_service_metadata = load_bento_service_metadata(saved_bento_path)
get_bento_response = self.yatai_service.GetBento(
GetBentoRequest(
bento_name=bento_service_metadata.name,
bento_version=bento_service_metadata.version,
))
if get_bento_response.status.status_code == status_pb2.Status.OK:
raise BentoMLException(
"BentoService bundle {}:{} already registered in repository. Reset "
"BentoService version with BentoService#set_version or bypass BentoML's"
" model registry feature with BentoService#save_to_dir".format(
bento_service_metadata.name,
bento_service_metadata.version))
elif get_bento_response.status.status_code != status_pb2.Status.NOT_FOUND:
raise BentoMLException(
'Failed accessing YataiService. {error_code}:'
'{error_message}'.format(
error_code=Status.Name(
get_bento_response.status.status_code),
error_message=get_bento_response.status.error_message,
))
request = AddBentoRequest(
bento_name=bento_service_metadata.name,
bento_version=bento_service_metadata.version,
)
response = self.yatai_service.AddBento(request)
if response.status.status_code != status_pb2.Status.OK:
raise BentoMLException(
"Error adding BentoService bundle to repository: {}:{}".format(
Status.Name(response.status.status_code),
response.status.error_message,
))
if response.uri.type == BentoUri.LOCAL:
if os.path.exists(response.uri.uri):
# due to copytree dst must not already exist
shutil.rmtree(response.uri.uri)
shutil.copytree(saved_bento_path, response.uri.uri)
self._update_bento_upload_progress(bento_service_metadata)
logger.info(
"BentoService bundle '%s:%s' saved to: %s",
bento_service_metadata.name,
bento_service_metadata.version,
response.uri.uri,
)
# Return URI to saved bento in repository storage
return response.uri.uri
elif response.uri.type == BentoUri.S3:
self._update_bento_upload_progress(bento_service_metadata,
UploadStatus.UPLOADING, 0)
fileobj = io.BytesIO()
with tarfile.open(mode="w:gz", fileobj=fileobj) as tar:
tar.add(saved_bento_path, arcname=bento_service_metadata.name)
fileobj.seek(0, 0)
files = {
'file': ('dummy', fileobj)
} # dummy file name because file name
# has been generated when getting the pre-signed signature.
data = json.loads(response.uri.additional_fields)
uri = data.pop('url')
http_response = requests.post(uri, data=data, files=files)
if http_response.status_code != 204:
self._update_bento_upload_progress(bento_service_metadata,
UploadStatus.ERROR)
raise BentoMLException(
f"Error saving BentoService bundle to S3. "
f"{http_response.status_code}: {http_response.text}")
self._update_bento_upload_progress(bento_service_metadata)
logger.info(
"Successfully saved BentoService bundle '%s:%s' to S3: %s",
bento_service_metadata.name,
bento_service_metadata.version,
response.uri.uri,
)
return response.uri.uri
else:
raise BentoMLException(
f"Error saving Bento to target repository, URI type {response.uri.type}"
f" at {response.uri.uri} not supported")
def deploy_bentoml(
clipper_conn, bundle_path, api_name, model_name=None, labels=None, build_envs=None
):
"""Deploy bentoml bundle to clipper cluster
Args:
clipper_conn(clipper_admin.ClipperConnection): Clipper connection instance
bundle_path(str): Path to the saved BentomlService bundle.
api_name(str): name of the api that will be used as prediction function for
clipper cluster
model_name(str): Model's name for clipper cluster
labels(:obj:`list(str)`, optional): labels for clipper model
Returns:
tuple: Model name and model version that deployed to clipper
"""
track("clipper-deploy", {'bento_service_bundle_path': bundle_path})
build_envs = {} if build_envs is None else build_envs
# docker is required to build clipper model image
ensure_docker_available_or_raise()
if not clipper_conn.connected:
raise BentoMLException(
"No connection to Clipper cluster. CallClipperConnection.connect to "
"connect to an existing cluster or ClipperConnnection.start_clipper to "
"create a new one"
)
bento_service_metadata = load_bento_service_metadata(bundle_path)
try:
api_metadata = next(
(api for api in bento_service_metadata.apis if api.name == api_name)
)
except StopIteration:
raise BentoMLException(
"Can't find API '{}' in BentoService bundle {}".format(
api_name, bento_service_metadata.name
)
)
if api_metadata.handler_type not in HANDLER_TYPE_TO_INPUT_TYPE:
raise BentoMLException(
"Only BentoService APIs using ClipperHandler can be deployed to Clipper"
)
input_type = HANDLER_TYPE_TO_INPUT_TYPE[api_metadata.handler_type]
model_name = model_name or get_clipper_compatiable_string(
bento_service_metadata.name + "-" + api_metadata.name
)
model_version = get_clipper_compatiable_string(bento_service_metadata.version)
with TempDirectory() as tempdir:
entry_py_content = CLIPPER_ENTRY.format(api_name=api_name)
model_path = os.path.join(tempdir, "bento")
shutil.copytree(bundle_path, model_path)
with open(os.path.join(tempdir, "clipper_entry.py"), "w") as f:
f.write(entry_py_content)
if bento_service_metadata.env.python_version.startswith("3.6"):
base_image = "clipper/python36-closure-container:0.4.1"
elif bento_service_metadata.env.python_version.startswith("2.7"):
base_image = "clipper/python-closure-container:0.4.1"
else:
raise BentoMLException(
"Python version {} is not supported in Clipper".format(
bento_service_metadata.env.python_version
)
)
docker_content = CLIPPER_DOCKERFILE.format(
model_name=model_name,
model_version=model_version,
base_image=base_image,
pip_index_url=build_envs.get("PIP_INDEX_URL", ""),
pip_trusted_url=build_envs.get("PIP_TRUSTED_HOST", ""),
)
with open(os.path.join(tempdir, "Dockerfile-clipper"), "w") as f:
f.write(docker_content)
docker_api = docker.APIClient()
clipper_model_docker_image_tag = "clipper-model-{}:{}".format(
bento_service_metadata.name.lower(), bento_service_metadata.version
)
for line in docker_api.build(
path=tempdir,
dockerfile="Dockerfile-clipper",
tag=clipper_model_docker_image_tag,
):
process_docker_api_line(line)
logger.info(
"Successfully built docker image %s for Clipper deployment",
clipper_model_docker_image_tag,
)
clipper_conn.deploy_model(
name=model_name,
version=model_version,
input_type=input_type,
image=clipper_model_docker_image_tag,
labels=labels,
)
track("clipper-deploy-success", {'bento_service_bundle_path': bundle_path})
return model_name, model_version
