def from_archive(cls, path):
from bentoml.archive import load_bentoml_config
# TODO: add model.env.verify() to check dependencies and python version etc
if cls._bento_archive_path is not None and cls._bento_archive_path != path:
raise BentoMLException(
"Loaded BentoArchive(from {}) can't be loaded again from a different"
"archive path {}".format(cls._bento_archive_path, path))
if is_s3_url(path):
temporary_path = tempfile.mkdtemp()
download_from_s3(path, temporary_path)
# Use loacl temp path for the following loading operations
path = temporary_path
artifacts_path = path
# For pip installed BentoService, artifacts directory is located at
# 'package_path/artifacts/', but for loading from BentoArchive, it is
# in 'path/{service_name}/artifacts/'
if not os.path.isdir(os.path.join(path, 'artifacts')):
artifacts_path = os.path.join(path, cls.name())
bentoml_config = load_bentoml_config(path)
# TODO: check archive type and allow loading archive only
if bentoml_config['service_name'] != cls.name():
raise BentoMLException(
'BentoService name does not match with BentoML Archive in path: {}'
.format(path))
artifacts = ArtifactCollection.load(artifacts_path,
cls._artifacts_spec)
svc = cls(artifacts)
return svc
def load(bento_service_cls, path=None):
# TODO: add model.env.verify() to check dependencies and python version etc
if bento_service_cls._bento_module_path is not None:
# When calling load from pip installled bento model, use installed
# python package for loading and the same path for '/artifacts'
# TODO: warn user that 'path' parameter is ignored if it's not None here
path = bento_service_cls._bento_module_path
artifacts_path = path
else:
if path is None:
raise BentoMLException("Loading path is required for BentoArchive: {}.".format(
bento_service_cls.name()))
# When calling load on generated archive directory, look for /artifacts
# directory under module sub-directory
if is_s3_url(path):
temporary_path = tempfile.mkdtemp()
download_from_s3(path, temporary_path)
# Use loacl temp path for the following loading operations
path = temporary_path
artifacts_path = os.path.join(path, bento_service_cls.name())
bentoml_config = load_bentoml_config(path)
bento_service = bento_service_cls.load(artifacts_path)
bento_service._version = bentoml_config['service_version']
return bento_service
def __init__(self,
archive_path,
api_name,
region=None,
instance_count=None,
instance_type=None):
if which('docker') is None:
raise ValueError(
'docker is not installed, please install docker and then try again'
)
super(SagemakerDeployment, self).__init__(archive_path)
self.region = DEFAULT_REGION if region is None else region
self.instance_count = DEFAULT_INSTANCE_COUNT if instance_count is None else instance_count
self.instant_type = DEFAULT_INSTANCE_TYPE if instance_type is None else instance_type
apis = self.bento_service.get_service_apis()
if api_name:
self.api = next(item for item in apis if item.name == api_name)
elif len(apis) == 1:
self.api = apis[0]
else:
raise BentoMLException(
'Please specify api-name, when more than one API is present in the archive'
)
self.sagemaker_client = boto3.client('sagemaker',
region_name=self.region)
self.model_name = generate_aws_compatible_string(
'bentoml-' + self.bento_service.name + '-' +
self.bento_service.version)
self.endpoint_config_name = generate_aws_compatible_string(
self.bento_service.name + '-' + self.bento_service.version +
'-configuration')
def load_bento_service_class(archive_path):
"""
Load a BentoService class from saved archive in given path
:param archive_path: A BentoArchive path generated from BentoService.save call
or the path to pip installed BentoArchive directory
:return: BentoService class
"""
if is_s3_url(archive_path):
tempdir = tempfile.mkdtemp()
download_from_s3(archive_path, tempdir)
archive_path = tempdir
config = load_bentoml_config(archive_path)
# Load target module containing BentoService class from given path
module_file_path = os.path.join(archive_path, config['service_name'],
config['module_file'])
if not os.path.isfile(module_file_path):
# Try loading without service_name prefix, for loading from a installed PyPi
module_file_path = os.path.join(archive_path, config['module_file'])
if not os.path.isfile(module_file_path):
raise BentoMLException(
'Can not locate module_file {} in archive {}'.format(
config['module_file'], archive_path))
# Prepend archive_path to sys.path for loading extra python dependencies
sys.path.insert(0, archive_path)
module_name = config['module_name']
if module_name in sys.modules:
# module already loaded, TODO: add warning
module = sys.modules[module_name]
elif sys.version_info >= (3, 5):
import importlib.util
spec = importlib.util.spec_from_file_location(module_name,
module_file_path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
elif sys.version_info >= (3, 3):
from importlib.machinery import SourceFileLoader
# pylint:disable=deprecated-method
module = SourceFileLoader(module_name,
module_file_path).load_module(module_name)
# pylint:enable=deprecated-method
else:
import imp
module = imp.load_source(module_name, module_file_path)
# Remove archive_path from sys.path to avoid import naming conflicts
sys.path.remove(archive_path)
model_service_class = module.__getattribute__(config['service_name'])
# Set _bento_archive_path, which tells BentoService where to load its artifacts
model_service_class._bento_archive_path = archive_path
# Set cls._version, service instance can access it via svc.version
model_service_class._bento_service_version = config['service_version']
return model_service_class
def check_status(self):
"""Check deployment status for the bentoml service.
return True, if it is active else return false
"""
apis = self.bento_service.get_service_apis()
config = {
"service": self.bento_service.name,
"provider": {
"region": self.region,
"stage": self.stage
},
"functions": {}
}
if self.platform == 'google-python':
config['provider']['name'] = 'google'
for api in apis:
config['functions'][api.name] = {
'handler': api.name,
'events': [{
'http': 'path'
}]
}
elif self.platform == 'aws-lambda' or self.platform == 'aws-lambda-py2':
config['provider']['name'] = 'aws'
for api in apis:
config['functions'][api.name] = {
'handler':
'handler.' + api.name,
'events': [{
'http': {
"path": '/' + api.name,
"method": 'post'
}
}]
}
else:
raise BentoMLException(
'check serverless does not support platform %s at the moment' %
self.platform)
yaml = YAML()
with TempDirectory() as tempdir:
saved_path = os.path.join(tempdir, 'serverless.yml')
yaml.dump(config, Path(saved_path))
with subprocess.Popen(['serverless', 'info'],
cwd=tempdir,
stdout=PIPE,
stderr=PIPE) as proc:
# We don't use the parse_response function here.
# Instead of raising error, we will just return false
content = proc.stdout.read().decode('utf-8')
response = content.strip().split('\n')
logger.debug('Serverless response: %s', '\n'.join(response))
error = [s for s in response if 'Serverless Error' in s]
if error:
return False, '\n'.join(response)
else:
return True, '\n'.join(response)
def parse_serverless_response(serverless_response):
"""Parse serverless response string, raise error if it is a serverless error,
otherwise, return information.
"""
str_list = serverless_response.strip().split('\n')
error = [s for s in str_list if 'Serverless Error' in s]
if error:
error_pos = str_list.index(error[0])
error_message = str_list[error_pos + 1]
raise BentoMLException(error_message)
return str_list
def check_deployment_status(archive_path, platform, region, stage, api_name):
if platform in SERVERLESS_PLATFORMS:
deployment = ServerlessDeployment(archive_path, platform, region, stage)
elif platform == 'aws-sagemaker':
deployment = SagemakerDeployment(archive_path, api_name, region)
else:
raise BentoMLException('check deployment status with --platform=%s' % platform +
'is not supported in the current version of BentoML')
deployment.check_status()
return
def deploy(archive_path, platform, region, stage, api_name, instance_type, instance_count):
if platform in SERVERLESS_PLATFORMS:
deployment = ServerlessDeployment(archive_path, platform, region, stage)
elif platform == 'aws-sagemaker':
deployment = SagemakerDeployment(archive_path, api_name, region, instance_count,
instance_type)
else:
raise BentoMLException('Deploying with "--platform=%s" is not supported ' % platform +
'in the current version of BentoML')
output_path = deployment.deploy()
_echo('Deploy to {platform} complete!'.format(platform=platform))
_echo(
'Deployment archive is saved at {output_path}'.format(output_path=output_path))
return
def delete_deployment(archive_path, platform, region, stage, api_name):
if platform in SERVERLESS_PLATFORMS:
deployment = ServerlessDeployment(archive_path, platform, region, stage)
elif platform == 'aws-sagemaker':
deployment = SagemakerDeployment(archive_path, api_name, region)
else:
raise BentoMLException('Remove deployment with --platform=%s' % platform +
'is not supported in the current version of BentoML')
result = deployment.delete()
if result:
_echo(
'Delete {platform} deployment successful'.format(platform=platform))
else:
_echo(
'Delete {platform} deployment unsuccessful'.format(platform=platform),
CLICK_COLOR_ERROR)
return
def deploy(archive_path, platform, region, stage):
if platform in SERVERLESS_PLATFORMS:
output_path = deploy_with_serverless(platform, archive_path, {
"region": region,
"stage": stage
})
click.echo('BentoML: ', nl=False)
click.secho(
'Deploy to {platform} complete!'.format(platform=platform),
fg='green')
click.secho('Deployment archive is saved at {output_path}'.format(
output_path=output_path),
fg='green')
return
else:
raise BentoMLException(
'Deploying with "--platform={platform}" is not supported in the current version of BentoML'
.format(platform=platform))
def generate_serverless_bundle(bento_service, platform, archive_path,
additional_options):
check_serverless_compatiable_version()
provider = SERVERLESS_PROVIDER[platform]
output_path = generate_bentoml_deployment_snapshot_path(
bento_service.name, platform)
Path(output_path).mkdir(parents=True, exist_ok=False)
# Calling serverless command to generate templated project
subprocess.call([
'serverless', 'create', '--template', provider, '--name',
bento_service.name
],
cwd=output_path)
if platform == 'google-python':
create_gcp_function_bundle(bento_service, output_path,
additional_options)
elif platform == 'aws-lambda' or platform == 'aws-lambda-py2':
# Installing two additional plugins to make it works for AWS lambda
# serverless-python-requirements will packaging required python modules, and automatically
# compress and create layer
subprocess.call([
'serverless', 'plugin', 'install', '-n',
'serverless-python-requirements'
],
cwd=output_path)
subprocess.call([
'serverless', 'plugin', 'install', '-n', 'serverless-apigw-binary'
],
cwd=output_path)
create_aws_lambda_bundle(bento_service, output_path,
additional_options)
else:
raise BentoMLException(
("{provider} is not supported in current version of BentoML",
provider))
shutil.copy(os.path.join(archive_path, 'requirements.txt'), output_path)
model_serivce_archive_path = os.path.join(output_path, bento_service.name)
shutil.copytree(archive_path, model_serivce_archive_path)
return os.path.realpath(output_path)
def handle_aws_lambda_event(self, event, func):
try:
import cv2
except ImportError:
raise ImportError(
"opencv-python package is required to use ImageHandler")
if event['headers'].get('Content-Type',
None) in ACCEPTED_CONTENT_TYPES:
nparr = np.fromstring(base64.b64decode(event['body']), np.uint8)
image = cv2.imdecode(nparr, cv2.IMREAD_COLOR)
else:
raise BentoMLException(
"BentoML currently doesn't support Content-Type: {content_type} for AWS Lambda"
.format(content_type=event['headers']['Content-Type']))
result = func(image)
result = get_output_str(result, event['headers'].get('output', 'json'))
return {'statusCode': 200, 'body': result}
def __init__(self, artifacts=None, env=None):
if artifacts is None:
if self._bento_archive_path:
artifacts = ArtifactCollection.load(self._bento_archive_path,
self.__class__._artifacts_spec)
else:
raise BentoMLException("Must provide artifacts or set cls._bento_archive_path"
"before instantiating a BentoService class")
# TODO: validate artifacts arg matches self.__class__._artifacts_spec definition
if isinstance(artifacts, ArtifactCollection):
self._artifacts = artifacts
else:
self._artifacts = ArtifactCollection()
for artifact in artifacts:
self._artifacts[artifact.name] = artifact
self._init_env(env)
self._config_service_apis()
self.name = self.__class__.name()
def delete(self):
"""Delete Sagemaker endpoint for the bentoml service.
It will also delete the model or the endpoint configuration.
return: Boolean, True if the deletion is successful
"""
if not self.check_status()[0]:
raise BentoMLException(
'No active AWS Sagemaker deployment for service %s' %
self.bento_service.name)
delete_endpoint_response = self.sagemaker_client.delete_endpoint(
EndpointName=self.bento_service.name)
logger.info('AWS delete endpoint response: %s',
delete_endpoint_response)
if delete_endpoint_response['ResponseMetadata'][
'HTTPStatusCode'] == 200:
# We will also try to delete both model and endpoint configuration for user.
# Since they are not critical, even they failed, we will still count delete deployment
# a success
delete_model_response = self.sagemaker_client.delete_model(
ModelName=self.model_name)
logger.info('AWS delete model response: %s', delete_model_response)
if delete_model_response['ResponseMetadata'][
'HTTPStatusCode'] != 200:
logger.error('Encounter error when deleting model: %s',
delete_model_response)
delete_endpoint_config_response = self.sagemaker_client.delete_endpoint_config(
EndpointConfigName=self.endpoint_config_name)
logger.info('AWS delete endpoint config response: %s',
delete_endpoint_config_response)
if delete_endpoint_config_response['ResponseMetadata'][
'HTTPStatusCode'] != 200:
logger.error(
'Encounter error when deleting endpoint configuration: %s',
delete_endpoint_config_response)
return True
else:
return False
def delete(self):
is_active, _ = self.check_status()
if not is_active:
raise BentoMLException('No active deployment for service %s' %
self.bento_service.name)
if self.platform == 'google-python':
provider_name = 'google'
elif self.platform == 'aws-lambda' or self.platform == 'aws-lambda-py2':
provider_name = 'aws'
config = {
"service": self.bento_service.name,
"provider": {
"name": provider_name,
"region": self.region,
"stage": self.stage
}
}
yaml = YAML()
with TempDirectory() as tempdir:
saved_path = os.path.join(tempdir, 'serverless.yml')
yaml.dump(config, Path(saved_path))
with subprocess.Popen(['serverless', 'remove'],
cwd=tempdir,
stdout=PIPE,
stderr=PIPE) as proc:
response = parse_serverless_response(
proc.stdout.read().decode('utf-8'))
logger.debug('Serverless response: %s', '\n'.join(response))
if self.platform == 'google-python':
# TODO: Add check for Google's response
return True
elif self.platform == 'aws-lambda' or self.platform == 'aws-lambda-py2':
if 'Serverless: Stack removal finished...' in response:
return True
else:
return False
def copy_used_py_modules(target_module, destination):
"""
bundle given module, and all its dependencies within top level package,
and copy all source files to destination path, essentially creating
a source distribution of target_module
"""
# When target_module is a string, try import it
if isinstance(target_module, string_types):
try:
target_module = importlib.import_module(target_module)
except ImportError:
pass
target_module = inspect.getmodule(target_module)
# When target module is defined in interactive session, we can not easily
# get the class definition into a python module file and distribute it
if target_module.__name__ == '__main__' and not hasattr(
target_module, '__file__'):
raise BentoMLException(
"Custom BentoModel class can not be defined in Python interactive REPL, try "
"writing the class definition to a file and import it.")
try:
target_module_name = target_module.__spec__.name
except AttributeError:
target_module_name = target_module.__name__
target_module_file = _get_module_src_file(target_module)
if target_module_name == '__main__':
# Assuming no relative import in this case
target_module_file_name = os.path.split(target_module_file)[1]
target_module_name = target_module_file_name[:-3] # remove '.py'
# Find all modules must be imported for target module to run
finder = ModuleFinder()
# NOTE: This method could take a few seconds to run
try:
finder.run_script(target_module_file)
except SyntaxError:
# For package with conditional import that may only work with py2
# or py3, ModuleFinder#run_script will try to compile the source
# with current python version. And that may result in SyntaxError.
pass
# extra site-packages or dist-packages directory
site_or_dist_package_path = [
f for f in sys.path if f.endswith('-packages')
]
# prefix used to find installed Python library
site_or_dist_package_path += [sys.prefix]
# prefix used to find machine-specific Python library
try:
site_or_dist_package_path += [sys.base_prefix]
except AttributeError:
# ignore when in PY2 there is no sys.base_prefix
pass
# Look for dependencies that are not distributed python package, but users'
# local python code, all other dependencies must be defined with @env
# decorator when creating a new BentoService class
user_packages_and_modules = {}
for name, module in iteritems(finder.modules):
if name == 'bentoml' or name.startswith('bentoml.'):
# Remove BentoML library from dependent modules list
break
if hasattr(module, '__file__') and module.__file__ is not None:
module_src_file = _get_module_src_file(module)
is_module_in_site_or_dist_package = False
for path in site_or_dist_package_path:
if module_src_file.startswith(path):
is_module_in_site_or_dist_package = True
break
if not is_module_in_site_or_dist_package:
user_packages_and_modules[name] = module
# Remove "__main__" module, if target module is loaded as __main__, it should
# be in module_files as (module_name, module_file) in current context
if '__main__' in user_packages_and_modules:
del user_packages_and_modules['__main__']
# Lastly, add target module itself
user_packages_and_modules[target_module_name] = target_module
for module_name, module in iteritems(user_packages_and_modules):
module_file = _get_module_src_file(module)
relative_path = _get_module_relative_file_path(module_name,
module_file)
target_file = os.path.join(destination, relative_path)
# Create target directory if not exist
Path(os.path.dirname(target_file)).mkdir(parents=True, exist_ok=True)
# Copy module file to BentoArchive for distribution
copyfile(module_file, target_file)
for root, _, files in os.walk(destination):
if '__init__.py' not in files:
Path(os.path.join(root, '__init__.py')).touch()
target_module_relative_path = _get_module_relative_file_path(
target_module_name, target_module_file)
return target_module_name, target_module_relative_path
def copy_used_py_modules(target_module, destination):
"""
bundle given module, and all its dependencies within top level package,
and copy all source files to destination path, essentially creating
a source distribution of target_module
"""
if isinstance(target_module, string_types):
target_module = importlib.import_module(target_module)
else:
target_module = inspect.getmodule(target_module)
if target_module.__name__ == '__main__' and not target_module.__file__:
raise BentoMLException(
"Custom BentoModel class can not be defined in Python interactive REPL, try "
"writing the class definition to a file and import, e.g. my_bentoml_model.py"
)
try:
target_module_name = target_module.__spec__.name
except AttributeError:
target_module_name = target_module.__name__
# TODO: handle when __main__ script filename starts with numbers
target_module_file = _get_module_src_file(target_module)
# TODO: Remove this two lines?
if target_module_name == '__main__':
target_module_name = target_module_file[:-3].replace(os.sep, '.')
# Find all modules must be imported for target module to run
finder = ModuleFinder()
# NOTE: This method could take a few seconds to run
try:
finder.run_script(target_module_file)
except SyntaxError:
# For package with conditional import that may only work with py2
# or py3, ModuleFinder#run_script will try to compile the source
# with current python version. And that may result in SyntaxError.
pass
# Remove dependencies not in current project source code
# all third party dependencies must be defined in BentoEnv when creating model
user_packages_and_modules = {}
site_or_dist_package_path = [
f for f in sys.path if f.endswith('-packages')
] + [sys.prefix]
for name, module in iteritems(finder.modules):
if module.__file__ is not None:
module_src_file = _get_module_src_file(module)
is_module_in_site_or_dist_package = False
for path in site_or_dist_package_path:
if module_src_file.startswith(path):
is_module_in_site_or_dist_package = True
break
if not is_module_in_site_or_dist_package:
user_packages_and_modules[name] = module
# Remove "__main__" module, if target module is loaded as __main__, it should
# be in module_files as (module_name, module_file) in current context
if '__main__' in user_packages_and_modules:
del user_packages_and_modules['__main__']
# Lastly, add target module itself
user_packages_and_modules[target_module_name] = target_module
for module_name, module in iteritems(user_packages_and_modules):
module_file = _get_module_src_file(module)
with open(module_file, "rb") as f:
src_code = f.read()
if not os.path.isabs(module_file):
# For modules within current top level package, module_file here should be a
# relative path to the src file
target_file = os.path.join(destination, module_file)
elif os.path.split(module_file)[1] == '__init__.py':
# for module a.b.c in 'some_path/a/b/c/__init__.py', copy file to
# 'destination/a/b/c/__init__.py'
target_file = os.path.join(destination,
module_name.replace('.', os.sep),
'__init__.py')
else:
# for module a.b.c in 'some_path/a/b/c.py', copy file to 'destination/a/b/c.py'
target_file = os.path.join(
destination,
module_name.replace('.', os.sep) + '.py')
target_path = os.path.dirname(target_file)
Path(target_path).mkdir(parents=True, exist_ok=True)
with open(target_file, 'wb') as f:
f.write(src_code)
for root, _, files in os.walk(destination):
if '__init__.py' not in files:
Path(os.path.join(root, '__init__.py')).touch()
return target_module_name, target_module_file
def version(self):
try:
return self.__class__._bento_service_version
except AttributeError:
raise BentoMLException(
"Only BentoService loaded from archive has version attribute")
def _generate_bundle(self):
output_path = generate_bentoml_deployment_snapshot_path(
self.bento_service.name, self.bento_service.version, self.platform)
Path(output_path).mkdir(parents=True, exist_ok=False)
# Calling serverless command to generate templated project
with subprocess.Popen([
'serverless', 'create', '--template', self.provider, '--name',
self.bento_service.name
],
cwd=output_path,
stdout=PIPE,
stderr=PIPE) as proc:
response = parse_serverless_response(
proc.stdout.read().decode('utf-8'))
logger.debug('Serverless response: %s', '\n'.join(response))
if self.platform == 'google-python':
create_gcp_function_bundle(self.bento_service, output_path,
self.region, self.stage)
elif self.platform == 'aws-lambda' or self.platform == 'aws-lambda-py2':
# Installing two additional plugins to make it works for AWS lambda
# serverless-python-requirements will packaging required python modules,
# and automatically compress and create layer
with subprocess.Popen([
'serverless', 'plugin', 'install', '-n',
'serverless-python-requirements'
],
cwd=output_path,
stdout=PIPE,
stderr=PIPE) as proc:
response = parse_serverless_response(
proc.stdout.read().decode('utf-8'))
logger.debug('Serverless response: %s', '\n'.join(response))
with subprocess.Popen([
'serverless', 'plugin', 'install', '-n',
'serverless-apigw-binary'
],
cwd=output_path,
stdout=PIPE,
stderr=PIPE) as proc:
response = parse_serverless_response(
proc.stdout.read().decode('utf-8'))
logger.debug('Serverless response: %s', '\n'.join(response))
create_aws_lambda_bundle(self.bento_service, output_path,
self.region, self.stage)
else:
raise BentoMLException(
"%s is not supported in current version of BentoML" %
self.provider)
shutil.copy(os.path.join(self.archive_path, 'requirements.txt'),
output_path)
model_serivce_archive_path = os.path.join(output_path,
self.bento_service.name)
shutil.copytree(self.archive_path, model_serivce_archive_path)
return os.path.realpath(output_path)