def from_archive(cls, path): from bentoml.archive import load_bentoml_config # TODO: add model.env.verify() to check dependencies and python version etc if cls._bento_archive_path is not None and cls._bento_archive_path != path: raise BentoMLException( "Loaded BentoArchive(from {}) can't be loaded again from a different" "archive path {}".format(cls._bento_archive_path, path)) if is_s3_url(path): temporary_path = tempfile.mkdtemp() download_from_s3(path, temporary_path) # Use loacl temp path for the following loading operations path = temporary_path artifacts_path = path # For pip installed BentoService, artifacts directory is located at # 'package_path/artifacts/', but for loading from BentoArchive, it is # in 'path/{service_name}/artifacts/' if not os.path.isdir(os.path.join(path, 'artifacts')): artifacts_path = os.path.join(path, cls.name()) bentoml_config = load_bentoml_config(path) # TODO: check archive type and allow loading archive only if bentoml_config['service_name'] != cls.name(): raise BentoMLException( 'BentoService name does not match with BentoML Archive in path: {}' .format(path)) artifacts = ArtifactCollection.load(artifacts_path, cls._artifacts_spec) svc = cls(artifacts) return svc
def load(bento_service_cls, path=None): # TODO: add model.env.verify() to check dependencies and python version etc if bento_service_cls._bento_module_path is not None: # When calling load from pip installled bento model, use installed # python package for loading and the same path for '/artifacts' # TODO: warn user that 'path' parameter is ignored if it's not None here path = bento_service_cls._bento_module_path artifacts_path = path else: if path is None: raise BentoMLException("Loading path is required for BentoArchive: {}.".format( bento_service_cls.name())) # When calling load on generated archive directory, look for /artifacts # directory under module sub-directory if is_s3_url(path): temporary_path = tempfile.mkdtemp() download_from_s3(path, temporary_path) # Use loacl temp path for the following loading operations path = temporary_path artifacts_path = os.path.join(path, bento_service_cls.name()) bentoml_config = load_bentoml_config(path) bento_service = bento_service_cls.load(artifacts_path) bento_service._version = bentoml_config['service_version'] return bento_service
def __init__(self, archive_path, api_name, region=None, instance_count=None, instance_type=None): if which('docker') is None: raise ValueError( 'docker is not installed, please install docker and then try again' ) super(SagemakerDeployment, self).__init__(archive_path) self.region = DEFAULT_REGION if region is None else region self.instance_count = DEFAULT_INSTANCE_COUNT if instance_count is None else instance_count self.instant_type = DEFAULT_INSTANCE_TYPE if instance_type is None else instance_type apis = self.bento_service.get_service_apis() if api_name: self.api = next(item for item in apis if item.name == api_name) elif len(apis) == 1: self.api = apis[0] else: raise BentoMLException( 'Please specify api-name, when more than one API is present in the archive' ) self.sagemaker_client = boto3.client('sagemaker', region_name=self.region) self.model_name = generate_aws_compatible_string( 'bentoml-' + self.bento_service.name + '-' + self.bento_service.version) self.endpoint_config_name = generate_aws_compatible_string( self.bento_service.name + '-' + self.bento_service.version + '-configuration')
def load_bento_service_class(archive_path): """ Load a BentoService class from saved archive in given path :param archive_path: A BentoArchive path generated from BentoService.save call or the path to pip installed BentoArchive directory :return: BentoService class """ if is_s3_url(archive_path): tempdir = tempfile.mkdtemp() download_from_s3(archive_path, tempdir) archive_path = tempdir config = load_bentoml_config(archive_path) # Load target module containing BentoService class from given path module_file_path = os.path.join(archive_path, config['service_name'], config['module_file']) if not os.path.isfile(module_file_path): # Try loading without service_name prefix, for loading from a installed PyPi module_file_path = os.path.join(archive_path, config['module_file']) if not os.path.isfile(module_file_path): raise BentoMLException( 'Can not locate module_file {} in archive {}'.format( config['module_file'], archive_path)) # Prepend archive_path to sys.path for loading extra python dependencies sys.path.insert(0, archive_path) module_name = config['module_name'] if module_name in sys.modules: # module already loaded, TODO: add warning module = sys.modules[module_name] elif sys.version_info >= (3, 5): import importlib.util spec = importlib.util.spec_from_file_location(module_name, module_file_path) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) elif sys.version_info >= (3, 3): from importlib.machinery import SourceFileLoader # pylint:disable=deprecated-method module = SourceFileLoader(module_name, module_file_path).load_module(module_name) # pylint:enable=deprecated-method else: import imp module = imp.load_source(module_name, module_file_path) # Remove archive_path from sys.path to avoid import naming conflicts sys.path.remove(archive_path) model_service_class = module.__getattribute__(config['service_name']) # Set _bento_archive_path, which tells BentoService where to load its artifacts model_service_class._bento_archive_path = archive_path # Set cls._version, service instance can access it via svc.version model_service_class._bento_service_version = config['service_version'] return model_service_class
def check_status(self): """Check deployment status for the bentoml service. return True, if it is active else return false """ apis = self.bento_service.get_service_apis() config = { "service": self.bento_service.name, "provider": { "region": self.region, "stage": self.stage }, "functions": {} } if self.platform == 'google-python': config['provider']['name'] = 'google' for api in apis: config['functions'][api.name] = { 'handler': api.name, 'events': [{ 'http': 'path' }] } elif self.platform == 'aws-lambda' or self.platform == 'aws-lambda-py2': config['provider']['name'] = 'aws' for api in apis: config['functions'][api.name] = { 'handler': 'handler.' + api.name, 'events': [{ 'http': { "path": '/' + api.name, "method": 'post' } }] } else: raise BentoMLException( 'check serverless does not support platform %s at the moment' % self.platform) yaml = YAML() with TempDirectory() as tempdir: saved_path = os.path.join(tempdir, 'serverless.yml') yaml.dump(config, Path(saved_path)) with subprocess.Popen(['serverless', 'info'], cwd=tempdir, stdout=PIPE, stderr=PIPE) as proc: # We don't use the parse_response function here. # Instead of raising error, we will just return false content = proc.stdout.read().decode('utf-8') response = content.strip().split('\n') logger.debug('Serverless response: %s', '\n'.join(response)) error = [s for s in response if 'Serverless Error' in s] if error: return False, '\n'.join(response) else: return True, '\n'.join(response)
def parse_serverless_response(serverless_response): """Parse serverless response string, raise error if it is a serverless error, otherwise, return information. """ str_list = serverless_response.strip().split('\n') error = [s for s in str_list if 'Serverless Error' in s] if error: error_pos = str_list.index(error[0]) error_message = str_list[error_pos + 1] raise BentoMLException(error_message) return str_list
def check_deployment_status(archive_path, platform, region, stage, api_name): if platform in SERVERLESS_PLATFORMS: deployment = ServerlessDeployment(archive_path, platform, region, stage) elif platform == 'aws-sagemaker': deployment = SagemakerDeployment(archive_path, api_name, region) else: raise BentoMLException('check deployment status with --platform=%s' % platform + 'is not supported in the current version of BentoML') deployment.check_status() return
def deploy(archive_path, platform, region, stage, api_name, instance_type, instance_count): if platform in SERVERLESS_PLATFORMS: deployment = ServerlessDeployment(archive_path, platform, region, stage) elif platform == 'aws-sagemaker': deployment = SagemakerDeployment(archive_path, api_name, region, instance_count, instance_type) else: raise BentoMLException('Deploying with "--platform=%s" is not supported ' % platform + 'in the current version of BentoML') output_path = deployment.deploy() _echo('Deploy to {platform} complete!'.format(platform=platform)) _echo( 'Deployment archive is saved at {output_path}'.format(output_path=output_path)) return
def delete_deployment(archive_path, platform, region, stage, api_name): if platform in SERVERLESS_PLATFORMS: deployment = ServerlessDeployment(archive_path, platform, region, stage) elif platform == 'aws-sagemaker': deployment = SagemakerDeployment(archive_path, api_name, region) else: raise BentoMLException('Remove deployment with --platform=%s' % platform + 'is not supported in the current version of BentoML') result = deployment.delete() if result: _echo( 'Delete {platform} deployment successful'.format(platform=platform)) else: _echo( 'Delete {platform} deployment unsuccessful'.format(platform=platform), CLICK_COLOR_ERROR) return
def deploy(archive_path, platform, region, stage): if platform in SERVERLESS_PLATFORMS: output_path = deploy_with_serverless(platform, archive_path, { "region": region, "stage": stage }) click.echo('BentoML: ', nl=False) click.secho( 'Deploy to {platform} complete!'.format(platform=platform), fg='green') click.secho('Deployment archive is saved at {output_path}'.format( output_path=output_path), fg='green') return else: raise BentoMLException( 'Deploying with "--platform={platform}" is not supported in the current version of BentoML' .format(platform=platform))
def generate_serverless_bundle(bento_service, platform, archive_path, additional_options): check_serverless_compatiable_version() provider = SERVERLESS_PROVIDER[platform] output_path = generate_bentoml_deployment_snapshot_path( bento_service.name, platform) Path(output_path).mkdir(parents=True, exist_ok=False) # Calling serverless command to generate templated project subprocess.call([ 'serverless', 'create', '--template', provider, '--name', bento_service.name ], cwd=output_path) if platform == 'google-python': create_gcp_function_bundle(bento_service, output_path, additional_options) elif platform == 'aws-lambda' or platform == 'aws-lambda-py2': # Installing two additional plugins to make it works for AWS lambda # serverless-python-requirements will packaging required python modules, and automatically # compress and create layer subprocess.call([ 'serverless', 'plugin', 'install', '-n', 'serverless-python-requirements' ], cwd=output_path) subprocess.call([ 'serverless', 'plugin', 'install', '-n', 'serverless-apigw-binary' ], cwd=output_path) create_aws_lambda_bundle(bento_service, output_path, additional_options) else: raise BentoMLException( ("{provider} is not supported in current version of BentoML", provider)) shutil.copy(os.path.join(archive_path, 'requirements.txt'), output_path) model_serivce_archive_path = os.path.join(output_path, bento_service.name) shutil.copytree(archive_path, model_serivce_archive_path) return os.path.realpath(output_path)
def handle_aws_lambda_event(self, event, func): try: import cv2 except ImportError: raise ImportError( "opencv-python package is required to use ImageHandler") if event['headers'].get('Content-Type', None) in ACCEPTED_CONTENT_TYPES: nparr = np.fromstring(base64.b64decode(event['body']), np.uint8) image = cv2.imdecode(nparr, cv2.IMREAD_COLOR) else: raise BentoMLException( "BentoML currently doesn't support Content-Type: {content_type} for AWS Lambda" .format(content_type=event['headers']['Content-Type'])) result = func(image) result = get_output_str(result, event['headers'].get('output', 'json')) return {'statusCode': 200, 'body': result}
def __init__(self, artifacts=None, env=None): if artifacts is None: if self._bento_archive_path: artifacts = ArtifactCollection.load(self._bento_archive_path, self.__class__._artifacts_spec) else: raise BentoMLException("Must provide artifacts or set cls._bento_archive_path" "before instantiating a BentoService class") # TODO: validate artifacts arg matches self.__class__._artifacts_spec definition if isinstance(artifacts, ArtifactCollection): self._artifacts = artifacts else: self._artifacts = ArtifactCollection() for artifact in artifacts: self._artifacts[artifact.name] = artifact self._init_env(env) self._config_service_apis() self.name = self.__class__.name()
def delete(self): """Delete Sagemaker endpoint for the bentoml service. It will also delete the model or the endpoint configuration. return: Boolean, True if the deletion is successful """ if not self.check_status()[0]: raise BentoMLException( 'No active AWS Sagemaker deployment for service %s' % self.bento_service.name) delete_endpoint_response = self.sagemaker_client.delete_endpoint( EndpointName=self.bento_service.name) logger.info('AWS delete endpoint response: %s', delete_endpoint_response) if delete_endpoint_response['ResponseMetadata'][ 'HTTPStatusCode'] == 200: # We will also try to delete both model and endpoint configuration for user. # Since they are not critical, even they failed, we will still count delete deployment # a success delete_model_response = self.sagemaker_client.delete_model( ModelName=self.model_name) logger.info('AWS delete model response: %s', delete_model_response) if delete_model_response['ResponseMetadata'][ 'HTTPStatusCode'] != 200: logger.error('Encounter error when deleting model: %s', delete_model_response) delete_endpoint_config_response = self.sagemaker_client.delete_endpoint_config( EndpointConfigName=self.endpoint_config_name) logger.info('AWS delete endpoint config response: %s', delete_endpoint_config_response) if delete_endpoint_config_response['ResponseMetadata'][ 'HTTPStatusCode'] != 200: logger.error( 'Encounter error when deleting endpoint configuration: %s', delete_endpoint_config_response) return True else: return False
def delete(self): is_active, _ = self.check_status() if not is_active: raise BentoMLException('No active deployment for service %s' % self.bento_service.name) if self.platform == 'google-python': provider_name = 'google' elif self.platform == 'aws-lambda' or self.platform == 'aws-lambda-py2': provider_name = 'aws' config = { "service": self.bento_service.name, "provider": { "name": provider_name, "region": self.region, "stage": self.stage } } yaml = YAML() with TempDirectory() as tempdir: saved_path = os.path.join(tempdir, 'serverless.yml') yaml.dump(config, Path(saved_path)) with subprocess.Popen(['serverless', 'remove'], cwd=tempdir, stdout=PIPE, stderr=PIPE) as proc: response = parse_serverless_response( proc.stdout.read().decode('utf-8')) logger.debug('Serverless response: %s', '\n'.join(response)) if self.platform == 'google-python': # TODO: Add check for Google's response return True elif self.platform == 'aws-lambda' or self.platform == 'aws-lambda-py2': if 'Serverless: Stack removal finished...' in response: return True else: return False
def copy_used_py_modules(target_module, destination): """ bundle given module, and all its dependencies within top level package, and copy all source files to destination path, essentially creating a source distribution of target_module """ # When target_module is a string, try import it if isinstance(target_module, string_types): try: target_module = importlib.import_module(target_module) except ImportError: pass target_module = inspect.getmodule(target_module) # When target module is defined in interactive session, we can not easily # get the class definition into a python module file and distribute it if target_module.__name__ == '__main__' and not hasattr( target_module, '__file__'): raise BentoMLException( "Custom BentoModel class can not be defined in Python interactive REPL, try " "writing the class definition to a file and import it.") try: target_module_name = target_module.__spec__.name except AttributeError: target_module_name = target_module.__name__ target_module_file = _get_module_src_file(target_module) if target_module_name == '__main__': # Assuming no relative import in this case target_module_file_name = os.path.split(target_module_file)[1] target_module_name = target_module_file_name[:-3] # remove '.py' # Find all modules must be imported for target module to run finder = ModuleFinder() # NOTE: This method could take a few seconds to run try: finder.run_script(target_module_file) except SyntaxError: # For package with conditional import that may only work with py2 # or py3, ModuleFinder#run_script will try to compile the source # with current python version. And that may result in SyntaxError. pass # extra site-packages or dist-packages directory site_or_dist_package_path = [ f for f in sys.path if f.endswith('-packages') ] # prefix used to find installed Python library site_or_dist_package_path += [sys.prefix] # prefix used to find machine-specific Python library try: site_or_dist_package_path += [sys.base_prefix] except AttributeError: # ignore when in PY2 there is no sys.base_prefix pass # Look for dependencies that are not distributed python package, but users' # local python code, all other dependencies must be defined with @env # decorator when creating a new BentoService class user_packages_and_modules = {} for name, module in iteritems(finder.modules): if name == 'bentoml' or name.startswith('bentoml.'): # Remove BentoML library from dependent modules list break if hasattr(module, '__file__') and module.__file__ is not None: module_src_file = _get_module_src_file(module) is_module_in_site_or_dist_package = False for path in site_or_dist_package_path: if module_src_file.startswith(path): is_module_in_site_or_dist_package = True break if not is_module_in_site_or_dist_package: user_packages_and_modules[name] = module # Remove "__main__" module, if target module is loaded as __main__, it should # be in module_files as (module_name, module_file) in current context if '__main__' in user_packages_and_modules: del user_packages_and_modules['__main__'] # Lastly, add target module itself user_packages_and_modules[target_module_name] = target_module for module_name, module in iteritems(user_packages_and_modules): module_file = _get_module_src_file(module) relative_path = _get_module_relative_file_path(module_name, module_file) target_file = os.path.join(destination, relative_path) # Create target directory if not exist Path(os.path.dirname(target_file)).mkdir(parents=True, exist_ok=True) # Copy module file to BentoArchive for distribution copyfile(module_file, target_file) for root, _, files in os.walk(destination): if '__init__.py' not in files: Path(os.path.join(root, '__init__.py')).touch() target_module_relative_path = _get_module_relative_file_path( target_module_name, target_module_file) return target_module_name, target_module_relative_path
def copy_used_py_modules(target_module, destination): """ bundle given module, and all its dependencies within top level package, and copy all source files to destination path, essentially creating a source distribution of target_module """ if isinstance(target_module, string_types): target_module = importlib.import_module(target_module) else: target_module = inspect.getmodule(target_module) if target_module.__name__ == '__main__' and not target_module.__file__: raise BentoMLException( "Custom BentoModel class can not be defined in Python interactive REPL, try " "writing the class definition to a file and import, e.g. my_bentoml_model.py" ) try: target_module_name = target_module.__spec__.name except AttributeError: target_module_name = target_module.__name__ # TODO: handle when __main__ script filename starts with numbers target_module_file = _get_module_src_file(target_module) # TODO: Remove this two lines? if target_module_name == '__main__': target_module_name = target_module_file[:-3].replace(os.sep, '.') # Find all modules must be imported for target module to run finder = ModuleFinder() # NOTE: This method could take a few seconds to run try: finder.run_script(target_module_file) except SyntaxError: # For package with conditional import that may only work with py2 # or py3, ModuleFinder#run_script will try to compile the source # with current python version. And that may result in SyntaxError. pass # Remove dependencies not in current project source code # all third party dependencies must be defined in BentoEnv when creating model user_packages_and_modules = {} site_or_dist_package_path = [ f for f in sys.path if f.endswith('-packages') ] + [sys.prefix] for name, module in iteritems(finder.modules): if module.__file__ is not None: module_src_file = _get_module_src_file(module) is_module_in_site_or_dist_package = False for path in site_or_dist_package_path: if module_src_file.startswith(path): is_module_in_site_or_dist_package = True break if not is_module_in_site_or_dist_package: user_packages_and_modules[name] = module # Remove "__main__" module, if target module is loaded as __main__, it should # be in module_files as (module_name, module_file) in current context if '__main__' in user_packages_and_modules: del user_packages_and_modules['__main__'] # Lastly, add target module itself user_packages_and_modules[target_module_name] = target_module for module_name, module in iteritems(user_packages_and_modules): module_file = _get_module_src_file(module) with open(module_file, "rb") as f: src_code = f.read() if not os.path.isabs(module_file): # For modules within current top level package, module_file here should be a # relative path to the src file target_file = os.path.join(destination, module_file) elif os.path.split(module_file)[1] == '__init__.py': # for module a.b.c in 'some_path/a/b/c/__init__.py', copy file to # 'destination/a/b/c/__init__.py' target_file = os.path.join(destination, module_name.replace('.', os.sep), '__init__.py') else: # for module a.b.c in 'some_path/a/b/c.py', copy file to 'destination/a/b/c.py' target_file = os.path.join( destination, module_name.replace('.', os.sep) + '.py') target_path = os.path.dirname(target_file) Path(target_path).mkdir(parents=True, exist_ok=True) with open(target_file, 'wb') as f: f.write(src_code) for root, _, files in os.walk(destination): if '__init__.py' not in files: Path(os.path.join(root, '__init__.py')).touch() return target_module_name, target_module_file
def version(self): try: return self.__class__._bento_service_version except AttributeError: raise BentoMLException( "Only BentoService loaded from archive has version attribute")
def _generate_bundle(self): output_path = generate_bentoml_deployment_snapshot_path( self.bento_service.name, self.bento_service.version, self.platform) Path(output_path).mkdir(parents=True, exist_ok=False) # Calling serverless command to generate templated project with subprocess.Popen([ 'serverless', 'create', '--template', self.provider, '--name', self.bento_service.name ], cwd=output_path, stdout=PIPE, stderr=PIPE) as proc: response = parse_serverless_response( proc.stdout.read().decode('utf-8')) logger.debug('Serverless response: %s', '\n'.join(response)) if self.platform == 'google-python': create_gcp_function_bundle(self.bento_service, output_path, self.region, self.stage) elif self.platform == 'aws-lambda' or self.platform == 'aws-lambda-py2': # Installing two additional plugins to make it works for AWS lambda # serverless-python-requirements will packaging required python modules, # and automatically compress and create layer with subprocess.Popen([ 'serverless', 'plugin', 'install', '-n', 'serverless-python-requirements' ], cwd=output_path, stdout=PIPE, stderr=PIPE) as proc: response = parse_serverless_response( proc.stdout.read().decode('utf-8')) logger.debug('Serverless response: %s', '\n'.join(response)) with subprocess.Popen([ 'serverless', 'plugin', 'install', '-n', 'serverless-apigw-binary' ], cwd=output_path, stdout=PIPE, stderr=PIPE) as proc: response = parse_serverless_response( proc.stdout.read().decode('utf-8')) logger.debug('Serverless response: %s', '\n'.join(response)) create_aws_lambda_bundle(self.bento_service, output_path, self.region, self.stage) else: raise BentoMLException( "%s is not supported in current version of BentoML" % self.provider) shutil.copy(os.path.join(self.archive_path, 'requirements.txt'), output_path) model_serivce_archive_path = os.path.join(output_path, self.bento_service.name) shutil.copytree(self.archive_path, model_serivce_archive_path) return os.path.realpath(output_path)