def __init__( self, bento_bundle_path, outbound_host="localhost", outbound_port=None, outbound_workers=1, mb_max_batch_size: int = None, mb_max_latency: int = None, ): self.outbound_host = outbound_host self.outbound_port = outbound_port self.outbound_workers = outbound_workers self.mb_max_batch_size = mb_max_batch_size self.mb_max_latency = mb_max_latency self.batch_handlers = dict() self._outbound_sema = None # the semaphore to limit outbound connections self.bento_service_metadata_pb = load_bento_service_metadata( bento_bundle_path) self.setup_routes_from_pb(self.bento_service_metadata_pb) if psutil.POSIX: import resource self.CONNECTION_LIMIT = resource.getrlimit( resource.RLIMIT_NOFILE)[0] else: self.CONNECTION_LIMIT = 1024 logger.info( "Your system nofile limit is %d, which means each instance of microbatch " "service is able to hold this number of connections at same time. " "You can increase the number of file descriptors for the server process, " "or launch more microbatch instances to accept more concurrent connection.", self.CONNECTION_LIMIT, )
def test_auto_artifact_dependencies(): clf = svm.SVC(gamma='scale') iris = datasets.load_iris() X, y = iris.data, iris.target clf.fit(X, y) # Create a iris classifier service iris_classifier_service = IrisClassifier() # Pack it with the newly trained model artifact iris_classifier_service.pack('model', clf) # Save the prediction service to a BentoService bundle saved_path = iris_classifier_service.save() with open(os.path.join(saved_path, 'requirements.txt')) as f: requirements_txt_content = f.read() dependencies = requirements_txt_content.split('\n') dependencies = [dep.split('==')[0] for dep in dependencies] assert 'scikit-learn' in dependencies assert 'bentoml' in dependencies # Test that dependencies also wrote to BentoServiceMetadat config file bs_matadata = load_bento_service_metadata(saved_path) dependencies = bs_matadata.env.pip_dependencies.split('\n') dependencies = [dep.split('==')[0] for dep in dependencies] assert 'scikit-learn' in dependencies assert 'bentoml' in dependencies delete_saved_bento_service(iris_classifier_service.name, iris_classifier_service.version)
def _get_bento_service_event_properties_from_bundle_path( bundle_path, properties=None): from bentoml.saved_bundle import load_bento_service_metadata bento_service_metadata = load_bento_service_metadata(bundle_path) return _bento_service_metadata_to_event_properties(bento_service_metadata, properties)
def test_auto_artifact_dependencies(): clf = _fit_clf() # Create a iris classifier service iris_classifier_service = IrisClassifier() # Pack it with the newly trained model artifact iris_classifier_service.pack('model', clf) # Save the prediction service to a BentoService bundle saved_path = iris_classifier_service.save() # parse generated requirements.txt dependencies = _dependencies_to_requirements( _parse_dependencies(saved_path)) _assert_in_dependencies(['scikit-learn', 'bentoml'], dependencies) # Test that dependencies also wrote to BentoServiceMetadata config file bs_metadata = load_bento_service_metadata(saved_path) dependencies = bs_metadata.env.pip_packages dependencies = _dependencies_to_requirements(dependencies) _assert_in_dependencies(['scikit-learn', 'bentoml'], dependencies) # Clean up delete_saved_bento_service(iris_classifier_service.name, iris_classifier_service.version)
def info(bento=None): """ List all APIs defined in the BentoService loaded from saved bundle """ saved_bundle_path = resolve_bundle_path(bento, pip_installed_bundle_path) bento_service_metadata_pb = load_bento_service_metadata(saved_bundle_path) output = json.dumps(ProtoMessageToDict(bento_service_metadata_pb), indent=2) _echo(output)
def containerize(bento, push, tag, build_arg, yatai_url): """Containerize specified BentoService. BENTO is the target BentoService to be containerized, referenced by its name and version in format of name:version. For example: "iris_classifier:v1.2.0" `bentoml containerize` command also supports the use of the `latest` tag which will automatically use the last built version of your Bento. You can provide a tag for the image built by Bento using the `--tag` flag. Additionally, you can provide a `--push` flag, which will push the built image to the Docker repository specified by the image tag. You can also prefixing the tag with a hostname for the repository you wish to push to. e.g. `bentoml containerize IrisClassifier:latest --push --tag repo-address.com:username/iris` would build a Docker image called `username/iris:latest` and push that to docker repository at repo-address.com. By default, the `containerize` command will use the current credentials provided by Docker daemon. """ saved_bundle_path = resolve_bundle_path(bento, pip_installed_bundle_path, yatai_url) _echo(f"Found Bento: {saved_bundle_path}") # fmt: off bento_metadata: "BentoServiceMetadata" = load_bento_service_metadata( saved_bundle_path) # noqa: E501 # fmt: on bento_tag = f'{bento_metadata.name}:{bento_metadata.version}' yatai_client: "YataiClient" = get_yatai_client(yatai_url) docker_build_args = {} if build_arg: for arg in build_arg: key, value = arg.split("=", 1) docker_build_args[key] = value if yatai_url is not None: spinner_message = f'Sending containerize RPC to YataiService at {yatai_url}' else: spinner_message = ( f'Containerizing {bento_tag} with local YataiService and docker ' f'daemon from local environment') with Spinner(spinner_message): tag: str = yatai_client.repository.containerize( bento=bento_tag, tag=tag, build_args=docker_build_args, push=push, ) _echo(f'\nBuild container image: {tag}', CLI_COLOR_SUCCESS)
def __init__( self, bento_bundle_path, outbound_host="localhost", outbound_port=None, outbound_workers: int = Provide[ BentoMLContainer.config.api_server.workers], mb_max_batch_size: int = Provide[ BentoMLContainer.config.marshal_server.max_batch_size], mb_max_latency: int = Provide[ BentoMLContainer.config.marshal_server.max_latency], request_header_flag: str = Provide[ BentoMLContainer.config.marshal_server.request_header_flag], max_request_size: int = Provide[ BentoMLContainer.config.api_server.max_request_size], outbound_unix_socket: str = None, enable_microbatch: bool = Provide[ BentoMLContainer.config.api_server.enable_microbatch], ): self._client = None self.outbound_unix_socket = outbound_unix_socket self.outbound_host = outbound_host self.outbound_port = outbound_port self.outbound_workers = outbound_workers self.mb_max_batch_size = mb_max_batch_size self.mb_max_latency = mb_max_latency self.batch_handlers = dict() self._outbound_sema = None # the semaphore to limit outbound connections self.request_header_flag = request_header_flag self.max_request_size = max_request_size self.bento_service_metadata_pb = load_bento_service_metadata( bento_bundle_path) if enable_microbatch: self.setup_routes_from_pb(self.bento_service_metadata_pb) if psutil.POSIX: import resource self.CONNECTION_LIMIT = resource.getrlimit( resource.RLIMIT_NOFILE)[0] else: self.CONNECTION_LIMIT = 1024 logger.info( "Your system nofile limit is %d, which means each instance of microbatch " "service is able to hold this number of connections at same time. " "You can increase the number of file descriptors for the server process, " "or launch more microbatch instances to accept more concurrent connection.", self.CONNECTION_LIMIT, )
def test_save_duplicated_bento_exception_raised(example_bento_service_class): test_model = TestModel() svc = example_bento_service_class() svc.pack("model", test_model) saved_path = svc.save() svc_metadata = load_bento_service_metadata(saved_path) assert svc.version == svc_metadata.version with pytest.raises(BentoMLException): with patch.object(bentoml.BentoService, 'save_to_dir') as save_to_dir_method: # attempt to save again svc.save() save_to_dir_method.assert_not_called() # reset svc version svc.set_version() saved_path = svc.save() svc_metadata_new = load_bento_service_metadata(saved_path) assert svc.version == svc_metadata_new.version delete_saved_bento_service(svc_metadata.name, svc_metadata.version) delete_saved_bento_service(svc_metadata_new.name, svc_metadata_new.version)
def test_requirements_txt_file(): clf = _fit_clf() iris_classifier_service = IrisClassifierPipEnv() iris_classifier_service.pack('model', clf) saved_path = iris_classifier_service.save() dependencies = _dependencies_to_requirements( _parse_dependencies(saved_path)) _assert_in_dependencies( ['scikit-learn', 'azure-cli', 'psycopg2-binary', 'bentoml'], dependencies) bs_metadata = load_bento_service_metadata(saved_path) requirements_txt = bs_metadata.env.requirements_txt requirements_content = open("./tests/pipenv_requirements.txt", "r").read() assert requirements_txt == requirements_content delete_saved_bento_service(iris_classifier_service.name, iris_classifier_service.version)
def containerize(bento, push, tag, build_arg, username, password): """Containerize specified BentoService. BENTO is the target BentoService to be containerized, referenced by its name and version in format of name:version. For example: "iris_classifier:v1.2.0" `bentoml containerize` command also supports the use of the `latest` tag which will automatically use the last built version of your Bento. You can provide a tag for the image built by Bento using the `--docker-image-tag` flag. Additionally, you can provide a `--push` flag, which will push the built image to the Docker repository specified by the image tag. You can also prefixing the tag with a hostname for the repository you wish to push to. e.g. `bentoml containerize IrisClassifier:latest --push --tag username/iris` would build a Docker image called `username/iris:latest` and push that to Docker Hub. By default, the `containerize` command will use the credentials provided by Docker. You may provide your own through `--username` and `--password`. """ saved_bundle_path = resolve_bundle_path(bento, pip_installed_bundle_path) _echo(f"Found Bento: {saved_bundle_path}") bento_metadata = load_bento_service_metadata(saved_bundle_path) name = to_valid_docker_image_name(bento_metadata.name) version = to_valid_docker_image_version(bento_metadata.version) if not tag: _echo("Tag not specified, using tag parsed from " f"BentoService: '{name}:{version}'") tag = f"{name}:{version}" if ":" not in tag: _echo( "Image version not specified, using version parsed " f"from BentoService: '{version}'", CLI_COLOR_WARNING, ) tag = f"{tag}:{version}" docker_build_args = {} if build_arg: for arg in build_arg: key, value = arg.split("=") docker_build_args[key] = value import docker docker_api = docker.APIClient() try: with Spinner(f"Building Docker image {tag} from {bento} \n"): for line in echo_docker_api_result( docker_api.build( path=saved_bundle_path, tag=tag, decode=True, buildargs=docker_build_args, )): _echo(line) except docker.errors.APIError as error: raise CLIException(f'Could not build Docker image: {error}') _echo( f'Finished building {tag} from {bento}', CLI_COLOR_SUCCESS, ) if push: auth_config_payload = ({ "username": username, "password": password } if username or password else None) try: with Spinner(f"Pushing docker image to {tag}\n"): for line in echo_docker_api_result( docker_api.push( repository=tag, stream=True, decode=True, auth_config=auth_config_payload, )): _echo(line) _echo( f'Pushed {tag} to {name}', CLI_COLOR_SUCCESS, ) except (docker.errors.APIError, BentoMLException) as error: raise CLIException(f'Could not push Docker image: {error}')
def upload_from_dir(self, saved_bento_path, labels=None): bento_service_metadata = load_bento_service_metadata(saved_bento_path) if labels: _validate_labels(labels) bento_service_metadata.labels.update(labels) get_bento_response = self.yatai_service.GetBento( GetBentoRequest( bento_name=bento_service_metadata.name, bento_version=bento_service_metadata.version, )) if get_bento_response.status.status_code == status_pb2.Status.OK: raise BentoMLException( "BentoService bundle {}:{} already registered in repository. Reset " "BentoService version with BentoService#set_version or bypass BentoML's" " model registry feature with BentoService#save_to_dir".format( bento_service_metadata.name, bento_service_metadata.version)) elif get_bento_response.status.status_code != status_pb2.Status.NOT_FOUND: raise BentoMLException( 'Failed accessing YataiService. {error_code}:' '{error_message}'.format( error_code=Status.Name( get_bento_response.status.status_code), error_message=get_bento_response.status.error_message, )) request = AddBentoRequest( bento_name=bento_service_metadata.name, bento_version=bento_service_metadata.version, ) response = self.yatai_service.AddBento(request) if response.status.status_code != status_pb2.Status.OK: raise BentoMLException( "Error adding BentoService bundle to repository: {}:{}".format( Status.Name(response.status.status_code), response.status.error_message, )) if response.uri.type == BentoUri.LOCAL: if os.path.exists(response.uri.uri): # due to copytree dst must not already exist shutil.rmtree(response.uri.uri) shutil.copytree(saved_bento_path, response.uri.uri) self._update_bento_upload_progress(bento_service_metadata) logger.info( "BentoService bundle '%s:%s' saved to: %s", bento_service_metadata.name, bento_service_metadata.version, response.uri.uri, ) # Return URI to saved bento in repository storage return response.uri.uri elif response.uri.type == BentoUri.S3 or response.uri.type == BentoUri.GCS: uri_type = 'S3' if response.uri.type == BentoUri.S3 else 'GCS' self._update_bento_upload_progress(bento_service_metadata, UploadStatus.UPLOADING, 0) fileobj = io.BytesIO() with tarfile.open(mode="w:gz", fileobj=fileobj) as tar: tar.add(saved_bento_path, arcname=bento_service_metadata.name) fileobj.seek(0, 0) if response.uri.type == BentoUri.S3: http_response = requests.put(response.uri.s3_presigned_url, data=fileobj) elif response.uri.type == BentoUri.GCS: http_response = requests.put(response.uri.gcs_presigned_url, data=fileobj) if http_response.status_code != 200: self._update_bento_upload_progress(bento_service_metadata, UploadStatus.ERROR) raise BentoMLException( f"Error saving BentoService bundle to {uri_type}." f"{http_response.status_code}: {http_response.text}") self._update_bento_upload_progress(bento_service_metadata) logger.info( "Successfully saved BentoService bundle '%s:%s' to {uri_type}: %s", bento_service_metadata.name, bento_service_metadata.version, response.uri.uri, ) return response.uri.uri else: raise BentoMLException( f"Error saving Bento to target repository, URI type {response.uri.type}" f" at {response.uri.uri} not supported")
def deploy_bentoml(clipper_conn, bundle_path, api_name, model_name=None, labels=None, build_envs=None): """Deploy bentoml bundle to clipper cluster Args: clipper_conn(clipper_admin.ClipperConnection): Clipper connection instance bundle_path(str): Path to the saved BentomlService bundle. api_name(str): name of the api that will be used as prediction function for clipper cluster model_name(str): Model's name for clipper cluster labels(:obj:`list(str)`, optional): labels for clipper model Returns: tuple: Model name and model version that deployed to clipper """ track("clipper-deploy", {'bento_service_bundle_path': bundle_path}) build_envs = {} if build_envs is None else build_envs # docker is required to build clipper model image ensure_docker_available_or_raise() if not clipper_conn.connected: raise BentoMLException( "No connection to Clipper cluster. CallClipperConnection.connect to " "connect to an existing cluster or ClipperConnection.start_clipper to " "create a new one") bento_service_metadata = load_bento_service_metadata(bundle_path) try: api_metadata = next((api for api in bento_service_metadata.apis if api.name == api_name)) except StopIteration: raise BentoMLException( "Can't find API '{}' in BentoService bundle {}".format( api_name, bento_service_metadata.name)) if api_metadata.input_type not in ADAPTER_TYPE_TO_INPUT_TYPE: raise BentoMLException( "Only BentoService APIs using ClipperInput can be deployed to Clipper" ) input_type = ADAPTER_TYPE_TO_INPUT_TYPE[api_metadata.input_type] model_name = model_name or get_clipper_compatible_string( bento_service_metadata.name + "-" + api_metadata.name) model_version = get_clipper_compatible_string( bento_service_metadata.version) with TempDirectory() as tempdir: entry_py_content = CLIPPER_ENTRY.format(api_name=api_name) build_path = os.path.join(tempdir, "bento") shutil.copytree(bundle_path, build_path) with open(os.path.join(build_path, "clipper_entry.py"), "w") as f: f.write(entry_py_content) if bento_service_metadata.env.python_version.startswith("3.6"): base_image = "clipper/python36-closure-container:0.4.1" elif bento_service_metadata.env.python_version.startswith("2.7"): base_image = "clipper/python-closure-container:0.4.1" else: raise BentoMLException( "Python version {} is not supported in Clipper".format( bento_service_metadata.env.python_version)) docker_content = CLIPPER_DOCKERFILE.format( model_name=model_name, model_version=model_version, base_image=base_image, pip_index_url=build_envs.get("PIP_INDEX_URL", ""), pip_trusted_url=build_envs.get("PIP_TRUSTED_HOST", ""), ) with open(os.path.join(build_path, "Dockerfile-clipper"), "w") as f: f.write(docker_content) docker_api = docker.APIClient() clipper_model_docker_image_tag = "clipper-model-{}:{}".format( bento_service_metadata.name.lower(), bento_service_metadata.version) for line in docker_api.build( path=build_path, dockerfile="Dockerfile-clipper", tag=clipper_model_docker_image_tag, ): process_docker_api_line(line) logger.info( "Successfully built docker image %s for Clipper deployment", clipper_model_docker_image_tag, ) clipper_conn.deploy_model( name=model_name, version=model_version, input_type=input_type, image=clipper_model_docker_image_tag, labels=labels, ) track("clipper-deploy-success", {'bento_service_bundle_path': bundle_path}) return model_name, model_version
def upload_from_dir(self, saved_bento_path: str, labels: Dict = None) -> "BentoUri": from bentoml.yatai.db.stores.label import _validate_labels bento_service_metadata = load_bento_service_metadata(saved_bento_path) if labels: _validate_labels(labels) bento_service_metadata.labels.update(labels) get_bento_response = self.yatai_service.GetBento( GetBentoRequest( bento_name=bento_service_metadata.name, bento_version=bento_service_metadata.version, )) if get_bento_response.status.status_code == status_pb2.Status.OK: raise BentoMLException( "BentoService bundle {}:{} already registered in repository. Reset " "BentoService version with BentoService#set_version or bypass BentoML's" " model registry feature with BentoService#save_to_dir".format( bento_service_metadata.name, bento_service_metadata.version)) elif get_bento_response.status.status_code != status_pb2.Status.NOT_FOUND: raise BentoMLException( 'Failed accessing YataiService. {error_code}:' '{error_message}'.format( error_code=Status.Name( get_bento_response.status.status_code), error_message=get_bento_response.status.error_message, )) request = AddBentoRequest( bento_name=bento_service_metadata.name, bento_version=bento_service_metadata.version, ) response = self.yatai_service.AddBento(request) if response.status.status_code != status_pb2.Status.OK: raise BentoMLException( "Error adding BentoService bundle to repository: {}:{}".format( Status.Name(response.status.status_code), response.status.error_message, )) if response.uri.type == BentoUri.LOCAL: # When using Yatai backed by File System repository, # if Yatai is a local instance, copy the files directly. # Otherwise, use UploadBento RPC to stream files to remote Yatai server if is_remote_yatai(self.yatai_service): self._upload_bento( bento_service_metadata.name, bento_service_metadata.version, saved_bento_path, ) update_bento_service = UpdateBentoRequest( bento_name=bento_service_metadata.name, bento_version=bento_service_metadata.version, service_metadata=bento_service_metadata, ) self.yatai_service.UpdateBento(update_bento_service) else: if os.path.exists(response.uri.uri): raise BentoMLException( f'Bento bundle directory {response.uri.uri} already exist' ) shutil.copytree(saved_bento_path, response.uri.uri) upload_status = UploadStatus.DONE self._update_bento_upload_progress(bento_service_metadata, status=upload_status) logger.info( "BentoService bundle '%s:%s' saved to: %s", bento_service_metadata.name, bento_service_metadata.version, response.uri.uri, ) # Return URI to saved bento in repository storage return response.uri.uri elif response.uri.type == BentoUri.S3 or response.uri.type == BentoUri.GCS: uri_type = 'S3' if response.uri.type == BentoUri.S3 else 'GCS' self._update_bento_upload_progress(bento_service_metadata, UploadStatus.UPLOADING, 0) fileobj = io.BytesIO() with tarfile.open(mode="w:gz", fileobj=fileobj) as tar: tar.add(saved_bento_path, arcname=bento_service_metadata.name) fileobj.seek(0, 0) if response.uri.type == BentoUri.S3: http_response = requests.put(response.uri.s3_presigned_url, data=fileobj) elif response.uri.type == BentoUri.GCS: http_response = requests.put(response.uri.gcs_presigned_url, data=fileobj) if http_response.status_code != 200: self._update_bento_upload_progress(bento_service_metadata, UploadStatus.ERROR) raise BentoMLException( f"Error saving BentoService bundle to {uri_type}." f"{http_response.status_code}: {http_response.text}") self._update_bento_upload_progress(bento_service_metadata) logger.info( "Successfully saved BentoService bundle '%s:%s' to {uri_type}: %s", bento_service_metadata.name, bento_service_metadata.version, response.uri.uri, ) return response.uri.uri else: raise BentoMLException( f"Error saving Bento to target repository, URI type {response.uri.type}" f" at {response.uri.uri} not supported")
def __init__( self, bento_bundle_path: str = Provide[BentoMLContainer.bundle_path], outbound_host: str = Provide[BentoMLContainer.forward_host], outbound_port: int = Provide[BentoMLContainer.forward_port], outbound_workers: int = Provide[BentoMLContainer.api_server_workers], mb_max_batch_size: int = Provide[ BentoMLContainer.config.bento_server.microbatch.max_batch_size ], mb_max_latency: int = Provide[ BentoMLContainer.config.bento_server.microbatch.max_latency ], max_request_size: int = Provide[ BentoMLContainer.config.bento_server.max_request_size ], outbound_unix_socket: str = None, enable_access_control: bool = Provide[ BentoMLContainer.config.bento_server.cors.enabled ], access_control_allow_origin: Optional[str] = Provide[ BentoMLContainer.config.bento_server.cors.access_control_allow_origin ], access_control_options: Optional["ResourceOptions"] = Provide[ BentoMLContainer.access_control_options ], timeout: int = Provide[BentoMLContainer.config.bento_server.timeout], tracer=Provide[BentoMLContainer.tracer], ): self._conn: Optional["BaseConnector"] = None self._client: Optional["ClientSession"] = None self.outbound_unix_socket = outbound_unix_socket self.outbound_host = outbound_host self.outbound_port = outbound_port self.outbound_workers = outbound_workers self.mb_max_batch_size = mb_max_batch_size self.mb_max_latency = mb_max_latency self.batch_handlers = dict() self._outbound_sema = None # the semaphore to limit outbound connections self._cleanup_tasks = None self.max_request_size = max_request_size self.tracer = tracer self.enable_access_control = enable_access_control self.access_control_allow_origin = access_control_allow_origin self.access_control_options = access_control_options self.bento_service_metadata_pb = load_bento_service_metadata(bento_bundle_path) self.setup_routes_from_pb(self.bento_service_metadata_pb) self.timeout = timeout if psutil.POSIX: import resource self.CONNECTION_LIMIT = resource.getrlimit(resource.RLIMIT_NOFILE)[0] else: self.CONNECTION_LIMIT = 1024 logger.info( "Your system nofile limit is %d, which means each instance of microbatch " "service is able to hold this number of connections at same time. " "You can increase the number of file descriptors for the server process, " "or launch more microbatch instances to accept more concurrent connection.", self.CONNECTION_LIMIT, )