def __init__( self, bento_bundle_path, outbound_host="localhost", outbound_port=None, outbound_workers=1, ): self.outbound_host = outbound_host self.outbound_port = outbound_port self.outbound_workers = outbound_workers self.batch_handlers = dict() self._outbound_sema = None # the semaphore to limit outbound connections self.bento_service_metadata_pb = load_bento_service_metadata( bento_bundle_path) self.setup_routes_from_pb(self.bento_service_metadata_pb) self.CONNECTION_LIMIT = resource.getrlimit(resource.RLIMIT_NOFILE)[0] logger.info( "Your system nofile limit is %d, which means each instance of microbatch " "service is able to hold this number of connections at same time. " "You can increase the number of file descriptors for the server process, " "or launch more microbatch instances to accept more concurrent connection.", self.CONNECTION_LIMIT, )
def _get_bento_service_event_properties_from_bundle_path( bundle_path, properties=None): from bentoml.bundler import load_bento_service_metadata bento_service_metadata = load_bento_service_metadata(bundle_path) return _bento_service_metadata_to_event_properties(bento_service_metadata, properties)
def serve(port, bento=None, with_conda=False, enable_microbatch=False): track_cli('serve') bento_service_bundle_path = resolve_bundle_path( bento, pip_installed_bundle_path) bento_service = load(bento_service_bundle_path) if with_conda: run_with_conda_env( bento_service_bundle_path, 'bentoml serve {bento} --port {port} {flags}'.format( bento=bento_service_bundle_path, port=port, flags="--enable-microbatch" if enable_microbatch else "", ), ) return if enable_microbatch: with reserve_free_port() as api_server_port: # start server right after port released # to reduce potential race bento_service_metadata_pb = load_bento_service_metadata( bento_service_bundle_path) marshal_server = MarshalServer(port=port, target_host="localhost", target_port=api_server_port) marshal_server.setup_routes_from_pb(bento_service_metadata_pb) api_server = BentoAPIServer(bento_service, port=api_server_port) marshal_server.async_start() api_server.start() else: api_server = BentoAPIServer(bento_service, port=port) api_server.start()
def test_auto_artifact_dependencies(): clf = svm.SVC(gamma='scale') iris = datasets.load_iris() X, y = iris.data, iris.target clf.fit(X, y) # Create a iris classifier service iris_classifier_service = IrisClassifier() # Pack it with the newly trained model artifact iris_classifier_service.pack('model', clf) # Save the prediction service to a BentoService bundle saved_path = iris_classifier_service.save() with open(os.path.join(saved_path, 'requirements.txt')) as f: requirements_txt_content = f.read() dependencies = requirements_txt_content.split('\n') dependencies = [dep.split('==')[0] for dep in dependencies] assert 'scikit-learn' in dependencies assert 'bentoml' in dependencies # Test that dependencies also wrote to BentoServiceMetadat config file bs_matadata = load_bento_service_metadata(saved_path) dependencies = bs_matadata.env.pip_dependencies.split('\n') dependencies = [dep.split('==')[0] for dep in dependencies] assert 'scikit-learn' in dependencies assert 'bentoml' in dependencies delete_saved_bento_service(iris_classifier_service.name, iris_classifier_service.version)
def info(bundle_path=bundle_path): """ List all APIs defined in the BentoService loaded from saved bundle """ track_cli('info') bento_service_metadata_pb = load_bento_service_metadata(bundle_path) output = json.dumps(ProtoMessageToDict(bento_service_metadata_pb), indent=2) _echo(output)
def load(self): server = MarshalServer(self.target_host, self.target_port, port=self.port) bento_service_metadata_pb = load_bento_service_metadata( self.bento_service_bundle_path) server.setup_routes_from_pb(bento_service_metadata_pb) return server.marshal_app.make_app()
def test_save_duplicated_bento_exception_raised(tmpdir, test_bento_service_class): test_model = TestModel() svc = test_bento_service_class() svc.pack("model", test_model) saved_path = svc.save(str(tmpdir)) svc_metadata = load_bento_service_metadata(saved_path) assert svc.version == svc_metadata.version with pytest.raises(BentoMLException): with patch.object(bentoml.BentoService, 'save_to_dir') as save_to_dir_method: # attempt to save again svc.save(str(tmpdir)) save_to_dir_method.assert_not_called() # reset svc version svc.set_version() saved_path = svc.save(str(tmpdir)) svc_metadata = load_bento_service_metadata(saved_path) assert svc.version == svc_metadata.version
def _upload_bento_service(self, saved_bento_path): bento_service_metadata = load_bento_service_metadata(saved_bento_path) get_bento_response = self.yatai_service.GetBento( GetBentoRequest( bento_name=bento_service_metadata.name, bento_version=bento_service_metadata.version, )) if get_bento_response.status.status_code == status_pb2.Status.OK: raise BentoMLException( "BentoService bundle {}:{} already registered in repository. Reset " "BentoService version with BentoService#set_version or bypass BentoML's" " model registry feature with BentoService#save_to_dir".format( bento_service_metadata.name, bento_service_metadata.version)) elif get_bento_response.status.status_code != status_pb2.Status.NOT_FOUND: raise BentoMLException( 'Failed accessing YataiService. {error_code}:' '{error_message}'.format( error_code=Status.Name( get_bento_response.status.status_code), error_message=get_bento_response.status.error_message, )) request = AddBentoRequest( bento_name=bento_service_metadata.name, bento_version=bento_service_metadata.version, ) response = self.yatai_service.AddBento(request) if response.status.status_code != status_pb2.Status.OK: raise BentoMLException( "Error adding BentoService bundle to repository: {}:{}".format( Status.Name(response.status.status_code), response.status.error_message, )) if response.uri.type == BentoUri.LOCAL: if os.path.exists(response.uri.uri): # due to copytree dst must not already exist shutil.rmtree(response.uri.uri) shutil.copytree(saved_bento_path, response.uri.uri) self._update_bento_upload_progress(bento_service_metadata) logger.info( "BentoService bundle '%s:%s' saved to: %s", bento_service_metadata.name, bento_service_metadata.version, response.uri.uri, ) # Return URI to saved bento in repository storage return response.uri.uri elif response.uri.type == BentoUri.S3: self._update_bento_upload_progress(bento_service_metadata, UploadStatus.UPLOADING, 0) fileobj = io.BytesIO() with tarfile.open(mode="w:gz", fileobj=fileobj) as tar: tar.add(saved_bento_path, arcname=bento_service_metadata.name) fileobj.seek(0, 0) files = { 'file': ('dummy', fileobj) } # dummy file name because file name # has been generated when getting the pre-signed signature. data = json.loads(response.uri.additional_fields) uri = data.pop('url') http_response = requests.post(uri, data=data, files=files) if http_response.status_code != 204: self._update_bento_upload_progress(bento_service_metadata, UploadStatus.ERROR) raise BentoMLException( f"Error saving BentoService bundle to S3. " f"{http_response.status_code}: {http_response.text}") self._update_bento_upload_progress(bento_service_metadata) logger.info( "Successfully saved BentoService bundle '%s:%s' to S3: %s", bento_service_metadata.name, bento_service_metadata.version, response.uri.uri, ) return response.uri.uri else: raise BentoMLException( f"Error saving Bento to target repository, URI type {response.uri.type}" f" at {response.uri.uri} not supported")
def deploy_bentoml( clipper_conn, bundle_path, api_name, model_name=None, labels=None, build_envs=None ): """Deploy bentoml bundle to clipper cluster Args: clipper_conn(clipper_admin.ClipperConnection): Clipper connection instance bundle_path(str): Path to the saved BentomlService bundle. api_name(str): name of the api that will be used as prediction function for clipper cluster model_name(str): Model's name for clipper cluster labels(:obj:`list(str)`, optional): labels for clipper model Returns: tuple: Model name and model version that deployed to clipper """ track("clipper-deploy", {'bento_service_bundle_path': bundle_path}) build_envs = {} if build_envs is None else build_envs # docker is required to build clipper model image ensure_docker_available_or_raise() if not clipper_conn.connected: raise BentoMLException( "No connection to Clipper cluster. CallClipperConnection.connect to " "connect to an existing cluster or ClipperConnnection.start_clipper to " "create a new one" ) bento_service_metadata = load_bento_service_metadata(bundle_path) try: api_metadata = next( (api for api in bento_service_metadata.apis if api.name == api_name) ) except StopIteration: raise BentoMLException( "Can't find API '{}' in BentoService bundle {}".format( api_name, bento_service_metadata.name ) ) if api_metadata.handler_type not in HANDLER_TYPE_TO_INPUT_TYPE: raise BentoMLException( "Only BentoService APIs using ClipperHandler can be deployed to Clipper" ) input_type = HANDLER_TYPE_TO_INPUT_TYPE[api_metadata.handler_type] model_name = model_name or get_clipper_compatiable_string( bento_service_metadata.name + "-" + api_metadata.name ) model_version = get_clipper_compatiable_string(bento_service_metadata.version) with TempDirectory() as tempdir: entry_py_content = CLIPPER_ENTRY.format(api_name=api_name) model_path = os.path.join(tempdir, "bento") shutil.copytree(bundle_path, model_path) with open(os.path.join(tempdir, "clipper_entry.py"), "w") as f: f.write(entry_py_content) if bento_service_metadata.env.python_version.startswith("3.6"): base_image = "clipper/python36-closure-container:0.4.1" elif bento_service_metadata.env.python_version.startswith("2.7"): base_image = "clipper/python-closure-container:0.4.1" else: raise BentoMLException( "Python version {} is not supported in Clipper".format( bento_service_metadata.env.python_version ) ) docker_content = CLIPPER_DOCKERFILE.format( model_name=model_name, model_version=model_version, base_image=base_image, pip_index_url=build_envs.get("PIP_INDEX_URL", ""), pip_trusted_url=build_envs.get("PIP_TRUSTED_HOST", ""), ) with open(os.path.join(tempdir, "Dockerfile-clipper"), "w") as f: f.write(docker_content) docker_api = docker.APIClient() clipper_model_docker_image_tag = "clipper-model-{}:{}".format( bento_service_metadata.name.lower(), bento_service_metadata.version ) for line in docker_api.build( path=tempdir, dockerfile="Dockerfile-clipper", tag=clipper_model_docker_image_tag, ): process_docker_api_line(line) logger.info( "Successfully built docker image %s for Clipper deployment", clipper_model_docker_image_tag, ) clipper_conn.deploy_model( name=model_name, version=model_version, input_type=input_type, image=clipper_model_docker_image_tag, labels=labels, ) track("clipper-deploy-success", {'bento_service_bundle_path': bundle_path}) return model_name, model_version