def resolve_bundle_path(bento, pip_installed_bundle_path): if pip_installed_bundle_path: assert ( bento is None ), "pip installed BentoService commands should not have Bento argument" return pip_installed_bundle_path if os.path.isdir(bento) or is_s3_url(bento) or is_gcs_url(bento): # saved_bundle already support loading local, s3 path and gcs path return bento elif ":" in bento: # assuming passing in BentoService in the form of Name:Version tag yatai_client = get_default_yatai_client() name, version = bento.split(':') get_bento_result = yatai_client.repository.get(name, version) if get_bento_result.status.status_code != yatai_proto.status_pb2.Status.OK: error_code, error_message = status_pb_to_error_code_and_message( get_bento_result.status) raise BentoMLException( f'BentoService {name}:{version} not found - ' f'{error_code}:{error_message}') if get_bento_result.bento.uri.s3_presigned_url: # Use s3 presigned URL for downloading the repository if it is presented return get_bento_result.bento.uri.s3_presigned_url if get_bento_result.bento.uri.gcs_presigned_url: return get_bento_result.bento.uri.gcs_presigned_url else: return get_bento_result.bento.uri.uri else: raise BentoMLException( f'BentoService "{bento}" not found - either specify the file path of ' f'the BentoService saved bundle, or the BentoService id in the form of ' f'"name:version"')
def resolve_bundle_path( bento: str, pip_installed_bundle_path: Optional[str] = None, yatai_url: Optional[str] = None, ) -> str: from bentoml.exceptions import BentoMLException from bentoml.yatai.client import get_yatai_client if pip_installed_bundle_path: assert ( bento is None ), "pip installed BentoService commands should not have Bento argument" return pip_installed_bundle_path if os.path.isdir(bento) or is_s3_url(bento) or is_gcs_url(bento): # saved_bundle already support loading local, s3 path and gcs path return bento elif ":" in bento: # assuming passing in BentoService in the form of Name:Version tag yatai_client = get_yatai_client(yatai_url) bento_pb = yatai_client.repository.get(bento) return resolve_bento_bundle_uri(bento_pb) else: raise BentoMLException( f'BentoService "{bento}" not found - either specify the file path of ' f"the BentoService saved bundle, or the BentoService id in the form of " f'"name:version"')
def resolve_bundle_path(bento, pip_installed_bundle_path, yatai_url=None): from bentoml.yatai.client import get_yatai_client from bentoml.exceptions import BentoMLException if pip_installed_bundle_path: assert ( bento is None ), "pip installed BentoService commands should not have Bento argument" return pip_installed_bundle_path if os.path.isdir(bento) or is_s3_url(bento) or is_gcs_url(bento): # saved_bundle already support loading local, s3 path and gcs path return bento elif ":" in bento: # assuming passing in BentoService in the form of Name:Version tag yatai_client = get_yatai_client(yatai_url) bento_pb = yatai_client.repository.get(bento) if bento_pb.uri.s3_presigned_url: # Use s3 presigned URL for downloading the repository if it is presented return bento_pb.uri.s3_presigned_url if bento_pb.uri.gcs_presigned_url: return bento_pb.uri.gcs_presigned_url else: return bento_pb.uri.uri else: raise BentoMLException( f'BentoService "{bento}" not found - either specify the file path of ' f"the BentoService saved bundle, or the BentoService id in the form of " f'"name:version"')
def yatai_service_start( db_url, repo_base_url, grpc_port, ui_port, ui, web_prefix_path, repository_type, file_system_directory, s3_url, s3_endpoint_url, gcs_url, ): from bentoml.utils.s3 import is_s3_url from bentoml.utils.gcs import is_gcs_url if repo_base_url: logger.warning( "Option --repo-base-url has been deprecated but is still supported " "in the current release. Consider using --repository-type and its " "corresponding options in the upcoming releases. ") if is_s3_url(repo_base_url): repository_type = YATAI_REPOSITORY_S3 s3_url = repo_base_url elif is_gcs_url(repo_base_url): repository_type = YATAI_REPOSITORY_GCS gcs_url = repo_base_url else: repository_type = YATAI_REPOSITORY_FILE_SYSTEM file_system_directory = repo_base_url if repository_type == YATAI_REPOSITORY_S3 and s3_url is None: logger.error("'--s3-url' must be specified for S3 repository type") return elif repository_type == YATAI_REPOSITORY_GCS and gcs_url is None: logger.error( "'--gcs-url' must be specified for GCS repository type") return elif (repository_type == YATAI_REPOSITORY_FILE_SYSTEM and file_system_directory is None): logger.error( "'--file-system-directory' must be specified for file system " "repository type") return else: start_yatai_service_grpc_server( db_url=db_url, grpc_port=grpc_port, ui_port=ui_port, with_ui=ui, base_url=web_prefix_path, repository_type=repository_type, file_system_directory=file_system_directory, s3_url=s3_url, s3_endpoint_url=s3_endpoint_url, gcs_url=gcs_url, )
def __init__(self, base_url=None, s3_endpoint_url=None): """ :param base_url: either a local file system path or a s3-compatible path such as s3://my-bucket/some-prefix/ :param s3_endpoint_url: configuring S3Repository to talk to a specific s3 endpoint """ if base_url is None: base_url = config().get('default_repository_base_url') if is_s3_url(base_url): self._repo = S3Repository(base_url, s3_endpoint_url) elif is_gcs_url(base_url): self._repo = GCSRepository(base_url) else: self._repo = LocalRepository(base_url)
def _resolve_remote_bundle_path(bundle_path): if is_s3_url(bundle_path): import boto3 parsed_url = urlparse(bundle_path) bucket_name = parsed_url.netloc object_name = parsed_url.path.lstrip('/') s3 = boto3.client('s3') fileobj = io.BytesIO() s3.download_fileobj(bucket_name, object_name, fileobj) fileobj.seek(0, 0) elif is_gcs_url(bundle_path): try: from google.cloud import storage except ImportError: raise BentoMLException( '"google-cloud-storage" package is required. You can install it with ' 'pip: "pip install google-cloud-storage"' ) gcs = storage.Client() fileobj = io.BytesIO() gcs.download_blob_to_file(bundle_path, fileobj) fileobj.seek(0, 0) elif _is_http_url(bundle_path): import requests response = requests.get(bundle_path) if response.status_code != 200: raise BentoMLException( f"Error retrieving BentoService bundle. " f"{response.status_code}: {response.text}" ) fileobj = io.BytesIO() fileobj.write(response.content) fileobj.seek(0, 0) else: raise BentoMLException(f"Saved bundle path: '{bundle_path}' is not supported") with tarfile.open(mode="r:gz", fileobj=fileobj) as tar: with tempfile.TemporaryDirectory() as tmpdir: filename = tar.getmembers()[0].name tar.extractall(path=tmpdir) yield os.path.join(tmpdir, filename)
def _is_remote_path(bundle_path): return ( is_s3_url(bundle_path) or is_gcs_url(bundle_path) or _is_http_url(bundle_path) )
def _is_remote_path(bundle_path): return isinstance(bundle_path, str) and (is_s3_url(bundle_path) or is_gcs_url(bundle_path) or _is_http_url(bundle_path))