def _upload_obj(
obj: bytes, bucket: str, key: str, client: boto3_session.client = None
) -> bool:
if not client:
session = boto3_session()
client = session.client("s3")
client.upload_fileobj(obj, bucket, key)
return True
def _s3_upload(file_obj: BinaryIO,
bucket: str,
key: str,
client: boto3_session.client = None) -> bool:
if not client:
session = boto3_session()
client = session.client("s3")
client.upload_fileobj(file_obj, bucket, key)
return True
def _s3_download(
bucket: str, key: str, out_key: str, client: boto3_session.client = None
) -> str:
if not client:
session = boto3_session()
client = session.client("s3")
client.download_file(bucket, key, out_key)
return out_key
def _aws_put_data(
key: str,
bucket: str,
body: BinaryIO,
options: Dict = {},
client: boto3_session.client = None,
) -> str:
if not client:
session = boto3_session()
client = session.client("s3")
client.put_object(Bucket=bucket, Key=key, Body=body, **options)
return key
def _aws_get_data(key, bucket, client: boto3_session.client = None) -> bytes:
if not client:
session = boto3_session()
client = session.client("s3")
response = client.get_object(Bucket=bucket, Key=key)
return response["Body"].read()
def _aws_put_data(key: str,
bucket: str,
body: bytes,
client: boto3_session.client = None,
**kwargs) -> str:
if not client:
session = boto3_session()
client = session.client("s3")
try:
client.put_object(Bucket=bucket, Key=key, Body=body, **kwargs)
except ClientError as e:
status_code = e.response["ResponseMetadata"]["HTTPStatusCode"]
exc = _HTTP_EXCEPTIONS.get(status_code, MosaicError)
raise exc(e.response["Error"]["Message"]) from e
return key
def s3_get_object(bucket, key, client: boto3_session.client = None) -> bytes:
"""GetObject from S3."""
if not client:
session = boto3_session()
client = session.client("s3")
response = client.get_object(Bucket=bucket, Key=key)
return response["Body"].read()
def _aws_head_object(key, bucket, client: boto3_session.client = None) -> bool:
if not client:
session = boto3_session()
client = session.client("s3")
try:
return client.head_object(Bucket=bucket, Key=key)
except ClientError:
return False
def _aws_get_data(key, bucket, client: boto3_session.client = None) -> bytes:
if not client:
session = boto3_session()
client = session.client("s3")
try:
response = client.get_object(Bucket=bucket, Key=key)
except ClientError as e:
status_code = e.response["ResponseMetadata"]["HTTPStatusCode"]
exc = _HTTP_EXCEPTIONS.get(status_code, MosaicError)
raise exc(e.response["Error"]["Message"]) from e
return response["Body"].read()
def _aws_head_object(url: str, client: boto3_session.client = None) -> bool:
if not client:
session = boto3_session()
client = session.client("s3")
parsed = urlparse(url)
bucket = parsed.netloc
key = parsed.path.strip("/")
try:
return client.head_object(Bucket=bucket, Key=key)
except ClientError:
return False
def aws_get_object(
bucket: str,
key: str,
request_pays: bool = False,
client: boto3_session.client = None,
) -> bytes:
"""AWS s3 get object content."""
if not client:
session = boto3_session()
client = session.client("s3")
params = {"Bucket": bucket, "Key": key}
if request_pays:
params["RequestPayer"] = "requester"
response = client.get_object(**params)
return response["Body"].read()
def aws_get_object(
bucket: str,
key: str,
request_pays: bool = False,
client: boto3_session.client = None,
) -> bytes:
"""AWS s3 get object content."""
if not client:
session = boto3_session()
endpoint_url = os.environ.get("AWS_S3_ENDPOINT", None)
client = session.client("s3", endpoint_url=endpoint_url)
params = {"Bucket": bucket, "Key": key}
if request_pays or os.environ.get("AWS_REQUEST_PAYER",
"").lower() == "requester":
params["RequestPayer"] = "requester"
response = client.get_object(**params)
return response["Body"].read()
def get_all_s3_buckets(s3_client: Session.client) -> dict:
# method 1
return s3_client.list_buckets()