def is_s3_key_exists(logger, s3_resource, s3_bucket, s3_key):
"""
Checks the existence of S3 key.
:param logger: logger
:param s3_resource: S3 resource
:param s3_bucket: S3 bucket
:param s3_key: S3 key
:type logger: logging.Logger
:type s3_resource: boto3.resources.factory.s3.ServiceResource
:type s3_bucket: str
:type s3_key: str
:returns: True if key exists, False if key does not exist, None if unable to determine
:rtype: bool
"""
try:
logger.debug(
"Checking the existence of s3://{s3_bucket}/{s3_key}".format(
s3_bucket=s3_bucket, s3_key=s3_key))
objs = list(
s3_resource.Bucket(s3_bucket).objects.filter(Prefix=s3_key))
return len(objs) > 0 and objs[0].key == s3_key
except ClientError as e:
if represent_int(e.response["Error"]["Code"]):
error_code = int(e.response["Error"]["Code"])
if error_code == 403:
logger.debug("Bucket {s3_bucket} is forbidden".format(
s3_bucket=s3_bucket))
return False
logger.warning("Unable to determine S3 key {s3_key} - {error}".format(
s3_key=s3_key, error=e.__str__()))
def delete_s3_prefix(logger, s3_resource, s3_bucket, s3_prefix):
"""
Deletes an S3 prefix.
:param logger: logger
:param s3_resource: S3 resource
:param s3_bucket: S3 bucket
:param s3_prefix: S3 prefix
:type logger: logging.Logger
:type s3_resource: boto3.resources.factory.s3.ServiceResource
:type s3_bucket: str
:type s3_prefix: str
"""
try:
logger.debug("Deleting s3://{s3_bucket}/{s3_prefix}/".format(
s3_bucket=s3_bucket, s3_prefix=s3_prefix))
s3_resource.Bucket(s3_bucket).objects.filter(Prefix=s3_prefix).delete()
s3_resource.Object(bucket_name=s3_bucket, key=s3_prefix).delete()
except ClientError as e:
if represent_int(e.response["Error"]["Code"]):
error_code = int(e.response["Error"]["Code"])
if error_code == 403:
logger.debug(
"s3://{s3_bucket}/{s3_prefix}/ is forbidden".format(
s3_bucket=s3_bucket, s3_prefix=s3_prefix))
return
elif error_code == 404:
logger.debug(
"s3://{s3_bucket}/{s3_prefix}/ does not exist".format(
s3_bucket=s3_bucket, s3_prefix=s3_prefix))
return
logger.warning(
"Unable to delete S3 prefix s3://{s3_bucket}/{s3_prefix} - {error}"
.format(s3_bucket=s3_bucket,
s3_prefix=s3_prefix,
error=e.__str__()))
def get_s3_object_size(logger, s3_resource, s3_bucket, s3_key):
"""
Gets S3 object size.
:param logger: logger
:param s3_resource: S3 resource
:param s3_bucket: S3 bucket
:param s3_prefix: S3 prefix
:param s3_keys: list of S3 keys
:type logger: logging.Logger
:type s3_resource: boto3.resources.factory.s3.ServiceResource
:type s3_bucket: str
:type s3_prefix: str
:type s3_keys: list
:returns: size of S3 object in bytes
:rtype: int
"""
try:
return s3_resource.meta.client.head_object(Bucket=s3_bucket,
Key=s3_key)["ContentLength"]
except ClientError as e:
if represent_int(e.response["Error"]["Code"]):
error_code = int(e.response["Error"]["Code"])
if error_code == 403:
logger.debug("s3://{s3_bucket}/{s3_key} is forbidden".format(
s3_bucket=s3_bucket, s3_key=s3_key))
return
elif error_code == 404:
logger.debug("s3://{s3_bucket}/{s3_key} does not exist".format(
s3_bucket=s3_bucket, s3_key=s3_key))
return
logger.warning(
"Unable to determine get size of s3://{s3_bucket}/{s3_key} - {error}"
.format(s3_bucket=s3_bucket, s3_key=s3_key, error=e.__str__()))
def is_s3_bucket_exists(logger, s3_resource, s3_bucket):
"""
Checks the existence of S3 bucket.
:param logger: logger
:param s3_resource: S3 resource
:param s3_bucket: S3 bucket
:type logger: logging.Logger
:type s3_resource: boto3.resources.factory.s3.ServiceResource
:type s3_bucket: str
:returns: True if bucket exists, False if bucket does not exist or forbidden, None if unable to determine
:rtype: bool
"""
try:
logger.debug("Checking the existence of s3://{s3_bucket}".format(
s3_bucket=s3_bucket))
s3_resource.meta.client.head_bucket(Bucket=s3_bucket)
return True
except ClientError as e:
if represent_int(e.response["Error"]["Code"]):
error_code = int(e.response["Error"]["Code"])
if error_code == 403:
logger.debug("Bucket {s3_bucket} is forbidden".format(
s3_bucket=s3_bucket))
return False
elif error_code == 404:
logger.debug("Bucket {s3_bucket} does not exist".format(
s3_bucket=s3_bucket))
return False
logger.warning(
"Unable to determine S3 bucket {s3_bucket} - {error}".format(
s3_bucket=s3_bucket, error=e.__str__()))
def test_represent_int_true(self):
valid_str_representations = ["0", "123"]
for s in valid_str_representations:
self.assertTrue(represent_int(s))
def test_represent_int_false(self):
invalid_str_representations = ["a", "1a", "a1"]
for s in invalid_str_representations:
self.assertFalse(represent_int(s))
def list_s3_keys(logger,
s3_resource,
s3_bucket,
s3_prefix="",
search="Contents"):
"""
Lists S3 keys for a given S3 bucket and S3 prefix.
:param logger: logger
:param s3_resource: S3 resource
:param s3_bucket: S3 bucket
:param s3_prefix: S3 prefix
:param search: JMESPath search string
:type logger: logging.Logger
:type s3_resource: boto3.resources.factory.s3.ServiceResource
:type s3_bucket: str
:type s3_prefix: str
:type search: str
:returns: list of S3 keys
:rtype: list
"""
try:
if s3_prefix and s3_prefix[0] == "/":
s3_prefix = s3_prefix[1:]
if s3_prefix and s3_prefix[len(s3_prefix) - 1] != "/":
s3_prefix = "{s3_prefix}/".format(s3_prefix=s3_prefix)
paginator = s3_resource.meta.client.get_paginator("list_objects_v2")
page_iterator = paginator.paginate(Bucket=s3_bucket, Prefix=s3_prefix)
if search is not None:
page_iterator = page_iterator.search(search)
s3_keys = []
for key_data in page_iterator:
if key_data is not None:
if not key_data["Key"].endswith("/"):
s3_keys.append(key_data["Key"])
return s3_keys
except ClientError as e:
if represent_int(e.response["Error"]["Code"]):
error_code = int(e.response["Error"]["Code"])
if error_code == 403:
logger.debug(
"s3://{s3_bucket}/{s3_prefix} is forbidden".format(
s3_bucket=s3_bucket, s3_prefix=s3_prefix))
return
elif error_code == 404:
logger.debug(
"s3://{s3_bucket}/{s3_prefix} does not exist".format(
s3_bucket=s3_bucket, s3_prefix=s3_prefix))
return
logger.warning(
"Unable to list S3 keys from s3://{s3_bucket}/{s3_prefix} - {error}"
.format(s3_bucket=s3_bucket,
s3_prefix=s3_prefix,
error=e.__str__()))
def download_s3_object(logger,
s3_resource,
f,
s3_bucket,
s3_key,
callback=None):
"""
Downloads file object from S3.
:param logger: logger
:param s3_resource: S3 resource
:param f: file path or file object
:param s3_bucket: S3 bucket
:param s3_key: S3 key
:param callback: callback for monitoring progress
:type logger: logging.Logger
:type s3_resource: boto3.resources.factory.s3.ServiceResource
:type f: str | io.BufferedWriter
:type s3_bucket: str
:type s3_key: str
:type callback: typing.Callables
"""
try:
if isinstance(f, str):
logger.debug("Downloading s3://{s3_bucket}/{s3_key} to {f}".format(
s3_bucket=s3_bucket, s3_key=s3_key, f=f))
s3_resource.meta.client.download_file(s3_bucket,
s3_key,
f,
Callback=callback)
elif isinstance(f, BufferedWriter):
logger.debug(
"Downloading s3://{s3_bucket}/{s3_key} to a file object".
format(s3_bucket=s3_bucket, s3_key=s3_key))
s3_resource.meta.client.download_fileobj(s3_bucket,
s3_key,
f,
Callback=callback)
else:
logger.error(
"Invalid input type for download - {input_type}".format(
input_type=type(f).__name__))
except ClientError as e:
if represent_int(e.response["Error"]["Code"]):
error_code = int(e.response["Error"]["Code"])
if error_code == 403:
logger.debug(
"S3 object s3://{s3_bucket}/{s3_key} is forbidden".format(
s3_bucket=s3_bucket, s3_key=s3_key))
return
elif error_code == 404:
logger.debug(
"S3 object s3://{s3_bucket}/{s3_key} does not exist".
format(s3_bucket=s3_bucket, s3_key=s3_key))
return
logger.warning(
"Unable to download s3://{s3_bucket}/{s3_key} - {error}".format(
s3_bucket=s3_bucket, s3_key=s3_key, error=e.__str__()))
