def remove_all_tags(s3_bucket, key, region=DEFAULT_REGION, version=None):
"""
Removes all tags from an item.
"""
payload = _build_version_payload(s3_bucket, key, version)
payload["Tagging"] = {"TagSet": []}
get_client("s3", region).put_object_tagging(**payload)
def create_log_stream(log_group_name, log_stream_name, region=DEFAULT_REGION):
"""
Creates a log stream if one does not exist.
"""
stream = get_log_stream(log_group_name, log_stream_name, region)
if not stream:
LOGGER.info("Creating log stream %s in group %s" % (log_stream_name, log_group_name))
get_client("logs", region=region).create_log_stream(
logGroupName=log_group_name,
logStreamName=log_stream_name
)
send_log(log_group_name, log_stream_name, "Starting Execution", region)
def save_to_s3(contents,
bucket,
prefix,
region=DEFAULT_REGION,
encryption="AES256"):
"""
Writes the contents of the file into the data lake.
"""
LOGGER.info("Writing object to bucket %s with prefix of %s." %
(bucket, prefix))
payload = {"Body": contents, "Bucket": bucket, "Key": prefix}
if encryption:
payload["ServerSideEncryption"] = encryption
get_client("s3", region).put_object(**payload)
def read_encrypted_value(property_key, region=DEFAULT_REGION):
"""
Reads an encrypted value from ssm.
"""
LOG.info("Reading encrypted value from SSM parameter %s." % property_key)
result = get_client("ssm", region).get_parameter(Name=property_key,
WithDecryption=True)
return result["Parameter"]["Value"]
def read_from_s3(bucket, prefix, region=DEFAULT_REGION):
"""
Read a object from S3
"""
LOGGER.info("Reading object from bucket %s with prefix of %s." %
(bucket, prefix))
obj = get_client("s3", region).get_object(Bucket=bucket, Key=prefix)
return obj
def update_object_tag(s3_bucket,
key,
tag_name,
tag_value,
region=DEFAULT_REGION,
version=None): # pylint: disable=too-many-arguments
"""
Updates an object tag
"""
LOGGER.info("Adding tag %s with value %s to element s3://%s/%s" \
% (tag_name, tag_value, key, s3_bucket))
response = get_object_tags(s3_bucket, key, region, version)
tag_set = response["TagSet"]
tag = _get_tag_element(tag_set, tag_name)
if tag:
tag["Value"] = tag_value
else:
tag_set.append({"Key": tag_name, "Value": tag_value})
payload = _build_version_payload(s3_bucket, key, version)
payload["Tagging"] = {"TagSet": tag_set}
get_client("s3", region).put_object_tagging(**payload)
def send_log(log_group_name, log_stream_name, message, region=DEFAULT_REGION):
"""
Sends a cloudwatch log event.
"""
try:
token = get_log_stream(log_group_name, log_stream_name, region)["uploadSequenceToken"]
except KeyError:
LOGGER.info("Token is missing, is either a new stream or a missing stream!")
token = None
payload = {
"logGroupName": log_group_name,
"logStreamName": log_stream_name,
"logEvents": [{
"timestamp": int(round(time.time() * 1000)),
"message": message
}]
}
if token:
# On a newly created stream the token will be missing and you cannot
# send a none value.
payload["sequenceToken"] = token
get_client("logs", region=region).put_log_events(**payload)
def copy(
source_bucket,
source_prefix,
target_bucket,
target_prefix, # pylint: disable=too-many-arguments
region=DEFAULT_REGION,
encryption="AES256"):
"""
Copies an item from a bucket to another.
"""
LOGGER.info("Coyping object from bucket s3://%s/%s to s3://%s/%s" \
% (source_bucket, source_prefix, target_bucket, target_prefix))
payload = {
"Bucket": target_bucket,
"CopySource": {
"Bucket": source_bucket,
"Key": source_prefix
},
"Key": target_prefix
}
if encryption:
payload["ServerSideEncryption"] = encryption
get_client("s3", region).copy_object(**payload)
def get_log_stream(log_group_name, log_stream_name, region=DEFAULT_REGION):
"""
Gets the log stream message.
"""
response = get_client("logs", region=region).describe_log_streams(
logGroupName=log_group_name,
logStreamNamePrefix=log_stream_name,
)
try:
log_streams = response["logStreams"]
for stream in log_streams:
if log_stream_name == stream["logStreamName"]:
return stream
except KeyError:
# Key doesn't exist if the stream doesn't exist.
pass
return None
def delete(bucket, prefix, region=DEFAULT_REGION):
"""
Deletes an object from a bucket.
"""
LOGGER.info("Deleting object from bucket s3://%s/%s." % (bucket, prefix))
get_client("s3", region).delete_object(Bucket=bucket, Key=prefix)
def get_object_tags(s3_bucket, key, region=DEFAULT_REGION, version=None):
"""
Gets the tags for an object.
"""
payload = _build_version_payload(s3_bucket, key, version)
return get_client("s3", region).get_object_tagging(**payload)