def upload_full_image(fileId):
"""
Upload a full-resolution image to Amazon S3.
"""
l.debug("Transferring full-resolution photo %s to Amazon S3.", fileId)
dest = str(sender) + '/full/' + fileId
src = 'full/' + fileId
saved = s3.save(src, dest, verbose=True)
if saved:
l.info("Full-resolution photo %s transferred to Amazon S3.", fileId)
return saved
else:
l.error("Failed to transfer full-resolution photo %s to Amazon S3.", fileId)
return False
def upload_full_image(fileId):
"""
Upload a full-resolution image to Amazon S3.
"""
l.debug("Transferring full-resolution photo %s to Amazon S3.", fileId)
dest = str(sender) + '/full/' + fileId
src = 'full/' + fileId
saved = s3.save(src, dest, verbose=True)
if saved:
l.info("Full-resolution photo %s transferred to Amazon S3.", fileId)
return saved
else:
l.error("Failed to transfer full-resolution photo %s to Amazon S3.",
fileId)
return False
def transfer_to_s3(file_name):
"""
Transfer a thumbnail photo to Amazon S3 and return the S3 Key object.
"""
l.debug("Transferring thumbnail photo %s to Amazon S3 as %s.", file_name, key)
dest = str(sender) + '/thumbs/' + key
src = 'thumbs/' + key
thumb = generate_thumb(file_name, 'thumbs/' + key)
saved = s3.save(thumb, dest)
if saved:
l.info("Thumbnail photo %s transferred to Amazon S3 as %s", file_name, key)
return saved
else:
l.warning("Sending thumbnail photo %s to Amazon S3 failed. Schedling for resending.",
file_name)
sql = db.connect()
thumbs = sql['thumbs']
thumbs.insert(dict(key=key, src=thumb, dest=dest))
return False
def transfer_to_s3(file_name):
"""
Transfer a thumbnail photo to Amazon S3 and return the S3 Key object.
"""
l.debug("Transferring thumbnail photo %s to Amazon S3 as %s.", file_name,
key)
dest = str(sender) + '/thumbs/' + key
src = 'thumbs/' + key
thumb = generate_thumb(file_name, 'thumbs/' + key)
saved = s3.save(thumb, dest)
if saved:
l.info("Thumbnail photo %s transferred to Amazon S3 as %s", file_name,
key)
return saved
else:
l.warning(
"Sending thumbnail photo %s to Amazon S3 failed. Schedling for resending.",
file_name)
sql = db.connect()
thumbs = sql['thumbs']
thumbs.insert(dict(key=key, src=thumb, dest=dest))
return False
def execute(queue_url, message_body, receipt_handle):
logger.info("Message received")
queue = get_queue(queue_url)
msg = queue.Message(receipt_handle)
try:
# Parse and validate incoming message
validate_message(message_body)
body = json.loads(message_body)
session = get_session(body.get("RoleArn"))
client = session.client("s3")
cols, object_path, job_id = itemgetter('Columns', 'Object',
'JobId')(body)
input_bucket, input_key = parse_s3_url(object_path)
validate_bucket_versioning(client, input_bucket)
creds = session.get_credentials().get_frozen_credentials()
s3 = s3fs.S3FileSystem(key=creds.access_key,
secret=creds.secret_key,
token=creds.token,
default_cache_type='none',
requester_pays=True,
default_fill_cache=False,
version_aware=True)
# Download the object in-memory and convert to PyArrow NativeFile
logger.info("Downloading and opening %s object in-memory", object_path)
with s3.open(object_path, "rb") as f:
source_version = f.version_id
logger.info("Using object version %s as source", source_version)
infile = load_parquet(f)
# Write new file in-memory
logger.info("Generating new parquet file without matches")
out_sink, stats = delete_matches_from_file(infile, cols)
if stats["DeletedRows"] == 0:
raise ValueError(
"The object {} was processed successfully but no rows required deletion"
.format(object_path))
with pa.BufferReader(out_sink.getvalue()) as output_buf:
new_version = save(s3, client, output_buf, input_bucket, input_key,
source_version)
logger.info("New object version: %s", new_version)
verify_object_versions_integrity(client, input_bucket, input_key,
source_version, new_version)
if body.get("DeleteOldVersions"):
logger.info(
"Deleting object {} versions older than version {}".format(
input_key, new_version))
delete_old_versions(client, input_bucket, input_key, new_version)
msg.delete()
emit_deletion_event(body, stats)
except (KeyError, ArrowException) as e:
err_message = "Parquet processing error: {}".format(str(e))
handle_error(msg, message_body, err_message)
except IOError as e:
err_message = "Unable to retrieve object: {}".format(str(e))
handle_error(msg, message_body, err_message)
except MemoryError as e:
err_message = "Insufficient memory to work on object: {}".format(
str(e))
handle_error(msg, message_body, err_message)
except ClientError as e:
err_message = "ClientError: {}".format(str(e))
if e.operation_name == "PutObjectAcl":
err_message += ". Redacted object uploaded successfully but unable to restore WRITE ACL"
if e.operation_name == "ListObjectVersions":
err_message += ". Could not verify redacted object version integrity"
handle_error(msg, message_body, err_message)
except ValueError as e:
err_message = "Unprocessable message: {}".format(str(e))
handle_error(msg, message_body, err_message)
except DeleteOldVersionsError as e:
err_message = "Unable to delete previous versions: {}".format(str(e))
handle_error(msg, message_body, err_message)
except IntegrityCheckFailedError as e:
err_description, client, bucket, key, version_id = e.args
err_message = "Object version integrity check failed: {}".format(
err_description)
handle_error(msg, message_body, err_message)
rollback_object_version(
client,
bucket,
key,
version_id,
on_error=lambda err: handle_error(None, "{}", err,
"ObjectRollbackFailed", False))
except Exception as e:
err_message = "Unknown error during message processing: {}".format(
str(e))
handle_error(msg, message_body, err_message)
l.info("Uploaded %d full-resolution file(s) to Amazon S3.",
len(response))
elif response == False:
# API error
pass
else:
l.info("No full-resolution photos have been requested.")
# Then, send thumbnails that failed to send earlier
thumbs = sql['thumbs']
if len(thumbs) > 0:
l.info("Attempting to resend %d thumbnail photos to Amazon S3.",
len(thumbs))
successes = list()
for thumb in thumbs:
saved = s3.save(thumb['src'], thumb['dest'])
if saved:
l.info("Thumbnail photo %s transferred to Amazon S3 as %s",
thumb['src'], thumb['dest'])
successes.append(thumb['key'])
server.post_thumb(thumb['src'], saved, thumb['key'])
else:
l.warning(
"Sending thumbnail photo %s to Amazon S3 failed again.",
thumb['src'])
# Delete sent photos from the database
for success in successes:
thumbs.delete(key=success)
l.debug(
"Deleted thumbnail photo %s from the failures database.",
success)
l.info("Uploaded %d full-resolution file(s) to Amazon S3.",
len(response))
elif response == False:
# API error
pass
else:
l.info("No full-resolution photos have been requested.")
# Then, send thumbnails that failed to send earlier
thumbs = sql['thumbs']
if len(thumbs) > 0:
l.info("Attempting to resend %d thumbnail photos to Amazon S3.",
len(thumbs))
successes = list()
for thumb in thumbs:
saved = s3.save(thumb['src'], thumb['dest'])
if saved:
l.info("Thumbnail photo %s transferred to Amazon S3 as %s",
thumb['src'], thumb['dest'])
successes.append(thumb['key'])
server.post_thumb(thumb['src'], saved, thumb['key'])
else:
l.warning("Sending thumbnail photo %s to Amazon S3 failed again.",
thumb['src'])
# Delete sent photos from the database
for success in successes:
thumbs.delete(key=success)
l.debug("Deleted thumbnail photo %s from the failures database.", success)
else:
l.debug("No thumbnail photos need to be resent to Amazon S3.")