def main():
# connect to SQS
sqs = aws.get_sqs()
queue_name = aws.SQS_JOB_RESULTS_QUEUE
queue = aws.get_queue(sqs, queue_name)
# loop for SQS messages
while True:
print("Asking SQS for up to 10 messages...")
# Get messages
messages = queue.receive_messages(WaitTimeSeconds=10)
if len(messages) > 0:
print("Received {0} messages...".format(str(len(messages))))
# Iterate each message
for message in messages:
# get job data
job_id = json.loads(json.loads(message.body)['Message'])
print("Job id: " + str(job_id))
# connect to the database
table_name = aws.DYNAMODB_ANNOTATIONS_TABLE
table = aws.get_table(table_name)
# get annotation details
annotation_details = aws.get_annotation_details(table, job_id)
# send notification using ses
print("Notifying user job complete...")
ses = aws.get_ses()
aws.notify_user(ses, annotation_details)
# Delete the message from the queue
print ("Deleting message...")
message.delete()
def main():
# connect to SQS
sqs = aws.get_sqs()
queue_name = aws.SQS_THAW_REQUESTS_QUEUE
queue = aws.get_queue(sqs, queue_name)
# loop for SQS messages
while True:
print("Asking SQS for up to 10 messages...")
# Get messages
messages = queue.receive_messages(WaitTimeSeconds=10)
if len(messages) > 0:
print("Received {0} messages...".format(str(len(messages))))
# Iterate each message
for message in messages:
# get user id
user_id = json.loads(json.loads(message.body)['Message'])
print("User id: " + str(user_id))
# get glacier connection
glacier = aws.get_glacier_client()
# connect to the database
table_name = aws.DYNAMODB_ANNOTATIONS_TABLE
table = aws.get_table(table_name)
# get annotations for user
annotations = aws.get_annotations(table, user_id)
print("Identified annotations in db for user.")
# loop through each annotation
for annotation in annotations:
# get the job id
job_id = annotation['job_id']
print('Evaluating job id: ' + str(job_id))
# set user role to premium
user_role = 'premium_user'
aws.set_user_role(table, job_id, user_role)
# check if annotation is archived in glacier
if annotation['result_file_location'] == 'Glacier':
# create a request to thaw
print("Data for this job is in Glacier.")
archive_id = annotation['results_file_archive_id']
tier = 'Expedited' # will auto-try Standard if Expeditied fails
aws.request_restore(glacier, archive_id, tier)
# update annotations db - show that file is being restored
result_file_location = 'restoring from archive...'
aws.set_result_file_location(table, job_id,
result_file_location)
else:
print("Data for this job is not in Glacier.")
# Delete the message from the queue (only if processed)
print("Deleting message...")
message.delete()
def main():
# connect to SQS
sqs = aws.get_sqs()
queue_name = aws.SQS_RESTORE_RESULTS_QUEUE
queue = aws.get_queue(sqs, queue_name)
# loop for SQS messages
while True:
print("Asking SQS for up to 10 messages...")
# Get messages
messages = queue.receive_messages(WaitTimeSeconds=10)
if len(messages) > 0:
print("Received {0} messages...".format(str(len(messages))))
# Iterate each message
for message in messages:
# get glacier restore job id
message_data = json.loads(json.loads(message.body)['Message'])
restore_job_id = message_data['JobId']
print("Restore job id: " + restore_job_id)
# get archive data
glacier = aws.get_glacier_client()
archive_data = aws.get_archive_data(glacier, restore_job_id)
# get original annotation job id - stored as archive description
job_id = archive_data['archiveDescription']
# get annotations table
table_name = aws.DYNAMODB_ANNOTATIONS_TABLE
table = aws.get_table(table_name)
# get annotation details for job
annotation_details = aws.get_annotation_details(table, job_id)
# put archive data in S3
s3 = aws.get_s3()
bucket = aws.S3_RESULTS_BUCKET
key = annotation_details['s3_key_result_file']
data = archive_data['body']
aws.put_data_in_S3(s3, bucket, key, data)
# update annotations db - set result file location as S3
result_file_location = 'S3'
aws.set_result_file_location(table, job_id,
result_file_location)
print("Restore process complete.")
# Delete the message from the queue (only if processed)
print("Deleting message...")
message.delete()
def main():
# connect to SQS
sqs = aws.get_sqs()
queue_name = aws.SQS_JOB_REQUESTS_QUEUE
queue = aws.get_queue(sqs, queue_name)
# loop for SQS messages
while True:
print("Asking SQS for up to 10 messages...")
# Get messages
messages = queue.receive_messages(WaitTimeSeconds=10)
if len(messages) > 0:
print("Received {0} messages...".format(str(len(messages))))
# Iterate each message
for message in messages:
job_data = json.loads(json.loads(message.body)['Message'])
print("Job data received:\n" + str(job_data))
print("Starting job...")
run_job(job_data)
# Delete the message from the queue
print("Deleting message...")
message.delete()
def main():
# connect to SQS
sqs = aws.get_sqs()
queue_name =aws.SQS_ARCHIVE_REQUESTS_QUEUE
queue = aws.get_queue(sqs, queue_name)
# loop for SQS messages
while True:
print("Asking SQS for up to 10 messages...")
# Get messages
messages = queue.receive_messages(WaitTimeSeconds=10)
if len(messages) > 0:
print("Received {0} messages...".format(str(len(messages))))
# Iterate each message
for message in messages:
# get job id
job_id = json.loads(json.loads(message.body)['Message'])
print("Job id: " + str(job_id))
# connect to the database
table_name = aws.DYNAMODB_ANNOTATIONS_TABLE
table = aws.get_table(table_name)
# get annotation details
annotation_details = aws.get_annotation_details(table, job_id)
# make sure user role is still free
if not is_free_user(annotation_details):
print("User role is no longer free.")
# Delete the message from the queue (no need to archive)
print ("Deleting message...")
message.delete()
# check if ready to be archived
elif not is_time_to_archive(annotation_details):
print("Not enough time has elapsed to archive this file.")
else:
# claim the archive job
if aws.claim_archive_job(table, job_id):
print("Archiving result file...")
# get s3
s3 = aws.get_s3()
# get Glacier vault
glacier_resource = aws.get_glacier_resource()
vault = aws.get_vault(glacier_resource)
# archive the result file
archive_id = aws.archive_result_file(s3, vault, annotation_details)
print("Result file archived")
# update archive status
result_file_location = 'Glacier'
aws.set_result_file_location(table, job_id, result_file_location)
aws.set_archive_id(table, job_id, archive_id)
print("Archive status updated in annotations database.")
# delete s3 result file
bucket = annotation_details['s3_results_bucket']
key = annotation_details['s3_key_result_file']
aws.delete_s3_file(s3, bucket, key)
print("S3 result file deleted")
# Delete the message from the queue (only if processed)
print ("Deleting message...")
message.delete()