def account():
""" User account management """
# If request is sent via "POST"
if request.method == "POST":
# Check if form fields are not empty
if not request.form.get("email"):
flash("Must provide e-mail.", "danger")
return redirect(url_for('account'))
# Checks if provided email exists in database
cursor = conn.execute("SELECT * FROM users WHERE email=:email",
{"email": request.form.get("email")})
row = cursor.fetchone()
if row:
flash("E-mail is already being used.", "danger")
return redirect(url_for('account'))
user = {}
user["id"] = session["user_id"]
user["email"] = request.form.get("email")
# Update user info on database
update_account(conn, user)
# Redirects to user account
flash("E-mail updated!", "success")
return redirect(url_for('account'))
# Else, render account page
else:
user = get_user_profile(conn, session["user_id"])
return render_template("account.html", user=user)
def account_delete():
""" Enables a user to delete account """
if request.method == "POST":
# Delete post from database
delete_account(app, conn, session["user_id"])
# Forgets session id
session.clear()
# Redirects to register page
flash("Account deleted!", "success")
return redirect(url_for('register'))
else:
# Get user
user = get_user_profile(conn, session["user_id"])
content = {}
content["id"] = user["id"]
content["type"] = "Account"
content["delete_url"] = "/account/delete"
content["cancel_url"] = "/account"
return render_template("delete.html", content=content)
def main():
# config aws service
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sqs.html#SQS.Client.receive_message
try:
sqs = boto3.resource('sqs',region_name=AwsRegionName)
queue = sqs.get_queue_by_name(QueueName=AWS_SQS_JOB_COMPLETE_QUEUE)
except botocore.exceptions.UnknownServiceError as e:
print("UnknownServiceError.")
return
except boto3.exceptions.ResourceNotExistsError as e:
print("ResourceNotExistsError.")
return
except:
print("UnknownError.")
return
while True:
# Get messages
messages = queue.receive_messages(
AttributeNames=['All'],
MaxNumberOfMessages=10,
WaitTimeSeconds=5,
)
if len(messages) > 0:
print("Received {0} messages...".format(str(len(messages))))
# Iterate each message
for message in messages:
# Parse JSON message (going two levels deep to get the embedded message)
sqs_body = json.loads(message.body)
msg_body = json.loads(sqs_body['Message'])
# extract job information from msg_body
job_id = msg_body['job_id']
user_id = msg_body['user_id']
job_detail_url = JOB_DETAIL_URL_PREFIX + job_id
user_profile = helpers.get_user_profile(id = user_id, db_name =config['postgresql']['DB_NAME'])
recipient_email = user_profile['email']
recipient_name = user_profile['name']
# send email
sender_email = SENDER_EMAIL
email_subject = "Job Completed"
email_body = "Hi {},\n\n The job {} has completed. Check detail here: {}".format(recipient_name, job_id, job_detail_url)
# send the email
helpers.send_email_ses(recipients=recipient_email, sender=sender_email, subject=email_subject, body=email_body)
# Delete the message from the queue
print ("Deleting message...")
message.delete()
def user_profile():
"""Get a Steam user's profile information from Steam API as JSON."""
# Make sure (Steam) API key is set, just in case something has gone horribly wrong
if not os.environ.get("API_KEY"):
raise RuntimeError("API Key not set.")
# Make sure an id was provided
if not request.args.get("id"):
# If wasn't, return an error value (HTTP "No Content")
return "", NO_CONTENT
return get_user_profile(request.args.get("id"))
def profile(userid):
""" Render profile page """
# Get user profile
user = get_user_profile(conn, userid)
# Get connection between profile user and logged user
connection = get_connection(conn, session["user_id"], int(userid))
# Get location
location = get_location(conn, userid)
# Get user occupations
occupations = get_all_occupations(conn, userid)
# Get instruments
instruments = get_all_instruments(conn, userid)
# Get pages
pages = get_all_pages(conn, userid)
return render_template("profile.html", user=user, connection=connection, location=location, occupations=occupations, instruments=instruments, pages=pages)
def profile_edit(userid):
""" Handles profile update """
if request.method == "POST":
# Check if name fields are not empty
if not request.form.get("first"):
flash("Must provide first name.", "danger")
return redirect("/profile/" + userid + "/edit")
if not request.form.get("last"):
flash("Must provide last name.", "danger")
return redirect("/profile/" + userid + "/edit")
# Creates profile object and stores form data and session id
profile = {}
profile["id"] = userid
# Get form data
profile["first"] = request.form.get("first")
profile["last"] = request.form.get("last")
profile["birth"] = request.form.get("birth")
profile["gender"] = request.form.get("gender")
profile["bio"] = request.form.get("bio")
if profile["bio"] == '':
bio = None
# Checks if the user provided a picture, saves new picture, deletes old picture and stores new name
if "picture" in request.files:
picture = request.files["picture"]
picture_name = save_picture(app, picture)
old_picture_name = conn.execute("SELECT picture FROM profiles WHERE id=:userid", {'userid': session['user_id']}).fetchone()[0]
if old_picture_name != 'default.jpg':
delete_picture(app, old_picture_name)
else:
picture_name = conn.execute("SELECT picture FROM profiles WHERE id=:userid", {'userid': session['user_id']}).fetchone()[0]
profile["picture"] = picture_name
# Update database with form data
update_profile(conn, profile)
# Update indexes virtual table
update_indexes_user(conn, session["user_id"], profile["first"], profile["last"])
# Redirect user to profile page
flash("Your profile has been updated.", "success")
return redirect("/profile/" + userid + "/edit")
else:
if str(session["user_id"]) != userid:
return redirect("/profile/" + userid + "/edit")
# Get all user info for requested profile
user = get_user_profile(conn, userid)
# Location
location = get_location(conn, userid)
# User occupations
occupations = get_all_occupations(conn, session["user_id"])
# Get instruments and proficiency levels
instruments = get_all_instruments(conn, session["user_id"])
proficiency = get_proficiency(conn)
# Get pages and platforms
pages = get_all_pages(conn, session["user_id"])
platforms = get_platforms(conn)
return render_template("profile_edit.html", user=user, location=location, occupations=occupations, instruments=instruments, proficiency=proficiency, pages=pages, platforms=platforms)
def main():
# config aws service
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sqs.html#SQS.Client.receive_message
try:
s3 = boto3.client('s3', region_name=AwsRegionName)
glacier = boto3.client('glacier', region_name=AwsRegionName)
sqs = boto3.resource('sqs', region_name=AwsRegionName)
queue = sqs.get_queue_by_name(QueueName=SQS_Archive_Name)
dynamo = boto3.resource('dynamodb', region_name=AwsRegionName)
table = table = dynamo.Table(TableName)
except botocore.exceptions.UnknownServiceError as e:
print("UnknownServiceError.")
return
except boto3.exceptions.ResourceNotExistsError as e:
print("ResourceNotExistsError.")
return
except botocore.exceptions.ClientError as e:
print("TableNotExistsError.")
return
except:
print("UnknownError.")
return
while True:
# Get messages
messages = queue.receive_messages(
AttributeNames=['All'],
MaxNumberOfMessages=10,
WaitTimeSeconds=5,
)
if len(messages) > 0:
print("Received {0} messages...".format(str(len(messages))))
# Iterate each message
for message in messages:
# Parse JSON message (going two levels deep to get the embedded message)
sqs_body = json.loads(message.body)
msg_body = json.loads(sqs_body['Message'])
# extract job information from msg_body
job_id = msg_body['job_id']
user_id = msg_body['user_id']
s3_key_result_file = msg_body['s3_key_result_file']
print("jobid: ", job_id)
# get user profile
user_profile = helpers.get_user_profile(
id=user_id, db_name=config['postgresql']['DB_NAME'])
print("profile: ", user_profile)
print(type(user_profile))
print("role: ", user_profile['role'])
# premium user
if user_profile['role'] == "premium_user":
print("Deleting message...")
message.delete()
continue
# get result file from s3
results_bucket = Results_Bucket
try:
result_file = s3.get_object(Bucket=results_bucket,
Key=s3_key_result_file)
log_content = result_file['Body'].read()
except botocore.exceptions.ClientError as e:
print(e)
return
# save file to glacier
try:
glacier_vault_name = Glacier_Vault_Name
response = glacier.upload_archive(
vaultName=glacier_vault_name, body=log_content)
print(response)
results_file_archive_id = response['ResponseMetadata'][
'HTTPHeaders']['x-amz-archive-id']
except:
print("Cannot upload file to glacier.")
return
# update database
try:
response = table.update_item(
Key={
'job_id': job_id,
},
UpdateExpression=
"set results_file_archive_id =:i, archived=:a",
ExpressionAttributeValues={
':i': results_file_archive_id,
':a': True,
},
ReturnValues="UPDATED_NEW")
except botocore.exceptions.ClientError as e:
print("Cannot update db.")
except:
print("Unknow Error.")
# delete the result file from s3
try:
s3.delete_object(Bucket=results_bucket,
Key=s3_key_result_file)
print("result deleted from s3")
except ClientError as e:
print(e)
# Delete the message from the queue
print("Deleting message...")
message.delete()
def relocate_result():
# Connect to SQS and get the archive message queue, and connect to s3
try:
sqs = boto3.client('sqs', region_name=config['aws']['AwsRegionName'])
queue_url = config['aws']['SQS_ARCHIVE_URL']
except boto3.exceptions.ResourceNotExistsError as e:
print(e)
while True:
# long polling the messages with wait time seconds set to 20s
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sqs.html#SQS.Client.receive_message
try:
result_response = sqs.receive_message(
QueueUrl=queue_url,
AttributeNames=['SentTimestamp'],
MaxNumberOfMessages=5,
WaitTimeSeconds=20)
except botocore.errorfactory.QueueDoesNotExist as e:
print(e)
continue
try:
messages = result_response['Messages']
if len(messages) == 0:
continue
except KeyError:
continue
for message in messages:
user_id, receipt_handle, job_id, s3_key_annot_file = extract_info(
message)
# retrieve user related info
profile = helpers.get_user_profile(
id=user_id, db_name=config['postgre']['DB_NAME'])
# check if the user is still a free user
if profile['role'] == "premium_user":
delete_message(sqs, queue_url, receipt_handle)
continue
# retreive the result file
# Create a session client to the S3 service
s3 = boto3.client('s3', region_name=config['aws']['AwsRegionName'])
bucket_name = config['aws']['AWS_S3_RESULTS_BUCKET']
# get the log file's object and read it
try:
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.get_object
result_file = s3.get_object(Bucket=bucket_name,
Key=s3_key_annot_file)
content = result_file['Body'].read()
except botocore.exceptions.ClientError as e:
print(e)
# relocate the result file from s3 to glacier
try:
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/glacier.html#Glacier.Client.upload_archive
glacier = boto3.client(
'glacier', region_name=config['aws']['AwsRegionName'])
vault = config['aws']['VAULT_NAME']
response = glacier.upload_archive(vaultName=vault,
body=content)
archive_id = response['ResponseMetadata']['HTTPHeaders'][
'x-amz-archive-id']
except (glacier.exceptions.ResourceNotFoundException, glacier.exceptions.InvalidParameterValueException,\
glacier.exceptions.MissingParameterValueException,glacier.exceptions.RequestTimeoutException,\
glacier.exceptions.ServiceUnavailableException) as e:
print(e)
# update the status in dynamodb (add archive_id attribute and set the result_file existence attribute)
# connect to the dynamoDB
try:
dynamodb = boto3.resource(
'dynamodb', region_name=config['aws']['AwsRegionName'])
table_name = config['aws']['DYNAMODB_TABLE_NAME']
ann_table = dynamodb.Table(table_name)
except (ClientError, boto3.exceptions.ResourceNotExistsError) as e:
print(e)
try:
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#DynamoDB.Client.update_item
response = ann_table.update_item(
Key={'job_id': job_id},
UpdateExpression=
"set results_file_archive_id =:r, existed = :e",
ExpressionAttributeValues={
':r': archive_id,
':e': "False"
},
)
except (ClientError) as e:
print(e.response['Error']['Message'])
continue
# delete the result file in s3
try:
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.delete_object
s3.delete_object(Bucket=bucket_name, Key=s3_key_annot_file)
except ClientError as e:
print(e)
# delete the message
delete_message(sqs, queue_url, receipt_handle)
def main(argv=None):
# Connect to SQS and get the message queue
sqs = boto3.resource('sqs', region_name=config['aws']['AwsRegionName'])
queue = sqs.get_queue_by_name(QueueName=config['sqs']['QueueName'])
# Dynamodb for accessing data
dynamodb = boto3.resource('dynamodb',
region_name=config['aws']['AwsRegionName'])
table = dynamodb.Table(config['db']['Name'])
# Poll the message queue in a loop
while True:
# Attempt to read a message from the queue
'''
Reference:
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sqs.html#SQS.Queue.receive_messages
'''
messages = queue.receive_messages(
AttributeNames=['All'],
MaxNumberOfMessages=int(config['sqs']['MaxNumMsg']),
VisibilityTimeout=int(config['sqs']['VisibilityTimeout']),
WaitTimeSeconds=int(config['sqs']['WaitTime']))
# If message read, extract job parameters from the msg body
if len(messages) > 0:
print("Received {0} messages...".format(str(len(messages))))
# Iterate each message and send notification Emails
for msg in messages:
# Get job_id from msg
job_id = json.loads(msg.body)['Message']
# Get user_id from database
'''
Reference:
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#DynamoDB.Table.get_item
'''
try:
response = table.get_item(Key={'job_id': job_id}, )
except ClientError as err1:
logging.error(err1)
print(err1)
except KeyError as err2:
logging.error(err2)
print(err2)
# Get information from dynamodb
s3_results_bucket = response['Item']['s3_results_bucket']
s3_key_log_file = response['Item']['s3_key_log_file']
s3_key_result_file = response['Item']['s3_key_result_file']
user_id = response['Item']['user_id']
# Get recipients Email
user_profile = helpers.get_user_profile(user_id)
email = user_profile['email']
# Send notification Email
msg_subject = config['msg']['Msg1'] + " " + job_id
msg_body = config['msg']['Msg1'] + '\n' \
+ 'job_id: ' + job_id + '\n' \
+ 'user_id: ' + user_id + '\n' \
+ 's3_results_bucket: ' + s3_results_bucket + '\n' \
+ 's3_key_log_file: ' + s3_key_log_file + '\n' \
+ 's3_key_result_file: '+ s3_key_result_file
helpers.send_email_ses(recipients=email,
subject=msg_subject,
body=msg_body)
# Delete the message from the queue
try:
msg.delete()
except Exception as e:
logging.error(e)
print(e)
def archive_results():
# Connect to SQS and get the results
sqs = boto3.resource('sqs', region_name=config['aws']['AwsRegionName'])
queue = sqs.get_queue_by_name(QueueName=config['sqs']['ArchiveQueueName'])
# Dynamodb Table
dynamodb = boto3.resource('dynamodb',
region_name=config['aws']['AwsRegionName'])
table = dynamodb.Table(config['dynamodb']['TableName'])
# Poll the result queue in a loop
while True:
# Attempt to read a message from the queue
'''
Reference:
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sqs.html#SQS.Queue.receive_messages
'''
messages = queue.receive_messages(
AttributeNames=['All'],
MaxNumberOfMessages=int(config['sqs']['MaxNumberOfMessages']),
VisibilityTimeout=int(config['sqs']['VisibilityTimeout']),
WaitTimeSeconds=int(config['sqs']['WaitTimeSeconds']))
# If message read, extract job parameters from the msg body
if len(messages) > 0:
print("Received {0} messages...".format(str(len(messages))))
# Iterate through each message and archive result for free user
for msg in messages:
# Get job_id from msg
job_id = json.loads(msg.body)['Message']
# Get user_id from database
'''
Reference:
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#DynamoDB.Table.get_item
'''
try:
response = table.get_item(Key={'job_id': job_id})
except ClientError as err:
print(err)
# Get user_id from dynamodb and get_user_profile
user_id = response['Item']['user_id']
s3_key_result_file = response['Item']['s3_key_result_file']
user_profile = helpers.get_user_profile(user_id)
# Archive free user data (result file) to Glacier
if user_profile['role'] == 'free_user':
# Archive result files 5 mins later
'''
Reference:
https://docs.python.org/3/library/sched.html
'''
ct = float(response['Item']['ct'])
at = ct + float(
config['parameters']['FreeUserDataRetention'])
s = sched.scheduler(time.time, time.sleep)
s.enterabs(at, 1, archive_job,
(job_id, s3_key_result_file, table))
s.run()
# Delete the message from the queue
try:
msg.delete()
except Exception as err:
print(err)
def poll_result_queue():
# Connect to SQS and get the message queue, and connect to s3
try:
sqs = boto3.client('sqs', region_name=config['aws']['AwsRegionName'])
queue_url = config['aws']['SQS_RESULT_URL']
except boto3.exceptions.ResourceNotExistsError as e:
print(e)
while True:
# long polling the messages with wait time seconds set to 20s
try:
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sqs.html#SQS.Client.receive_message
result_response = sqs.receive_message(
QueueUrl=queue_url,
AttributeNames=[
'SentTimestamp'
],
MaxNumberOfMessages=5,
WaitTimeSeconds=20
)
except botocore.errorfactory.QueueDoesNotExist as e:
print(e)
continue
try:
messages = result_response['Messages']
if len(messages) ==0:
continue
except KeyError:
continue
for message in messages:
user_id,receipt_handle, job_id = extract_info(message)
# retrieve user related info
profile = helpers.get_user_profile(id = user_id, db_name =config['postgre']['DB_NAME'])
print(profile)
print(type(profile))
user_email = profile['email']
user_name = profile['name']
print(user_id+" "+job_id+" "+user_name+" "+user_email)
# construct email related info
sender = config['aws']['EMAIL_SENDER']
recipients = user_email
subject = "Job Finished"
detail_url = "https://xsunan.ucmpcs.org/annotations/{}".format(job_id)
body = "Dear {},\nYour submitted job {} has completed.\
You can check via the below website.\n{}".format(user_name, job_id,detail_url)
# send the email
helpers.send_email_ses(recipients=recipients, sender = sender, subject=subject, body = body)
# delete the message
try:
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sqs.html#SQS.Client.delete_message
dele_rsp = sqs.delete_message(
QueueUrl = queue_url,
ReceiptHandle = receipt_handle
)
except (boto3.exceptions.ClientError,SQS.Client.exceptions.InvalidIdFormat, SQS.Client.exceptions.ReceiptHandleIsInvalid) as e:
print(e)