def test_uploads(self):
"""Simply tests the backend functions in `lib.uploads`.
Also tests `posts.get_upload` since this is only a simple wrapper
around the backend function.
"""
test_upload_dir = 'tests/upload_test_files/'
test_upload_files = [
join(test_upload_dir, f) for f in listdir(test_upload_dir)
if isfile(join(test_upload_dir, f))
]
# Create a GridFS object to test image deletion
grid = gridfs.GridFS(m.db, collection='uploads')
# Test each file in the upload directory
for f in test_upload_files:
uuid = get_uuid()
image = io.BytesIO(
open(f).read()
)
filename = process_upload(uuid, image)
self.assertEqual(filename, '{}.png'.format(uuid))
# Get the upload these are designed for being served directly by
# Flask. This is a Flask/Werkzeug response object
image = get_upload(filename)
self.assertTrue(grid.exists({'filename': filename}))
self.assertEqual(image.headers['Content-Type'], 'image/png')
# Test deletion
# Ensure file is present (it will be)
self.assertTrue(grid.exists({'filename': filename}))
# Delete the file and ensure it is not there through GridFS
delete_upload(filename)
# Ensure the file has gone
self.assertFalse(grid.exists({'filename': filename}))
# Ensure that if we load a non-image file a None value is returned
uuid = get_uuid()
image = io.BytesIO()
self.assertIsNone(process_upload(uuid, image))
def create_account(username, email, password):
"""Creates a new user account.
:param username: The new users user name
:type username: str
:param email: The new users e-mail address
:type email: str
:param password: The new users password un-hashed
:type password: str
:returns: The UID of the new user
:rtype: str or None
"""
username = username.lower()
email = email.lower()
try:
if check_username(username) and check_username_pattern(username) and \
check_email(email) and check_email_pattern(email):
# Get a new UUID for the user
uid = get_uuid()
user = {
'_id': uid,
'username': username.lower(),
'email': email.lower(),
'password': generate_password(password,
method='pbkdf2:sha256:2000',
salt_length=20),
'created': timestamp(),
'last_login': -1,
'active': False,
'banned': False,
'op': False,
'muted': False,
'about': "",
'score': 0,
'alerts_last_checked': -1,
# Set the TTL for a newly created user, this has to be Datetime
# object for MongoDB to recognise it. This is removed on
# activation.
'ttl': datetime.utcnow()
}
# Set all the tips for new users
for tip_name in k.VALID_TIP_NAMES:
user['tip_{}'.format(tip_name)] = True
# Insert the new user in to Mongo. If this fails a None will be
# returned
result = m.db.users.insert(user)
return uid if result else None
except DuplicateKeyError: # pragma: no cover
# Oh no something went wrong. Pass over it. A None will be returned.
pass
return None
def create_account(username, email, password):
"""Creates a new user account.
:param username: The new users user name
:type username: str
:param email: The new users e-mail address
:type email: str
:param password: The new users password un-hashed
:type password: str
:returns: The UID of the new user
:rtype: str or None
"""
username = username.lower()
email = email.lower()
try:
if (
check_username(username)
and check_username_pattern(username)
and check_email(email)
and check_email_pattern(email)
):
# Get a new UUID for the user
uid = get_uuid()
user = {
"_id": uid,
"username": username.lower(),
"email": email.lower(),
"password": generate_password(password, method="pbkdf2:sha256:2000", salt_length=20),
"created": timestamp(),
"last_login": -1,
"active": False,
"banned": False,
"op": False,
"muted": False,
"about": "",
"score": 0,
"alerts_last_checked": -1,
# Set the TTL for a newly created user, this has to be Datetime
# object for MongoDB to recognise it. This is removed on
# activation.
"ttl": datetime.utcnow(),
}
# Insert the new user in to Mongo. If this fails a None will be
# returned
result = m.db.users.insert(user)
return uid if result else None
except DuplicateKeyError: # pragma: no cover
# Oh no something went wrong. Pass over it. A None will be returned.
pass
return None
def create_post(uid, body):
"""Creates a new post
"""
# Get a new UUID for the pid
pid = get_uuid()
# Hash form for posts
# TODO this needs expanding to include some form of image upload hook
post = {
'pid': pid,
'uid': uid,
'body': body,
'created': timestamp(),
'score': 0
}
# Add post
r.hmset(K.POST.format(pid), post)
# Add post to users post list
r.lpush(K.USER_POSTS.format(uid), pid)
# Add post to authors feed
r.lpush(K.USER_FEED.format(uid), pid)
# Ensure the feed does not grow to large
r.ltrim(K.USER_FEED.format(uid), 0, 999)
# Append to all followers feeds
populate_feeds(uid, pid)
# Subscribe the poster to there post
subscribe(uid, pid, SubscriptionReasons.POSTER)
# TAGGING
# Create alert manager and alert
alert = TaggingAlert(uid, pid)
# Alert tagees
tagees = parse_tags(body)
# Store a list of uids which need to alerted to the tagging
tagees_to_alert = []
for tagee in tagees:
# Don't allow tagging yourself
if tagee[0] != uid:
# Subscribe the tagee to the alert
subscribe(tagee[0], pid, SubscriptionReasons.TAGEE)
# Add the tagee's uid to the list to alert them
tagees_to_alert.append(tagee[0])
# Alert the required tagees
AlertManager().alert(alert, tagees_to_alert)
return pid
def process_upload(upload, collection='uploads', image_size=(1280, 720),
thumbnail=True):
"""Processes the uploaded images in the posts and also the users avatars.
This should be extensible in future to support the uploading of videos,
audio, etc...
:param _id: The ID for the post, user or anything you want as the filename
:type _id: str
:param upload: The uploaded Werkzeug FileStorage object
:type upload: ``Werkzeug.datastructures.FileStorage``
:param collection: The GridFS collection to upload the file too.
:type collection: str
:param image_size: The max height and width for the upload
:type image_size: Tuple length 2 of int
:param thumbnail: Is the image to have it's aspect ration kept?
:type thumbnail: bool
"""
try:
# StringIO to take the uploaded image to transport to GridFS
output = io.BytesIO()
# All images are passed through PIL and turned in to PNG files.
# Will change if they need thumbnailing or resizing.
img = PILImage.open(upload)
if thumbnail:
img.thumbnail(image_size, PILImage.ANTIALIAS)
else:
# Pillow `resize` returns an image unlike thumbnail
img = img.resize(image_size, PILImage.ANTIALIAS)
img.save(output, format='PNG', quality=100)
# Return the file pointer to the start
output.seek(0)
# Create a new file name <uuid>.<upload_extension>
filename = '{0}.{1}'.format(get_uuid(), 'png')
# Place file inside GridFS
m.save_file(filename, output, base=collection)
return filename
except (IOError):
# File will not have been uploaded
return None
def create_user(username, email, password):
"""Creates a user account
"""
username = username.lower()
email = email.lower()
if check_username(username) and check_email(email) and \
check_username_pattern(username) and check_email_pattern(email):
# Create the user lookup keys. This LUA script ensures
# that the name can not be taken at the same time causing a race
# condition. This is also passed a UUID and will only return it if
# successful
uid = L.create_user(keys=[K.UID_USERNAME.format(username),
K.UID_EMAIL.format(email)],
args=[get_uuid()])
# Create user dictionary ready for HMSET only if uid is not None
# This will only be None in the event of a race condition which we cant
# really test for.
if uid is not None: # pragma: no branch
user = {
'uid': uid,
'username': username,
'email': email,
'password': generate_password(password),
'created': timestamp(),
'last_login': -1,
'active': 0,
'banned': 0,
'op': 0,
'muted': 0,
'about': "",
'score': 0,
'alerts_last_checked': 0
}
r.hmset(K.USER.format(uid), user)
# Set the TTL for the user account
r.expire(K.USER.format(uid), K.EXPIRE_24HRS)
return uid
# If none of this worked return nothing
return None
def create_post(user_id, username, body, reply_to=None, upload=None,
permission=k.PERM_PUBLIC):
"""Creates a new post
This handled both posts and what used to be called comments. If the
reply_to field is not None then the post will be treat as a comment.
You will need to make sure the reply_to post exists.
:param user_id: The user id of the user posting the post
:type user_id: str
:param username: The user name of the user posting (saves a lookup)
:type username: str
:param body: The content of the post
:type body: str
:param reply_to: The post id of the post this is a reply to if any
:type reply_to: str
:param upload:
:returns: The post id of the new post
:param permission: Who can see/interact with the post you are posting
:type permission: int
:rtype: str or None
"""
# Get a new UUID for the post_id ("_id" in MongoDB)
post_id = get_uuid()
# Get the timestamp, we will use this to populate users feeds
post_time = timestamp()
post = {
'_id': post_id, # Newly created post id
'user_id': user_id, # User id of the poster
'username': username, # Username of the poster
'body': body, # Body of the post
'created': post_time, # Unix timestamp for this moment in time
'score': 0, # Atomic score counter
}
if reply_to is not None:
# If the is a reply it must have this property
post['reply_to'] = reply_to
else:
# Replies don't need a comment count
post['comment_count'] = 0
# Set the permission a user needs to view
post['permission'] = permission
# TODO: Make the upload process better at dealing with issues
if upload:
# If there is an upload along with this post it needs to go for
# processing.
# process_upload() can throw an Exception of UploadError. We will let
# it fall through as a 500 is okay I think.
# TODO: Turn this in to a Queue task at some point
filename = process_upload(upload)
if filename is not None:
# If the upload process was okay attach the filename to the doc
post['upload'] = filename
else:
# Stop the image upload process here if something went wrong.
return None
# Process everything thats needed in a post
links, mentions, hashtags = parse_post(body)
# Only add the fields if we need too.
if links:
post['links'] = links
if mentions:
post['mentions'] = mentions
if hashtags:
post['hashtags'] = hashtags
# Add the post to the database
# If the post isn't stored, result will be None
result = m.db.posts.insert(post)
# Only carry out the rest of the actions if the insert was successful
if result:
if reply_to is None:
# Add post to authors feed
r.zadd(k.USER_FEED.format(user_id), post_time, post_id)
# Ensure the feed does not grow to large
r.zremrangebyrank(k.USER_FEED.format(user_id), 0, -1000)
# Subscribe the poster to there post
subscribe(user_id, post_id, SubscriptionReasons.POSTER)
# Alert everyone tagged in the post
alert_tagees(mentions, user_id, post_id)
# Append to all followers feeds or approved followers based
# on the posts permission
if permission < k.PERM_APPROVED:
populate_followers_feeds(user_id, post_id, post_time)
else:
populate_approved_followers_feeds(user_id, post_id, post_time)
else:
# To reduce database look ups on the read path we will increment
# the reply_to's comment count.
m.db.posts.update({'_id': reply_to},
{'$inc': {'comment_count': 1}})
# Alert all subscribers to the post that a new comment has been
# added. We do this before subscribing anyone new
alert = CommentingAlert(user_id, reply_to)
subscribers = []
# Iterate through subscribers and let them know about the comment
for subscriber_id in get_subscribers(reply_to):
# Ensure we don't get alerted for our own comments
if subscriber_id != user_id:
subscribers.append(subscriber_id)
# Push the comment alert out to all subscribers
AlertManager().alert(alert, subscribers)
# Subscribe the user to the post, will not change anything if they
# are already subscribed
subscribe(user_id, reply_to, SubscriptionReasons.COMMENTER)
# Alert everyone tagged in the post
alert_tagees(mentions, user_id, reply_to)
return post_id
# If there was a problem putting the post in to Mongo we will return None
return None # pragma: no cover
def process_upload(upload, collection='uploads', image_size=(1280, 720),
thumbnail=True):
"""Processes the uploaded images in the posts and also the users avatars.
This should be extensible in future to support the uploading of videos,
audio, etc...
:param _id: The ID for the post, user or anything you want as the filename
:type _id: str
:param upload: The uploaded Werkzeug FileStorage object
:type upload: ``Werkzeug.datastructures.FileStorage``
:param collection: The GridFS collection to upload the file too.
:type collection: str
:param image_size: The max height and width for the upload
:type image_size: Tuple length 2 of int
:param thumbnail: Is the image to have it's aspect ration kept?
:type thumbnail: bool
"""
try:
# StringIO to take the uploaded image to transport to GridFS
output = io.BytesIO()
# All images are passed through Wand and turned in to PNG files.
# Unless the image is a GIF and its format is kept
# Will change if they need thumbnailing or resizing.
img = Image(file=upload)
# If the input file if a GIF then we need to know
gif = True if img.format == 'GIF' else False
animated_gif = True if gif and len(img.sequence) > 1 else False
# Check the exif data.
# If there is an orientation then transform the image so that
# it is always looking up.
try:
exif_data = {
k[5:]: v
for k, v in img.metadata.items()
if k.startswith('exif:')
}
orientation = exif_data.get('Orientation')
orientation = int(orientation)
if orientation: # pragma: no branch
if orientation == 2:
img.flop()
elif orientation == 3:
img.rotate(180)
elif orientation == 4:
img.flip()
elif orientation == 5:
img.flip()
img.rotate(90)
elif orientation == 6:
img.rotate(90)
elif orientation == 7:
img.flip()
img.rotate(270)
elif orientation == 8:
img.rotate(270)
except (AttributeError, TypeError, AttributeError):
pass
if thumbnail:
# If the GIF was known to be animated then save the animated
# then cycle through the frames, transforming them and save the
# output
if animated_gif:
animated_image = Image()
for frame in img.sequence:
frame.transform(resize='{0}x{1}>'.format(*image_size))
# Directly append the frame to the output image
animated_image.sequence.append(frame)
animated_output = io.BytesIO()
animated_output.format = 'GIF'
animated_image.save(file=animated_output)
animated_output.seek(0)
img.transform(resize='{0}x{1}>'.format(*image_size))
else:
# Just sample the image to the correct size
img.sample(*image_size)
# Turn off animated GIF
animated_gif = False
img.format = 'PNG'
uuid = get_uuid()
filename = '{0}.{1}'.format(uuid, 'png')
# Return the file pointer to the start
img.save(file=output)
output.seek(0)
# Place file inside GridFS
m.save_file(filename, output, base=collection)
animated_filename = ''
if animated_gif:
animated_filename = '{0}.{1}'.format(uuid, 'gif')
m.save_file(animated_filename, animated_output, base=collection)
return filename, animated_filename
except (IOError, MissingDelegateError):
# File will not have been uploaded
return None, None
def create_account(username, email, password):
"""Creates a new user account.
:param username: The new users user name
:type username: str
:param email: The new users e-mail address
:type email: str
:param password: The new users password un-hashed
:type password: str
:returns: The UID of the new user
:rtype: str or None
"""
username = username.lower()
email = email.lower()
try:
if check_username(username) and check_username_pattern(username) and \
check_email(email) and check_email_pattern(email):
# Get a new UUID for the user
uid = get_uuid()
user = {
'_id':
uid,
'username':
username.lower(),
'email':
email.lower(),
'password':
generate_password(password,
method='pbkdf2:sha256:2000',
salt_length=20),
'created':
timestamp(),
'last_login':
-1,
'active':
False,
'banned':
False,
'op':
False,
'muted':
False,
'about':
"",
'score':
0,
'alerts_last_checked':
-1,
# Set the TTL for a newly created user, this has to be Datetime
# object for MongoDB to recognise it. This is removed on
# activation.
'ttl':
datetime.utcnow()
}
# Set all the tips for new users
for tip_name in k.VALID_TIP_NAMES:
user['tip_{}'.format(tip_name)] = True
# Insert the new user in to Mongo. If this fails a None will be
# returned
result = m.db.users.insert(user)
return uid if result else None
except DuplicateKeyError: # pragma: no cover
# Oh no something went wrong. Pass over it. A None will be returned.
pass
return None
def __init__(self, user_id):
self.alert_id = get_uuid()
self.timestamp = timestamp()
self.user_id = user_id
def create_comment(uid, pid, body):
"""Create a new comment
"""
# Get a new UUID for the cid
cid = get_uuid()
# Form for comment hash
comment = {
'cid': cid,
'uid': uid,
'pid': pid,
'body': body,
'created': timestamp(),
'score': 0
}
# Add comment
r.hmset(K.COMMENT.format(cid), comment)
# Add comment to posts comment list
r.lpush(K.POST_COMMENTS.format(pid), cid)
# Add comment to users comment list
# This may seem redundant but it allows for perfect account deletion
# Please see Issue #3 on Github
r.lpush(K.USER_COMMENTS.format(uid), cid)
# COMMENT ALERTING
# Alert all subscribers to the post that a new comment has been added.
# We do this before subscribing anyone new
# Create alert manager and alert
alert = CommentingAlert(uid, pid)
subscribers = []
# Iterate through subscribers and let them know about the comment
for subscriber in get_subscribers(pid):
# Ensure we don't get alerted for our own comments
if subscriber != uid:
subscribers.append(subscriber)
# Push the comment alert out to all subscribers
AlertManager().alert(alert, subscribers)
# Subscribe the user to the post, will not change anything if they are
# already subscribed
subscribe(uid, pid, SubscriptionReasons.COMMENTER)
# TAGGING
# Create alert
alert = TaggingAlert(uid, pid)
# Subscribe tagees
tagees = parse_tags(body)
tagees_to_alert = []
for tagee in tagees:
# Don't allow tagging yourself
if tagee[0] != uid:
subscribe(tagee[0], pid, SubscriptionReasons.TAGEE)
tagees_to_alert.append(tagee[0])
# Get an alert manager to notify all tagees
AlertManager().alert(alert, tagees_to_alert)
return cid
def process_upload(upload,
collection='uploads',
image_size=(1280, 720),
thumbnail=True):
"""Processes the uploaded images in the posts and also the users avatars.
This should be extensible in future to support the uploading of videos,
audio, etc...
:param _id: The ID for the post, user or anything you want as the filename
:type _id: str
:param upload: The uploaded Werkzeug FileStorage object
:type upload: ``Werkzeug.datastructures.FileStorage``
:param collection: The GridFS collection to upload the file too.
:type collection: str
:param image_size: The max height and width for the upload
:type image_size: Tuple length 2 of int
:param thumbnail: Is the image to have it's aspect ration kept?
:type thumbnail: bool
"""
try:
# StringIO to take the uploaded image to transport to GridFS
output = io.BytesIO()
# All images are passed through PIL and turned in to PNG files.
# Will change if they need thumbnailing or resizing.
img = PILImage.open(upload)
# Check the exif data.
# If there is an orientation then transform the image so that
# it is always looking up.
try:
exif_data = {
ExifTags.TAGS[k]: v
for k, v in img._getexif().items() if k in ExifTags.TAGS
}
orientation = exif_data.get('Orientation')
if orientation: # pragma: no branch
if orientation == 2:
img = img.transpose(PILImage.FLIP_LEFT_RIGHT)
if orientation == 3:
img = img.transpose(PILImage.ROTATE_180)
elif orientation == 4:
img = img.transpose(PILImage.FLIP_TOP_BOTTOM)
elif orientation == 5:
img = img.transpose(PILImage.FLIP_TOP_BOTTOM)
img = img.transpose(PILImage.ROTATE_270)
elif orientation == 6:
img = img.transpose(PILImage.ROTATE_270)
elif orientation == 7:
img = img.transpose(PILImage.FLIP_TOP_BOTTOM)
img = img.transpose(PILImage.ROTATE_90)
elif orientation == 8:
img = img.transpose(PILImage.ROTATE_90)
except AttributeError:
pass
if thumbnail:
img.thumbnail(image_size, PILImage.ANTIALIAS)
else:
# Pillow `resize` returns an image unlike thumbnail
img = img.resize(image_size, PILImage.ANTIALIAS)
img.save(output, format='PNG', quality=100)
# Return the file pointer to the start
output.seek(0)
# Create a new file name <uuid>.<upload_extension>
filename = '{0}.{1}'.format(get_uuid(), 'png')
# Place file inside GridFS
m.save_file(filename, output, base=collection)
return filename
except (IOError):
# File will not have been uploaded
return None
def __init__(self, user_id):
self.alert_id = get_uuid()
self.timestamp = timestamp()
self.user_id = user_id