def upload_from_www(form):
"""Upload a photo from the Internet.
This requires the `form` object, but not necessarily Flask's. It just has to
be a dict with the form keys for the upload.
Returns the same structure as `upload_from_pc()`.
"""
url = form.get("url")
if not url or not allowed_filetype(url):
return dict(success=False, error="Invalid file extension.")
# Make a temp filename for it.
filetype = url.rsplit(".", 1)[1]
tempfile = "{}/rophako-photo-{}.{}".format(Config.site.tempdir, int(time.time()), filetype)
logger.debug("Save incoming photo to: {}".format(tempfile))
# Grab the file.
try:
data = requests.get(url).content
except:
return dict(success=False, error="Failed to get that URL.")
fh = open(tempfile, "wb")
fh.write(data)
fh.close()
# All good so far. Process the photo.
return process_photo(form, tempfile)
def crop_photo(key, x, y, length):
"""Change the crop coordinates of a photo and re-crop it."""
index = get_index()
if not key in index["map"]:
raise Exception("Can't crop photo: doesn't exist!")
album = index["map"][key]
# Sanity check.
if not album in index["albums"]:
raise Exception("Can't find photo in album!")
logger.debug("Recropping photo {}".format(key))
# Delete all the images except the large one.
for size in ["thumb", "avatar"]:
pic = index["albums"][album][key][size]
logger.debug("Delete {} size: {}".format(size, pic))
os.unlink(os.path.join(Config.photo.root_private, pic))
# Regenerate all the thumbnails.
large = index["albums"][album][key]["large"]
source = os.path.join(Config.photo.root_private, large)
for size in ["thumb", "avatar"]:
pic = resize_photo(source, size, crop=dict(
x=x,
y=y,
length=length,
))
index["albums"][album][key][size] = pic
# Save changes.
write_index(index)
def crop_photo(key, x, y, length):
"""Change the crop coordinates of a photo and re-crop it."""
index = get_index()
if not key in index["map"]:
raise Exception("Can't crop photo: doesn't exist!")
album = index["map"][key]
# Sanity check.
if not album in index["albums"]:
raise Exception("Can't find photo in album!")
logger.debug("Recropping photo {}".format(key))
# Delete all the images except the large one.
for size in ["thumb", "avatar"]:
pic = index["albums"][album][key][size]
logger.debug("Delete {} size: {}".format(size, pic))
os.unlink(os.path.join(Config.photo.root_private, pic))
# Regenerate all the thumbnails.
large = index["albums"][album][key]["large"]
source = os.path.join(Config.photo.root_private, large)
for size in ["thumb", "avatar"]:
pic = resize_photo(source, size, crop=dict(
x=x,
y=y,
length=length,
))
index["albums"][album][key][size] = pic
# Save changes.
write_index(index)
def commit(document, data, cache=True):
"""Insert/update a document in the DB."""
# Only allow one commit at a time.
lock = lock_cache(document)
# Need to create the file?
path = mkpath(document)
if not os.path.isfile(path):
parts = path.split("/")
parts.pop() # Remove the file part
directory = list()
# Create all the folders.
for part in parts:
directory.append(part)
segment = "/".join(directory)
if len(segment) > 0 and not os.path.isdir(segment):
logger.debug("JsonDB: mkdir {}".format(segment))
os.mkdir(segment, 0o755)
# Write the JSON.
write_json(path, data)
# Update the cached document.
if cache:
set_cache(document, data, expires=cache_lifetime)
set_cache(document+"_mtime", time.time(), expires=cache_lifetime)
# Release the lock.
unlock_cache(lock)
def commit(document, data, cache=True):
"""Insert/update a document in the DB."""
# Only allow one commit at a time.
lock = lock_cache(document)
# Need to create the file?
path = mkpath(document)
if not os.path.isfile(path):
parts = path.split("/")
parts.pop() # Remove the file part
directory = list()
# Create all the folders.
for part in parts:
directory.append(part)
segment = "/".join(directory)
if len(segment) > 0 and not os.path.isdir(segment):
logger.debug("JsonDB: mkdir {}".format(segment))
os.mkdir(segment, 0o755)
# Write the JSON.
write_json(path, data)
# Update the cached document.
if cache:
set_cache(document, data, expires=cache_lifetime)
set_cache(document + "_mtime", time.time(), expires=cache_lifetime)
# Release the lock.
unlock_cache(lock)
def delete(document):
"""Delete a document from the DB."""
path = mkpath(document)
if os.path.isfile(path):
logger.debug("Delete DB document: {}".format(path))
os.unlink(path)
del_cache(document)
def upload_from_www(form):
"""Upload a photo from the Internet.
This requires the `form` object, but not necessarily Flask's. It just has to
be a dict with the form keys for the upload.
Returns the same structure as `upload_from_pc()`.
"""
url = form.get("url")
if not url or not allowed_filetype(url):
return dict(success=False, error="Invalid file extension.")
# Make a temp filename for it.
filetype = url.rsplit(".", 1)[1]
tempfile = "{}/rophako-photo-{}.{}".format(Config.site.tempdir,
int(time.time()), filetype)
logger.debug("Save incoming photo to: {}".format(tempfile))
# Grab the file.
try:
data = requests.get(url).content
except:
return dict(success=False, error="Failed to get that URL.")
fh = open(tempfile, "wb")
fh.write(data)
fh.close()
# All good so far. Process the photo.
return process_photo(form, tempfile)
def get_next_id(index):
"""Get the next free ID for a blog post."""
logger.debug("Getting next available blog ID number")
sort = sorted(index.keys(), key=lambda x: int(x))
next_id = 1
if len(sort) > 0:
next_id = int(sort[-1]) + 1
logger.debug("Highest post ID is: {}".format(next_id))
# Sanity check!
if next_id in index:
raise Exception("Failed to get_next_id for the blog. Chosen ID is still in the index!")
return next_id
def get_next_id(index):
"""Get the next free ID for a blog post."""
logger.debug("Getting next available blog ID number")
sort = sorted(index.keys(), key=lambda x: int(x))
next_id = 1
if len(sort) > 0:
next_id = int(sort[-1]) + 1
logger.debug("Highest post ID is: {}".format(next_id))
# Sanity check!
if next_id in index:
raise Exception(
"Failed to get_next_id for the blog. Chosen ID is still in the index!"
)
return next_id
def get_cache(key):
"""Get a cached item."""
key = Config.db.redis_prefix + key
value = None
client = get_redis()
if not client:
return
try:
value = client.get(key)
if value:
value = json.loads(value)
except:
logger.debug("Redis exception: couldn't get_cache {}".format(key))
value = None
return value
def lock_cache(key, timeout=5, expire=20):
"""Cache level 'file locking' implementation.
The `key` will be automatically suffixed with `_lock`.
The `timeout` is the max amount of time to wait for a lock.
The `expire` is how long a lock may exist before it's considered stale.
Returns True on success, None on failure to acquire lock."""
client = get_redis()
if not client:
return
# Take the lock.
lock = client.lock(key, timeout=expire)
lock.acquire()
logger.debug("Cache lock acquired: {}, expires in {}s".format(key, expire))
return lock
def lock_cache(key, timeout=5, expire=20):
"""Cache level 'file locking' implementation.
The `key` will be automatically suffixed with `_lock`.
The `timeout` is the max amount of time to wait for a lock.
The `expire` is how long a lock may exist before it's considered stale.
Returns True on success, None on failure to acquire lock."""
client = get_redis()
if not client:
return
# Take the lock.
lock = client.lock(key, timeout=expire)
lock.acquire()
logger.debug("Cache lock acquired: {}, expires in {}s".format(key, expire))
return lock
def write_json(path, data):
"""Write a JSON document."""
path = str(path)
# Don't allow any fishy looking paths.
if ".." in path:
logger.error("ERROR: JsonDB tried to write a path with two dots: {}".format(path))
raise Exception()
logger.debug("JsonDB: WRITE > {}".format(path))
# Open and lock the file.
fh = codecs.open(path, 'w', 'utf-8')
flock(fh, LOCK_EX)
# Write it.
fh.write(json.dumps(data, indent=4, separators=(',', ': ')))
# Unlock and close.
flock(fh, LOCK_UN)
fh.close()
def upload_from_pc(request):
"""Upload a photo from the user's filesystem.
This requires the Flask `request` object. Returns a dict with the following
keys:
* success: True || False
* error: if unsuccessful
* photo: if successful
"""
form = request.form
count = 0
status = None
for upload in reversed(request.files.getlist("file")):
count += 1
# Make a temp filename for it.
filetype = upload.filename.rsplit(".", 1)[-1]
if not allowed_filetype(upload.filename):
return dict(success=False, error="Unsupported file extension.")
tempfile = "{}/rophako-photo-{}.{}".format(Config.site.tempdir,
int(time.time()), filetype)
logger.debug("Save incoming photo to: {}".format(tempfile))
upload.save(tempfile)
# All good so far. Process the photo.
status = process_photo(form, tempfile)
if not status["success"]:
return status
# Multi upload?
if count > 1:
status["multi"] = True
else:
status["multi"] = False
return status
def upload_from_pc(request):
"""Upload a photo from the user's filesystem.
This requires the Flask `request` object. Returns a dict with the following
keys:
* success: True || False
* error: if unsuccessful
* photo: if successful
"""
form = request.form
count = 0
status = None
for upload in reversed(request.files.getlist("file")):
count += 1
# Make a temp filename for it.
filetype = upload.filename.rsplit(".", 1)[-1]
if not allowed_filetype(upload.filename):
return dict(success=False, error="Unsupported file extension.")
tempfile = "{}/rophako-photo-{}.{}".format(Config.site.tempdir, int(time.time()), filetype)
logger.debug("Save incoming photo to: {}".format(tempfile))
upload.save(tempfile)
# All good so far. Process the photo.
status = process_photo(form, tempfile)
if not status["success"]:
return status
# Multi upload?
if count > 1:
status["multi"] = True
else:
status["multi"] = False
return status
def write_json(path, data):
"""Write a JSON document."""
path = str(path)
# Don't allow any fishy looking paths.
if ".." in path:
logger.error(
"ERROR: JsonDB tried to write a path with two dots: {}".format(
path))
raise Exception()
logger.debug("JsonDB: WRITE > {}".format(path))
# Open and lock the file.
fh = codecs.open(path, 'w', 'utf-8')
flock(fh, LOCK_EX)
# Write it.
fh.write(json.dumps(data, indent=4, separators=(',', ': ')))
# Unlock and close.
flock(fh, LOCK_UN)
fh.close()
def get(document, cache=True):
"""Get a specific document from the DB."""
logger.debug("JsonDB: GET {}".format(document))
# Exists?
if not exists(document):
logger.debug("Requested document doesn't exist")
return None
path = mkpath(document)
stat = os.stat(path)
# Do we have it cached?
data = get_cache(document) if cache else None
if data:
# Check if the cache is fresh.
if stat.st_mtime > get_cache(document+"_mtime"):
del_cache(document)
del_cache(document+"_mtime")
else:
return data
# Get a lock for reading.
lock = lock_cache(document)
# Get the JSON data.
data = read_json(path)
# Unlock!
unlock_cache(lock)
# Cache and return it.
if cache:
set_cache(document, data, expires=cache_lifetime)
set_cache(document+"_mtime", stat.st_mtime, expires=cache_lifetime)
return data
def get(document, cache=True):
"""Get a specific document from the DB."""
logger.debug("JsonDB: GET {}".format(document))
# Exists?
if not exists(document):
logger.debug("Requested document doesn't exist")
return None
path = mkpath(document)
stat = os.stat(path)
# Do we have it cached?
data = get_cache(document) if cache else None
if data:
# Check if the cache is fresh.
if stat.st_mtime > get_cache(document + "_mtime"):
del_cache(document)
del_cache(document + "_mtime")
else:
return data
# Get a lock for reading.
lock = lock_cache(document)
# Get the JSON data.
data = read_json(path)
# Unlock!
unlock_cache(lock)
# Cache and return it.
if cache:
set_cache(document, data, expires=cache_lifetime)
set_cache(document + "_mtime", stat.st_mtime, expires=cache_lifetime)
return data
def post_entry(post_id, fid, epoch, author, subject, avatar, categories,
privacy, ip, emoticons, comments, format, body):
"""Post (or update) a blog entry."""
# Fetch the index.
index = get_index()
# Editing an existing post?
if not post_id:
post_id = get_next_id(index)
logger.debug("Posting blog post ID {}".format(post_id))
# Get a unique friendly ID.
if not fid:
# The default friendly ID = the subject.
fid = subject.lower()
fid = re.sub(r'[^A-Za-z0-9]', '-', fid)
fid = re.sub(r'\-+', '-', fid)
fid = fid.strip("-")
logger.debug("Chosen friendly ID: {}".format(fid))
# Make sure the friendly ID is unique!
if len(fid):
test = fid
loop = 1
logger.debug("Verifying the friendly ID is unique: {}".format(fid))
while True:
collision = False
for k, v in index.items():
# Skip the same post, for updates.
if k == post_id: continue
if v["fid"] == test:
# Not unique.
loop += 1
test = fid + "_" + unicode(loop)
collision = True
logger.debug("Collision with existing post {}: {}".format(k, v["fid"]))
break
# Was there a collision?
if collision:
continue # Try again.
# Nope!
break
fid = test
# DB body for the post.
db = dict(
fid = fid,
ip = ip,
time = epoch or int(time.time()),
categories = categories,
sticky = False, # TODO: implement sticky
comments = comments,
emoticons = emoticons,
avatar = avatar,
privacy = privacy or "public",
author = author,
subject = subject,
format = format,
body = body,
)
# Write the post.
JsonDB.commit("blog/entries/{}".format(post_id), db)
# Update the index cache.
update_index(post_id, db, index)
return post_id, fid
def get_user(uid=None, username=None):
"""Get a user's DB file, or None if not found."""
if username:
uid = get_uid(username)
logger.debug("get_user: resolved username {} to UID {}".format(username, uid))
return JsonDB.get("users/by-id/{}".format(uid))
def unlock_cache(lock):
"""Release the lock on a cache key."""
if lock:
lock.release()
logger.debug("Cache lock released")
def post_entry(post_id, fid, epoch, author, subject, avatar, categories,
privacy, ip, emoticons, comments, format, body):
"""Post (or update) a blog entry."""
# Fetch the index.
index = get_index()
# Editing an existing post?
if not post_id:
post_id = get_next_id(index)
logger.debug("Posting blog post ID {}".format(post_id))
# Get a unique friendly ID.
if not fid:
# The default friendly ID = the subject.
fid = subject.lower()
fid = re.sub(r'[^A-Za-z0-9]', '-', fid)
fid = re.sub(r'\-+', '-', fid)
fid = fid.strip("-")
logger.debug("Chosen friendly ID: {}".format(fid))
# Make sure the friendly ID is unique!
if len(fid):
test = fid
loop = 1
logger.debug("Verifying the friendly ID is unique: {}".format(fid))
while True:
collision = False
for k, v in index.items():
# Skip the same post, for updates.
if k == post_id: continue
if v["fid"] == test:
# Not unique.
loop += 1
test = fid + "_" + unicode(loop)
collision = True
logger.debug("Collision with existing post {}: {}".format(
k, v["fid"]))
break
# Was there a collision?
if collision:
continue # Try again.
# Nope!
break
fid = test
# DB body for the post.
db = dict(
fid=fid,
ip=ip,
time=epoch or int(time.time()),
categories=categories,
sticky=False, # TODO: implement sticky
comments=comments,
emoticons=emoticons,
avatar=avatar,
privacy=privacy or "public",
author=author,
subject=subject,
format=format,
body=body,
)
# Write the post.
JsonDB.commit("blog/entries/{}".format(post_id), db)
# Update the index cache.
update_index(post_id, db, index)
return post_id, fid
def unlock_cache(lock):
"""Release the lock on a cache key."""
if lock:
lock.release()
logger.debug("Cache lock released")
def process_photo(form, filename):
"""Formats an incoming photo."""
# Resize the photo to each of the various sizes and collect their names.
sizes = dict()
for size in PHOTO_SCALES.keys():
sizes[size] = resize_photo(filename, size)
# Remove the temp file.
os.unlink(filename)
# What album are the photos going to?
album = form.get("album", "")
new_album = form.get("new-album", None)
new_desc = form.get("new-description", None)
if album == "" and new_album:
album = new_album
# Sanitize the name.
album = sanitize_name(album)
if album == "":
logger.warning("Album name didn't pass sanitization! Fall back to default album name.")
album = Config.photo.default_album
# Make up a unique public key for this set of photos.
key = random_hash()
while photo_exists(key):
key = random_hash()
logger.debug("Photo set public key: {}".format(key))
# Get the album index to manipulate ordering.
index = get_index()
# Update the photo data.
if not album in index["albums"]:
index["albums"][album] = {}
if not "settings" in index:
index["settings"] = dict()
if not album in index["settings"]:
index["settings"][album] = {
"format": "classic",
"description": new_desc,
}
index["albums"][album][key] = dict(
ip=remote_addr(),
author=g.info["session"]["uid"],
uploaded=int(time.time()),
caption=form.get("caption", ""),
description=form.get("description", ""),
**sizes
)
# Maintain a photo map to album.
index["map"][key] = album
# Add this pic to the front of the album.
if not album in index["photo-order"]:
index["photo-order"][album] = []
index["photo-order"][album].insert(0, key)
# If this is a new album, add it to the front of the album ordering.
if not album in index["album-order"]:
index["album-order"].insert(0, album)
# Set the album cover for a new album.
if not album in index["covers"] or len(index["covers"][album]) == 0:
index["covers"][album] = key
# Save changes to the index.
write_index(index)
return dict(success=True, photo=key)
def resize_photo(filename, size, crop=None):
"""Resize a photo from the target filename into the requested size.
Optionally the photo can be cropped with custom parameters.
"""
# Find the file type.
filetype = filename.rsplit(".", 1)[1]
if filetype == "jpeg": filetype = "jpg"
# Open the image.
img = Image.open(filename)
# Make up a unique filename.
outfile = random_name(filetype)
target = os.path.join(Config.photo.root_private, outfile)
logger.debug("Output file for {} scale: {}".format(size, target))
# Get the image's dimensions.
orig_width, orig_height = img.size
new_width = PHOTO_SCALES[size]
logger.debug("Original photo dimensions: {}x{}".format(orig_width, orig_height))
# For the large version, only scale it, don't crop it.
if size == "large":
# Do we NEED to scale it?
if orig_width <= new_width:
logger.debug("Don't need to scale down the large image!")
img.save(target)
return outfile
# Scale it down.
ratio = float(new_width) / float(orig_width)
new_height = int(float(orig_height) * float(ratio))
logger.debug("New image dimensions: {}x{}".format(new_width, new_height))
img = img.resize((new_width, new_height), Image.ANTIALIAS)
img.save(target)
return outfile
# For all other versions, crop them into a square.
x, y, length = 0, 0, 0
# Use 0,0 and find the shortest dimension for the length.
if orig_width > orig_height:
length = orig_height
else:
length = orig_width
# Did they give us crop coordinates?
if crop is not None:
x = crop["x"]
y = crop["y"]
if crop["length"] > 0:
length = crop["length"]
# Adjust the coords if they're impossible.
if x < 0:
logger.warning("X-Coord is less than 0; fixing!")
x = 0
if y < 0:
logger.warning("Y-Coord is less than 0; fixing!")
y = 0
if x > orig_width:
logger.warning("X-Coord is greater than image width; fixing!")
x = orig_width - length
if x < 0: x = 0
if y > orig_height:
logger.warning("Y-Coord is greater than image height; fixing!")
y = orig_height - length
if y < 0: y = 0
# Make sure the crop box fits.
if x + length > orig_width:
diff = x + length - orig_width
logger.warning("Crop box is outside the right edge of the image by {}px; fixing!".format(diff))
length -= diff
if y + length > orig_height:
diff = y + length - orig_height
logger.warning("Crop box is outside the bottom edge of the image by {}px; fixing!".format(diff))
length -= diff
# Do we need to scale?
if new_width == length:
logger.debug("Image doesn't need to be cropped or scaled!")
img.save(target)
return outfile
# Crop to the requested box.
logger.debug("Cropping the photo")
img = img.crop((x, y, x+length, y+length))
# Scale it to the proper dimensions.
img = img.resize((new_width, new_width), Image.ANTIALIAS)
img.save(target)
return outfile
def resize_photo(filename, size, crop=None):
"""Resize a photo from the target filename into the requested size.
Optionally the photo can be cropped with custom parameters.
"""
# Find the file type.
filetype = filename.rsplit(".", 1)[1]
if filetype == "jpeg": filetype = "jpg"
# Open the image.
img = Image.open(filename)
# Make up a unique filename.
outfile = random_name(filetype)
target = os.path.join(Config.photo.root_private, outfile)
logger.debug("Output file for {} scale: {}".format(size, target))
# Get the image's dimensions.
orig_width, orig_height = img.size
new_width = PHOTO_SCALES[size]
logger.debug("Original photo dimensions: {}x{}".format(
orig_width, orig_height))
# For the large version, only scale it, don't crop it.
if size == "large":
# Do we NEED to scale it?
if orig_width <= new_width:
logger.debug("Don't need to scale down the large image!")
img.save(target)
return outfile
# Scale it down.
ratio = float(new_width) / float(orig_width)
new_height = int(float(orig_height) * float(ratio))
logger.debug("New image dimensions: {}x{}".format(
new_width, new_height))
img = img.resize((new_width, new_height), Image.ANTIALIAS)
img.save(target)
return outfile
# For all other versions, crop them into a square.
x, y, length = 0, 0, 0
# Use 0,0 and find the shortest dimension for the length.
if orig_width > orig_height:
length = orig_height
else:
length = orig_width
# Did they give us crop coordinates?
if crop is not None:
x = crop["x"]
y = crop["y"]
if crop["length"] > 0:
length = crop["length"]
# Adjust the coords if they're impossible.
if x < 0:
logger.warning("X-Coord is less than 0; fixing!")
x = 0
if y < 0:
logger.warning("Y-Coord is less than 0; fixing!")
y = 0
if x > orig_width:
logger.warning("X-Coord is greater than image width; fixing!")
x = orig_width - length
if x < 0: x = 0
if y > orig_height:
logger.warning("Y-Coord is greater than image height; fixing!")
y = orig_height - length
if y < 0: y = 0
# Make sure the crop box fits.
if x + length > orig_width:
diff = x + length - orig_width
logger.warning(
"Crop box is outside the right edge of the image by {}px; fixing!".
format(diff))
length -= diff
if y + length > orig_height:
diff = y + length - orig_height
logger.warning(
"Crop box is outside the bottom edge of the image by {}px; fixing!"
.format(diff))
length -= diff
# Do we need to scale?
if new_width == length:
logger.debug("Image doesn't need to be cropped or scaled!")
img.save(target)
return outfile
# Crop to the requested box.
logger.debug("Cropping the photo")
img = img.crop((x, y, x + length, y + length))
# Scale it to the proper dimensions.
img = img.resize((new_width, new_width), Image.ANTIALIAS)
img.save(target)
return outfile
def process_photo(form, filename):
"""Formats an incoming photo."""
# Resize the photo to each of the various sizes and collect their names.
sizes = dict()
for size in PHOTO_SCALES.keys():
sizes[size] = resize_photo(filename, size)
# Remove the temp file.
os.unlink(filename)
# What album are the photos going to?
album = form.get("album", "")
new_album = form.get("new-album", None)
new_desc = form.get("new-description", None)
if album == "" and new_album:
album = new_album
# Sanitize the name.
album = sanitize_name(album)
if album == "":
logger.warning(
"Album name didn't pass sanitization! Fall back to default album name."
)
album = Config.photo.default_album
# Make up a unique public key for this set of photos.
key = random_hash()
while photo_exists(key):
key = random_hash()
logger.debug("Photo set public key: {}".format(key))
# Get the album index to manipulate ordering.
index = get_index()
# Update the photo data.
if not album in index["albums"]:
index["albums"][album] = {}
if not "settings" in index:
index["settings"] = dict()
if not album in index["settings"]:
index["settings"][album] = {
"format": "classic",
"description": new_desc,
}
index["albums"][album][key] = dict(ip=remote_addr(),
author=g.info["session"]["uid"],
uploaded=int(time.time()),
caption=form.get("caption", ""),
description=form.get("description", ""),
**sizes)
# Maintain a photo map to album.
index["map"][key] = album
# Add this pic to the front of the album.
if not album in index["photo-order"]:
index["photo-order"][album] = []
index["photo-order"][album].insert(0, key)
# If this is a new album, add it to the front of the album ordering.
if not album in index["album-order"]:
index["album-order"].insert(0, album)
# Set the album cover for a new album.
if not album in index["covers"] or len(index["covers"][album]) == 0:
index["covers"][album] = key
# Save changes to the index.
write_index(index)
return dict(success=True, photo=key)
