def upload_from_www(form): """Upload a photo from the Internet. This requires the `form` object, but not necessarily Flask's. It just has to be a dict with the form keys for the upload. Returns the same structure as `upload_from_pc()`. """ url = form.get("url") if not url or not allowed_filetype(url): return dict(success=False, error="Invalid file extension.") # Make a temp filename for it. filetype = url.rsplit(".", 1)[1] tempfile = "{}/rophako-photo-{}.{}".format(Config.site.tempdir, int(time.time()), filetype) logger.debug("Save incoming photo to: {}".format(tempfile)) # Grab the file. try: data = requests.get(url).content except: return dict(success=False, error="Failed to get that URL.") fh = open(tempfile, "wb") fh.write(data) fh.close() # All good so far. Process the photo. return process_photo(form, tempfile)
def crop_photo(key, x, y, length): """Change the crop coordinates of a photo and re-crop it.""" index = get_index() if not key in index["map"]: raise Exception("Can't crop photo: doesn't exist!") album = index["map"][key] # Sanity check. if not album in index["albums"]: raise Exception("Can't find photo in album!") logger.debug("Recropping photo {}".format(key)) # Delete all the images except the large one. for size in ["thumb", "avatar"]: pic = index["albums"][album][key][size] logger.debug("Delete {} size: {}".format(size, pic)) os.unlink(os.path.join(Config.photo.root_private, pic)) # Regenerate all the thumbnails. large = index["albums"][album][key]["large"] source = os.path.join(Config.photo.root_private, large) for size in ["thumb", "avatar"]: pic = resize_photo(source, size, crop=dict( x=x, y=y, length=length, )) index["albums"][album][key][size] = pic # Save changes. write_index(index)
def commit(document, data, cache=True): """Insert/update a document in the DB.""" # Only allow one commit at a time. lock = lock_cache(document) # Need to create the file? path = mkpath(document) if not os.path.isfile(path): parts = path.split("/") parts.pop() # Remove the file part directory = list() # Create all the folders. for part in parts: directory.append(part) segment = "/".join(directory) if len(segment) > 0 and not os.path.isdir(segment): logger.debug("JsonDB: mkdir {}".format(segment)) os.mkdir(segment, 0o755) # Write the JSON. write_json(path, data) # Update the cached document. if cache: set_cache(document, data, expires=cache_lifetime) set_cache(document+"_mtime", time.time(), expires=cache_lifetime) # Release the lock. unlock_cache(lock)
def commit(document, data, cache=True): """Insert/update a document in the DB.""" # Only allow one commit at a time. lock = lock_cache(document) # Need to create the file? path = mkpath(document) if not os.path.isfile(path): parts = path.split("/") parts.pop() # Remove the file part directory = list() # Create all the folders. for part in parts: directory.append(part) segment = "/".join(directory) if len(segment) > 0 and not os.path.isdir(segment): logger.debug("JsonDB: mkdir {}".format(segment)) os.mkdir(segment, 0o755) # Write the JSON. write_json(path, data) # Update the cached document. if cache: set_cache(document, data, expires=cache_lifetime) set_cache(document + "_mtime", time.time(), expires=cache_lifetime) # Release the lock. unlock_cache(lock)
def delete(document): """Delete a document from the DB.""" path = mkpath(document) if os.path.isfile(path): logger.debug("Delete DB document: {}".format(path)) os.unlink(path) del_cache(document)
def get_next_id(index): """Get the next free ID for a blog post.""" logger.debug("Getting next available blog ID number") sort = sorted(index.keys(), key=lambda x: int(x)) next_id = 1 if len(sort) > 0: next_id = int(sort[-1]) + 1 logger.debug("Highest post ID is: {}".format(next_id)) # Sanity check! if next_id in index: raise Exception("Failed to get_next_id for the blog. Chosen ID is still in the index!") return next_id
def get_next_id(index): """Get the next free ID for a blog post.""" logger.debug("Getting next available blog ID number") sort = sorted(index.keys(), key=lambda x: int(x)) next_id = 1 if len(sort) > 0: next_id = int(sort[-1]) + 1 logger.debug("Highest post ID is: {}".format(next_id)) # Sanity check! if next_id in index: raise Exception( "Failed to get_next_id for the blog. Chosen ID is still in the index!" ) return next_id
def get_cache(key): """Get a cached item.""" key = Config.db.redis_prefix + key value = None client = get_redis() if not client: return try: value = client.get(key) if value: value = json.loads(value) except: logger.debug("Redis exception: couldn't get_cache {}".format(key)) value = None return value
def lock_cache(key, timeout=5, expire=20): """Cache level 'file locking' implementation. The `key` will be automatically suffixed with `_lock`. The `timeout` is the max amount of time to wait for a lock. The `expire` is how long a lock may exist before it's considered stale. Returns True on success, None on failure to acquire lock.""" client = get_redis() if not client: return # Take the lock. lock = client.lock(key, timeout=expire) lock.acquire() logger.debug("Cache lock acquired: {}, expires in {}s".format(key, expire)) return lock
def write_json(path, data): """Write a JSON document.""" path = str(path) # Don't allow any fishy looking paths. if ".." in path: logger.error("ERROR: JsonDB tried to write a path with two dots: {}".format(path)) raise Exception() logger.debug("JsonDB: WRITE > {}".format(path)) # Open and lock the file. fh = codecs.open(path, 'w', 'utf-8') flock(fh, LOCK_EX) # Write it. fh.write(json.dumps(data, indent=4, separators=(',', ': '))) # Unlock and close. flock(fh, LOCK_UN) fh.close()
def upload_from_pc(request): """Upload a photo from the user's filesystem. This requires the Flask `request` object. Returns a dict with the following keys: * success: True || False * error: if unsuccessful * photo: if successful """ form = request.form count = 0 status = None for upload in reversed(request.files.getlist("file")): count += 1 # Make a temp filename for it. filetype = upload.filename.rsplit(".", 1)[-1] if not allowed_filetype(upload.filename): return dict(success=False, error="Unsupported file extension.") tempfile = "{}/rophako-photo-{}.{}".format(Config.site.tempdir, int(time.time()), filetype) logger.debug("Save incoming photo to: {}".format(tempfile)) upload.save(tempfile) # All good so far. Process the photo. status = process_photo(form, tempfile) if not status["success"]: return status # Multi upload? if count > 1: status["multi"] = True else: status["multi"] = False return status
def write_json(path, data): """Write a JSON document.""" path = str(path) # Don't allow any fishy looking paths. if ".." in path: logger.error( "ERROR: JsonDB tried to write a path with two dots: {}".format( path)) raise Exception() logger.debug("JsonDB: WRITE > {}".format(path)) # Open and lock the file. fh = codecs.open(path, 'w', 'utf-8') flock(fh, LOCK_EX) # Write it. fh.write(json.dumps(data, indent=4, separators=(',', ': '))) # Unlock and close. flock(fh, LOCK_UN) fh.close()
def get(document, cache=True): """Get a specific document from the DB.""" logger.debug("JsonDB: GET {}".format(document)) # Exists? if not exists(document): logger.debug("Requested document doesn't exist") return None path = mkpath(document) stat = os.stat(path) # Do we have it cached? data = get_cache(document) if cache else None if data: # Check if the cache is fresh. if stat.st_mtime > get_cache(document+"_mtime"): del_cache(document) del_cache(document+"_mtime") else: return data # Get a lock for reading. lock = lock_cache(document) # Get the JSON data. data = read_json(path) # Unlock! unlock_cache(lock) # Cache and return it. if cache: set_cache(document, data, expires=cache_lifetime) set_cache(document+"_mtime", stat.st_mtime, expires=cache_lifetime) return data
def get(document, cache=True): """Get a specific document from the DB.""" logger.debug("JsonDB: GET {}".format(document)) # Exists? if not exists(document): logger.debug("Requested document doesn't exist") return None path = mkpath(document) stat = os.stat(path) # Do we have it cached? data = get_cache(document) if cache else None if data: # Check if the cache is fresh. if stat.st_mtime > get_cache(document + "_mtime"): del_cache(document) del_cache(document + "_mtime") else: return data # Get a lock for reading. lock = lock_cache(document) # Get the JSON data. data = read_json(path) # Unlock! unlock_cache(lock) # Cache and return it. if cache: set_cache(document, data, expires=cache_lifetime) set_cache(document + "_mtime", stat.st_mtime, expires=cache_lifetime) return data
def post_entry(post_id, fid, epoch, author, subject, avatar, categories, privacy, ip, emoticons, comments, format, body): """Post (or update) a blog entry.""" # Fetch the index. index = get_index() # Editing an existing post? if not post_id: post_id = get_next_id(index) logger.debug("Posting blog post ID {}".format(post_id)) # Get a unique friendly ID. if not fid: # The default friendly ID = the subject. fid = subject.lower() fid = re.sub(r'[^A-Za-z0-9]', '-', fid) fid = re.sub(r'\-+', '-', fid) fid = fid.strip("-") logger.debug("Chosen friendly ID: {}".format(fid)) # Make sure the friendly ID is unique! if len(fid): test = fid loop = 1 logger.debug("Verifying the friendly ID is unique: {}".format(fid)) while True: collision = False for k, v in index.items(): # Skip the same post, for updates. if k == post_id: continue if v["fid"] == test: # Not unique. loop += 1 test = fid + "_" + unicode(loop) collision = True logger.debug("Collision with existing post {}: {}".format(k, v["fid"])) break # Was there a collision? if collision: continue # Try again. # Nope! break fid = test # DB body for the post. db = dict( fid = fid, ip = ip, time = epoch or int(time.time()), categories = categories, sticky = False, # TODO: implement sticky comments = comments, emoticons = emoticons, avatar = avatar, privacy = privacy or "public", author = author, subject = subject, format = format, body = body, ) # Write the post. JsonDB.commit("blog/entries/{}".format(post_id), db) # Update the index cache. update_index(post_id, db, index) return post_id, fid
def get_user(uid=None, username=None): """Get a user's DB file, or None if not found.""" if username: uid = get_uid(username) logger.debug("get_user: resolved username {} to UID {}".format(username, uid)) return JsonDB.get("users/by-id/{}".format(uid))
def unlock_cache(lock): """Release the lock on a cache key.""" if lock: lock.release() logger.debug("Cache lock released")
def post_entry(post_id, fid, epoch, author, subject, avatar, categories, privacy, ip, emoticons, comments, format, body): """Post (or update) a blog entry.""" # Fetch the index. index = get_index() # Editing an existing post? if not post_id: post_id = get_next_id(index) logger.debug("Posting blog post ID {}".format(post_id)) # Get a unique friendly ID. if not fid: # The default friendly ID = the subject. fid = subject.lower() fid = re.sub(r'[^A-Za-z0-9]', '-', fid) fid = re.sub(r'\-+', '-', fid) fid = fid.strip("-") logger.debug("Chosen friendly ID: {}".format(fid)) # Make sure the friendly ID is unique! if len(fid): test = fid loop = 1 logger.debug("Verifying the friendly ID is unique: {}".format(fid)) while True: collision = False for k, v in index.items(): # Skip the same post, for updates. if k == post_id: continue if v["fid"] == test: # Not unique. loop += 1 test = fid + "_" + unicode(loop) collision = True logger.debug("Collision with existing post {}: {}".format( k, v["fid"])) break # Was there a collision? if collision: continue # Try again. # Nope! break fid = test # DB body for the post. db = dict( fid=fid, ip=ip, time=epoch or int(time.time()), categories=categories, sticky=False, # TODO: implement sticky comments=comments, emoticons=emoticons, avatar=avatar, privacy=privacy or "public", author=author, subject=subject, format=format, body=body, ) # Write the post. JsonDB.commit("blog/entries/{}".format(post_id), db) # Update the index cache. update_index(post_id, db, index) return post_id, fid
def process_photo(form, filename): """Formats an incoming photo.""" # Resize the photo to each of the various sizes and collect their names. sizes = dict() for size in PHOTO_SCALES.keys(): sizes[size] = resize_photo(filename, size) # Remove the temp file. os.unlink(filename) # What album are the photos going to? album = form.get("album", "") new_album = form.get("new-album", None) new_desc = form.get("new-description", None) if album == "" and new_album: album = new_album # Sanitize the name. album = sanitize_name(album) if album == "": logger.warning("Album name didn't pass sanitization! Fall back to default album name.") album = Config.photo.default_album # Make up a unique public key for this set of photos. key = random_hash() while photo_exists(key): key = random_hash() logger.debug("Photo set public key: {}".format(key)) # Get the album index to manipulate ordering. index = get_index() # Update the photo data. if not album in index["albums"]: index["albums"][album] = {} if not "settings" in index: index["settings"] = dict() if not album in index["settings"]: index["settings"][album] = { "format": "classic", "description": new_desc, } index["albums"][album][key] = dict( ip=remote_addr(), author=g.info["session"]["uid"], uploaded=int(time.time()), caption=form.get("caption", ""), description=form.get("description", ""), **sizes ) # Maintain a photo map to album. index["map"][key] = album # Add this pic to the front of the album. if not album in index["photo-order"]: index["photo-order"][album] = [] index["photo-order"][album].insert(0, key) # If this is a new album, add it to the front of the album ordering. if not album in index["album-order"]: index["album-order"].insert(0, album) # Set the album cover for a new album. if not album in index["covers"] or len(index["covers"][album]) == 0: index["covers"][album] = key # Save changes to the index. write_index(index) return dict(success=True, photo=key)
def resize_photo(filename, size, crop=None): """Resize a photo from the target filename into the requested size. Optionally the photo can be cropped with custom parameters. """ # Find the file type. filetype = filename.rsplit(".", 1)[1] if filetype == "jpeg": filetype = "jpg" # Open the image. img = Image.open(filename) # Make up a unique filename. outfile = random_name(filetype) target = os.path.join(Config.photo.root_private, outfile) logger.debug("Output file for {} scale: {}".format(size, target)) # Get the image's dimensions. orig_width, orig_height = img.size new_width = PHOTO_SCALES[size] logger.debug("Original photo dimensions: {}x{}".format(orig_width, orig_height)) # For the large version, only scale it, don't crop it. if size == "large": # Do we NEED to scale it? if orig_width <= new_width: logger.debug("Don't need to scale down the large image!") img.save(target) return outfile # Scale it down. ratio = float(new_width) / float(orig_width) new_height = int(float(orig_height) * float(ratio)) logger.debug("New image dimensions: {}x{}".format(new_width, new_height)) img = img.resize((new_width, new_height), Image.ANTIALIAS) img.save(target) return outfile # For all other versions, crop them into a square. x, y, length = 0, 0, 0 # Use 0,0 and find the shortest dimension for the length. if orig_width > orig_height: length = orig_height else: length = orig_width # Did they give us crop coordinates? if crop is not None: x = crop["x"] y = crop["y"] if crop["length"] > 0: length = crop["length"] # Adjust the coords if they're impossible. if x < 0: logger.warning("X-Coord is less than 0; fixing!") x = 0 if y < 0: logger.warning("Y-Coord is less than 0; fixing!") y = 0 if x > orig_width: logger.warning("X-Coord is greater than image width; fixing!") x = orig_width - length if x < 0: x = 0 if y > orig_height: logger.warning("Y-Coord is greater than image height; fixing!") y = orig_height - length if y < 0: y = 0 # Make sure the crop box fits. if x + length > orig_width: diff = x + length - orig_width logger.warning("Crop box is outside the right edge of the image by {}px; fixing!".format(diff)) length -= diff if y + length > orig_height: diff = y + length - orig_height logger.warning("Crop box is outside the bottom edge of the image by {}px; fixing!".format(diff)) length -= diff # Do we need to scale? if new_width == length: logger.debug("Image doesn't need to be cropped or scaled!") img.save(target) return outfile # Crop to the requested box. logger.debug("Cropping the photo") img = img.crop((x, y, x+length, y+length)) # Scale it to the proper dimensions. img = img.resize((new_width, new_width), Image.ANTIALIAS) img.save(target) return outfile
def resize_photo(filename, size, crop=None): """Resize a photo from the target filename into the requested size. Optionally the photo can be cropped with custom parameters. """ # Find the file type. filetype = filename.rsplit(".", 1)[1] if filetype == "jpeg": filetype = "jpg" # Open the image. img = Image.open(filename) # Make up a unique filename. outfile = random_name(filetype) target = os.path.join(Config.photo.root_private, outfile) logger.debug("Output file for {} scale: {}".format(size, target)) # Get the image's dimensions. orig_width, orig_height = img.size new_width = PHOTO_SCALES[size] logger.debug("Original photo dimensions: {}x{}".format( orig_width, orig_height)) # For the large version, only scale it, don't crop it. if size == "large": # Do we NEED to scale it? if orig_width <= new_width: logger.debug("Don't need to scale down the large image!") img.save(target) return outfile # Scale it down. ratio = float(new_width) / float(orig_width) new_height = int(float(orig_height) * float(ratio)) logger.debug("New image dimensions: {}x{}".format( new_width, new_height)) img = img.resize((new_width, new_height), Image.ANTIALIAS) img.save(target) return outfile # For all other versions, crop them into a square. x, y, length = 0, 0, 0 # Use 0,0 and find the shortest dimension for the length. if orig_width > orig_height: length = orig_height else: length = orig_width # Did they give us crop coordinates? if crop is not None: x = crop["x"] y = crop["y"] if crop["length"] > 0: length = crop["length"] # Adjust the coords if they're impossible. if x < 0: logger.warning("X-Coord is less than 0; fixing!") x = 0 if y < 0: logger.warning("Y-Coord is less than 0; fixing!") y = 0 if x > orig_width: logger.warning("X-Coord is greater than image width; fixing!") x = orig_width - length if x < 0: x = 0 if y > orig_height: logger.warning("Y-Coord is greater than image height; fixing!") y = orig_height - length if y < 0: y = 0 # Make sure the crop box fits. if x + length > orig_width: diff = x + length - orig_width logger.warning( "Crop box is outside the right edge of the image by {}px; fixing!". format(diff)) length -= diff if y + length > orig_height: diff = y + length - orig_height logger.warning( "Crop box is outside the bottom edge of the image by {}px; fixing!" .format(diff)) length -= diff # Do we need to scale? if new_width == length: logger.debug("Image doesn't need to be cropped or scaled!") img.save(target) return outfile # Crop to the requested box. logger.debug("Cropping the photo") img = img.crop((x, y, x + length, y + length)) # Scale it to the proper dimensions. img = img.resize((new_width, new_width), Image.ANTIALIAS) img.save(target) return outfile
def process_photo(form, filename): """Formats an incoming photo.""" # Resize the photo to each of the various sizes and collect their names. sizes = dict() for size in PHOTO_SCALES.keys(): sizes[size] = resize_photo(filename, size) # Remove the temp file. os.unlink(filename) # What album are the photos going to? album = form.get("album", "") new_album = form.get("new-album", None) new_desc = form.get("new-description", None) if album == "" and new_album: album = new_album # Sanitize the name. album = sanitize_name(album) if album == "": logger.warning( "Album name didn't pass sanitization! Fall back to default album name." ) album = Config.photo.default_album # Make up a unique public key for this set of photos. key = random_hash() while photo_exists(key): key = random_hash() logger.debug("Photo set public key: {}".format(key)) # Get the album index to manipulate ordering. index = get_index() # Update the photo data. if not album in index["albums"]: index["albums"][album] = {} if not "settings" in index: index["settings"] = dict() if not album in index["settings"]: index["settings"][album] = { "format": "classic", "description": new_desc, } index["albums"][album][key] = dict(ip=remote_addr(), author=g.info["session"]["uid"], uploaded=int(time.time()), caption=form.get("caption", ""), description=form.get("description", ""), **sizes) # Maintain a photo map to album. index["map"][key] = album # Add this pic to the front of the album. if not album in index["photo-order"]: index["photo-order"][album] = [] index["photo-order"][album].insert(0, key) # If this is a new album, add it to the front of the album ordering. if not album in index["album-order"]: index["album-order"].insert(0, album) # Set the album cover for a new album. if not album in index["covers"] or len(index["covers"][album]) == 0: index["covers"][album] = key # Save changes to the index. write_index(index) return dict(success=True, photo=key)