def get_visitor_details():
"""Retrieve detailed visitor information for the frontend."""
result = {
"traffic": [], # Historical traffic data
"most_unique": ["0000-00-00", 0], # Day with the most unique
"most_hits": ["0000-00-00", 0], # Day with the most hits
"oldest": None, # Oldest day on record.
}
# List all the documents.
hits = JsonDB.list_docs("traffic/hits")
for date in sorted(hits):
if date == "total": continue
if not result["oldest"]:
result["oldest"] = date
# Get the DBs.
hits_db = JsonDB.get("traffic/hits/{}".format(date), cache=False)
uniq_db = JsonDB.get("traffic/unique/{}".format(date), cache=False)
# Most we've seen?
if hits_db["hits"] > result["most_hits"][1]:
result["most_hits"] = [date, hits_db["hits"]]
if len(uniq_db.keys()) > result["most_unique"][1]:
result["most_unique"] = [date, len(uniq_db.keys())]
result["traffic"].append(
dict(
date=date,
hits=hits_db["hits"],
unique=len(uniq_db.keys()),
))
return result
def get_index():
"""Get the blog index.
The index is the cache of available blog posts. It has the format:
```
{
'post_id': {
fid: Friendly ID for the blog post (for URLs)
time: epoch time of the post
sticky: the stickiness of the post (shows first on global views)
author: the author user ID of the post
categories: [ list of categories ]
privacy: the privacy setting
subject: the post subject
},
...
}
```
"""
# Index doesn't exist?
if not JsonDB.exists("blog/index"):
return rebuild_index()
db = JsonDB.get("blog/index")
# Hide any private posts if we aren't logged in.
if not g.info["session"]["login"]:
for post_id, data in db.items():
if data["privacy"] == "private":
del db[post_id]
return db
def get_index():
"""Get the blog index.
The index is the cache of available blog posts. It has the format:
```
{
'post_id': {
fid: Friendly ID for the blog post (for URLs)
time: epoch time of the post
sticky: the stickiness of the post (shows first on global views)
author: the author user ID of the post
categories: [ list of categories ]
privacy: the privacy setting
subject: the post subject
},
...
}
```
"""
# Index doesn't exist?
if not JsonDB.exists("blog/index"):
return rebuild_index()
db = JsonDB.get("blog/index")
# Hide any private posts if we aren't logged in.
if not g.info["session"]["login"]:
for post_id, data in db.items():
if data["privacy"] == "private":
del db[post_id]
return db
def get_page(name):
"""Get a Wiki page. Returns `None` if the page isn't found."""
name = name.strip("/") # Remove any surrounding slashes.
path = "wiki/pages/{}".format(name)
if not JsonDB.exists(path):
return None
# TODO: case insensitive page names...
db = JsonDB.get(path)
return db
def rebuild_index():
"""Rebuild the index.json if it goes missing."""
index = {}
entries = JsonDB.list_docs("blog/entries")
for post_id in entries:
db = JsonDB.get("blog/entries/{}".format(post_id))
update_index(post_id, db, index, False)
JsonDB.commit("blog/index", index)
return index
def rebuild_visitor_stats():
"""Recalculate the total unique/hits based on daily info."""
total_unique = {}
total_hits = 0
# Tally them all up!
for date in JsonDB.list_docs("traffic/unique"):
if date == "total":
continue
db = JsonDB.get("traffic/unique/{}".format(date), cache=False)
total_unique.update(db)
for date in JsonDB.list_docs("traffic/hits"):
if date == "total":
continue
db = JsonDB.get("traffic/hits/{}".format(date), cache=False)
total_hits += db.get("hits", 0)
# Write the outputs.
JsonDB.commit("traffic/unique/total", total_unique)
JsonDB.commit("traffic/hits/total", dict(hits=total_hits))
def get_page(name):
"""Get a Wiki page. Returns `None` if the page isn't found."""
name = name.strip("/") # Remove any surrounding slashes.
path = "wiki/pages/{}".format(name)
if not JsonDB.exists(path):
return None
# TODO: case insensitive page names...
db = JsonDB.get(path)
return db
def rebuild_index():
"""Rebuild the index.json if it goes missing."""
index = {}
entries = JsonDB.list_docs("blog/entries")
for post_id in entries:
db = JsonDB.get("blog/entries/{}".format(post_id))
update_index(post_id, db, index, False)
JsonDB.commit("blog/index", index)
return index
def get_index():
"""Get the photo album index, or a new empty DB if it doesn't exist."""
if JsonDB.exists("photos/index"):
return JsonDB.get("photos/index")
return {
"albums": {}, # Album data
"map": {}, # Map photo keys to albums
"covers": {}, # Album cover photos
"photo-order": {}, # Ordering of photos in albums
"album-order": [], # Ordering of albums themselves
}
def get_index():
"""Get the photo album index, or a new empty DB if it doesn't exist."""
if JsonDB.exists("photos/index"):
return JsonDB.get("photos/index")
return {
"albums": {}, # Album data
"map": {}, # Map photo keys to albums
"covers": {}, # Album cover photos
"photo-order": {}, # Ordering of photos in albums
"album-order": [], # Ordering of albums themselves
}
def get_entry(post_id):
"""Load a full blog entry."""
if not JsonDB.exists("blog/entries/{}".format(post_id)):
return None
db = JsonDB.get("blog/entries/{}".format(post_id))
# If no FID, set it to the ID.
if len(db["fid"]) == 0:
db["fid"] = str(post_id)
# If no "format" option, set it to HTML (legacy)
if db.get("format", "") == "":
db["format"] = "html"
return db
def get_entry(post_id):
"""Load a full blog entry."""
if not JsonDB.exists("blog/entries/{}".format(post_id)):
return None
db = JsonDB.get("blog/entries/{}".format(post_id))
# If no FID, set it to the ID.
if len(db["fid"]) == 0:
db["fid"] = str(post_id)
# If no "format" option, set it to HTML (legacy)
if db.get("format", "") == "":
db["format"] = "html"
return db
def legacy_download():
form = None
if request.method == "POST":
form = request.form
else:
# CNET links to the MS-DOS download using semicolon delimiters in the
# query string. Fix that if detected.
query = request.query_string.decode()
if not '&' in query and ';' in query:
url = re.sub(r';|%3b', '&', request.url, flags=re.IGNORECASE)
return redirect(url)
form = request.args
method = form.get("method", "index")
project = form.get("project", "")
filename = form.get("file", "")
root = "/home/kirsle/www/projects"
if project and filename:
# Filter the sections.
project = re.sub(r'[^A-Za-z0-9]', '', project) # Project name is alphanumeric only.
filename = re.sub(r'[^A-Za-z0-9\-_\.]', '', filename)
# Check that all the files exist.
if os.path.isdir(os.path.join(root, project)) and os.path.isfile(os.path.join(root, project, filename)):
# Hit counters.
hits = { "hits": 0 }
db = "data/downloads/{}-{}".format(project, filename)
if JsonDB.exists(db.format(project, filename)):
hits = JsonDB.get(db)
# Actually getting the file?
if method == "get":
# Up the hit counter.
hits["hits"] += 1
JsonDB.commit(db, hits)
g.info["method"] = method
g.info["project"] = project
g.info["file"] = filename
g.info["hits"] = hits["hits"]
return template("download.html")
flash("The file or project wasn't found.")
return redirect(url_for("index"))
def get_index(drafts=False):
"""Get the blog index.
The index is the cache of available blog posts. It has the format:
```
{
'post_id': {
fid: Friendly ID for the blog post (for URLs)
time: epoch time of the post
sticky: the stickiness of the post (shows first on global views)
author: the author user ID of the post
categories: [ list of categories ]
privacy: the privacy setting
subject: the post subject
},
...
}
```
Args:
drafts (bool): Whether to allow draft posts to be included in the index
(for logged-in users only).
"""
# Index doesn't exist?
if not JsonDB.exists("blog/index"):
return rebuild_index()
db = JsonDB.get("blog/index")
# Filter out posts that shouldn't be visible (draft/private)
posts = list(db.keys())
for post_id in posts:
privacy = db[post_id]["privacy"]
# Drafts are hidden universally so they can't be seen on any of the
# normal blog routes.
if privacy == "draft":
if drafts is False or not g.info["session"]["login"]:
del db[post_id]
# Private posts are only visible to logged in users.
elif privacy == "private" and not g.info["session"]["login"]:
del db[post_id]
return db
def main():
if len(sys.argv) == 1:
print("Usage: {} <path/to/static/photos>".format(__file__))
sys.exit(1)
photo_root = sys.argv[1]
db = JsonDB.get("photos/index")
photos = set()
for album in db["albums"]:
for key, data in db["albums"][album].iteritems():
for img in ["large", "thumb", "avatar"]:
photos.add(data[img])
# Get all the images and compare.
for img in glob.glob("{}/*.*".format(photo_root)):
fname = img.split("/")[-1]
if not fname in photos:
print("Orphan:", fname)
def get_drafts():
"""Get the draft blog posts.
Drafts are hidden from all places of the blog, just like private posts are
(for non-logged-in users), so get_index() skips drafts and therefore
resolve_id, etc. does too, making them invisible on the normal blog pages.
This function is like get_index() except it *only* returns the drafts.
"""
# Index doesn't exist?
if not JsonDB.exists("blog/index"):
return rebuild_index()
db = JsonDB.get("blog/index")
# Filter out only the draft posts.
return {
key: data for key, data in db.items() if data["privacy"] == "draft"
}
def main():
if len(sys.argv) == 1:
print("Usage: {} <path/to/static/photos>".format(__file__))
sys.exit(1)
photo_root = sys.argv[1]
db = JsonDB.get("photos/index")
photos = set()
for album in db["albums"]:
for key, data in db["albums"][album].iteritems():
for img in ["large", "thumb", "avatar"]:
photos.add(data[img])
# Get all the images and compare.
for img in glob.glob("{}/*.*".format(photo_root)):
fname = img.split("/")[-1]
if not fname in photos:
print("Orphan:", fname)
def legacy_download():
form = None
if request.method == "POST":
form = request.form
else:
form = request.args
method = form.get("method", "index")
project = form.get("project", "")
filename = form.get("file", "")
root = "/home/kirsle/www/projects"
if project and filename:
# Filter the sections.
project = re.sub(r'[^A-Za-z0-9]', '',
project) # Project name is alphanumeric only.
filename = re.sub(r'[^A-Za-z0-9\-_\.]', '', filename)
# Check that all the files exist.
if os.path.isdir(os.path.join(root, project)) and os.path.isfile(
os.path.join(root, project, filename)):
# Hit counters.
hits = {"hits": 0}
db = "data/downloads/{}-{}".format(project, filename)
if JsonDB.exists(db.format(project, filename)):
hits = JsonDB.get(db)
# Actually getting the file?
if method == "get":
# Up the hit counter.
hits["hits"] += 1
JsonDB.commit(db, hits)
g.info["method"] = method
g.info["project"] = project
g.info["file"] = filename
g.info["hits"] = hits["hits"]
return template("download.html")
flash("The file or project wasn't found.")
return redirect(url_for("index"))
def get_private():
"""Get only the private blog posts.
Since you can view only drafts, it made sense to have an easy way to view
only private posts, too.
This function is like get_index() except it *only* returns the private
posts. It doesn't check for logged-in users, because the routes that view
all private posts are login_required anyway.
"""
# Index doesn't exist?
if not JsonDB.exists("blog/index"):
return rebuild_index()
db = JsonDB.get("blog/index")
# Filter out only the draft posts.
return {
key: data for key, data in db.items() if data["privacy"] == "private"
}
def legacy_download():
form = None
if request.method == "POST":
form = request.form
else:
form = request.args
method = form.get("method", "index")
project = form.get("project", "")
filename = form.get("file", "")
root = "/home/kirsle/www/projects"
if project and filename:
# Filter the sections.
project = re.sub(r'[^A-Za-z0-9]', '', project) # Project name is alphanumeric only.
filename = re.sub(r'[^A-Za-z0-9\-_\.]', '', filename)
# Check that all the files exist.
if os.path.isdir(os.path.join(root, project)) and os.path.isfile(os.path.join(root, project, filename)):
# Hit counters.
hits = { "hits": 0 }
db = "data/downloads/{}-{}".format(project, filename)
if JsonDB.exists(db.format(project, filename)):
hits = JsonDB.get(db)
# Actually getting the file?
if method == "get":
# Up the hit counter.
hits["hits"] += 1
JsonDB.commit(db, hits)
g.info["method"] = method
g.info["project"] = project
g.info["file"] = filename
g.info["hits"] = hits["hits"]
return template("download.html")
flash("The file or project wasn't found.")
return redirect(url_for("index"))
def get_referrers(recent=25):
"""Retrieve the referrer details. Returns results in this format:
```
{
referrers: [
["http://...", 20], # Pre-sorted by number of hits
],
recent: [ recent list ]
}
```
"""
db = []
if JsonDB.exists("traffic/referrers"):
db = JsonDB.get("traffic/referrers", cache=False)
# Count the links.
unique = dict()
for link in db:
if not link in unique:
unique[link] = 1
else:
unique[link] += 1
# Sort them by popularity.
result = dict(
referrers=[],
recent=[],
)
sorted_links = sorted(unique.keys(), key=lambda x: unique[x], reverse=True)
for link in sorted_links:
result["referrers"].append([link, unique[link]])
recent = 0 - recent
result["recent"] = db[recent:]
result["recent"].reverse()
return result
def log_referrer(request, link):
"""Double check the referring URL."""
# Ignore if same domain.
hostname = server_name()
if link.startswith("http://{}".format(hostname)) or \
link.startswith("https://{}".format(hostname)):
return None
# See if the URL really links back to us.
hostname = server_name()
try:
r = requests.get(
link,
timeout=5,
verify=False, # Don't do SSL verification
)
# Make sure the request didn't just redirect back to our main site
# (e.g. http://whatever.example.com wildcard may redirect back to
# http://example.com, and if that's us, don't log that!
if r.url.startswith("http://{}".format(hostname)) or \
r.url.startswith("https://{}".format(hostname)):
return None
# Look for our hostname in their page.
if hostname in r.text:
# Log it.
db = list()
if JsonDB.exists("traffic/referrers"):
# Don't cache the result -- the list can get huge!
db = JsonDB.get("traffic/referrers", cache=False)
db.append(link)
JsonDB.commit("traffic/referrers", db, cache=False)
return link
except:
pass
return None
def get_comments(thread):
"""Get the comment thread."""
doc = "comments/threads/{}".format(thread)
if JsonDB.exists(doc):
return JsonDB.get(doc)
return {}
def track_visit(request, session):
"""Main logic to track and log visitor details."""
# Get their tracking cookie value. The value will either be their HTTP
# referrer (if exists and valid) or else a "1".
cookie = session.get("tracking")
addr = remote_addr()
values = dict() # Returnable traffic values
# Log hit counts. We need four kinds:
# - Unique today - Unique total
# - Hits today - Hits total
today = pretty_time("%Y-%m-%d", time.time())
files = {
"unique/{}".format(today): "unique_today",
"unique/total": "unique_total",
"hits/{}".format(today): "hits_today",
"hits/total": "hits_total",
}
# Go through the hit count files. Update them only if their tracking
# cookie was not present.
for file, key in files.items():
dbfile = "traffic/{}".format(file)
if file.startswith("hits"):
# Hit file is just a simple counter.
db = dict(hits=0)
if JsonDB.exists(dbfile):
db = JsonDB.get(dbfile)
if db is None:
db = dict(hits=0)
# Update it?
if not cookie:
db["hits"] += 1
JsonDB.commit(dbfile, db)
# Store the copy.
values[key] = db["hits"]
else:
# Unique file is a collection of IP addresses.
db = dict()
if JsonDB.exists(dbfile):
db = JsonDB.get(dbfile)
if db is None:
db = dict()
# Update with their IP?
if not cookie and not addr in db:
db[addr] = time.time()
JsonDB.commit(dbfile, db)
# Store the copy.
values[key] = len(db.keys())
# Log their HTTP referrer.
referrer = "1"
if request.referrer:
# Branch and check this.
referrer = log_referrer(request, request.referrer)
if not referrer:
# Wasn't a valid referrer.
referrer = "1"
# Set their tracking cookie.
if not cookie:
cookie = referrer
session["tracking"] = cookie
return values
def get_subscribers(thread):
"""Get the subscribers to a comment thread."""
doc = "comments/subscribers/{}".format(thread)
if JsonDB.exists(doc):
return JsonDB.get(doc)
return {}
def get_comments(thread):
"""Get the comment thread."""
doc = "comments/threads/{}".format(thread)
if JsonDB.exists(doc):
return JsonDB.get(doc)
return {}
def get_user(uid=None, username=None):
"""Get a user's DB file, or None if not found."""
if username:
uid = get_uid(username)
logger.debug("get_user: resolved username {} to UID {}".format(username, uid))
return JsonDB.get("users/by-id/{}".format(uid))
def get_uid(username):
"""Turn a username into a user ID."""
db = JsonDB.get("users/by-name/{}".format(username))
if db:
return int(db["uid"])
return None
def get_subscribers(thread):
"""Get the subscribers to a comment thread."""
doc = "comments/subscribers/{}".format(thread)
if JsonDB.exists(doc):
return JsonDB.get(doc)
return {}