def render_page(content):
"""Render the Markdown content of a Wiki page, and support inter-page
linking with [[double braces]].
For simple links, just use the [[Page Name]]. To have a different link text
than the page name, use [[Link Text|Page Name]]."""
html = render_markdown(content)
# Look for [[double brackets]]
links = re.findall(r'\[\[(.+?)\]\]', html)
for match in links:
label = page = match
if "|" in match:
label, page = match.split("|", 2)
# Does the page exist?
output = '''<a href="{url}">{label}</a>'''
if not JsonDB.exists("wiki/pages/{}".format(page)):
output = '''<a href="{url}" class="wiki-broken">{label}</a>'''
html = html.replace(
"[[{}]]".format(match),
output.format(
url=url_for("wiki.view_page", name=name_to_url(page)),
label=label,
))
return html
def render_page(content):
"""Render the Markdown content of a Wiki page, and support inter-page
linking with [[double braces]].
For simple links, just use the [[Page Name]]. To have a different link text
than the page name, use [[Link Text|Page Name]]."""
html = render_markdown(content)
# Look for [[double brackets]]
links = re.findall(r'\[\[(.+?)\]\]', html)
for match in links:
label = page = match
if "|" in match:
label, page = match.split("|", 2)
# Does the page exist?
output = '''<a href="{url}">{label}</a>'''
if not JsonDB.exists("wiki/pages/{}".format(page)):
output = '''<a href="{url}" class="wiki-broken">{label}</a>'''
html = html.replace("[[{}]]".format(match),
output.format(
url=url_for("wiki.view_page", name=name_to_url(page)),
label=label,
)
)
return html
def get_index():
"""Get the blog index.
The index is the cache of available blog posts. It has the format:
```
{
'post_id': {
fid: Friendly ID for the blog post (for URLs)
time: epoch time of the post
sticky: the stickiness of the post (shows first on global views)
author: the author user ID of the post
categories: [ list of categories ]
privacy: the privacy setting
subject: the post subject
},
...
}
```
"""
# Index doesn't exist?
if not JsonDB.exists("blog/index"):
return rebuild_index()
db = JsonDB.get("blog/index")
# Hide any private posts if we aren't logged in.
if not g.info["session"]["login"]:
for post_id, data in db.items():
if data["privacy"] == "private":
del db[post_id]
return db
def get_index():
"""Get the blog index.
The index is the cache of available blog posts. It has the format:
```
{
'post_id': {
fid: Friendly ID for the blog post (for URLs)
time: epoch time of the post
sticky: the stickiness of the post (shows first on global views)
author: the author user ID of the post
categories: [ list of categories ]
privacy: the privacy setting
subject: the post subject
},
...
}
```
"""
# Index doesn't exist?
if not JsonDB.exists("blog/index"):
return rebuild_index()
db = JsonDB.get("blog/index")
# Hide any private posts if we aren't logged in.
if not g.info["session"]["login"]:
for post_id, data in db.items():
if data["privacy"] == "private":
del db[post_id]
return db
def delete_page(name):
"""Completely delete a wiki page."""
name = name.strip("/")
path = "wiki/pages/{}".format(name)
if JsonDB.exists(path):
logger.info("Delete Wiki page {}".format(name))
JsonDB.delete(path)
return True
def delete_page(name):
"""Completely delete a wiki page."""
name = name.strip("/")
path = "wiki/pages/{}".format(name)
if JsonDB.exists(path):
logger.info("Delete Wiki page {}".format(name))
JsonDB.delete(path)
return True
def get_page(name):
"""Get a Wiki page. Returns `None` if the page isn't found."""
name = name.strip("/") # Remove any surrounding slashes.
path = "wiki/pages/{}".format(name)
if not JsonDB.exists(path):
return None
# TODO: case insensitive page names...
db = JsonDB.get(path)
return db
def get_page(name):
"""Get a Wiki page. Returns `None` if the page isn't found."""
name = name.strip("/") # Remove any surrounding slashes.
path = "wiki/pages/{}".format(name)
if not JsonDB.exists(path):
return None
# TODO: case insensitive page names...
db = JsonDB.get(path)
return db
def get_index():
"""Get the photo album index, or a new empty DB if it doesn't exist."""
if JsonDB.exists("photos/index"):
return JsonDB.get("photos/index")
return {
"albums": {}, # Album data
"map": {}, # Map photo keys to albums
"covers": {}, # Album cover photos
"photo-order": {}, # Ordering of photos in albums
"album-order": [], # Ordering of albums themselves
}
def get_index():
"""Get the photo album index, or a new empty DB if it doesn't exist."""
if JsonDB.exists("photos/index"):
return JsonDB.get("photos/index")
return {
"albums": {}, # Album data
"map": {}, # Map photo keys to albums
"covers": {}, # Album cover photos
"photo-order": {}, # Ordering of photos in albums
"album-order": [], # Ordering of albums themselves
}
def add_subscriber(thread, email):
"""Add a subscriber to a thread."""
if not "@" in email:
return
# Sanity check: only subscribe to threads that exist.
if not JsonDB.exists("comments/threads/{}".format(thread)):
return
logger.info("Subscribe e-mail {} to thread {}".format(email, thread))
subs = get_subscribers(thread)
subs[email] = int(time.time())
write_subscribers(thread, subs)
def add_subscriber(thread, email):
"""Add a subscriber to a thread."""
if not "@" in email:
return
# Sanity check: only subscribe to threads that exist.
if not JsonDB.exists("comments/threads/{}".format(thread)):
return
logger.info("Subscribe e-mail {} to thread {}".format(email, thread))
subs = get_subscribers(thread)
subs[email] = int(time.time())
write_subscribers(thread, subs)
def get_entry(post_id):
"""Load a full blog entry."""
if not JsonDB.exists("blog/entries/{}".format(post_id)):
return None
db = JsonDB.get("blog/entries/{}".format(post_id))
# If no FID, set it to the ID.
if len(db["fid"]) == 0:
db["fid"] = str(post_id)
# If no "format" option, set it to HTML (legacy)
if db.get("format", "") == "":
db["format"] = "html"
return db
def get_entry(post_id):
"""Load a full blog entry."""
if not JsonDB.exists("blog/entries/{}".format(post_id)):
return None
db = JsonDB.get("blog/entries/{}".format(post_id))
# If no FID, set it to the ID.
if len(db["fid"]) == 0:
db["fid"] = str(post_id)
# If no "format" option, set it to HTML (legacy)
if db.get("format", "") == "":
db["format"] = "html"
return db
def legacy_download():
form = None
if request.method == "POST":
form = request.form
else:
# CNET links to the MS-DOS download using semicolon delimiters in the
# query string. Fix that if detected.
query = request.query_string.decode()
if not '&' in query and ';' in query:
url = re.sub(r';|%3b', '&', request.url, flags=re.IGNORECASE)
return redirect(url)
form = request.args
method = form.get("method", "index")
project = form.get("project", "")
filename = form.get("file", "")
root = "/home/kirsle/www/projects"
if project and filename:
# Filter the sections.
project = re.sub(r'[^A-Za-z0-9]', '', project) # Project name is alphanumeric only.
filename = re.sub(r'[^A-Za-z0-9\-_\.]', '', filename)
# Check that all the files exist.
if os.path.isdir(os.path.join(root, project)) and os.path.isfile(os.path.join(root, project, filename)):
# Hit counters.
hits = { "hits": 0 }
db = "data/downloads/{}-{}".format(project, filename)
if JsonDB.exists(db.format(project, filename)):
hits = JsonDB.get(db)
# Actually getting the file?
if method == "get":
# Up the hit counter.
hits["hits"] += 1
JsonDB.commit(db, hits)
g.info["method"] = method
g.info["project"] = project
g.info["file"] = filename
g.info["hits"] = hits["hits"]
return template("download.html")
flash("The file or project wasn't found.")
return redirect(url_for("index"))
def get_index(drafts=False):
"""Get the blog index.
The index is the cache of available blog posts. It has the format:
```
{
'post_id': {
fid: Friendly ID for the blog post (for URLs)
time: epoch time of the post
sticky: the stickiness of the post (shows first on global views)
author: the author user ID of the post
categories: [ list of categories ]
privacy: the privacy setting
subject: the post subject
},
...
}
```
Args:
drafts (bool): Whether to allow draft posts to be included in the index
(for logged-in users only).
"""
# Index doesn't exist?
if not JsonDB.exists("blog/index"):
return rebuild_index()
db = JsonDB.get("blog/index")
# Filter out posts that shouldn't be visible (draft/private)
posts = list(db.keys())
for post_id in posts:
privacy = db[post_id]["privacy"]
# Drafts are hidden universally so they can't be seen on any of the
# normal blog routes.
if privacy == "draft":
if drafts is False or not g.info["session"]["login"]:
del db[post_id]
# Private posts are only visible to logged in users.
elif privacy == "private" and not g.info["session"]["login"]:
del db[post_id]
return db
def unsubscribe(thread, email):
"""Unsubscribe an e-mail address from a thread.
If `thread` is `*`, the e-mail is unsubscribed from all threads."""
# Which threads to unsubscribe from?
threads = []
if thread == "*":
threads = JsonDB.list_docs("comments/subscribers")
else:
threads = [thread]
# Remove them as a subscriber.
for thread in threads:
if JsonDB.exists("comments/subscribers/{}".format(thread)):
logger.info("Unsubscribe e-mail address {} from comment thread {}".format(email, thread))
db = get_subscribers(thread)
del db[email]
write_subscribers(thread, db)
def get_drafts():
"""Get the draft blog posts.
Drafts are hidden from all places of the blog, just like private posts are
(for non-logged-in users), so get_index() skips drafts and therefore
resolve_id, etc. does too, making them invisible on the normal blog pages.
This function is like get_index() except it *only* returns the drafts.
"""
# Index doesn't exist?
if not JsonDB.exists("blog/index"):
return rebuild_index()
db = JsonDB.get("blog/index")
# Filter out only the draft posts.
return {
key: data for key, data in db.items() if data["privacy"] == "draft"
}
def legacy_download():
form = None
if request.method == "POST":
form = request.form
else:
form = request.args
method = form.get("method", "index")
project = form.get("project", "")
filename = form.get("file", "")
root = "/home/kirsle/www/projects"
if project and filename:
# Filter the sections.
project = re.sub(r'[^A-Za-z0-9]', '',
project) # Project name is alphanumeric only.
filename = re.sub(r'[^A-Za-z0-9\-_\.]', '', filename)
# Check that all the files exist.
if os.path.isdir(os.path.join(root, project)) and os.path.isfile(
os.path.join(root, project, filename)):
# Hit counters.
hits = {"hits": 0}
db = "data/downloads/{}-{}".format(project, filename)
if JsonDB.exists(db.format(project, filename)):
hits = JsonDB.get(db)
# Actually getting the file?
if method == "get":
# Up the hit counter.
hits["hits"] += 1
JsonDB.commit(db, hits)
g.info["method"] = method
g.info["project"] = project
g.info["file"] = filename
g.info["hits"] = hits["hits"]
return template("download.html")
flash("The file or project wasn't found.")
return redirect(url_for("index"))
def get_private():
"""Get only the private blog posts.
Since you can view only drafts, it made sense to have an easy way to view
only private posts, too.
This function is like get_index() except it *only* returns the private
posts. It doesn't check for logged-in users, because the routes that view
all private posts are login_required anyway.
"""
# Index doesn't exist?
if not JsonDB.exists("blog/index"):
return rebuild_index()
db = JsonDB.get("blog/index")
# Filter out only the draft posts.
return {
key: data for key, data in db.items() if data["privacy"] == "private"
}
def unsubscribe(thread, email):
"""Unsubscribe an e-mail address from a thread.
If `thread` is `*`, the e-mail is unsubscribed from all threads."""
# Which threads to unsubscribe from?
threads = []
if thread == "*":
threads = JsonDB.list_docs("comments/subscribers")
else:
threads = [thread]
# Remove them as a subscriber.
for thread in threads:
if JsonDB.exists("comments/subscribers/{}".format(thread)):
logger.info(
"Unsubscribe e-mail address {} from comment thread {}".format(
email, thread))
db = get_subscribers(thread)
del db[email]
write_subscribers(thread, db)
def legacy_download():
form = None
if request.method == "POST":
form = request.form
else:
form = request.args
method = form.get("method", "index")
project = form.get("project", "")
filename = form.get("file", "")
root = "/home/kirsle/www/projects"
if project and filename:
# Filter the sections.
project = re.sub(r'[^A-Za-z0-9]', '', project) # Project name is alphanumeric only.
filename = re.sub(r'[^A-Za-z0-9\-_\.]', '', filename)
# Check that all the files exist.
if os.path.isdir(os.path.join(root, project)) and os.path.isfile(os.path.join(root, project, filename)):
# Hit counters.
hits = { "hits": 0 }
db = "data/downloads/{}-{}".format(project, filename)
if JsonDB.exists(db.format(project, filename)):
hits = JsonDB.get(db)
# Actually getting the file?
if method == "get":
# Up the hit counter.
hits["hits"] += 1
JsonDB.commit(db, hits)
g.info["method"] = method
g.info["project"] = project
g.info["file"] = filename
g.info["hits"] = hits["hits"]
return template("download.html")
flash("The file or project wasn't found.")
return redirect(url_for("index"))
def get_referrers(recent=25):
"""Retrieve the referrer details. Returns results in this format:
```
{
referrers: [
["http://...", 20], # Pre-sorted by number of hits
],
recent: [ recent list ]
}
```
"""
db = []
if JsonDB.exists("traffic/referrers"):
db = JsonDB.get("traffic/referrers", cache=False)
# Count the links.
unique = dict()
for link in db:
if not link in unique:
unique[link] = 1
else:
unique[link] += 1
# Sort them by popularity.
result = dict(
referrers=[],
recent=[],
)
sorted_links = sorted(unique.keys(), key=lambda x: unique[x], reverse=True)
for link in sorted_links:
result["referrers"].append([link, unique[link]])
recent = 0 - recent
result["recent"] = db[recent:]
result["recent"].reverse()
return result
def log_referrer(request, link):
"""Double check the referring URL."""
# Ignore if same domain.
hostname = server_name()
if link.startswith("http://{}".format(hostname)) or \
link.startswith("https://{}".format(hostname)):
return None
# See if the URL really links back to us.
hostname = server_name()
try:
r = requests.get(
link,
timeout=5,
verify=False, # Don't do SSL verification
)
# Make sure the request didn't just redirect back to our main site
# (e.g. http://whatever.example.com wildcard may redirect back to
# http://example.com, and if that's us, don't log that!
if r.url.startswith("http://{}".format(hostname)) or \
r.url.startswith("https://{}".format(hostname)):
return None
# Look for our hostname in their page.
if hostname in r.text:
# Log it.
db = list()
if JsonDB.exists("traffic/referrers"):
# Don't cache the result -- the list can get huge!
db = JsonDB.get("traffic/referrers", cache=False)
db.append(link)
JsonDB.commit("traffic/referrers", db, cache=False)
return link
except:
pass
return None
def get_subscribers(thread):
"""Get the subscribers to a comment thread."""
doc = "comments/subscribers/{}".format(thread)
if JsonDB.exists(doc):
return JsonDB.get(doc)
return {}
def get_comments(thread):
"""Get the comment thread."""
doc = "comments/threads/{}".format(thread)
if JsonDB.exists(doc):
return JsonDB.get(doc)
return {}
def exists(uid=None, username=None):
"""Query whether a user ID or name exists."""
if uid:
return JsonDB.exists("users/by-id/{}".format(uid))
elif username:
return JsonDB.exists("users/by-name/{}".format(username.lower()))
def get_subscribers(thread):
"""Get the subscribers to a comment thread."""
doc = "comments/subscribers/{}".format(thread)
if JsonDB.exists(doc):
return JsonDB.get(doc)
return {}
def get_comments(thread):
"""Get the comment thread."""
doc = "comments/threads/{}".format(thread)
if JsonDB.exists(doc):
return JsonDB.get(doc)
return {}
def track_visit(request, session):
"""Main logic to track and log visitor details."""
# Get their tracking cookie value. The value will either be their HTTP
# referrer (if exists and valid) or else a "1".
cookie = session.get("tracking")
addr = remote_addr()
values = dict() # Returnable traffic values
# Log hit counts. We need four kinds:
# - Unique today - Unique total
# - Hits today - Hits total
today = pretty_time("%Y-%m-%d", time.time())
files = {
"unique/{}".format(today): "unique_today",
"unique/total": "unique_total",
"hits/{}".format(today): "hits_today",
"hits/total": "hits_total",
}
# Go through the hit count files. Update them only if their tracking
# cookie was not present.
for file, key in files.items():
dbfile = "traffic/{}".format(file)
if file.startswith("hits"):
# Hit file is just a simple counter.
db = dict(hits=0)
if JsonDB.exists(dbfile):
db = JsonDB.get(dbfile)
if db is None:
db = dict(hits=0)
# Update it?
if not cookie:
db["hits"] += 1
JsonDB.commit(dbfile, db)
# Store the copy.
values[key] = db["hits"]
else:
# Unique file is a collection of IP addresses.
db = dict()
if JsonDB.exists(dbfile):
db = JsonDB.get(dbfile)
if db is None:
db = dict()
# Update with their IP?
if not cookie and not addr in db:
db[addr] = time.time()
JsonDB.commit(dbfile, db)
# Store the copy.
values[key] = len(db.keys())
# Log their HTTP referrer.
referrer = "1"
if request.referrer:
# Branch and check this.
referrer = log_referrer(request, request.referrer)
if not referrer:
# Wasn't a valid referrer.
referrer = "1"
# Set their tracking cookie.
if not cookie:
cookie = referrer
session["tracking"] = cookie
return values