def delete(self, username, reponame):
# Check whether the key exists and if maybe the last change already is
# a delete, else insert a `CSet.DELETE` entry without any blob data.
key = self.get_query_argument("key")
if username != self.current_user.name:
raise HTTPError(403)
if not key:
raise HTTPError(400)
datestr = self.get_query_argument("datetime", None)
ts = datestr and date(datestr, QSDATEFMT) or now()
try:
repo = (Repo
.select(Repo.id)
.join(User)
.where((User.name == username) & (Repo.name == reponame))
.naive()
.get())
except Repo.DoesNotExist:
raise HTTPError(404)
sha = shasum(key.encode("utf-8"))
try:
last = (CSet
.select(CSet.time, CSet.type)
.where((CSet.repo == repo) & (CSet.hkey == sha))
.order_by(CSet.time.desc())
.limit(1)
.naive()
.get())
except CSet.DoesNotExist:
# No changeset was found for the given key -
# the resource does not exist.
raise HTTPError(400)
if not ts > last.time:
# Appended timestamps must be monotonically increasing!
raise HTTPError(400)
if last.type == CSet.DELETE:
# The resource was deleted already, return instantly.
return self.finish()
# Insert the new "delete" change.
CSet.create(repo=repo, hkey=sha, time=ts, type=CSet.DELETE, len=0)
def get(self, username, reponame):
try:
repo = (Repo.select().join(User).alias("user").where(
(User.name == username) & (Repo.name == reponame)).get())
title = repo.user.name + "/" + repo.name
timemap = self.get_query_argument("timemap", "false") == "true"
datetime = self.get_query_argument("datetime", None)
key = self.get_query_argument("key", None)
if key and not timemap:
self.render("repo/memento.html",
repo=repo,
key=key,
datetime=datetime)
elif key and timemap:
self.render("repo/history.html", repo=repo, key=key)
else:
cs = (CSet.select(fn.distinct(
CSet.hkey)).where(CSet.repo == repo).limit(5).alias("cs"))
samples = (HMap.select(HMap.val).join(
cs, on=(HMap.sha == cs.c.hkey_id)))
self.render("repo/show.html",
title=title,
repo=repo,
samples=list(samples))
except Repo.DoesNotExist:
raise HTTPError(404)
def get(self, username, reponame):
try:
repo = (Repo.select().join(User).alias("user")
.where((User.name == username) & (Repo.name == reponame))
.get())
title = repo.user.name + "/" + repo.name
timemap = self.get_query_argument("timemap", "false") == "true"
datetime = self.get_query_argument("datetime", None)
key = self.get_query_argument("key", None)
if key and not timemap:
self.render("repo/memento.html", repo=repo, key=key,
datetime=datetime)
elif key and timemap:
self.render("repo/history.html", repo=repo, key=key)
else:
cs = (CSet.select(fn.distinct(CSet.hkey))
.where(CSet.repo == repo).limit(5).alias("cs"))
samples = (HMap.select(HMap.val)
.join(cs, on=(HMap.sha == cs.c.hkey_id)))
self.render("repo/show.html", title=title, repo=repo,
samples=list(samples))
except Repo.DoesNotExist:
raise HTTPError(404)
def get(self, username, reponame):
try:
repo = (Repo.select().join(User).alias("user")
.where((User.name == username) & (Repo.name == reponame))
.get())
title = repo.user.name + "/" + repo.name
timemap = self.get_query_argument("timemap", "false") == "true"
datetime = self.get_query_argument("datetime", None)
key = self.get_query_argument("key", None)
index = self.get_query_argument("index", "false") == "true"
if self.get_query_argument("datetime", None):
datestr = self.get_query_argument("datetime")
try:
ts = date(datestr, QSDATEFMT)
except ValueError:
raise HTTPError(reason="Invalid format of datetime param", status_code=400)
elif "Accept-Datetime" in self.request.headers:
datestr = self.request.headers.get("Accept-Datetime")
ts = date(datestr, RFC1123DATEFMT)
else:
ts = now()
if key and not timemap:
chain = revision_logic.get_chain_at_ts(repo, key, ts)
# use ts of cset instead of now(), to make prev work
if len(chain) != 0:
ts = chain[-1].time
cs_prev = revision_logic.get_cset_prev_before_ts(repo, key, ts)
cs_next = revision_logic.get_cset_next_after_ts(repo, key, ts)
if cs_prev:
cs_prev_str = self.request.protocol + "://" + self.request.host + self.request.path + "?key=" + key + "&datetime=" + cs_prev.time.strftime(QSDATEFMT)
else:
cs_prev_str = ""
if cs_next:
cs_next_str = self.request.protocol + "://" + self.request.host + self.request.path + "?key=" + key + "&datetime=" + cs_next.time.strftime(QSDATEFMT)
else:
cs_next_str = ""
commit_message = revision_logic.get_commit_message(repo, key, ts)
self.render("repo/memento.html", repo=repo, key=key, datetime=datetime, cs_next_str=cs_next_str, cs_prev_str=cs_prev_str, commit_message=commit_message)
elif key and timemap:
self.render("repo/history.html", repo=repo, key=key)
elif index:
cs = (CSet.select(fn.distinct(CSet.hkey)).where((CSet.repo == repo) & (CSet.time <= ts)).alias("cs"))
key_count = (HMap.select(HMap.val).join(cs, on=(HMap.sha == cs.c.hkey_id))).count()
page = int(self.get_query_argument("page", "1"))
hm = revision_logic.get_repo_index(repo, ts, page)
self.render("repo/index.html", repo=repo, title=title, key_count=key_count, page_size=revision_logic.INDEX_PAGE_SIZE, hm=hm, current_page=page)
else:
hm = list(revision_logic.get_repo_index(repo, ts, 1, 5))
# cs = (CSet.select(fn.distinct(CSet.hkey)).where(CSet.repo == repo).limit(5).alias("cs"))
# samples = (HMap.select(HMap.val).join(cs, on=(HMap.sha == cs.c.hkey_id)))
self.render("repo/show.html", title=title, repo=repo, hm=hm)
except Repo.DoesNotExist:
raise HTTPError(reason="Repo not found.", status_code=404)
def __get_cset_at_ts(repo, sha, ts):
try:
cset = CSet.get(CSet.repo == repo, CSet.hkey == sha, CSet.time == ts)
except CSet.DoesNotExist:
return None
return cset
def __remove_csets(repo, sha):
# remove blobs
q_blobs = Blob.delete().where(Blob.repo == repo, Blob.hkey == sha)
q_blobs.execute()
# remove csets
q_csets = CSet.delete().where(CSet.repo == repo, CSet.hkey == sha)
q_csets.execute()
def __remove_csets_repo(repo):
# remove blobs
q_blobs = Blob.delete().where(Blob.repo == repo)
q_blobs.execute()
# remove csets
q_csets = CSet.delete().where(CSet.repo == repo)
q_csets.execute()
def delete(self, username, reponame):
# Check whether the key exists and if maybe the last change already is
# a delete, else insert a `CSet.DELETE` entry without any blob data.
key = self.get_query_argument("key")
if username != self.current_user.name:
raise HTTPError(403)
if not key:
raise HTTPError(400)
datestr = self.get_query_argument("datetime", None)
ts = datestr and date(datestr, QSDATEFMT) or now()
try:
repo = (Repo.select(Repo.id).join(
User).where((User.name == username)
& (Repo.name == reponame)).naive().get())
except Repo.DoesNotExist:
raise HTTPError(404)
sha = shasum(key.encode("utf-8"))
try:
last = (CSet.select(
CSet.time,
CSet.type).where((CSet.repo == repo)
& (CSet.hkey == sha)).order_by(
CSet.time.desc()).limit(1).naive().get())
except CSet.DoesNotExist:
# No changeset was found for the given key -
# the resource does not exist.
raise HTTPError(400)
if not ts > last.time:
# Appended timestamps must be monotonically increasing!
raise HTTPError(400)
if last.type == CSet.DELETE:
# The resource was deleted already, return instantly.
return self.finish()
# Insert the new "delete" change.
CSet.create(repo=repo, hkey=sha, time=ts, type=CSet.DELETE, len=0)
def __save_revision(repo, sha, chain, stmts, ts):
# this checks if timestamp is after the last cset of the chain, not if its after all csets for key.
# this allows pushing to any timestamp, if the chain is right
if chain and len(chain) > 0 and not ts > chain[-1].time:
# Appended timestamps must be monotonically increasing!
raise ValueError
if len(chain) == 0 or chain[0].type == CSet.DELETE:
# Provide dummy value for `patch` which is never stored.
# If we get here, we always store a snapshot later on!
patch = ""
else:
# Reconstruct the previous state of the resource
prev = __get_revision(repo, sha, chain)
if stmts == prev:
# No changes, nothing to be done. Bail out.
return None
patch = compress(join(
map(lambda s: "D " + s, prev - stmts) +
map(lambda s: "A " + s, stmts - prev), "\n"))
snapc = compress(join(stmts, "\n"))
# Calculate the accumulated size of the delta chain including
# the (potential) patch from the previous to the pushed state.
accumulated_len = reduce(lambda s, e: s + e.len, chain[1:], 0) + len(patch)
base_len = len(chain) > 0 and chain[0].len or 0 # base length
if (len(chain) == 0 or
chain[0].type == CSet.DELETE or
len(snapc) <= len(patch) or
SNAPF * base_len <= accumulated_len):
# Store the current state as a new snapshot
Blob.create(repo=repo, hkey=sha, time=ts, data=snapc)
CSet.create(repo=repo, hkey=sha, time=ts, type=CSet.SNAPSHOT,
len=len(snapc))
else:
# Store a directed delta between the previous and current state
Blob.create(repo=repo, hkey=sha, time=ts, data=patch)
CSet.create(repo=repo, hkey=sha, time=ts, type=CSet.DELTA,
len=len(patch))
return 0
def __remove_cset(repo, sha, ts):
# remove cset
try:
cset = CSet.get(CSet.repo == repo, CSet.hkey == sha, CSet.time == ts)
count = cset.delete_instance()
except CSet.DoesNotExist:
return None
# remove blob
try:
blob = Blob.get(Blob.repo == repo, Blob.hkey == sha, Blob.time == ts)
blob.delete_instance()
except Blob.DoesNotExist:
return None
def __save_revision_delete(repo, sha, ts):
chain = __get_chain_at_ts(repo, sha, ts)
if chain[-1]:
if not chain[-1].type == CSet.DELETE:
# only if there are csets before and the last is no delete
stmts_next = set()
cset_next = __get_cset_next_after_ts(repo, sha, ts)
if cset_next != None:
if cset_next.type == CSet.DELTA:
# If next changeset is Delta, keep next revision statements
chain_next = __get_chain_at_ts(repo, sha, cset_next.time)
stmts_next = __get_revision(repo, sha, chain_next)
elif cset_next.type == CSet.DELETE:
# If next changeset is Delete, remove it
__remove_cset(repo, sha, cset_next.time)
# Nothing to be done if next changeset is a Snapshot
if __get_cset_at_ts(repo, sha, ts):
# If there is a cset at the exact time, delete it
# (reconstruction of next Cset already happened)
__remove_revision(repo, sha, ts)
# Insert the new "delete" change
CSet.create(repo=repo, hkey=sha, time=ts, type=CSet.DELETE, len=0)
if cset_next != None:
if cset_next.type == CSet.DELTA:
# If next changeset is Delta, reconstruct
__remove_cset(repo, sha, cset_next.time)
chain = __get_chain_at_ts(repo, sha, cset_next.time)
__save_revision(repo, sha, chain, stmts_next, cset_next.time)
else:
# Nothing happens. Resource is already deleted. This is legitimate, so no exception needed
# TODO should this be announced to client somehow?
pass
else:
raise LookupError
def get_revision_count(): return CSet.select().count()
def put(self, username, reponame):
# Create a new revision of the resource specified by `key`.
fmt = self.request.headers.get("Content-Type", "application/n-triples")
key = self.get_query_argument("key", None)
if username != self.current_user.name:
raise HTTPError(403)
if not key:
raise HTTPError(400)
datestr = self.get_query_argument("datetime", None)
ts = datestr and date(datestr, QSDATEFMT) or now()
try:
repo = (Repo.select(Repo.id).join(
User).where((User.name == username)
& (Repo.name == reponame)).naive().get())
except Repo.DoesNotExist:
raise HTTPError(404)
sha = shasum(key.encode("utf-8"))
chain = list(
CSet.select(CSet.time, CSet.type, CSet.len).where(
(CSet.repo == repo) & (CSet.hkey == sha) & (CSet.time >= SQL(
"COALESCE((SELECT time FROM cset "
"WHERE repo_id = %s "
"AND hkey_id = %s "
"AND type != %s "
"ORDER BY time DESC "
"LIMIT 1), 0)", repo.id, sha, CSet.DELTA))).order_by(
CSet.time).naive())
if len(chain) > 0 and not ts > chain[-1].time:
# Appended timestamps must be monotonically increasing!
raise HTTPError(400)
if len(chain) == 0:
# Mapping for `key` likely does not exist:
# Store the SHA-to-KEY mapping in HMap,
# looking out for possible collisions.
try:
HMap.create(sha=sha, val=key)
except IntegrityError:
val = HMap.select(HMap.val).where(HMap.sha == sha).scalar()
if val != key:
raise HTTPError(500)
# Parse and normalize into a set of N-Quad lines
stmts = parse(self.request.body, fmt)
snapc = compress(join(stmts, "\n"))
if len(chain) == 0 or chain[0].type == CSet.DELETE:
# Provide dummy value for `patch` which is never stored.
# If we get here, we always store a snapshot later on!
patch = ""
else:
# Reconstruct the previous state of the resource
prev = set()
blobs = (Blob.select(Blob.data).where(
(Blob.repo == repo) & (Blob.hkey == sha)
& (Blob.time << map(lambda e: e.time, chain))).order_by(
Blob.time).naive())
for i, blob in enumerate(blobs.iterator()):
data = decompress(blob.data)
if i == 0:
# Base snapshot for the delta chain
prev.update(data.splitlines())
else:
for line in data.splitlines():
mode, stmt = line[0], line[2:]
if mode == "A":
prev.add(stmt)
else:
prev.discard(stmt)
if stmts == prev:
# No changes, nothing to be done. Bail out.
return self.finish()
patch = compress(
join(
map(lambda s: "D " + s, prev - stmts) +
map(lambda s: "A " + s, stmts - prev), "\n"))
# Calculate the accumulated size of the delta chain including
# the (potential) patch from the previous to the pushed state.
acclen = reduce(lambda s, e: s + e.len, chain[1:], 0) + len(patch)
blen = len(chain) > 0 and chain[0].len or 0 # base length
if (len(chain) == 0 or chain[0].type == CSet.DELETE
or len(snapc) <= len(patch) or SNAPF * blen <= acclen):
# Store the current state as a new snapshot
Blob.create(repo=repo, hkey=sha, time=ts, data=snapc)
CSet.create(repo=repo,
hkey=sha,
time=ts,
type=CSet.SNAPSHOT,
len=len(snapc))
else:
# Store a directed delta between the previous and current state
Blob.create(repo=repo, hkey=sha, time=ts, data=patch)
CSet.create(repo=repo,
hkey=sha,
time=ts,
type=CSet.DELTA,
len=len(patch))
def get(self, username, reponame):
timemap = self.get_query_argument("timemap", "false") == "true"
index = self.get_query_argument("index", "false") == "true"
key = self.get_query_argument("key", None)
if (index and timemap) or (index and key) or (timemap and not key):
raise HTTPError(400)
if self.get_query_argument("datetime", None):
datestr = self.get_query_argument("datetime")
ts = date(datestr, QSDATEFMT)
elif "Accept-Datetime" in self.request.headers:
datestr = self.request.headers.get("Accept-Datetime")
ts = date(datestr, RFC1123DATEFMT)
else:
ts = now()
try:
repo = (Repo.select(Repo.id).join(
User).where((User.name == username)
& (Repo.name == reponame)).naive().get())
except Repo.DoesNotExist:
raise HTTPError(404)
if key and not timemap:
# Recreate the resource for the given key in its latest state -
# if no `datetime` was provided - or in the state it was in at
# the time indicated by the passed `datetime` argument.
self.set_header("Content-Type", "application/n-quads")
self.set_header("Vary", "accept-datetime")
sha = shasum(key.encode("utf-8"))
# Fetch all relevant changes from the last "non-delta" onwards,
# ordered by time. The returned delta-chain consists of either:
# a snapshot followed by 0 or more deltas, or
# a single delete.
chain = list(
CSet.select(CSet.time, CSet.type).where((CSet.repo == repo) & (
CSet.hkey == sha) & (CSet.time <= ts) & (CSet.time >= SQL(
"COALESCE((SELECT time FROM cset "
"WHERE repo_id = %s "
"AND hkey_id = %s "
"AND time <= %s "
"AND type != %s "
"ORDER BY time DESC "
"LIMIT 1), 0)", repo.id, sha, ts, CSet.DELTA))).
order_by(CSet.time).naive())
if len(chain) == 0:
# A resource does not exist for the given key.
raise HTTPError(404)
timegate_url = (self.request.protocol + "://" + self.request.host +
self.request.path)
timemap_url = (self.request.protocol + "://" + self.request.host +
self.request.uri + "&timemap=true")
self.set_header(
"Link", '<%s>; rel="original"'
', <%s>; rel="timegate"'
', <%s>; rel="timemap"' % (key, timegate_url, timemap_url))
self.set_header("Memento-Datetime",
chain[-1].time.strftime(RFC1123DATEFMT))
if chain[0].type == CSet.DELETE:
# The last change was a delete. Return a 404 response with
# appropriate "Link" and "Memento-Datetime" headers.
raise HTTPError(404)
# Load the data required in order to restore the resource state.
blobs = (Blob.select(Blob.data).where(
(Blob.repo == repo) & (Blob.hkey == sha)
& (Blob.time << map(lambda e: e.time, chain))).order_by(
Blob.time).naive())
if len(chain) == 1:
# Special case, where we can simply return
# the blob data of the snapshot.
snap = blobs.first().data
return self.finish(decompress(snap))
stmts = set()
for i, blob in enumerate(blobs.iterator()):
data = decompress(blob.data)
if i == 0:
# Base snapshot for the delta chain
stmts.update(data.splitlines())
else:
for line in data.splitlines():
mode, stmt = line[0], line[2:]
if mode == "A":
stmts.add(stmt)
else:
stmts.discard(stmt)
self.write(join(stmts, "\n"))
elif key and timemap:
# Generate a timemap containing historic change information
# for the requested key. The timemap is in the default link-format
# or as JSON (http://mementoweb.org/guide/timemap-json/).
sha = shasum(key.encode("utf-8"))
csets = (CSet.select(
CSet.time).where((CSet.repo == repo)
& (CSet.hkey == sha)).order_by(
CSet.time.desc()).naive())
# TODO: Paginate?
csit = csets.iterator()
try:
first = csit.next()
except StopIteration:
# Resource for given key does not exist.
raise HTTPError(404)
req = self.request
base = req.protocol + "://" + req.host + req.path
accept = self.request.headers.get("Accept", "")
if "application/json" in accept or "*/*" in accept:
self.set_header("Content-Type", "application/json")
self.write('{"original_uri": ' + json_encode(key))
self.write(', "mementos": {"list":[')
m = ('{{"datetime": "{0}", "uri": "' + base + '?key=' +
url_escape(key) + '&datetime={1}"}}')
self.write(
m.format(first.time.isoformat(),
first.time.strftime(QSDATEFMT)))
for cs in csit:
self.write(', ' + m.format(cs.time.isoformat(),
cs.time.strftime(QSDATEFMT)))
self.write(']}')
self.write('}')
else:
m = (',\n'
'<' + base + '?key=' + url_escape(key) + '&datetime={0}>'
'; rel="memento"'
'; datetime="{1}"'
'; type="application/n-quads"')
self.set_header("Content-Type", "application/link-format")
self.write('<' + key + '>; rel="original"')
self.write(
m.format(first.time.strftime(QSDATEFMT),
first.time.strftime(RFC1123DATEFMT)))
for cs in csit:
self.write(
m.format(cs.time.strftime(QSDATEFMT),
cs.time.strftime(RFC1123DATEFMT)))
elif index:
# Generate an index of all URIs contained in the dataset at the
# provided point in time or in its current state.
self.set_header("Vary", "accept-datetime")
self.set_header("Content-Type", "text/plain")
page = int(self.get_query_argument("page", "1"))
# Subquery for selecting max. time per hkey group
mx = (CSet.select(
CSet.hkey,
fn.Max(CSet.time).alias("maxtime")).where(
(CSet.repo == repo) & (CSet.time <= ts)).group_by(
CSet.hkey).order_by(CSet.hkey).paginate(
page, INDEX_PAGE_SIZE).alias("mx"))
# Query for all the relevant csets (those with max. time values)
cs = (CSet.select(CSet.hkey, CSet.time).join(
mx,
on=((CSet.hkey == mx.c.hkey_id) & (CSet.time == mx.c.maxtime)
)).where((CSet.repo == repo)
& (CSet.type != CSet.DELETE)).alias("cs"))
# Join with the hmap table to retrieve the plain key values
hm = (HMap.select(HMap.val).join(
cs, on=(HMap.sha == cs.c.hkey_id)).naive())
for h in hm.iterator():
self.write(h.val + "\n")
else:
raise HTTPError(400)
def put(self, username, reponame):
# Create a new revision of the resource specified by `key`.
fmt = self.request.headers.get("Content-Type", "application/n-triples")
key = self.get_query_argument("key", None)
if username != self.current_user.name:
raise HTTPError(403)
if not key:
raise HTTPError(400)
datestr = self.get_query_argument("datetime", None)
ts = datestr and date(datestr, QSDATEFMT) or now()
try:
repo = (Repo
.select(Repo.id)
.join(User)
.where((User.name == username) & (Repo.name == reponame))
.naive()
.get())
except Repo.DoesNotExist:
raise HTTPError(404)
sha = shasum(key.encode("utf-8"))
chain = list(CSet
.select(CSet.time, CSet.type, CSet.len)
.where(
(CSet.repo == repo) &
(CSet.hkey == sha) &
(CSet.time >= SQL(
"COALESCE((SELECT time FROM cset "
"WHERE repo_id = %s "
"AND hkey_id = %s "
"AND type != %s "
"ORDER BY time DESC "
"LIMIT 1), 0)",
repo.id, sha, CSet.DELTA
)))
.order_by(CSet.time)
.naive())
if len(chain) > 0 and not ts > chain[-1].time:
# Appended timestamps must be monotonically increasing!
raise HTTPError(400)
if len(chain) == 0:
# Mapping for `key` likely does not exist:
# Store the SHA-to-KEY mapping in HMap,
# looking out for possible collisions.
try:
HMap.create(sha=sha, val=key)
except IntegrityError:
val = HMap.select(HMap.val).where(HMap.sha == sha).scalar()
if val != key:
raise HTTPError(500)
# Parse and normalize into a set of N-Quad lines
stmts = parse(self.request.body, fmt)
snapc = compress(join(stmts, "\n"))
if len(chain) == 0 or chain[0].type == CSet.DELETE:
# Provide dummy value for `patch` which is never stored.
# If we get here, we always store a snapshot later on!
patch = ""
else:
# Reconstruct the previous state of the resource
prev = set()
blobs = (Blob
.select(Blob.data)
.where(
(Blob.repo == repo) &
(Blob.hkey == sha) &
(Blob.time << map(lambda e: e.time, chain)))
.order_by(Blob.time)
.naive())
for i, blob in enumerate(blobs.iterator()):
data = decompress(blob.data)
if i == 0:
# Base snapshot for the delta chain
prev.update(data.splitlines())
else:
for line in data.splitlines():
mode, stmt = line[0], line[2:]
if mode == "A":
prev.add(stmt)
else:
prev.discard(stmt)
if stmts == prev:
# No changes, nothing to be done. Bail out.
return self.finish()
patch = compress(join(
map(lambda s: "D " + s, prev - stmts) +
map(lambda s: "A " + s, stmts - prev), "\n"))
# Calculate the accumulated size of the delta chain including
# the (potential) patch from the previous to the pushed state.
acclen = reduce(lambda s, e: s + e.len, chain[1:], 0) + len(patch)
blen = len(chain) > 0 and chain[0].len or 0 # base length
if (len(chain) == 0 or chain[0].type == CSet.DELETE or
len(snapc) <= len(patch) or SNAPF * blen <= acclen):
# Store the current state as a new snapshot
Blob.create(repo=repo, hkey=sha, time=ts, data=snapc)
CSet.create(repo=repo, hkey=sha, time=ts, type=CSet.SNAPSHOT,
len=len(snapc))
else:
# Store a directed delta between the previous and current state
Blob.create(repo=repo, hkey=sha, time=ts, data=patch)
CSet.create(repo=repo, hkey=sha, time=ts, type=CSet.DELTA,
len=len(patch))
