def resolve_document_using_git(git, docref):
document = docref_to_dict(docref)
kind = docref.kind
id = docref.id
get = lambda f, n=docref.name: call(
git + ["show", "--quiet", "--format=format:%s" % (f,), n],
do_check=False)
get = lambda a: git_show(git, docref.name, a)
if kind == "branches":
branches = [trim(a, prefix=" remotes/origin/")
for a in read_lines(call(git + ["branch", "-a"]))]
document["branches"] = []
for branch in branches:
document["branches"].append(docref_to_dict(BranchDocref(branch)))
elif kind == "branch":
sha = get1(
read_lines(
call(git + ["rev-parse", "remotes/origin/" + docref.name])))
document["commit"] = docref_to_dict(ShaDocRef("commit", sha))
elif kind == "commit":
document.update(
{"author": {"name": get("%an"),
"email": get("%ae"),
"date": get("%ai")},
"committer": {"name": get("%cn"),
"email": get("%ce"),
"date": get("%ci")},
"message": get("%B"),
"tree": docref_to_dict(ShaDocRef("tree", get("%T"))),
"parents": [],
})
for p in sorted(get("%P").split(" ")):
if p == "":
continue
document["parents"].append(
docref_to_dict(ShaDocRef("commit", p)))
elif kind == "tree":
document["children"] = []
for line in read_lines(call(git + ["ls-tree", docref.name])):
child_mode, child_kind, rest = line.split(" ", 2)
child_sha, child_basename = rest.split("\t", 1)
ref = {"child": docref_to_dict(ShaDocRef(child_kind, child_sha)),
"basename": child_basename,
"mode": octal_to_symbolic_mode(child_mode)}
document["children"].append(ref)
document["children"].sort(key=lambda a: a["child"]["sha"])
elif kind == "blob":
blob = call(git + ["show", docref.name], do_crlf_fix=False)
if is_text(blob):
document["encoding"] = "raw"
document["raw"] = blob
else:
document["encoding"] = "base64"
document["base64"] = base64.b64encode(blob)
else:
raise NotImplementedError(kind)
return document
def force_couchdb_put_with_rev(couchdb_url, *documents):
for document in documents:
doc_url = posixpath.join(couchdb_url, document["_id"]).encode("ascii")
i = 0
while True:
if i % 1000 == 0 and i != 0:
print i, "The race is on!"
old_doc = get(doc_url)
if (old_doc.get("error") == "not_found"
and old_doc.get("reason") == "missing"):
result = put(doc_url, document)
if result.get("error") is None:
break
else:
assert old_doc.get("error") is None, old_doc
rev = old_doc.pop("_rev")
if document == old_doc:
break
else:
d2 = dict(document)
d2["_rev"] = rev
result = put(doc_url, d2)
if result.get("error") is None:
break
i += 1
def blob_to_fs(git_couchdb_url, file_path, blob):
blob_data = get(posixpath.join(git_couchdb_url, blob))
if blob_data["encoding"] == "raw":
write_file(file_path, blob_data["raw"])
elif blob_data["encoding"] == "base64":
write_file(file_path, base64.b64decode(blob_data["base64"]))
else:
raise NotImplementedError(blob_data)
def fetch_all(resolve_document, couchdb_url, seeds):
to_fetch = list(seeds)
push = lambda x: to_fetch.append(x)
pop = lambda: to_fetch.pop()
def priority_sort_key(docref):
priority_items = ["commit", "tree"]
i = dict((a, idx) for (idx, a) in enumerate(priority_items)).get(
docref.kind, len(priority_items))
return (i, docref.name, docref)
def multipush(many, limit=None):
for i, item in enumerate(reversed(
sorted(many, key=priority_sort_key))):
if limit is not None and i > limit:
break
push(item)
multipush(seeds)
mutable_buffer = {}
local_buffer = {}
fetched = set()
for match in get(couchdb_url + "/_all_docs")["rows"]:
if not match["id"].startswith("git-"):
continue
docref = id_to_docref(match["id"])
if docref.kind not in MUTABLE_TYPES:
fetched.add(docref)
while len(to_fetch) > 0:
docref = pop()
if docref not in fetched:
document = local_buffer.get(docref)
if document is None:
print "get", len(to_fetch), docref
document = resolve_document(docref)
local_buffer[docref] = document
if docref.kind in ("branches", "branch"):
mutable_buffer[docref] = document
local_dependencies = set(find_dependencies(document)) - fetched
if len(local_dependencies) == 0:
force_couchdb_put(couchdb_url, document)
del local_buffer[docref]
fetched.add(docref)
print "put", len(to_fetch), docref
else:
push(docref)
multipush(local_dependencies)
if len(local_buffer) > BIG_NUMBER:
local_buffer.clear()
local_buffer.update(mutable_buffer)
assert BIG_NUMBER > 15
if len(to_fetch) > BIG_NUMBER:
to_keep = to_fetch[:-SMALL_NUMBER]
to_fetch[:] = []
multipush(seeds)
multipush(to_keep)
if len(to_fetch) > BIG_NUMBER:
print "ouch, lots of seeds?"
assert len(to_fetch) > 0
def sync_batch(git_couchdb_url, design_couchdb_url, branch, app_subdir):
if branch is None:
branches = [
b["_id"] for b in get(
posixpath.join(git_couchdb_url, "git-branches"))["branches"]]
else:
# TODO: Should really look for branches with this name using
# an index
branches = ["git-branch-" + branch]
with mkdtemp() as temp_dir:
for branch in branches:
basename = urllib.quote(branch, safe="")
local_dir = os.path.join(temp_dir, basename)
commit = get(posixpath.join(git_couchdb_url,
branch))["commit"]["_id"]
tree = get(posixpath.join(git_couchdb_url,
commit))["tree"]["_id"]
if app_subdir != ".":
# TODO: Support multi-level sub_dir
assert "/" not in app_subdir, app_subdir
for child in get(posixpath.join(git_couchdb_url,
tree))["children"]:
if child["basename"] == app_subdir:
assert child["child"]["type"] == "git-tree", child
tree = child["child"]["_id"]
break
else:
raise Exception("Missing child %r" % (app_subdir,))
doc_url = posixpath.join(design_couchdb_url,
"_design", url_quote(basename))
existing = get(doc_url)
if existing.get("couchapp_git_tree_id") == tree:
continue
print "Updating %r..." % (branch,)
tree_to_fs(git_couchdb_url, local_dir, tree)
write_file(os.path.join(local_dir, "couchapp_git_tree_id"),
tree)
if os.path.exists(os.path.join(local_dir, "_id")):
os.unlink(os.path.join(local_dir, "_id"))
couchapp(doc_url, local_dir)
print "...done %r" % (branch,)
def tree_to_fs(git_couchdb_url, local_dir, tree):
os.mkdir(local_dir)
tree_data = get(posixpath.join(git_couchdb_url, tree))
for entry in tree_data["children"]:
out_path = os.path.join(local_dir, entry["basename"])
if entry["child"]["type"] == "git-tree":
tree_to_fs(git_couchdb_url, out_path, entry["child"]["_id"])
elif entry["child"]["type"] == "git-blob":
blob_to_fs(git_couchdb_url, out_path, entry["child"]["_id"])
else:
raise NotImplementedError(entry)
mode = symbolic_to_octal_mode(entry["mode"])
def test(self):
# Clean up any partially aborted test runs
def mark_aborted(doc):
doc.update({"status": "aborted"})
queue_url = posixpath.join(self.couchdb_url, "test-queue")
queued_ids = set(i["_id"] for i in get(queue_url)["queue"])
pending_map_func = """\
function (doc) {
var prefix = "test-run-";
if (doc._id.substr(0, prefix.length) == prefix) {
if (doc.status == "pending") {
emit(null, doc._id);
}
}
}
"""
git_map_func = """\
function (doc) {
var prefix = "test-";
if (doc._id.substr(0, prefix.length) != prefix) {
emit(null, doc._id);
}
}
"""
design_doc = {
"language": "javascript",
"views": {
"pending-tests": {
"map": pending_map_func,
},
"git-documents": {
"map": git_map_func,
},
},
}
put_update(posixpath.join(self.couchdb_url, "_design/test"),
lambda a=None: design_doc)
result = get(posixpath.join(self.couchdb_url,
"_design/test/_view/pending-tests"))
pending_ids = set(i["id"] for i in result["rows"])
for missing_id in pending_ids - queued_ids:
put_update(posixpath.join(self.couchdb_url, url_quote(missing_id)),
mark_aborted)
# Add this test run to the queue on the shared database
def now():
return datetime.datetime.utcnow().isoformat() + "Z"
document = {
"request_time": now(),
"start_time": "",
"end_time": "",
"status": "pending",
}
result = post_new(self.couchdb_url, document,
id_template="test-run-%s")
my_url = posixpath.join(self.couchdb_url, url_quote(result["id"]))
def append_to_queue(queued):
queued.setdefault("queue", []).append(
{"_id": result["id"],
"request_time": document["request_time"],
})
def remove_from_queue(queued):
queue = queued.get("queue", [])
queue = [e for e in queue if e.get("_id") != result["id"]]
queued["queue"] = queue
put_update(queue_url, append_to_queue)
try:
while True:
queue = get(queue_url)
for i, entry in enumerate(queue.get("queue", [])):
if entry["_id"] == result["id"]:
my_index = i
break
else:
raise Exception("I seem to have been un queued")
# Is there a non-expired entry in the queue ahead of mine?
if my_index == 0:
def mark_running(job):
job.update(
{"status": "running",
"start_time": now()})
status = "aborted"
def mark_stopped(job):
job.update(
{"status": status,
"stop_time": now()})
# Mark the test as running
put_update(my_url, mark_running)
try:
try:
self._run_test()
except AssertionError:
status = "failed"
raise
except Exception:
status = "error"
raise
else:
status = "passed"
return
finally:
put_update(my_url, mark_stopped)
else:
# Set a timout for the next time to check, if necessary
print "sleeping..."
time.sleep(5)
finally:
# Mark the test as completed
put_update(queue_url, remove_from_queue)
def _run_test(self):
# Wipe out all git-related documents and the design document
result = get(posixpath.join(self.couchdb_url,
"_design/test/_view/git-documents"))
for item in result["rows"]:
delete(posixpath.join(self.couchdb_url, url_quote(item["id"])))
# Upload the gitbrowser to the couchdb design document
gitbrowser_source_path = os.path.join(self.source_path, "gitbrowser")
couchapp(posixpath.join(self.couchdb_url, "_design/gitbrowser"),
gitbrowser_source_path)
# Populate a test GIT repository
with mkdtemp() as temp_dir:
cwd_script = ["bash", "-c", 'cd "$1" && shift && exec "$@"', "-",
temp_dir]
subprocess.check_call(cwd_script + ["git", "init"])
## write file README
git = cwd_script + ["env",
"HOME=%s" % (temp_dir,),
"git"]
subprocess.check_call(git + ["config", "user.name", "Tester"])
subprocess.check_call(git + ["config", "user.email",
"*****@*****.**"])
path = os.path.join(temp_dir, "README")
with open(path, "wb") as fh:
fh.write("Initial version of text file\r\n")
subprocess.check_call(git + ["add", path])
subprocess.check_call(git + ["commit", "-m",
"Initial commit"])
## write file README
with open(path, "wb") as fh:
fh.write("""\
# Purpose
This is a minimal git repository for testing
# Example Markdown
Readme files like this are Markdown formatted:
* Bullet points
* Headings
* Code blocks
* _etc_
They can have hyperlinks:
* to other hosts like [google](http://www.google.com/)
* absolute paths, like the [hexdump file](/show/master/head/binary-file)
* relative paths, like the [child folder](subfolder)
""")
subprocess.check_call(git + ["add", path])
## write file binary
path = os.path.join(temp_dir, "binary-file")
with open(path, "wb") as fh:
fh.write("".join(chr(i) for i in range(2**8)))
subprocess.check_call(git + ["add", path])
## write python file
path = os.path.join(temp_dir, "example.py")
with open(path, "wb") as fh:
fh.write("""\
## Example\r\n\
#This is an example python file\r\n\
#\r\n\
# - Example bullet list\r\n\
# - _etc_\r\n\
#\r\n\
#Because of the blank line, this documentation block should not\r\n\
#have any code on the right and the following code block should not\r\n\
#have any documentation to its left.\r\n\
\r\n\
from __future__ import division\r\n\
import os, sys, unittest\r\n\
\r\n\
class SomeTest(unittest.TestCase):\r\n\
\r\n\
def testSomething(self):\r\n\
'''Just an example test'''\r\n\
self.fail("Just an example")\r\n\
\r\n\
## Main entry point\r\n\
#Some normal *Markdown* paragraph text, the example \r\n\
#heading should line up with the if statement.\r\n\
if __name__ == "__main__":\r\n\
# This comment, starting after column 0, is a normal comment shown\r\n\
# inline with the code instead of one that is shown in the left panel.\r\n\
unittest.main()\r\n\
""")
subprocess.check_call(git + ["add", path])
## write file .gitbrowser-project.json
path = os.path.join(temp_dir, ".gitbrowser-project.json")
with open(path, "wb") as fh:
fh.write('{"title": "Example Project"}')
subprocess.check_call(git + ["add", path])
## write file subfolder/README
path = os.path.join(temp_dir, "subfolder", "README")
os.makedirs(os.path.dirname(path))
with open(path, "wb") as fh:
fh.write("A file in a subfolder with a [parent link](..)")
subprocess.check_call(git + ["add", path])
subprocess.check_call(git + ["commit", "-m", "Update"])
# Copy the git repository to the couchdb
sync_script = os.path.join(self.source_path, "gitcouchdbsync.py")
subprocess.check_call(cwd_script + ["python", sync_script,
self.couchdb_url])
# Run selenium testing against the public URL
print "Uploaded. Running tests..."
self._go_to_the_selenium_stage()
def resolve_document_using_git(git, docref):
document = docref_to_dict(docref)
kind = docref.kind
id = docref.id
get = lambda a: git_show(git, docref.name, a)
if kind == "branches":
branches = set()
for line in read_lines(call(git + ["branch", "-a"])):
if line.startswith(" "):
line = trim(line, prefix=" ")
elif line.startswith("* "):
line = trim(line, prefix="* ")
if line == "(no branch)":
continue
else:
raise NotImplementedError(line)
if " -> " in line:
ba, bb = line.split(" -> ", 1)
branches.add(posixpath.basename(ba))
branches.add(posixpath.basename(bb))
else:
branches.add(posixpath.basename(line))
branches = list(sorted(b for b in branches if b != "HEAD"))
document["branches"] = []
for branch in branches:
document["branches"].append(docref_to_dict(BranchDocref(branch)))
elif kind == "branch":
sha = get1(
read_lines(
call(git + ["rev-parse", docref.name])))
document["commit"] = docref_to_dict(ShaDocRef("commit", sha))
elif kind == "commit":
document.update(
{"author": {"name": get("%an"),
"email": get("%ae"),
"date": get("%ai")},
"committer": {"name": get("%cn"),
"email": get("%ce"),
"date": get("%ci")},
"message": get("%B"),
"tree": docref_to_dict(ShaDocRef("tree", get("%T"))),
"parents": [],
})
for p in sorted(get("%P").split(" ")):
if p == "":
continue
document["parents"].append(
docref_to_dict(ShaDocRef("commit", p)))
elif kind == "tree":
document["children"] = []
for line in read_lines(call(git + ["ls-tree", docref.name])):
child_mode, child_kind, rest = line.split(" ", 2)
child_sha, child_basename = rest.split("\t", 1)
ref = {"child": docref_to_dict(ShaDocRef(child_kind, child_sha)),
"basename": child_basename,
"mode": octal_to_symbolic_mode(child_mode)}
document["children"].append(ref)
document["children"].sort(key=lambda a: a["child"]["sha"])
elif kind == "blob":
blob = call(git + ["show", docref.name], do_crlf_fix=False)
if is_text(blob):
document["encoding"] = "raw"
document["raw"] = blob
else:
document["encoding"] = "base64"
document["base64"] = base64.b64encode(blob)
else:
raise NotImplementedError(kind)
return document