def resolve_document_using_git(git, docref): document = docref_to_dict(docref) kind = docref.kind id = docref.id get = lambda f, n=docref.name: call( git + ["show", "--quiet", "--format=format:%s" % (f,), n], do_check=False) get = lambda a: git_show(git, docref.name, a) if kind == "branches": branches = [trim(a, prefix=" remotes/origin/") for a in read_lines(call(git + ["branch", "-a"]))] document["branches"] = [] for branch in branches: document["branches"].append(docref_to_dict(BranchDocref(branch))) elif kind == "branch": sha = get1( read_lines( call(git + ["rev-parse", "remotes/origin/" + docref.name]))) document["commit"] = docref_to_dict(ShaDocRef("commit", sha)) elif kind == "commit": document.update( {"author": {"name": get("%an"), "email": get("%ae"), "date": get("%ai")}, "committer": {"name": get("%cn"), "email": get("%ce"), "date": get("%ci")}, "message": get("%B"), "tree": docref_to_dict(ShaDocRef("tree", get("%T"))), "parents": [], }) for p in sorted(get("%P").split(" ")): if p == "": continue document["parents"].append( docref_to_dict(ShaDocRef("commit", p))) elif kind == "tree": document["children"] = [] for line in read_lines(call(git + ["ls-tree", docref.name])): child_mode, child_kind, rest = line.split(" ", 2) child_sha, child_basename = rest.split("\t", 1) ref = {"child": docref_to_dict(ShaDocRef(child_kind, child_sha)), "basename": child_basename, "mode": octal_to_symbolic_mode(child_mode)} document["children"].append(ref) document["children"].sort(key=lambda a: a["child"]["sha"]) elif kind == "blob": blob = call(git + ["show", docref.name], do_crlf_fix=False) if is_text(blob): document["encoding"] = "raw" document["raw"] = blob else: document["encoding"] = "base64" document["base64"] = base64.b64encode(blob) else: raise NotImplementedError(kind) return document
def get_release_sha1sums(release_data): sha1sums = {} release = get1(parse_control_file(release_data)) for line in read_lines(release["sha1"] + "\n"): sha1sum, size, relpath = line.split() sha1sums[relpath] = sha1sum return sha1sums
def git_to_couchdb_using_git(cache_root, git_url, couchdb_url): cache_dir = os.path.join(cache_root, encode_as_c_identifier(git_url)) git = ["bash", "-c", 'cd "$1" && shift && exec "$@"', "-", cache_dir, "git"] call(["mkdir", "-p", cache_dir]) call(git + ["init"]) for r in read_lines(call(git + ["remote"])): call(git + ["remote", "rm", r]) call(git + ["remote", "add", "origin", git_url]) call(git + ["fetch", "origin"], stdout=None, stderr=None) resolve_document = lambda d: resolve_document_using_git(git, d) fetch_all(resolve_document, couchdb_url, [BRANCHES_DOCREF])
def ubuntu_to_hg(hg_path, username, do_development=False): def hg(argv, **kwargs): kwargs.setdefault("cwd", hg_path) kwargs.setdefault("do_print", True) prefix = ["hg"] if username is not None and argv[0] in ("commit", "ci"): prefix.extend(["--config", "ui.username=%s" % (username,)]) return call(prefix + argv, **kwargs) with with_ubuntu_keyring() as gpg: meta_release_data = get( join(BASE_URL, "meta-release-development")).content meta_release = parse_control_file(meta_release_data) group_by(meta_release, lambda r: r["dist"]) if not os.path.exists(hg_path): os.makedirs(hg_path) hg(["init"]) branches = set([a.split()[0] for a in read_lines(hg(["branches"]))]) ok_branches = set() seen_supported_non_lts = False for release in meta_release: branch = "ubuntu_codename_%s" % (release["dist"],) is_lts = "LTS" in release["version"] is_supported = release["supported"] == "1" if is_supported and not is_lts: seen_supported_non_lts = True is_development = not is_supported and seen_supported_non_lts if not is_supported and not is_development: continue ok_branches.add(branch) if is_development and not do_development: continue done = set() if branch not in branches: hg(["update", "--clean", "--rev", "00"]) hg(["branch", "--force", branch]) else: hg(["update", "--clean", branch]) hg(["--config", "extensions.purge=", "purge", "--all"]) release_gpg_path = os.path.join(hg_path, "Release.gpg") release_path = os.path.join(hg_path, "Release") old_sha1sums = {} release_gpg_data = get(release["release-file"] + ".gpg").content if os.path.exists(release_gpg_path): if release_gpg_data == read_file(release_gpg_path): continue release_data = read_file(release_path) old_sha1sums = get_release_sha1sums(release_data) old_sha1sums["Release"] = hashlib.sha1( release_data).hexdigest() old_sha1sums["Release.gpg"] = hashlib.sha1( release_gpg_data).hexdigest() # for relpath in sorted(old_sha1sums): # if posixpath.dirname(relpath) == "Index": # index_data = read_file(os.path.join(hg_path, relpath)) # child_sha1sums = get_release_sha1sums(index_data) # for relpath2 in sorted(child_sha1sums): # relpath3 = posixpath.join( # posixpath.dirname(relpath), relpath2) # old_sha1sums[relpath3] = child_sha1sums[relpath2] release_data = get(release["release-file"]).content with open(release_gpg_path, "wb") as fh: fh.write(release_gpg_data) done.add("Release") with open(release_path, "wb") as fh: fh.write(release_data) done.add("Release.gpg") gpg(["--verify", release_gpg_path, release_path]) new_sha1sums = get_release_sha1sums(release_data) new_sha1sums["Release.gpg"] = hashlib.sha1( release_gpg_data).hexdigest() new_sha1sums["Release"] = hashlib.sha1( release_data).hexdigest() # for relpath in sorted(new_sha1sums): # if posixpath.basename(relpath) == "Index": # if new_sha1sums[relpath] == old_sha1sums.get(relpath): # index_data = read_file(os.path.join(hg_path, relpath)) # else: # index_data = get( # posixpath.join( # posixpath.dirname(release["Release-File"]), # relpath)).content # sha1sum = hashlib.sha1(index_data).hexdigest() # if sha1sum != new_sha1sums[relpath]: # raise Exception("sha1sum mismatch for %r: " # "got %s expecting %s" # % (url, sha1sum, # new_sha1sums[relpath])) # index_path = os.path.join(hg_path, relpath) # if not os.path.exists(os.path.dirname(index_path)): # os.makedirs(os.path.dirname(index_path)) # with open(index_path, "wb") as fh: # fh.write(index_data) # done.add(relpath) # child_sha1sums = get_release_sha1sums(index_data) # for relpath2 in sorted(child_sha1sums): # relpath3 = posixpath.join( # posixpath.dirname(relpath), relpath2) # new_sha1sums[relpath3] = child_sha1sums[relpath2] for relpath in old_sha1sums: if relpath in new_sha1sums: continue file_path = os.path.join(hg_path, relpath) call(["rm", "-rf", "--one-file-system", file_path]) for relpath in new_sha1sums: if relpath in old_sha1sums: if new_sha1sums[relpath] == old_sha1sums[relpath]: continue if (relpath.endswith(".gz") and trim(relpath, suffix=".gz") in new_sha1sums): continue if (relpath.endswith(".bz2") and trim(relpath, suffix=".bz2") in new_sha1sums): continue if relpath in done: continue file_path = os.path.join(hg_path, relpath) file_data = get_release_file( posixpath.join( posixpath.dirname(release["release-file"]), relpath)) sha1sum = hashlib.sha1(file_data).hexdigest() if sha1sum != new_sha1sums[relpath]: raise Exception("sha1sum mismatch for %r: " "got %s expecting %s" % (url, sha1sum, new_sha1sums[relpath])) if not os.path.exists(os.path.dirname(file_path)): os.makedirs(os.path.dirname(file_path)) with open(file_path, "wb") as fh: fh.write(file_data) hg(["addremove"]) if len(read_lines(hg(["status"]))) > 0: hg(["commit", "-m", "Update from upstream"]) for branch in branches: if branch == "default" or branch in ok_branches: continue hg(["update", "--clean", branch]) hg(["commit", "--close-branch", "-m", "Closing unsupported release"])
def resolve_document_using_git(git, docref): document = docref_to_dict(docref) kind = docref.kind id = docref.id get = lambda a: git_show(git, docref.name, a) if kind == "branches": branches = set() for line in read_lines(call(git + ["branch", "-a"])): if line.startswith(" "): line = trim(line, prefix=" ") elif line.startswith("* "): line = trim(line, prefix="* ") if line == "(no branch)": continue else: raise NotImplementedError(line) if " -> " in line: ba, bb = line.split(" -> ", 1) branches.add(posixpath.basename(ba)) branches.add(posixpath.basename(bb)) else: branches.add(posixpath.basename(line)) branches = list(sorted(b for b in branches if b != "HEAD")) document["branches"] = [] for branch in branches: document["branches"].append(docref_to_dict(BranchDocref(branch))) elif kind == "branch": sha = get1( read_lines( call(git + ["rev-parse", docref.name]))) document["commit"] = docref_to_dict(ShaDocRef("commit", sha)) elif kind == "commit": document.update( {"author": {"name": get("%an"), "email": get("%ae"), "date": get("%ai")}, "committer": {"name": get("%cn"), "email": get("%ce"), "date": get("%ci")}, "message": get("%B"), "tree": docref_to_dict(ShaDocRef("tree", get("%T"))), "parents": [], }) for p in sorted(get("%P").split(" ")): if p == "": continue document["parents"].append( docref_to_dict(ShaDocRef("commit", p))) elif kind == "tree": document["children"] = [] for line in read_lines(call(git + ["ls-tree", docref.name])): child_mode, child_kind, rest = line.split(" ", 2) child_sha, child_basename = rest.split("\t", 1) ref = {"child": docref_to_dict(ShaDocRef(child_kind, child_sha)), "basename": child_basename, "mode": octal_to_symbolic_mode(child_mode)} document["children"].append(ref) document["children"].sort(key=lambda a: a["child"]["sha"]) elif kind == "blob": blob = call(git + ["show", docref.name], do_crlf_fix=False) if is_text(blob): document["encoding"] = "raw" document["raw"] = blob else: document["encoding"] = "base64" document["base64"] = base64.b64encode(blob) else: raise NotImplementedError(kind) return document