def _init_common(index_path: Path, origin_location: str,
crates_root_path: Path) -> git.Repo:
if index_path.is_dir():
raise error.UsageError("index directory {} already exists".format(
repr(str(index_path))))
common.iprint("create index repository at {}:".format(repr(
str(index_path))))
index_path.mkdir(parents=True)
repo = git.Repo.init(str(index_path))
common.iprint(" remote add origin {}".format(origin_location))
repo.create_remote("origin", origin_location)
# Setup "HEAD" to new "working" branch.
working = git.Reference(repo, "refs/heads/working")
repo.head.set_reference(working)
# Setup default remote and merge branch for "working".
with repo.config_writer() as writer:
writer.set_value('branch "working"', "remote", "origin")
writer.set_value('branch "working"', "merge", "refs/heads/master")
if not crates_root_path.is_dir():
common.iprint("create crates directory at {}:".format(
repr(str(crates_root_path))))
crates_root_path.mkdir(parents=True)
return repo
def get_or_create(cls, repo):
ref_name = env.config["sync"]["ref"]
ref = git.Reference(repo, ref_name)
try:
ref.commit.tree["index/%s" % cls.name]
except KeyError:
return cls.create(repo)
return cls(repo)
def __init__(self, repo, process_name, commit_cls=sync_commit.Commit):
if not git.Reference(repo, self.process_path(process_name)).is_valid():
raise ValueError(
"No ref found in %s with path %s" %
(repo.working_dir, self.process_path(process_name)))
self.repo = repo
self._ref = str(process_name)
self._process_name = process_name.attach_ref(self)
self.commit_cls = commit_cls
def test_checkout_head(rc: ServerRepoCtx):
master: git.Reference = git.Reference(rc.repo, "refs/heads/master")
# FIXME: only touches paths recorded in index - stray worktree files not removed etc
# see IndexFile.checkout vs head.checkout
# FIXME: use force=True ?
#index: git.IndexFile = git.IndexFile.from_tree(rc.repo, master.commit.tree)
#index.checkout()
assert os_path_exists(str(rc.repodir / '.git'))
rc.repo.head.reset(master.commit, index=True, working_tree=True)
def test_commit_head(rc: ServerRepoCtx, client: flask.testing.FlaskClient):
master_tree: shahex = _get_master_tree_hex(client)
dummy: git.Actor = git.Actor('dummy', "*****@*****.**")
commit: git.Commit = git.Commit.create_from_tree(rc.repo,
master_tree,
message="",
author=dummy,
committer=dummy)
master: git.Reference = git.Reference(rc.repo, "refs/heads/master")
master.set_commit(commit)
def print_pq_status(self, out=sys.stdout):
with TablePrinter(out=out) as printrow:
for subtree in self.get_pq_subtrees():
if subtree.worktree:
branch_name = git.Reference(self,
subtree.worktree.branch).name
printrow(subtree.relpath, f"[editing: {branch_name}]")
else:
printrow(subtree.relpath, "[not editing]")
out.write("\n")
def prepare_project_for_test():
if will_record:
self.service.connect()
# let's create a project and add it to current repository
self.service.create(namespace, create_repository, add=True)
# make a modification, commit and push it
with open(os.path.join(self.repository.working_dir, 'first_file'),
'w') as test:
test.write(
'he who makes a beast of himself gets rid of the pain of being a man. Dr Johnson'
)
self.repository.git.config('user.name', 'travis')
self.repository.git.config('user.email', '*****@*****.**')
self.repository.git.add('first_file')
self.repository.git.commit(message='First commit')
if will_record:
self.repository.git.push(self.service.name, 'master')
# create a new branch
new_branch = self.repository.create_head(source_branch, 'HEAD')
self.repository.head.reference = new_branch
self.repository.head.reset(index=True, working_tree=True)
# make a modification, commit and push it to that branch
with open(
os.path.join(self.repository.working_dir,
'second_file'), 'w') as test:
test.write(
'La meilleure façon de ne pas avancer est de suivre une idée fixe. J.Prévert'
)
self.repository.git.add('second_file')
self.repository.git.commit(message='Second commit')
self.repository.git.push(service, source_branch)
else:
import git
self.service._extracts_ref = lambda *a: git.Reference(
self.service.repository,
'{}/{}'.format(namespace, repository),
check_path=False)
self.service.repository.head.reference = git.Head(
self.service.repository, path='refs/heads/pr-test')
existing_remotes = [r.name for r in self.repository.remotes]
if 'all' in existing_remotes:
r_all = self.repository.remote('all')
else:
r_all = self.repository.create_remote('all', '')
for name in ('github', 'gitlab', 'bitbucket', 'gogs', 'upstream'):
if name not in existing_remotes:
kw = dict(user=namespace, project=repository, host=name)
self.repository.create_remote(
name, 'git@{host}.com:{user}/{project}'.format(**kw))
r_all.add_url(
'git@{host}.com:{user}/{project}'.format(**kw))
yield
if will_record:
self.service.delete(create_repository)
def _upgrade_to_working(repo: git.Repo) -> None:
working = git.Reference(repo, "refs/heads/working")
if repo.head.reference.name != "working" and not working.is_valid():
# Time to upgrade working tree to use "working" branch.
common.eprint("""Upgrade index to use "working" branch as HEAD""")
if repo.head.reference.is_valid():
# Create "working" branch based on current HEAD.
common.iprint(
"""Checkout new "working" branch from current HEAD""")
repo.create_head("refs/heads/working", "HEAD")
repo.head.set_reference(working)
def __init__(self, repo, process_name):
assert process_name.obj_type == self.obj_type
self.repo = repo
self.pygit2_repo = pygit2_get(repo)
self.process_name = process_name
self.ref = git.Reference(repo, env.config["sync"]["ref"])
self.path = self.get_path(process_name)
self._data = self._load()
self._lock = None
self._updated = set()
self._deleted = set()
self._delete = False
def require_no_merge_conflict(self):
"""
Raises :exc:`MergeConflict` if the current working directory
contains a merge conflict.
"""
try:
git.Reference(self.repo, 'MERGE_HEAD', check_path=False).commit
# reference exists, so there is a merge conflict
raise MergeConflict()
except ValueError:
# no such reference, so there is no merge conflict
pass
def setup(self, repo):
data_ref = git.Reference(repo, self.config["sync"]["ref"])
if not data_ref.is_valid():
from . import base
with base.CommitBuilder(repo, "Create initial sync metadata",
ref=data_ref.path) as commit:
path = "_metadata"
data = json.dumps({"name": "wptsync"})
commit.add_tree({path: data})
from . import index
for idx in index.indicies:
idx.get_or_create(repo)
def _create(cls, repo, process_name, obj, commit_cls=sync_commit.Commit):
path = cls.process_path(process_name)
logger.debug("Creating ref %s" % path)
for ref in repo.refs:
if fnmatch(
ref.path, "refs/%s/%s" %
(cls.ref_prefix, process_name.name_filter())):
raise ValueError("Ref %s exists which conflicts with %s" %
(ref.path, path))
ref = git.Reference(repo, path)
ref.set_object(obj)
return cls(repo, process_name, commit_cls)
def create(cls, lock, repo, process_name, data, message="Sync data"):
assert process_name.obj_type == cls.obj_type
path = cls.get_path(process_name)
ref = git.Reference(repo, env.config["sync"]["ref"])
try:
ref.commit.tree[path]
except KeyError:
pass
else:
raise ValueError("%s already exists at path %s" % (cls.__name__, path))
with CommitBuilder(repo, message, ref=ref) as commit:
commit.add_tree({path: json.dumps(data)})
ProcessNameIndex(repo).insert(process_name)
return cls(repo, process_name)
def rename(self):
path = self.path
ref = self.ref
if not ref:
return
# This implicitly updates self.path
self._ref = str(self._process_name)
if self.path == path:
return
logger.debug("Renaming ref %s to %s" % (path, self.path))
new_ref = git.Reference(self.repo, self.path)
new_ref.set_object(ref.commit.hexsha)
logger.debug(
"Deleting ref %s pointing at %s.\n"
"To recreate this ref run `git update-ref %s %s`" %
(ref.path, ref.commit.hexsha, ref.path, ref.commit.hexsha))
ref.delete(self.repo, ref.path)
def preload_repo(
tupdater3_exe: str,
stage2_exe: str,
repodir_s: pathlib.Path
) -> ServerRepoCtx:
with git.Repo.init(str(repodir_s)) as repo:
rc = ServerRepoCtx(repodir_s, repo)
# BEG populate with dummy data
for ff in [("a.txt", b"aaa"), ("d/b.txt", b"bbb")]:
with _file_open_mkdirp(str(rc.repodir.joinpath(ff[0]))) as f:
f.write(ff[1])
rc.repo.index.add([ff[0]])
# END populate with dummy data
# BEG populate with executables
for gg in [("updater.exe", tupdater3_exe), ("stage2.exe", stage2_exe)]:
shutil_copyfile(gg[1], str(rc.repodir.joinpath(gg[0])))
rc.repo.index.add([gg[0]])
# END populate with executables
commit = rc.repo.index.commit("ccc")
ref_master = git.Reference(rc.repo, "refs/heads/master")
ref_master.set_object(commit)
def finish_pq(self, subtree, out=sys.stdout):
if not subtree.worktree:
print(f"{subtree.uiname} is not being edited")
return
wt = Repo(subtree.path)
ok = True
try:
relpath = relpath_nodots(wt.git_dir, self.git_dir)
except OSError:
ok = False
relpath = relpath.split(os.path.sep)
ok = ok and len(relpath) > 1 and relpath[0] == 'worktrees'
if not ok:
print(
f"the GIT_DIR for {subtree.uiname} is unexpectedly at {wt.git_dir}, cannot proceed"
)
return
os.unlink(os.path.join(subtree.path, '.git'))
shutil.rmtree(wt.git_dir)
self.delete_head(git.Reference(self, subtree.worktree.branch),
force=True)
def rc_s(customopt_tupdater2_exe: str, customopt_tupdater3_exe: str,
dirs: Dirs) -> ServerRepoCtx:
import shutil
repo = git.Repo.init(str(dirs.repodir_s))
rc = ServerRepoCtx(dirs.repodir_s, repo)
# BEG populate with dummy data
for ff in [("a.txt", b"aaa"), ("d/b.txt", b"bbb")]:
with _file_open_mkdirp(str(rc.repodir.joinpath(ff[0]))) as f:
f.write(ff[1])
rc.repo.index.add([ff[0]])
# END populate with dummy data
# BEG populate with tupdater
for gg in [("updater.exe", customopt_tupdater3_exe),
("stage2.exe", customopt_tupdater3_exe)]:
shutil.copyfile(gg[1], str(rc.repodir.joinpath(gg[0])))
rc.repo.index.add([gg[0]])
# END populate with tupdater
commit = rc.repo.index.commit("ccc")
ref_master = git.Reference(rc.repo, "refs/heads/master")
ref_master.set_object(commit)
yield rc
def test_delete(env, git_gecko, git_wpt, upstream_gecko_commit):
# Do some stuff to create an example sync
bug = 1234
test_changes = {"README": b"Change README\n"}
rev = upstream_gecko_commit(test_changes=test_changes,
bug=bug,
message=b"Change README")
update_repositories(git_gecko, git_wpt, wait_gecko_commit=rev)
_, _, _ = upstream.gecko_push(git_gecko,
git_wpt,
"autoland",
rev,
raise_on_error=True)
sync = upstream.UpstreamSync.for_bug(git_gecko, git_wpt, bug,
flat=True).pop()
process_name = sync.process_name
sync_path = "/".join(str(item) for item in process_name.as_tuple())
ref = git.Reference(git_gecko, "refs/syncs/data")
assert ref.commit.tree[sync_path]
gecko_ref = git.Reference(git_gecko, "refs/heads/%s" % sync_path)
assert gecko_ref.is_valid()
wpt_ref = git.Reference(git_wpt, "refs/heads/%s" % sync_path)
assert wpt_ref.is_valid()
with SyncLock.for_process(process_name) as lock:
with sync.as_mut(lock):
sync.delete()
for idx_cls in (index.SyncIndex, index.PrIdIndex, index.BugIdIndex):
idx = idx_cls(git_gecko)
assert not idx.get(idx.make_key(sync))
ref = git.Reference(git_gecko, "refs/syncs/data")
with pytest.raises(KeyError):
ref.commit.tree[sync_path]
ref = git.Reference(git_gecko, "refs/heads/%s" % sync_path)
assert not ref.is_valid()
ref = git.Reference(git_wpt, "refs/heads/%s" % sync_path)
assert not ref.is_valid()
def run():
# get args
parser = argparse__ArgumentParser()
parser.add_argument('--refname', nargs=1, required=True)
parser.add_argument('--stagedir', nargs=1, required=True)
parser.add_argument('--repodir', nargs=1, required=True)
args = parser.parse_args()
refname: str = args.refname[0]
stagedir: pathlib.Path = pathlib__Path(args.stagedir[0])
repodir: pathlib.Path = pathlib__Path(args.repodir[0])
assert stagedir.is_dir() and repodir.is_dir()
repo: git.Repo = repo_create_ensure(str(repodir))
# repo clean
repo.git.clean(d=True, f=True)
repo.git.rm('.', r=True)
# stage copy to repo
copy_fromdir_todir(srcdir=stagedir, dstdir=repodir)
# repo add files and commit
repo.git.add(A=True)
commit: git.Commit = repo.index.commit('ccc')
# repo set ref
ref = git.Reference(repo, refname)
ref.set_object(commit)
def create(
cls,
lock, # type: SyncLock
repo, # type: Repo
process_name, # type: ProcessName
data, # type: Dict[Text, Any]
message=u"Sync data", # type: Text
):
# type: (...) -> ProcessData
assert process_name.obj_type == cls.obj_type
path = cls.get_path(process_name)
ref = git.Reference(repo, env.config["sync"]["ref"])
try:
ref.commit.tree[path]
except KeyError:
pass
else:
raise ValueError("%s already exists at path %s" %
(cls.__name__, path))
with CommitBuilder(repo, message, ref=ref.path) as commit:
commit.add_tree({path: json.dumps(data).encode("utf8")})
ProcessNameIndex(repo).insert(process_name)
return cls(repo, process_name)
def refs_heads(refname):
ref = git.Reference(server_repo_ctx_get().repo, "refs/heads/" + refname)
commit = ref.commit
tree = commit.tree
#return flask_jsonify({ "tree": tree.hexsha })
return tree.hexsha
def commit(self, commit):
if isinstance(commit, sync_commit.Commit):
commit = commit.commit
ref = git.Reference(self.repo, self.path)
ref.set_object(commit)
def create(cls, repo, process_name, obj, commit_cls=sync_commit.Commit):
path = cls.process_path(process_name)
if git.Reference(repo, cls.process_path(process_name)).is_valid():
raise ValueError("Ref %s already exists" % path)
return cls._create(repo, process_name, obj, commit_cls)
def test_create(env, git_gecko):
TestIndex.create(git_gecko)
ref = git.Reference(git_gecko, env.config["sync"]["ref"])
assert ref.is_valid()
tree = ref.commit.tree["index/test"]
assert isinstance(tree, git.Tree)
def do_migrate(git_gecko, git_wpt, **kwargs):
assert False, "Running this is probably a bad idea"
# Migrate refs from the refs/<type>/<subtype>/<status>/<obj_id>[/<seq_id>] format
# to refs/<type>/<subtype>/<obj_id>/<seq_id>
from collections import defaultdict
from . import base
import pygit2
git2_gecko = pygit2.Repository(git_gecko.working_dir)
git2_wpt = pygit2.Repository(git_wpt.working_dir)
repo_map = {git_gecko: git2_gecko, git_wpt: git2_wpt}
rev_repo_map = {value: key for key, value in iteritems(repo_map)}
special = {}
sync_ref = re.compile("^refs/"
"(?P<reftype>[^/]+)/"
"(?P<obj_type>[^/]+)/"
"(?P<subtype>[^/]+)/"
"(?P<status>[^0-9/]+)/"
"(?P<obj_id>[0-9]+)"
"(?:/(?P<seq_id>[0-9]*))?$")
print("Updating refs")
seen = defaultdict(list)
total_refs = 0
processing_refs = 0
for ref in itertools.chain(git_gecko.refs, git_wpt.refs):
git2_repo = repo_map[ref.repo]
ref = git2_repo.lookup_reference(ref.path)
total_refs += 1
if ref.name in special:
continue
m = sync_ref.match(ref.name)
if not m:
continue
if m.group("reftype") not in ("heads", "syncs"):
continue
if m.group("obj_type") not in ("sync", "try"):
continue
processing_refs += 1
assert m.group("subtype") in ("upstream", "downstream", "landing")
assert int(m.group("obj_id")) > 0
new_ref = "refs/%s/%s/%s/%s/%s" % (
m.group("reftype"), m.group("obj_type"), m.group("subtype"),
m.group("obj_id"), m.group("seq_id") or "0")
seen[(git2_repo, new_ref)].append((ref, m.group("status")))
duplicate = {}
delete = set()
for (repo, new_ref), refs in iteritems(seen):
if len(refs) > 1:
# If we have multiple /syncs/ ref, but only one /heads/ ref, use the corresponding one
if new_ref.startswith("refs/syncs/"):
has_head = set()
no_head = set()
for ref, status in refs:
if "refs/heads/%s" % ref.name[len(
"refs/syncs/")] in repo.references:
has_head.add((ref.name, status))
else:
no_head.add((ref.name, status))
if len(has_head) == 1:
print(" Using %s from %s" %
(list(has_head)[0][0].path, " ".join(
ref.name for ref, _ in refs)))
refs[:] = list(has_head)
delete |= set((repo, ref_name) for ref_name, _ in no_head)
if len(refs) > 1:
# If we have a later status, prefer that over an earlier one
matches = {ref.name: sync_ref.match(ref.name) for ref, _ in refs}
by_status = {
matches[ref.name].group("status"): (ref, status)
for (ref, status) in refs
}
for target_status in [
"complete", "wpt-merged", "incomplete", "infra-fail"
]:
if target_status in by_status:
print(" Using %s from %s" %
(by_status[target_status][0].name, " ".join(
ref.name for ref, _ in refs)))
delete |= set((repo, ref.name) for ref, status in refs
if ref != by_status[target_status])
refs[:] = [by_status[target_status]]
if len(refs) > 1:
duplicate[(repo, new_ref)] = refs
if duplicate:
print(" ERROR! Got duplicate %s source refs" % len(duplicate))
for (repo, new_ref), refs in iteritems(duplicate):
print(" %s %s: %s" %
(repo.working_dir, new_ref, " ".join(ref.name
for ref, _ in refs)))
return
for (repo, new_ref), refs in iteritems(seen):
ref, _ = refs[0]
if ref.name.startswith("refs/syncs/sync/"):
if "refs/heads/%s" % ref.name[len("refs/syncs/"
):] not in repo.references:
# Try with the post-migration head
m = sync_ref.match(ref.name)
ref_path = "refs/heads/%s/%s/%s/%s" % (
m.group("obj_type"), m.group("subtype"), m.group("obj_id"),
m.group("seq_id"))
if ref_path not in repo.references:
print(" Missing head %s" % (ref.name))
created = 0
for i, ((repo, new_ref), refs) in enumerate(iteritems(seen)):
assert len(refs) == 1
ref, status = refs[0]
print("Updating %s" % ref.name)
print(" Moving %s to %s %d/%d" %
(ref.name, new_ref, i + 1, len(seen)))
if "/syncs/" in ref.name:
ref_obj = ref.peel().id
data = json.loads(repo[ref.peel().tree["data"].id].data)
if data.get("status") != status:
with base.CommitBuilder(rev_repo_map[repo],
"Add status",
ref=ref.name) as commit:
now_ref_obj = ref.peel().id
if ref_obj != now_ref_obj:
data = json.loads(
repo[ref.peel().tree["data"].id].data)
data["status"] = status
commit.add_tree({"data": json.dumps(data)})
print("Making commit")
commit = commit.get().sha1
else:
commit = ref.peel().id
print(" Got commit %s" % commit)
if new_ref not in repo.references:
print(" Rename %s %s" % (ref.name, new_ref))
repo.references.create(new_ref, commit)
created += 1
else:
print(" %s already exists" % new_ref)
delete.add((repo, ref.name))
for repo, ref_name in delete:
print(" Deleting %s" % ref_name)
repo.references.delete(ref_name)
print("%s total refs" % total_refs)
print("%s refs to process" % processing_refs)
print("%s refs to create" % created)
print("%s refs to delete" % len(delete))
print("Moving to single history")
# Migrate from refs/syncs/ to paths
sync_ref = re.compile("^refs/"
"syncs/"
"(?P<obj_type>[^/]*)/"
"(?P<subtype>[^/]*)/"
"(?P<obj_id>[^/]*)/"
"(?P<seq_id>[0-9]*)$")
delete = set()
initial_ref = git.Reference(git_gecko, "refs/syncs/data")
if initial_ref.is_valid():
existing_paths = {
item.path
for item in initial_ref.commit.tree.traverse()
}
else:
existing_paths = set()
for ref in git_gecko.refs:
m = sync_ref.match(ref.path)
if not m:
continue
path = "%s/%s/%s/%s" % (m.group("obj_type"), m.group("subtype"),
m.group("obj_id"), m.group("seq_id"))
if path not in existing_paths:
with base.CommitBuilder(git_gecko,
"Migrate %s to single ref for data" %
ref.path,
ref="refs/syncs/data") as commit:
data = json.load(ref.commit.tree["data"].data_stream)
print(" Moving path %s" % (path, ))
tree = {path: json.dumps(data)}
commit.add_tree(tree)
delete.add(ref.path)
git2_repo = repo_map[git_gecko]
for ref_name in delete:
git2_repo.references.delete(ref_name)
def ref(self):
# type: () -> Reference
if self.path in self.pygit2_repo.references:
return git.Reference(self.repo, self.path)
return None
def ref(self):
ref = git.Reference(self.repo, self.path)
if ref.is_valid():
return ref
return None
