def _init_common(index_path: Path, origin_location: str, crates_root_path: Path) -> git.Repo: if index_path.is_dir(): raise error.UsageError("index directory {} already exists".format( repr(str(index_path)))) common.iprint("create index repository at {}:".format(repr( str(index_path)))) index_path.mkdir(parents=True) repo = git.Repo.init(str(index_path)) common.iprint(" remote add origin {}".format(origin_location)) repo.create_remote("origin", origin_location) # Setup "HEAD" to new "working" branch. working = git.Reference(repo, "refs/heads/working") repo.head.set_reference(working) # Setup default remote and merge branch for "working". with repo.config_writer() as writer: writer.set_value('branch "working"', "remote", "origin") writer.set_value('branch "working"', "merge", "refs/heads/master") if not crates_root_path.is_dir(): common.iprint("create crates directory at {}:".format( repr(str(crates_root_path)))) crates_root_path.mkdir(parents=True) return repo
def get_or_create(cls, repo): ref_name = env.config["sync"]["ref"] ref = git.Reference(repo, ref_name) try: ref.commit.tree["index/%s" % cls.name] except KeyError: return cls.create(repo) return cls(repo)
def __init__(self, repo, process_name, commit_cls=sync_commit.Commit): if not git.Reference(repo, self.process_path(process_name)).is_valid(): raise ValueError( "No ref found in %s with path %s" % (repo.working_dir, self.process_path(process_name))) self.repo = repo self._ref = str(process_name) self._process_name = process_name.attach_ref(self) self.commit_cls = commit_cls
def test_checkout_head(rc: ServerRepoCtx): master: git.Reference = git.Reference(rc.repo, "refs/heads/master") # FIXME: only touches paths recorded in index - stray worktree files not removed etc # see IndexFile.checkout vs head.checkout # FIXME: use force=True ? #index: git.IndexFile = git.IndexFile.from_tree(rc.repo, master.commit.tree) #index.checkout() assert os_path_exists(str(rc.repodir / '.git')) rc.repo.head.reset(master.commit, index=True, working_tree=True)
def test_commit_head(rc: ServerRepoCtx, client: flask.testing.FlaskClient): master_tree: shahex = _get_master_tree_hex(client) dummy: git.Actor = git.Actor('dummy', "*****@*****.**") commit: git.Commit = git.Commit.create_from_tree(rc.repo, master_tree, message="", author=dummy, committer=dummy) master: git.Reference = git.Reference(rc.repo, "refs/heads/master") master.set_commit(commit)
def print_pq_status(self, out=sys.stdout): with TablePrinter(out=out) as printrow: for subtree in self.get_pq_subtrees(): if subtree.worktree: branch_name = git.Reference(self, subtree.worktree.branch).name printrow(subtree.relpath, f"[editing: {branch_name}]") else: printrow(subtree.relpath, "[not editing]") out.write("\n")
def prepare_project_for_test(): if will_record: self.service.connect() # let's create a project and add it to current repository self.service.create(namespace, create_repository, add=True) # make a modification, commit and push it with open(os.path.join(self.repository.working_dir, 'first_file'), 'w') as test: test.write( 'he who makes a beast of himself gets rid of the pain of being a man. Dr Johnson' ) self.repository.git.config('user.name', 'travis') self.repository.git.config('user.email', '*****@*****.**') self.repository.git.add('first_file') self.repository.git.commit(message='First commit') if will_record: self.repository.git.push(self.service.name, 'master') # create a new branch new_branch = self.repository.create_head(source_branch, 'HEAD') self.repository.head.reference = new_branch self.repository.head.reset(index=True, working_tree=True) # make a modification, commit and push it to that branch with open( os.path.join(self.repository.working_dir, 'second_file'), 'w') as test: test.write( 'La meilleure façon de ne pas avancer est de suivre une idée fixe. J.Prévert' ) self.repository.git.add('second_file') self.repository.git.commit(message='Second commit') self.repository.git.push(service, source_branch) else: import git self.service._extracts_ref = lambda *a: git.Reference( self.service.repository, '{}/{}'.format(namespace, repository), check_path=False) self.service.repository.head.reference = git.Head( self.service.repository, path='refs/heads/pr-test') existing_remotes = [r.name for r in self.repository.remotes] if 'all' in existing_remotes: r_all = self.repository.remote('all') else: r_all = self.repository.create_remote('all', '') for name in ('github', 'gitlab', 'bitbucket', 'gogs', 'upstream'): if name not in existing_remotes: kw = dict(user=namespace, project=repository, host=name) self.repository.create_remote( name, 'git@{host}.com:{user}/{project}'.format(**kw)) r_all.add_url( 'git@{host}.com:{user}/{project}'.format(**kw)) yield if will_record: self.service.delete(create_repository)
def _upgrade_to_working(repo: git.Repo) -> None: working = git.Reference(repo, "refs/heads/working") if repo.head.reference.name != "working" and not working.is_valid(): # Time to upgrade working tree to use "working" branch. common.eprint("""Upgrade index to use "working" branch as HEAD""") if repo.head.reference.is_valid(): # Create "working" branch based on current HEAD. common.iprint( """Checkout new "working" branch from current HEAD""") repo.create_head("refs/heads/working", "HEAD") repo.head.set_reference(working)
def __init__(self, repo, process_name): assert process_name.obj_type == self.obj_type self.repo = repo self.pygit2_repo = pygit2_get(repo) self.process_name = process_name self.ref = git.Reference(repo, env.config["sync"]["ref"]) self.path = self.get_path(process_name) self._data = self._load() self._lock = None self._updated = set() self._deleted = set() self._delete = False
def require_no_merge_conflict(self): """ Raises :exc:`MergeConflict` if the current working directory contains a merge conflict. """ try: git.Reference(self.repo, 'MERGE_HEAD', check_path=False).commit # reference exists, so there is a merge conflict raise MergeConflict() except ValueError: # no such reference, so there is no merge conflict pass
def setup(self, repo): data_ref = git.Reference(repo, self.config["sync"]["ref"]) if not data_ref.is_valid(): from . import base with base.CommitBuilder(repo, "Create initial sync metadata", ref=data_ref.path) as commit: path = "_metadata" data = json.dumps({"name": "wptsync"}) commit.add_tree({path: data}) from . import index for idx in index.indicies: idx.get_or_create(repo)
def _create(cls, repo, process_name, obj, commit_cls=sync_commit.Commit): path = cls.process_path(process_name) logger.debug("Creating ref %s" % path) for ref in repo.refs: if fnmatch( ref.path, "refs/%s/%s" % (cls.ref_prefix, process_name.name_filter())): raise ValueError("Ref %s exists which conflicts with %s" % (ref.path, path)) ref = git.Reference(repo, path) ref.set_object(obj) return cls(repo, process_name, commit_cls)
def create(cls, lock, repo, process_name, data, message="Sync data"): assert process_name.obj_type == cls.obj_type path = cls.get_path(process_name) ref = git.Reference(repo, env.config["sync"]["ref"]) try: ref.commit.tree[path] except KeyError: pass else: raise ValueError("%s already exists at path %s" % (cls.__name__, path)) with CommitBuilder(repo, message, ref=ref) as commit: commit.add_tree({path: json.dumps(data)}) ProcessNameIndex(repo).insert(process_name) return cls(repo, process_name)
def rename(self): path = self.path ref = self.ref if not ref: return # This implicitly updates self.path self._ref = str(self._process_name) if self.path == path: return logger.debug("Renaming ref %s to %s" % (path, self.path)) new_ref = git.Reference(self.repo, self.path) new_ref.set_object(ref.commit.hexsha) logger.debug( "Deleting ref %s pointing at %s.\n" "To recreate this ref run `git update-ref %s %s`" % (ref.path, ref.commit.hexsha, ref.path, ref.commit.hexsha)) ref.delete(self.repo, ref.path)
def preload_repo( tupdater3_exe: str, stage2_exe: str, repodir_s: pathlib.Path ) -> ServerRepoCtx: with git.Repo.init(str(repodir_s)) as repo: rc = ServerRepoCtx(repodir_s, repo) # BEG populate with dummy data for ff in [("a.txt", b"aaa"), ("d/b.txt", b"bbb")]: with _file_open_mkdirp(str(rc.repodir.joinpath(ff[0]))) as f: f.write(ff[1]) rc.repo.index.add([ff[0]]) # END populate with dummy data # BEG populate with executables for gg in [("updater.exe", tupdater3_exe), ("stage2.exe", stage2_exe)]: shutil_copyfile(gg[1], str(rc.repodir.joinpath(gg[0]))) rc.repo.index.add([gg[0]]) # END populate with executables commit = rc.repo.index.commit("ccc") ref_master = git.Reference(rc.repo, "refs/heads/master") ref_master.set_object(commit)
def finish_pq(self, subtree, out=sys.stdout): if not subtree.worktree: print(f"{subtree.uiname} is not being edited") return wt = Repo(subtree.path) ok = True try: relpath = relpath_nodots(wt.git_dir, self.git_dir) except OSError: ok = False relpath = relpath.split(os.path.sep) ok = ok and len(relpath) > 1 and relpath[0] == 'worktrees' if not ok: print( f"the GIT_DIR for {subtree.uiname} is unexpectedly at {wt.git_dir}, cannot proceed" ) return os.unlink(os.path.join(subtree.path, '.git')) shutil.rmtree(wt.git_dir) self.delete_head(git.Reference(self, subtree.worktree.branch), force=True)
def rc_s(customopt_tupdater2_exe: str, customopt_tupdater3_exe: str, dirs: Dirs) -> ServerRepoCtx: import shutil repo = git.Repo.init(str(dirs.repodir_s)) rc = ServerRepoCtx(dirs.repodir_s, repo) # BEG populate with dummy data for ff in [("a.txt", b"aaa"), ("d/b.txt", b"bbb")]: with _file_open_mkdirp(str(rc.repodir.joinpath(ff[0]))) as f: f.write(ff[1]) rc.repo.index.add([ff[0]]) # END populate with dummy data # BEG populate with tupdater for gg in [("updater.exe", customopt_tupdater3_exe), ("stage2.exe", customopt_tupdater3_exe)]: shutil.copyfile(gg[1], str(rc.repodir.joinpath(gg[0]))) rc.repo.index.add([gg[0]]) # END populate with tupdater commit = rc.repo.index.commit("ccc") ref_master = git.Reference(rc.repo, "refs/heads/master") ref_master.set_object(commit) yield rc
def test_delete(env, git_gecko, git_wpt, upstream_gecko_commit): # Do some stuff to create an example sync bug = 1234 test_changes = {"README": b"Change README\n"} rev = upstream_gecko_commit(test_changes=test_changes, bug=bug, message=b"Change README") update_repositories(git_gecko, git_wpt, wait_gecko_commit=rev) _, _, _ = upstream.gecko_push(git_gecko, git_wpt, "autoland", rev, raise_on_error=True) sync = upstream.UpstreamSync.for_bug(git_gecko, git_wpt, bug, flat=True).pop() process_name = sync.process_name sync_path = "/".join(str(item) for item in process_name.as_tuple()) ref = git.Reference(git_gecko, "refs/syncs/data") assert ref.commit.tree[sync_path] gecko_ref = git.Reference(git_gecko, "refs/heads/%s" % sync_path) assert gecko_ref.is_valid() wpt_ref = git.Reference(git_wpt, "refs/heads/%s" % sync_path) assert wpt_ref.is_valid() with SyncLock.for_process(process_name) as lock: with sync.as_mut(lock): sync.delete() for idx_cls in (index.SyncIndex, index.PrIdIndex, index.BugIdIndex): idx = idx_cls(git_gecko) assert not idx.get(idx.make_key(sync)) ref = git.Reference(git_gecko, "refs/syncs/data") with pytest.raises(KeyError): ref.commit.tree[sync_path] ref = git.Reference(git_gecko, "refs/heads/%s" % sync_path) assert not ref.is_valid() ref = git.Reference(git_wpt, "refs/heads/%s" % sync_path) assert not ref.is_valid()
def run(): # get args parser = argparse__ArgumentParser() parser.add_argument('--refname', nargs=1, required=True) parser.add_argument('--stagedir', nargs=1, required=True) parser.add_argument('--repodir', nargs=1, required=True) args = parser.parse_args() refname: str = args.refname[0] stagedir: pathlib.Path = pathlib__Path(args.stagedir[0]) repodir: pathlib.Path = pathlib__Path(args.repodir[0]) assert stagedir.is_dir() and repodir.is_dir() repo: git.Repo = repo_create_ensure(str(repodir)) # repo clean repo.git.clean(d=True, f=True) repo.git.rm('.', r=True) # stage copy to repo copy_fromdir_todir(srcdir=stagedir, dstdir=repodir) # repo add files and commit repo.git.add(A=True) commit: git.Commit = repo.index.commit('ccc') # repo set ref ref = git.Reference(repo, refname) ref.set_object(commit)
def create( cls, lock, # type: SyncLock repo, # type: Repo process_name, # type: ProcessName data, # type: Dict[Text, Any] message=u"Sync data", # type: Text ): # type: (...) -> ProcessData assert process_name.obj_type == cls.obj_type path = cls.get_path(process_name) ref = git.Reference(repo, env.config["sync"]["ref"]) try: ref.commit.tree[path] except KeyError: pass else: raise ValueError("%s already exists at path %s" % (cls.__name__, path)) with CommitBuilder(repo, message, ref=ref.path) as commit: commit.add_tree({path: json.dumps(data).encode("utf8")}) ProcessNameIndex(repo).insert(process_name) return cls(repo, process_name)
def refs_heads(refname): ref = git.Reference(server_repo_ctx_get().repo, "refs/heads/" + refname) commit = ref.commit tree = commit.tree #return flask_jsonify({ "tree": tree.hexsha }) return tree.hexsha
def commit(self, commit): if isinstance(commit, sync_commit.Commit): commit = commit.commit ref = git.Reference(self.repo, self.path) ref.set_object(commit)
def create(cls, repo, process_name, obj, commit_cls=sync_commit.Commit): path = cls.process_path(process_name) if git.Reference(repo, cls.process_path(process_name)).is_valid(): raise ValueError("Ref %s already exists" % path) return cls._create(repo, process_name, obj, commit_cls)
def test_create(env, git_gecko): TestIndex.create(git_gecko) ref = git.Reference(git_gecko, env.config["sync"]["ref"]) assert ref.is_valid() tree = ref.commit.tree["index/test"] assert isinstance(tree, git.Tree)
def do_migrate(git_gecko, git_wpt, **kwargs): assert False, "Running this is probably a bad idea" # Migrate refs from the refs/<type>/<subtype>/<status>/<obj_id>[/<seq_id>] format # to refs/<type>/<subtype>/<obj_id>/<seq_id> from collections import defaultdict from . import base import pygit2 git2_gecko = pygit2.Repository(git_gecko.working_dir) git2_wpt = pygit2.Repository(git_wpt.working_dir) repo_map = {git_gecko: git2_gecko, git_wpt: git2_wpt} rev_repo_map = {value: key for key, value in iteritems(repo_map)} special = {} sync_ref = re.compile("^refs/" "(?P<reftype>[^/]+)/" "(?P<obj_type>[^/]+)/" "(?P<subtype>[^/]+)/" "(?P<status>[^0-9/]+)/" "(?P<obj_id>[0-9]+)" "(?:/(?P<seq_id>[0-9]*))?$") print("Updating refs") seen = defaultdict(list) total_refs = 0 processing_refs = 0 for ref in itertools.chain(git_gecko.refs, git_wpt.refs): git2_repo = repo_map[ref.repo] ref = git2_repo.lookup_reference(ref.path) total_refs += 1 if ref.name in special: continue m = sync_ref.match(ref.name) if not m: continue if m.group("reftype") not in ("heads", "syncs"): continue if m.group("obj_type") not in ("sync", "try"): continue processing_refs += 1 assert m.group("subtype") in ("upstream", "downstream", "landing") assert int(m.group("obj_id")) > 0 new_ref = "refs/%s/%s/%s/%s/%s" % ( m.group("reftype"), m.group("obj_type"), m.group("subtype"), m.group("obj_id"), m.group("seq_id") or "0") seen[(git2_repo, new_ref)].append((ref, m.group("status"))) duplicate = {} delete = set() for (repo, new_ref), refs in iteritems(seen): if len(refs) > 1: # If we have multiple /syncs/ ref, but only one /heads/ ref, use the corresponding one if new_ref.startswith("refs/syncs/"): has_head = set() no_head = set() for ref, status in refs: if "refs/heads/%s" % ref.name[len( "refs/syncs/")] in repo.references: has_head.add((ref.name, status)) else: no_head.add((ref.name, status)) if len(has_head) == 1: print(" Using %s from %s" % (list(has_head)[0][0].path, " ".join( ref.name for ref, _ in refs))) refs[:] = list(has_head) delete |= set((repo, ref_name) for ref_name, _ in no_head) if len(refs) > 1: # If we have a later status, prefer that over an earlier one matches = {ref.name: sync_ref.match(ref.name) for ref, _ in refs} by_status = { matches[ref.name].group("status"): (ref, status) for (ref, status) in refs } for target_status in [ "complete", "wpt-merged", "incomplete", "infra-fail" ]: if target_status in by_status: print(" Using %s from %s" % (by_status[target_status][0].name, " ".join( ref.name for ref, _ in refs))) delete |= set((repo, ref.name) for ref, status in refs if ref != by_status[target_status]) refs[:] = [by_status[target_status]] if len(refs) > 1: duplicate[(repo, new_ref)] = refs if duplicate: print(" ERROR! Got duplicate %s source refs" % len(duplicate)) for (repo, new_ref), refs in iteritems(duplicate): print(" %s %s: %s" % (repo.working_dir, new_ref, " ".join(ref.name for ref, _ in refs))) return for (repo, new_ref), refs in iteritems(seen): ref, _ = refs[0] if ref.name.startswith("refs/syncs/sync/"): if "refs/heads/%s" % ref.name[len("refs/syncs/" ):] not in repo.references: # Try with the post-migration head m = sync_ref.match(ref.name) ref_path = "refs/heads/%s/%s/%s/%s" % ( m.group("obj_type"), m.group("subtype"), m.group("obj_id"), m.group("seq_id")) if ref_path not in repo.references: print(" Missing head %s" % (ref.name)) created = 0 for i, ((repo, new_ref), refs) in enumerate(iteritems(seen)): assert len(refs) == 1 ref, status = refs[0] print("Updating %s" % ref.name) print(" Moving %s to %s %d/%d" % (ref.name, new_ref, i + 1, len(seen))) if "/syncs/" in ref.name: ref_obj = ref.peel().id data = json.loads(repo[ref.peel().tree["data"].id].data) if data.get("status") != status: with base.CommitBuilder(rev_repo_map[repo], "Add status", ref=ref.name) as commit: now_ref_obj = ref.peel().id if ref_obj != now_ref_obj: data = json.loads( repo[ref.peel().tree["data"].id].data) data["status"] = status commit.add_tree({"data": json.dumps(data)}) print("Making commit") commit = commit.get().sha1 else: commit = ref.peel().id print(" Got commit %s" % commit) if new_ref not in repo.references: print(" Rename %s %s" % (ref.name, new_ref)) repo.references.create(new_ref, commit) created += 1 else: print(" %s already exists" % new_ref) delete.add((repo, ref.name)) for repo, ref_name in delete: print(" Deleting %s" % ref_name) repo.references.delete(ref_name) print("%s total refs" % total_refs) print("%s refs to process" % processing_refs) print("%s refs to create" % created) print("%s refs to delete" % len(delete)) print("Moving to single history") # Migrate from refs/syncs/ to paths sync_ref = re.compile("^refs/" "syncs/" "(?P<obj_type>[^/]*)/" "(?P<subtype>[^/]*)/" "(?P<obj_id>[^/]*)/" "(?P<seq_id>[0-9]*)$") delete = set() initial_ref = git.Reference(git_gecko, "refs/syncs/data") if initial_ref.is_valid(): existing_paths = { item.path for item in initial_ref.commit.tree.traverse() } else: existing_paths = set() for ref in git_gecko.refs: m = sync_ref.match(ref.path) if not m: continue path = "%s/%s/%s/%s" % (m.group("obj_type"), m.group("subtype"), m.group("obj_id"), m.group("seq_id")) if path not in existing_paths: with base.CommitBuilder(git_gecko, "Migrate %s to single ref for data" % ref.path, ref="refs/syncs/data") as commit: data = json.load(ref.commit.tree["data"].data_stream) print(" Moving path %s" % (path, )) tree = {path: json.dumps(data)} commit.add_tree(tree) delete.add(ref.path) git2_repo = repo_map[git_gecko] for ref_name in delete: git2_repo.references.delete(ref_name)
def ref(self): # type: () -> Reference if self.path in self.pygit2_repo.references: return git.Reference(self.repo, self.path) return None
def ref(self): ref = git.Reference(self.repo, self.path) if ref.is_valid(): return ref return None