def _assign_previous(self):
'''
Many commits in assign_dict were already assigned branches in a
previous push or pull. Remember and honor those.
'''
# Test-only mode skips this step and that's okay.
if not self.ctx:
return
for assign in self.assign_dict.values():
otl = ObjectType.commits_for_sha1(self.ctx, assign.sha1)
for ot in otl:
self._assign_branch(assign, ot.details.branch_id)
def _honor_original_git_parent_sequence(self):
'''
Reorder self.parent_commit_list to match the order its commits appear in
the original Git commit.
Any parents not listed in the original Git commit are appended to the
back of the list in preserved order.
'''
# +++ Cannot resequence a list without multiple elements.
if len(self.parent_commit_list) < 2:
return
want_seq = self._git_object_parent_list()
if not want_seq:
return
# Build the result in order.
keep_seq = []
rem_parent_list = copy.copy(self.parent_commit_list)
for w in want_seq:
if w in rem_parent_list:
keep_seq.append(w)
rem_parent_list.remove(w)
elif w in self.p2g.sha1_to_mark:
# try converting the SHA1 to a mark...
m = self.p2g.sha1_to_mark[w]
if m in rem_parent_list:
keep_seq.append(m)
rem_parent_list.remove(m)
# Convert sha1 to mark number(s), in case parent is part of this
# pull and not yet stored in git with a sha1.
commits = ObjectType.commits_for_sha1(self.ctx, w)
for commit in commits:
for mark in self.p2g.mark_list.cl_to_mark_list(
commit.details.changelist):
if mark in rem_parent_list:
keep_seq.append(mark)
rem_parent_list.remove(mark)
# Add any remainder to the end of the result.
keep_seq.extend(rem_parent_list)
self.parent_commit_list = keep_seq
def _honor_original_git_parent_sequence(self):
'''
Reorder self.parent_commit_list to match the order its commits appear in
the original Git commit.
Any parents not listed in the original Git commit are appended to the
back of the list in preserved order.
'''
# +++ Cannot resequence a list without multiple elements.
if len(self.parent_commit_list) < 2:
return
want_seq = self._git_object_parent_list()
if not want_seq:
return
# Build the result in order.
keep_seq = []
rem_parent_list = copy.copy(self.parent_commit_list)
for w in want_seq:
if w in rem_parent_list:
keep_seq.append(w)
rem_parent_list.remove(w)
elif w in self.p2g.sha1_to_mark:
# try converting the SHA1 to a mark...
m = self.p2g.sha1_to_mark[w]
if m in rem_parent_list:
keep_seq.append(m)
rem_parent_list.remove(m)
# Convert sha1 to mark number(s), in case parent is part of this
# pull and not yet stored in git with a sha1.
commits = ObjectType.commits_for_sha1(self.ctx, w)
for commit in commits:
for mark in self.p2g.mark_list.cl_to_mark_list(commit.details.changelist):
if mark in rem_parent_list:
keep_seq.append(mark)
rem_parent_list.remove(mark)
# Add any remainder to the end of the result.
keep_seq.extend(rem_parent_list)
self.parent_commit_list = keep_seq
def _can_use_as_parent(self, sha1):
"""If sha1 already copied to Git, return sha1.
If sha1 not yet copied, but its corresponding Perforce changelist is in
our MarkList of impending copies, return the mark for that changelist.
If not, return None
"""
# Already stored in Git from some previous push or pull?
if p4gf_util.sha1_exists(sha1):
return sha1
# Any of this commit's corresponding changelists
# about to be copied to Git in this pull?
for commit in ObjectType.commits_for_sha1(self.ctx, sha1):
ml = self.p2g.mark_list.cl_to_mark_list(commit.change_num)
if ml:
return ml[0]
return None
def _can_use_as_parent(self, sha1):
'''
If sha1 already copied to Git, return sha1.
If sha1 not yet copied, but its corresponding Perforce changelist is in
our MarkList of impending copies, return the mark for that changelist.
If not, return None
'''
# Already stored in Git from some previous push or pull?
if p4gf_util.sha1_exists(sha1):
return sha1
# Any of this commit's corresponding changelists
# about to be copied to Git in this pull?
for commit in ObjectType.commits_for_sha1(self.ctx, sha1):
ml = self.p2g.mark_list.cl_to_mark_list(commit.details.changelist)
if ml:
return ml[0]
return None
def main():
"""Do the thing."""
args = _argparse()
_log_config(args)
LOG.debug("args={}".format(args))
ctx = _create_ctx(p4port=args.p4port,
p4user=args.p4user,
repo_name=args.repo_name,
server_id=args.server_id)
LOG.info("P4PORT={}".format(ctx.p4.port))
LOG.info("P4USER={}".format(ctx.p4.user))
di_list = changes_to_desc_info_list(ctx, entire_depot=args.entire_depot)
# Load branch view definitions.
# Gives us Git branch names to go with pushed
# commmit sha1s.
branch_dict = ctx.branch_dict()
# Create index for fast dbid->branch lookups later.
dbid_to_branch = {
b.depot_branch.depot_branch_id: b
for b in branch_dict.values() if b.depot_branch
}
# Use DescInfo and ObjectType backup to
# find branch_id assignments.
for di in di_list:
di.branch_id = None
di.git_branch_name = None
change_num_str = str(di.change_num)
# Use depot branch ID recorded in DescInfo.
if di.depot_branch_id:
branch = dbid_to_branch.get(di.depot_branch_id)
if branch:
di.branch_id = branch.branch_id
di.git_branch_name = branch.git_branch_name
continue
# Fetch from ObjectType.
otl = ObjectType.commits_for_sha1(ctx, di.sha1)
for ot in otl:
if ot.change_num == change_num_str:
di.branch_id = ot.branch_id
if di.branch_id:
branch = branch_dict.get(di.branch_id)
if branch:
di.git_branch_name = branch.git_branch_name
continue
# Report time
attr_list = [
'change_num', 'sha1', 'push_state', 'branch_id', 'git_branch_name'
]
print("# " + "\t".join(attr_list))
for di in di_list:
if args.complete_only and di.push_state != "complete":
continue
v = [str(getattr(di, a)) for a in attr_list]
print("\t".join(v))
def process_tags(ctx, tags):
"""Add or remove tags objects from the Git Fusion mirror.
:param ctx: P4GF context with initialized pygit2 Repository.
:param tags: list of PreReceiveTuple objects for tags
"""
# pylint:disable=too-many-branches
if not tags:
LOG.debug("process_tags() no incoming tags to process")
return
# Re-sync the tags since preflight_tags() synced with a different temp client.
tags_path = "objects/repos/{repo}/tags".format(repo=ctx.config.repo_name)
with ctx.p4gf.at_exception_level(P4.P4.RAISE_NONE):
# Raises an exception when there are no files to sync?
ctx.p4gfrun('sync', '-q', "//{}/{}/...".format(ctx.p4gf.client,
tags_path))
# Decide what to do with the tag references.
tags_to_delete = []
tags_to_add = []
tags_to_edit = []
for prt in tags:
tag = prt.ref[10:]
if prt.old_sha1 == p4gf_const.NULL_COMMIT_SHA1:
if prt.new_sha1 == p4gf_const.NULL_COMMIT_SHA1:
# No idea how this happens, but it did, so guard against it.
continue
# Adding a new tag; if it references a commit, check that it
# exists; for other types, it is too costly to verify
# reachability from a known commit, so just ignore them.
obj = _get_tag_target(ctx.repo, prt.new_sha1)
is_commit = obj.type == pygit2.GIT_OBJ_COMMIT
if is_commit and not ObjectType.commits_for_sha1(
ctx, p4gf_pygit2.object_to_sha1(obj)):
LOG.debug("Tag '{}' of unknown commit {:7.7} not stored."
" Removing ref from git repo.".format(
tag, prt.new_sha1))
_remove_tag_ref(tag, prt.new_sha1)
continue
if obj.type == pygit2.GIT_OBJ_TREE:
continue
if obj.type == pygit2.GIT_OBJ_BLOB:
continue
_add_tag(ctx, tag, prt.new_sha1, tags_to_edit, tags_to_add)
elif prt.new_sha1 == p4gf_const.NULL_COMMIT_SHA1:
# Removing an existing tag
_remove_tag(ctx, tag, prt.old_sha1, tags_to_edit, tags_to_delete)
# Seemingly nothing to do.
if not tags_to_add and not tags_to_edit and not tags_to_delete:
LOG.debug("process_tags() mysteriously came up empty"
" - probably a tag of a non-existing commit.")
return
# Add and remove tags as appropriate, doing so in batches.
LOG.info(
"adding {} tags, removing {} tags, editing {} tags from Git mirror".
format(len(tags_to_add), len(tags_to_delete), len(tags_to_edit)))
desc = _("Git Fusion '{repo}' tag changes").format(
repo=ctx.config.repo_name)
with p4gf_util.NumberedChangelist(gfctx=ctx, description=desc) as nc:
while len(tags_to_add):
bite = tags_to_add[:_BITE_SIZE]
tags_to_add = tags_to_add[_BITE_SIZE:]
ctx.p4gfrun('add', '-t', 'binary+F', bite)
while len(tags_to_edit):
bite = tags_to_edit[:_BITE_SIZE]
tags_to_edit = tags_to_edit[_BITE_SIZE:]
ctx.p4gfrun('edit', '-k', bite)
while len(tags_to_delete):
bite = tags_to_delete[:_BITE_SIZE]
tags_to_delete = tags_to_delete[_BITE_SIZE:]
ctx.p4gfrun('delete', bite)
nc.submit()
if nc.submitted:
_write_last_copied_tag(ctx, nc.change_num)
LOG.debug("process_tags() complete")
def process_tags(ctx, prl):
"""
For each tag reference in the pre-receive-tuple list, add or remove the
object from the Git Fusion mirror.
Arguments:
ctx - P4GF context with initialized pygit2 Repository.
prl - list of PreReceiveTuple objects.
Returns None if successful and an error string otherwise.
"""
tags = [prt for prt in prl if prt.ref.startswith('refs/tags/')]
if not tags:
LOG.debug("process_tags() no incoming tags to process")
return None
# Screen the tags to ensure their names won't cause problems sometime
# in the future (i.e. when we create Perforce labels). Several of these
# characters are not allowed in Git tag names anyway, but better to
# check in case that changes in the future.
# In particular git disallows a leading '-', but we'll check for it anyway
# Otherwise allow internal '-'
regex = re.compile(r'[*@#,]|\.\.\.|%%')
for prt in tags:
tag = prt.ref[10:]
if regex.search(tag) or tag.startswith('-'):
return _("illegal characters (@#*,...%%) in tag name: '{}'").format(tag)
if not ctx.view_repo:
# In some cases the Git repository object is not yet created.
ctx.view_repo = pygit2.Repository(ctx.view_dirs.GIT_DIR)
LOG.debug("process_tags() beginning...")
tags_path = "objects/repos/{repo}/tags/...".format(repo=ctx.config.view_name)
with ctx.p4gf.at_exception_level(P4.P4.RAISE_NONE):
# Raises an exception when there are no files to sync?
ctx.p4gfrun(['sync', '-q', "//{}/{}/...".format(ctx.config.p4client_gf, tags_path)])
# Decide what to do with the tag references.
tags_to_delete = []
tags_to_add = []
tags_to_edit = []
for prt in tags:
tag = prt.ref[10:]
if prt.old_sha1 == p4gf_const.NULL_COMMIT_SHA1:
if prt.new_sha1 == p4gf_const.NULL_COMMIT_SHA1:
# No idea how this happens, but it did, so guard against it.
sys.stderr.write(_('Ignoring double-zero pre-receive-tuple line'))
continue
# Adding a new tag; if it references a commit, check that it
# exists; for other types, it is too costly to verify
# reachability from a known commit, so just ignore them.
obj = _get_tag_target(ctx.view_repo, prt.new_sha1)
is_commit = obj.type == pygit2.GIT_OBJ_COMMIT
if is_commit and not ObjectType.commits_for_sha1(ctx, obj.hex):
return _("Tag '{}' references unknown objects."
" Push commits before tags.").format(tag)
if obj.type == pygit2.GIT_OBJ_TREE:
sys.stderr.write(_("Tag '{}' of tree will not be stored in Perforce\n").format(tag))
continue
if obj.type == pygit2.GIT_OBJ_BLOB:
sys.stderr.write(_("Tag '{}' of blob will not be stored in Perforce\n").format(tag))
continue
_add_tag(ctx, tag, prt.new_sha1, tags_to_edit, tags_to_add)
elif prt.new_sha1 == p4gf_const.NULL_COMMIT_SHA1:
# Removing an existing tag
_remove_tag(ctx, tag, prt.old_sha1, tags_to_edit, tags_to_delete)
else:
# Older versions of Git allowed moving a tag reference, while
# newer ones seemingly do not. We will take the new behavior as
# the correct one and reject such changes.
return _('Updates were rejected because the tag already exists in the remote.')
# Seemingly nothing to do.
if not tags_to_add and not tags_to_edit and not tags_to_delete:
LOG.debug("process_tags() mysteriously came up empty")
return None
# Add and remove tags as appropriate, doing so in batches.
LOG.info("adding {} tags, removing {} tags from Git mirror".format(
len(tags_to_add), len(tags_to_delete)))
desc = _("Git Fusion '{repo}' tag changes").format(repo=ctx.config.view_name)
with p4gf_util.NumberedChangelist(gfctx=ctx, description=desc) as nc:
while len(tags_to_add):
bite = tags_to_add[:_BITE_SIZE]
tags_to_add = tags_to_add[_BITE_SIZE:]
ctx.p4gfrun(["add", "-t", "binary+F", bite])
while len(tags_to_edit):
bite = tags_to_edit[:_BITE_SIZE]
tags_to_edit = tags_to_edit[_BITE_SIZE:]
ctx.p4gfrun(["edit", "-k", bite])
while len(tags_to_delete):
bite = tags_to_delete[:_BITE_SIZE]
tags_to_delete = tags_to_delete[_BITE_SIZE:]
ctx.p4gfrun(["delete", bite])
nc.submit()
if nc.submitted:
_write_last_copied_tag(ctx, nc.change_num)
LOG.debug("process_tags() complete")
return None
def _git_branch_to_ot(self, branch):
"""Return the ObjectType of the most recent surviving
commit for the given branch.
Return None if branch is not visible to Git.
Returns the current branch head if the branch is already
pointing to some old commit/changelist from before the cutoff.
WARNING: Does not detect or counteract any branch deletion/creation
that occur after the cutoff. Branches deleted in soon-to-be-obliterated
history remain deleted after rollback. Branches created in soon-to-be-
obliterated history remain created, either empty or containing some
random reused branch content.
"""
# Ignore branches with no Git counterpart.
# (branch.sha1_for_branch() checks .git_branch_name,
# but not .deleted)
if branch.deleted or not branch.git_branch_name:
if branch.git_branch_name:
LOG.info("Skip Git ref {gbn}."
" Branch {branch_id} marked as deleted."
.format( branch_id = branch.branch_id
, gbn = branch.git_branch_name))
return None
# Before the rollback, is the Git head already
# positioned before/at the cutoff? If so,
# change nothing.
sha1 = branch.sha1_for_branch()
if not sha1:
# No such reference in local Git repo? Expected.
# git-branch-name defined from Perforce or other
# another Git Fusion server of this same repo, but
# either unpopulated, or not yet pulled to this
# Git Fusion server's repo.
LOG.info("Ignore Git ref {gbn}."
" Branch {branch_id} has no sha1 in local repo."
.format( branch_id = branch.branch_id
, gbn = branch.git_branch_name))
return None
# Lookup this sha1/branch and find its changelist
# number. If that's at/before cutoff, retain it.
otl = ObjectType.commits_for_sha1( ctx = self.ctx
, sha1 = sha1
, branch_id = branch.branch_id )
if otl:
ot = otl[0]
if ot and int(ot.change_num) <= int(self.change_num):
LOG.info("Ignore Git ref {gbn}."
" Branch {branch_id} has no sha1 in local repo."
.format( branch_id = branch.branch_id
, gbn = branch.git_branch_name))
return None
# If it's missing or after the cutoff, find the
# change at/before cutoff.
with self.ctx.switched_to_branch(branch):
r = self.ctx.p4run(
'changes'
, '-m1'
, self.ctx.client_view_path(change_num=self.change_num)
)
change_num = p4gf_util.first_value_for_key(r, key='change')
# No changelists at/before cutoff?
if not change_num:
LOG.info("Delete Git ref {gbn}."
" Branch {branch_id} has no changelists before"
" @{change_num}."
.format( branch_id = branch.branch_id
, gbn = branch.git_branch_name
, change_num = self.change_num ))
return self._create_ot_to_delete_git_ref(branch)
# Find corresponding Git commit sha1.
ot = ObjectType.change_num_to_commit( ctx = self.ctx
, change_num = change_num
, branch_id = branch.branch_id )
if not ot:
LOG.info("Delete Git ref {gbn}."
" Branch {branch_id} has changelist @{survive_cn} before"
" @{change_num}, but no corresponding Git commit."
.format( branch_id = branch.branch_id
, gbn = branch.git_branch_name
, change_num = self.change_num
, survive_cn = change_num ))
return self._create_ot_to_delete_git_ref(branch)
# Surviving changelist exists as a commit in local
# Git repo.
LOG.info("Move Git ref {gbn}."
" Branch {branch_id} has changelist @{survive_cn},"
" Git commit {sha1} at/before @{change_num}."
.format( branch_id = branch.branch_id
, gbn = branch.git_branch_name
, change_num = self.change_num
, survive_cn = change_num
, sha1 = ot.sha1 ))
return ot
def process_tags(ctx, prl):
"""
For each tag reference in the pre-receive-tuple list, add or remove the
object from the Git Fusion mirror.
Arguments:
ctx - P4GF context with initialized pygit2 Repository.
prl - list of PreReceiveTuple objects.
Returns None if successful and an error string otherwise.
"""
tags = [prt for prt in prl if prt.ref.startswith('refs/tags/')]
if not tags:
LOG.debug("process_tags() no incoming tags to process")
return None
# Screen the tags to ensure their names won't cause problems sometime
# in the future (i.e. when we create Perforce labels). Several of these
# characters are not allowed in Git tag names anyway, but better to
# check in case that changes in the future.
# In particular git disallows a leading '-', but we'll check for it anyway
# Otherwise allow internal '-'
regex = re.compile(r'[*@#,]|\.\.\.|%%')
for prt in tags:
tag = prt.ref[10:]
if regex.search(tag) or tag.startswith('-'):
return _(
"illegal characters (@#*,...%%) in tag name: '{}'").format(tag)
if not ctx.view_repo:
# In some cases the Git repository object is not yet created.
ctx.view_repo = pygit2.Repository(ctx.view_dirs.GIT_DIR)
LOG.debug("process_tags() beginning...")
tags_path = "objects/repos/{repo}/tags/...".format(
repo=ctx.config.view_name)
with ctx.p4gf.at_exception_level(P4.P4.RAISE_NONE):
# Raises an exception when there are no files to sync?
ctx.p4gfrun([
'sync', '-q', "//{}/{}/...".format(ctx.config.p4client_gf,
tags_path)
])
# Decide what to do with the tag references.
tags_to_delete = []
tags_to_add = []
tags_to_edit = []
for prt in tags:
tag = prt.ref[10:]
if prt.old_sha1 == p4gf_const.NULL_COMMIT_SHA1:
if prt.new_sha1 == p4gf_const.NULL_COMMIT_SHA1:
# No idea how this happens, but it did, so guard against it.
sys.stderr.write(
_('Ignoring double-zero pre-receive-tuple line'))
continue
# Adding a new tag; if it references a commit, check that it
# exists; for other types, it is too costly to verify
# reachability from a known commit, so just ignore them.
obj = _get_tag_target(ctx.view_repo, prt.new_sha1)
is_commit = obj.type == pygit2.GIT_OBJ_COMMIT
if is_commit and not ObjectType.commits_for_sha1(ctx, obj.hex):
return _("Tag '{}' references unknown objects."
" Push commits before tags.").format(tag)
if obj.type == pygit2.GIT_OBJ_TREE:
sys.stderr.write(
_("Tag '{}' of tree will not be stored in Perforce\n").
format(tag))
continue
if obj.type == pygit2.GIT_OBJ_BLOB:
sys.stderr.write(
_("Tag '{}' of blob will not be stored in Perforce\n").
format(tag))
continue
_add_tag(ctx, tag, prt.new_sha1, tags_to_edit, tags_to_add)
elif prt.new_sha1 == p4gf_const.NULL_COMMIT_SHA1:
# Removing an existing tag
_remove_tag(ctx, tag, prt.old_sha1, tags_to_edit, tags_to_delete)
else:
# Older versions of Git allowed moving a tag reference, while
# newer ones seemingly do not. We will take the new behavior as
# the correct one and reject such changes.
return _(
'Updates were rejected because the tag already exists in the remote.'
)
# Seemingly nothing to do.
if not tags_to_add and not tags_to_edit and not tags_to_delete:
LOG.debug("process_tags() mysteriously came up empty")
return None
# Add and remove tags as appropriate, doing so in batches.
LOG.info("adding {} tags, removing {} tags from Git mirror".format(
len(tags_to_add), len(tags_to_delete)))
desc = _("Git Fusion '{repo}' tag changes").format(
repo=ctx.config.view_name)
with p4gf_util.NumberedChangelist(gfctx=ctx, description=desc) as nc:
while len(tags_to_add):
bite = tags_to_add[:_BITE_SIZE]
tags_to_add = tags_to_add[_BITE_SIZE:]
ctx.p4gfrun(["add", "-t", "binary+F", bite])
while len(tags_to_edit):
bite = tags_to_edit[:_BITE_SIZE]
tags_to_edit = tags_to_edit[_BITE_SIZE:]
ctx.p4gfrun(["edit", "-k", bite])
while len(tags_to_delete):
bite = tags_to_delete[:_BITE_SIZE]
tags_to_delete = tags_to_delete[_BITE_SIZE:]
ctx.p4gfrun(["delete", bite])
nc.submit()
if nc.submitted:
_write_last_copied_tag(ctx, nc.change_num)
LOG.debug("process_tags() complete")
return None