def _add_commits_to_p4(self, ctx):
"""
Attempt to add a set of commits to the cache, retrying if there
is an exception. If it doesn't go on the second attempt,
raise the exception. Use numbered changelists to aid in recovery.
"""
for i in range(2):
try:
self._really_add_commits_to_p4(ctx)
ObjectType.reset_cache()
break
except P4Exception:
if i:
raise
def _add_commits_to_p4(self, ctx):
"""
Attempt to add a set of commits to the cache, retrying if there
is an exception. If it doesn't go on the second attempt,
raise the exception. Use numbered changelists to aid in recovery.
"""
for i in range(2):
try:
self._really_add_commits_to_p4(ctx)
ObjectType.reset_cache()
break
except P4Exception:
if i:
raise
def _parent_commit_for_first_lt_child(self):
'''
If "change" is going to be the first commit on a new Git branch,
and if current_branch is a view into a lightweight depot branch
based on some other branch at some other changelist number, return
that other branch@change as a sha1/commit that should be a
commit parent of "change".
If not, return None.
Required when the first commit in a lightweight branh is an add,
not edit, and thus has zero integ actions to connect it to the
parent branch. We must connect manually. Here.
'''
# Not copying from a lightweight branch?
if not self.current_branch.depot_branch:
return None
# Lightweight branch lacks a parent?
if not self.current_branch.depot_branch.parent_depot_branch_id_list:
return None
# Find a commit to go with the parent branch @ changelist
# upon which dest_db is based.
dest_db = self.current_branch.depot_branch
for par_cl in dest_db.parent_changelist_list:
# Mark for a change/commit we're about to copy to this repo?
ml = self.p2g.mark_list.cl_to_mark_list(par_cl)
if ml:
return ml[0]
# sha1 we've already copied to this repo?
commit = ObjectType.commit_for_change(self.ctx, par_cl)
if commit:
return commit.sha1
def get_last_change_for_commit(commit, ctx, branch_id=None):
"""Given a commit SHA1, find the latest corresponding Perforce change.
Note that a Git commit may correspond to several Perforce changes.
"""
return ObjectType.change_for_sha1(ctx,
commit,
branch_id)
def greatest_lesser_change_for_branch(ctx, branch, change_num):
"""Find the change for the branch that is no higher than change_num.
:param ctx: Git Fusion context
:param branch: branch for which to find highest change
:param change_num: the high water mark of changes to find
:return: ObjectType or None
"""
results = None
pattern = p4gf_const.P4GF_P4KEY_INDEX_OT.format(
repo_name=ctx.config.repo_name, change_num='*', branch_id=branch.branch_id)
LOG.debug("greatest_lesser_change_for_branch() pattern %s", pattern)
d = p4gf_p4key.get_all(ctx.p4gf, pattern)
if d:
for key, value in d.items():
mk = p4gf_object_type_util.KEY_BRANCH_REGEX.search(key)
if not mk:
LOG.debug("ignoring unexpected p4key: %s", key)
continue
branch_id = mk.group('branch_id')
cl = int(mk.group('change_num'))
if cl <= int(change_num):
# Ensure we keep the highest change found so far for this branch.
if results and int(results.change_num) > cl:
continue
LOG.debug("greatest_lesser_change_for_branch() candidate %s, %s, %s",
branch_id, cl, value)
results = ObjectType.create_commit(sha1=value, repo_name=ctx.config.repo_name,
change_num=cl, branch_id=branch_id)
LOG.debug("greatest_lesser_change_for_branch() returning %s", results)
return results
def _calc_repairs(ctx):
"""
Scan Perforce for Git commit data and Perforce changelist descriptions,
calculate which Perforce changelists need more data copied from Git
backing store //.git-fusion/objects/...
"""
# Load repo's entire set of Commit/Changelist metadata
# into memory.
LOG.info("Fetching list of Git commits/changelists from %s/objects/...",
p4gf_const.objects_root())
r = ctx.p4run(
'files', '{root}/repos/{repo}/commits/...'.format(
root=p4gf_const.objects_root(), repo=ctx.config.repo_name))
# 'p4 print' each Git commit from its backup in
# //.git-fusion/objects/...
LOG.info("Fetched commit objects: {ct}".format(ct=len(r)))
for rr in r:
depot_path = rr.get('depotFile')
if not depot_path:
continue
ot = ObjectType.commit_from_filepath(depot_path)
SHA1_TO_OTL[ot.sha1].append(ot)
LOG.debug('p4 print {}'.format(depot_path))
blob_raw = p4gf_util.print_depot_path_raw(ctx.p4, depot_path)
blob = p4gf_util.bytes_to_git_object(blob_raw)
par_list = commit_to_parent_list(blob)
SHA1_TO_PAR_SHA1_LIST[ot.sha1] = par_list
LOG.debug("{sha1:7.7} parents={par}".format(
sha1=ot.sha1, par=[p4gf_util.abbrev(p) for p in par_list]))
# Loop through changelists, comparing against
# backup and calculating if additional data
# needs to be copied to its changelist description.
return _calc_repairs_loop(ctx)
def _parent_commit_for_first_lt_child(self):
"""If "change" is going to be the first commit on a new Git branch,
and if current_branch is a view into a lightweight depot branch
based on some other branch at some other changelist number, return
that other branch@change as a sha1/commit that should be a
commit parent of "change".
If not, return None.
Required when the first commit in a lightweight branh is an add,
not edit, and thus has zero integ actions to connect it to the
parent branch. We must connect manually. Here.
"""
# Not copying from a lightweight branch?
if not self.current_branch.depot_branch:
return None
# Lightweight branch lacks a parent?
if not self.current_branch.depot_branch.parent_depot_branch_id_list:
return None
# Find a commit to go with the parent branch @ changelist
# upon which dest_db is based.
dest_db = self.current_branch.depot_branch
for par_cl in dest_db.parent_changelist_list:
# Mark for a change/commit we're about to copy to this repo?
ml = self.p2g.mark_list.cl_to_mark_list(par_cl)
if ml:
return ml[0]
# sha1 we've already copied to this repo?
commit = ObjectType.change_num_to_commit(self.ctx, par_cl)
if commit:
return commit.sha1
def _really_add_commits_to_p4(self, ctx):
"""actually run p4 add, submit to create mirror files in .git-fusion"""
desc = _("Git Fusion '{view}' copied to Git.").format(
view=ctx.config.view_name)
with p4gf_util.NumberedChangelist(gfctx=ctx, description=desc) as nc:
with Timer(ADD_SUBMIT):
LOG.debug("adding {0} commits to .git-fusion...".format(
len(self.commits.commits)))
# build list of objects to add, extracting them from git
add_files = [
self.__add_object_to_p4(ctx, go)
for go in self.commits.commits.values()
]
add_files = GitMirror.optimize_objects_to_add_to_p4(
ctx, add_files)
if not (len(add_files) or self.depot_branch_info_list
or self.branch_list):
# Avoid a blank line in output by printing something
ProgressReporter.write(
_('No Git objects to submit to Perforce'))
LOG.debug("_really_add_objects_to_p4() nothing to add...")
return
with Timer(P4_ADD):
files_added = self.add_objects_to_p4_2(ctx, add_files)
depot_branch_infos_added = \
self._add_depot_branch_infos_to_p4(ctx)
config2_added = self._add_branch_defs_to_p4(ctx)
cldfs_added = self._add_cldfs_to_p4(ctx)
with Timer(P4_SUBMIT):
if (files_added or depot_branch_infos_added
or config2_added or cldfs_added):
ProgressReporter.increment(
_('Submitting new Git commit objects to Perforce'))
r = nc.submit()
ObjectType.update_indexes(ctx, r)
else:
ProgressReporter.write(
_('No new Git objects to submit to Perforce'))
LOG.debug("ignoring empty change list...")
def add_commit(self, sha1, details):
"""skip over duplicate objects (e.g. tree shared by commits)"""
key = sha1
if details.branch_id:
key += ',' + details.branch_id
if key not in self.commits:
commit = ObjectType(sha1, "commit", details)
self.commits[key] = commit
def _really_add_commits_to_p4(self, ctx):
"""actually run p4 add, submit to create mirror files in .git-fusion"""
desc = _("Git Fusion '{view}' copied to Git.").format(view=ctx.config.view_name)
with p4gf_util.NumberedChangelist(gfctx=ctx, description=desc) as nc:
with Timer(ADD_SUBMIT):
LOG.debug("adding {0} commits to .git-fusion...".
format(len(self.commits.commits)))
# build list of objects to add, extracting them from git
add_files = [self.__add_object_to_p4(ctx, go)
for go in self.commits.commits.values()]
add_files = GitMirror.optimize_objects_to_add_to_p4(ctx, add_files)
if not ( len(add_files)
or self.depot_branch_info_list
or self.branch_list ):
# Avoid a blank line in output by printing something
ProgressReporter.write(_('No Git objects to submit to Perforce'))
LOG.debug("_really_add_objects_to_p4() nothing to add...")
return
with Timer(P4_ADD):
files_added = self.add_objects_to_p4_2(ctx, add_files)
depot_branch_infos_added = \
self._add_depot_branch_infos_to_p4(ctx)
config2_added = self._add_branch_defs_to_p4(ctx)
cldfs_added = self._add_cldfs_to_p4(ctx)
with Timer(P4_SUBMIT):
if ( files_added
or depot_branch_infos_added
or config2_added
or cldfs_added ):
ProgressReporter.increment(
_('Submitting new Git commit objects to Perforce'))
r = nc.submit()
ObjectType.update_indexes(ctx, r)
else:
ProgressReporter.write(
_('No new Git objects to submit to Perforce'))
LOG.debug("ignoring empty change list...")
def _create_ot_to_delete_git_ref(self, branch):
"""Return a fake ObjectType instance that tells us to delete this
branch's Git reference later.
Use _is_ot_to_delete_git_ref() to detect these fake instances.
"""
return ObjectType.create_commit(
sha1 = p4gf_const.NULL_COMMIT_SHA1
, repo_name = self.ctx.config.repo_name
, change_num = 0
, branch_id = branch.branch_id )
def _assign_previous(self):
'''
Many commits in assign_dict were already assigned branches in a
previous push or pull. Remember and honor those.
'''
# Test-only mode skips this step and that's okay.
if not self.ctx:
return
for assign in self.assign_dict.values():
otl = ObjectType.commits_for_sha1(self.ctx, assign.sha1)
for ot in otl:
self._assign_branch(assign, ot.details.branch_id)
def _add_parent_changes(self):
"""If desc_info has parent-changes, add those to parent_commit_list.
Return True if any parent added to self.parent_commit_list,
False if nothing added.
"""
if self.desc_info and self.desc_info.parent_changes:
# orphan git commits with no parents
if 'None' in self.desc_info.parent_changes:
self.parent_commit_list = []
self.is_git_orphan = True
return True
add_list = []
for sha1 in self.desc_info.parents:
changes = self.desc_info.parent_changes.get(sha1, None)
if changes is None:
LOG.error(
'DescInfo missing parent changes for {}'.format(sha1))
return False
for cl in changes:
ml = self.p2g.mark_list.cl_to_mark_list(str(cl))
if ml:
add_list.append(ml[0])
LOG.debug3(
'_add_parent_changes() @{} using mark {}'.format(
cl, ml[0]))
break
commit = ObjectType.change_num_to_commit(self.ctx, cl)
if commit and p4gf_util.sha1_exists(commit.sha1):
add_list.append(commit.sha1)
LOG.debug3(
'_add_parent_changes() @{} using SHA1 {}'.format(
cl, commit.sha1))
break
else:
# Fell off the end of the "for cl" list without finding the
# parent in our repo.
#
# DescInfo lists a parent commit sha1/change_num that does
# not intersect this repo. This is okay: just means the
# changelist came from some other repo that can see some
# depot paths that this current repo cannot.
LOG.debug3('_add_parent_changes() skipping, could not find'
' parent SHA1 {} cl {}'.format(sha1, changes))
self.omitted_di_parents = True
if add_list:
self.parent_commit_list.extend(add_list)
return True
return False
def _find_commit_files(path, client_name):
"""Generator function that walks a directory tree, returning each commit
file found for the given client.
Arguments:
path -- root of directory tree to walk.
client_name -- name of client for which to find commits.
"""
for root, _dirs, files in os.walk(path):
for fyle in files:
fpath = os.path.join(root, fyle)
# Convert the object file path to an ObjectType, but don't
# let those silly non-P4GF objects stop us.
ot = ObjectType.commit_from_filepath(fpath)
if ot and ot.applies_to_view(client_name):
yield fpath
def _find_commit_files(path, client_name):
"""Generator function that walks a directory tree, returning each commit
file found for the given client.
Arguments:
path -- root of directory tree to walk.
client_name -- name of client for which to find commits.
"""
for root, _dirs, files in os.walk(path):
for fyle in files:
fpath = os.path.join(root, fyle)
# Convert the object file path to an ObjectType, but don't
# let those silly non-P4GF objects stop us.
ot = ObjectType.commit_from_filepath(fpath)
if ot and ot.applies_to_view(client_name):
yield fpath
def _honor_original_git_parent_sequence(self):
'''
Reorder self.parent_commit_list to match the order its commits appear in
the original Git commit.
Any parents not listed in the original Git commit are appended to the
back of the list in preserved order.
'''
# +++ Cannot resequence a list without multiple elements.
if len(self.parent_commit_list) < 2:
return
want_seq = self._git_object_parent_list()
if not want_seq:
return
# Build the result in order.
keep_seq = []
rem_parent_list = copy.copy(self.parent_commit_list)
for w in want_seq:
if w in rem_parent_list:
keep_seq.append(w)
rem_parent_list.remove(w)
elif w in self.p2g.sha1_to_mark:
# try converting the SHA1 to a mark...
m = self.p2g.sha1_to_mark[w]
if m in rem_parent_list:
keep_seq.append(m)
rem_parent_list.remove(m)
# Convert sha1 to mark number(s), in case parent is part of this
# pull and not yet stored in git with a sha1.
commits = ObjectType.commits_for_sha1(self.ctx, w)
for commit in commits:
for mark in self.p2g.mark_list.cl_to_mark_list(
commit.details.changelist):
if mark in rem_parent_list:
keep_seq.append(mark)
rem_parent_list.remove(mark)
# Add any remainder to the end of the result.
keep_seq.extend(rem_parent_list)
self.parent_commit_list = keep_seq
def _honor_original_git_parent_sequence(self):
'''
Reorder self.parent_commit_list to match the order its commits appear in
the original Git commit.
Any parents not listed in the original Git commit are appended to the
back of the list in preserved order.
'''
# +++ Cannot resequence a list without multiple elements.
if len(self.parent_commit_list) < 2:
return
want_seq = self._git_object_parent_list()
if not want_seq:
return
# Build the result in order.
keep_seq = []
rem_parent_list = copy.copy(self.parent_commit_list)
for w in want_seq:
if w in rem_parent_list:
keep_seq.append(w)
rem_parent_list.remove(w)
elif w in self.p2g.sha1_to_mark:
# try converting the SHA1 to a mark...
m = self.p2g.sha1_to_mark[w]
if m in rem_parent_list:
keep_seq.append(m)
rem_parent_list.remove(m)
# Convert sha1 to mark number(s), in case parent is part of this
# pull and not yet stored in git with a sha1.
commits = ObjectType.commits_for_sha1(self.ctx, w)
for commit in commits:
for mark in self.p2g.mark_list.cl_to_mark_list(commit.details.changelist):
if mark in rem_parent_list:
keep_seq.append(mark)
rem_parent_list.remove(mark)
# Add any remainder to the end of the result.
keep_seq.extend(rem_parent_list)
self.parent_commit_list = keep_seq
def compare_change_num(self, change_num):
"""Compare one Perforce changelist with its corresponding Git
commit.
"""
ot = ObjectType.change_num_to_commit(self.ctx, change_num)
if not ot:
key_pattern = (p4gf_const.P4GF_P4KEY_INDEX_OT.format(
repo_name=self.ctx.config.repo_name,
change_num=change_num,
branch_id="*"))
raise RuntimeError("No Git commit for @{change_num}."
"\nNo 'p4 keys -e {key_pattern}".format(
change_num=change_num,
key_pattern=key_pattern))
branch = self.ctx.branch_dict().get(ot.branch_id)
if not branch:
raise RuntimeError("No branch view defined for {}".format(
ot.branch_id))
self._compare_ot(ot)
def _can_use_as_parent(self, sha1):
"""If sha1 already copied to Git, return sha1.
If sha1 not yet copied, but its corresponding Perforce changelist is in
our MarkList of impending copies, return the mark for that changelist.
If not, return None
"""
# Already stored in Git from some previous push or pull?
if p4gf_util.sha1_exists(sha1):
return sha1
# Any of this commit's corresponding changelists
# about to be copied to Git in this pull?
for commit in ObjectType.commits_for_sha1(self.ctx, sha1):
ml = self.p2g.mark_list.cl_to_mark_list(commit.change_num)
if ml:
return ml[0]
return None
def _can_use_as_parent(self, sha1):
'''
If sha1 already copied to Git, return sha1.
If sha1 not yet copied, but its corresponding Perforce changelist is in
our MarkList of impending copies, return the mark for that changelist.
If not, return None
'''
# Already stored in Git from some previous push or pull?
if p4gf_util.sha1_exists(sha1):
return sha1
# Any of this commit's corresponding changelists
# about to be copied to Git in this pull?
for commit in ObjectType.commits_for_sha1(self.ctx, sha1):
ml = self.p2g.mark_list.cl_to_mark_list(commit.details.changelist)
if ml:
return ml[0]
return None
def _calc_merge_parent_list(self):
'''
Return a list of marks/sha1s to use as parents for a Git merge commit.
Returns a 2-tuple of ( [mark/commit parent], [parent branch id] ).
'''
parent_commit_id = []
parent_branch_id = []
for branch_id, cl in self.branch_id_to_changelist_num.items():
# Changelists from before the start of history cannot be
# parents. No merging from beyond the event horizon.
if cl < self.p2g.rev_range.begin_change_num:
continue
# Each Perforce commit maps to zero or more Git commits, one per
# branch that intersects the integ source files from that commit.
# Do we have any pending commits for this changelist?
ml = self.p2g.mark_list.cl_to_mark_list(str(cl))
for mark in ml:
### Must only include mark if mark associated with branch.
### Must only include one mark here.
### But that requires adding more branch/mark tracking than we
### want to add until I have a test that proves we need it.
if not mark in parent_commit_id:
parent_commit_id.append(mark)
parent_branch_id.append(branch_id)
# Do we have any existing commits for this changelist?
commit = ObjectType.commit_for_change(self.ctx, cl)
# Does this Git commit occur in our Git repo at this
# changelist number?
if (commit and
(not commit.sha1 in parent_commit_id) and
p4gf_util.sha1_exists(commit.sha1) ):
parent_commit_id.append(commit.sha1)
parent_branch_id.append(branch_id)
return (parent_commit_id, parent_branch_id)
def _calc_merge_parent_list(self):
'''
Return a list of marks/sha1s to use as parents for a Git merge commit.
Returns a 2-tuple of ( [mark/commit parent], [parent branch id] ).
'''
parent_commit_id = []
parent_branch_id = []
for branch_id, cl in self.branch_id_to_changelist_num.items():
# Changelists from before the start of history cannot be
# parents. No merging from beyond the event horizon.
if cl < self.p2g.rev_range.begin_change_num:
continue
# Each Perforce commit maps to zero or more Git commits, one per
# branch that intersects the integ source files from that commit.
# Do we have any pending commits for this changelist?
ml = self.p2g.mark_list.cl_to_mark_list(str(cl))
for mark in ml:
### Must only include mark if mark associated with branch.
### Must only include one mark here.
### But that requires adding more branch/mark tracking than we
### want to add until I have a test that proves we need it.
if not mark in parent_commit_id:
parent_commit_id.append(mark)
parent_branch_id.append(branch_id)
# Do we have any existing commits for this changelist?
commit = ObjectType.commit_for_change(self.ctx, cl)
# Does this Git commit occur in our Git repo at this
# changelist number?
if (commit and (not commit.sha1 in parent_commit_id)
and p4gf_util.sha1_exists(commit.sha1)):
parent_commit_id.append(commit.sha1)
parent_branch_id.append(branch_id)
return (parent_commit_id, parent_branch_id)
def _git_object_parent_list(self):
"""Fetch the list of parent commits from our git object mirror of the
original Git commit that created our current Perforce changelist.
Return None if no such commit found.
Return empty list [] if commit found but it lacked parents
(orphan/first commit in a chain of commits).
"""
# Find corresponding Git commit object for this changelist. Ignore the
# sha1 in the changelist DescInfo: it's only there for Git-to-Perforce
# changelists, not changelists that originated in Perforce.
commit = ObjectType.change_num_to_commit(self.ctx,
self.p4change.change)
if not commit:
# Cached object is missing, see if the change description has
# what we need (in the 'parents' field of the Git desc info).
if self.desc_info and self.desc_info.parents:
LOG.debug2(
'_git_object_parent_list() parents from change: {}'.format(
self.desc_info.parents))
return self.desc_info.parents
else:
return None
depot_path = commit.to_depot_path()
commit_text = p4gf_util.depot_path_to_git_object(
self.ctx.p4gf, depot_path)
# Parse out the parent list.
parent_list = []
for line in commit_text.splitlines():
if line.startswith(b'parent '):
sha1 = line[len(b'parent '):].decode().strip()
parent_list.append(sha1)
elif not len(line.strip()):
# Done with header. Stop scanning.
break
return parent_list
def _git_object_parent_list(self):
'''
Fetch the list of parent commits from our git object mirror of the
original Git commit that created our current Perforce changelist.
Return None if no such commit found.
Return empty list [] if commit found but it lacked parents
(orphan/first commit in a chain of commits).
'''
# Find corresponding Git commit object for this changelist. Ignore the
# sha1 in the changelist DescInfo: it's only there for Git-to-Perforce
# changelists, not changelists that originated in Perforce.
commit = ObjectType.commit_for_change(self.ctx, self.p4change.change)
if not commit:
# Cached object is missing, see if the change description has
# what we need (in the 'parents' field of the Git desc info).
if self.desc_info and self.desc_info.parents:
LOG.debug2('_git_object_parent_list() parents from change: {}'.format(
self.desc_info.parents))
return self.desc_info.parents
else:
return None
depot_path = commit.to_depot_path()
commit_text = p4gf_util.depot_path_to_git_object( self.ctx.p4gf
, depot_path )
# Parse out the parent list.
parent_list = []
for line in commit_text.splitlines():
if line.startswith(b'parent '):
sha1 = line[len(b'parent '):].decode().strip()
parent_list.append(sha1)
elif not len(line.strip()):
# Done with header. Stop scanning.
break
return parent_list
def process_tags(ctx, tags):
"""Add or remove tags objects from the Git Fusion mirror.
:param ctx: P4GF context with initialized pygit2 Repository.
:param tags: list of PreReceiveTuple objects for tags
"""
# pylint:disable=too-many-branches
if not tags:
LOG.debug("process_tags() no incoming tags to process")
return
# Re-sync the tags since preflight_tags() synced with a different temp client.
tags_path = "objects/repos/{repo}/tags".format(repo=ctx.config.repo_name)
with ctx.p4gf.at_exception_level(P4.P4.RAISE_NONE):
# Raises an exception when there are no files to sync?
ctx.p4gfrun('sync', '-q', "//{}/{}/...".format(ctx.p4gf.client,
tags_path))
# Decide what to do with the tag references.
tags_to_delete = []
tags_to_add = []
tags_to_edit = []
for prt in tags:
tag = prt.ref[10:]
if prt.old_sha1 == p4gf_const.NULL_COMMIT_SHA1:
if prt.new_sha1 == p4gf_const.NULL_COMMIT_SHA1:
# No idea how this happens, but it did, so guard against it.
continue
# Adding a new tag; if it references a commit, check that it
# exists; for other types, it is too costly to verify
# reachability from a known commit, so just ignore them.
obj = _get_tag_target(ctx.repo, prt.new_sha1)
is_commit = obj.type == pygit2.GIT_OBJ_COMMIT
if is_commit and not ObjectType.commits_for_sha1(
ctx, p4gf_pygit2.object_to_sha1(obj)):
LOG.debug("Tag '{}' of unknown commit {:7.7} not stored."
" Removing ref from git repo.".format(
tag, prt.new_sha1))
_remove_tag_ref(tag, prt.new_sha1)
continue
if obj.type == pygit2.GIT_OBJ_TREE:
continue
if obj.type == pygit2.GIT_OBJ_BLOB:
continue
_add_tag(ctx, tag, prt.new_sha1, tags_to_edit, tags_to_add)
elif prt.new_sha1 == p4gf_const.NULL_COMMIT_SHA1:
# Removing an existing tag
_remove_tag(ctx, tag, prt.old_sha1, tags_to_edit, tags_to_delete)
# Seemingly nothing to do.
if not tags_to_add and not tags_to_edit and not tags_to_delete:
LOG.debug("process_tags() mysteriously came up empty"
" - probably a tag of a non-existing commit.")
return
# Add and remove tags as appropriate, doing so in batches.
LOG.info(
"adding {} tags, removing {} tags, editing {} tags from Git mirror".
format(len(tags_to_add), len(tags_to_delete), len(tags_to_edit)))
desc = _("Git Fusion '{repo}' tag changes").format(
repo=ctx.config.repo_name)
with p4gf_util.NumberedChangelist(gfctx=ctx, description=desc) as nc:
while len(tags_to_add):
bite = tags_to_add[:_BITE_SIZE]
tags_to_add = tags_to_add[_BITE_SIZE:]
ctx.p4gfrun('add', '-t', 'binary+F', bite)
while len(tags_to_edit):
bite = tags_to_edit[:_BITE_SIZE]
tags_to_edit = tags_to_edit[_BITE_SIZE:]
ctx.p4gfrun('edit', '-k', bite)
while len(tags_to_delete):
bite = tags_to_delete[:_BITE_SIZE]
tags_to_delete = tags_to_delete[_BITE_SIZE:]
ctx.p4gfrun('delete', bite)
nc.submit()
if nc.submitted:
_write_last_copied_tag(ctx, nc.change_num)
LOG.debug("process_tags() complete")
def _git_branch_to_ot(self, branch):
"""Return the ObjectType of the most recent surviving
commit for the given branch.
Return None if branch is not visible to Git.
Returns the current branch head if the branch is already
pointing to some old commit/changelist from before the cutoff.
WARNING: Does not detect or counteract any branch deletion/creation
that occur after the cutoff. Branches deleted in soon-to-be-obliterated
history remain deleted after rollback. Branches created in soon-to-be-
obliterated history remain created, either empty or containing some
random reused branch content.
"""
# Ignore branches with no Git counterpart.
# (branch.sha1_for_branch() checks .git_branch_name,
# but not .deleted)
if branch.deleted or not branch.git_branch_name:
if branch.git_branch_name:
LOG.info("Skip Git ref {gbn}."
" Branch {branch_id} marked as deleted."
.format( branch_id = branch.branch_id
, gbn = branch.git_branch_name))
return None
# Before the rollback, is the Git head already
# positioned before/at the cutoff? If so,
# change nothing.
sha1 = branch.sha1_for_branch()
if not sha1:
# No such reference in local Git repo? Expected.
# git-branch-name defined from Perforce or other
# another Git Fusion server of this same repo, but
# either unpopulated, or not yet pulled to this
# Git Fusion server's repo.
LOG.info("Ignore Git ref {gbn}."
" Branch {branch_id} has no sha1 in local repo."
.format( branch_id = branch.branch_id
, gbn = branch.git_branch_name))
return None
# Lookup this sha1/branch and find its changelist
# number. If that's at/before cutoff, retain it.
otl = ObjectType.commits_for_sha1( ctx = self.ctx
, sha1 = sha1
, branch_id = branch.branch_id )
if otl:
ot = otl[0]
if ot and int(ot.change_num) <= int(self.change_num):
LOG.info("Ignore Git ref {gbn}."
" Branch {branch_id} has no sha1 in local repo."
.format( branch_id = branch.branch_id
, gbn = branch.git_branch_name))
return None
# If it's missing or after the cutoff, find the
# change at/before cutoff.
with self.ctx.switched_to_branch(branch):
r = self.ctx.p4run(
'changes'
, '-m1'
, self.ctx.client_view_path(change_num=self.change_num)
)
change_num = p4gf_util.first_value_for_key(r, key='change')
# No changelists at/before cutoff?
if not change_num:
LOG.info("Delete Git ref {gbn}."
" Branch {branch_id} has no changelists before"
" @{change_num}."
.format( branch_id = branch.branch_id
, gbn = branch.git_branch_name
, change_num = self.change_num ))
return self._create_ot_to_delete_git_ref(branch)
# Find corresponding Git commit sha1.
ot = ObjectType.change_num_to_commit( ctx = self.ctx
, change_num = change_num
, branch_id = branch.branch_id )
if not ot:
LOG.info("Delete Git ref {gbn}."
" Branch {branch_id} has changelist @{survive_cn} before"
" @{change_num}, but no corresponding Git commit."
.format( branch_id = branch.branch_id
, gbn = branch.git_branch_name
, change_num = self.change_num
, survive_cn = change_num ))
return self._create_ot_to_delete_git_ref(branch)
# Surviving changelist exists as a commit in local
# Git repo.
LOG.info("Move Git ref {gbn}."
" Branch {branch_id} has changelist @{survive_cn},"
" Git commit {sha1} at/before @{change_num}."
.format( branch_id = branch.branch_id
, gbn = branch.git_branch_name
, change_num = self.change_num
, survive_cn = change_num
, sha1 = ot.sha1 ))
return ot
def process_tags(ctx, prl):
"""
For each tag reference in the pre-receive-tuple list, add or remove the
object from the Git Fusion mirror.
Arguments:
ctx - P4GF context with initialized pygit2 Repository.
prl - list of PreReceiveTuple objects.
Returns None if successful and an error string otherwise.
"""
tags = [prt for prt in prl if prt.ref.startswith('refs/tags/')]
if not tags:
LOG.debug("process_tags() no incoming tags to process")
return None
# Screen the tags to ensure their names won't cause problems sometime
# in the future (i.e. when we create Perforce labels). Several of these
# characters are not allowed in Git tag names anyway, but better to
# check in case that changes in the future.
# In particular git disallows a leading '-', but we'll check for it anyway
# Otherwise allow internal '-'
regex = re.compile(r'[*@#,]|\.\.\.|%%')
for prt in tags:
tag = prt.ref[10:]
if regex.search(tag) or tag.startswith('-'):
return _("illegal characters (@#*,...%%) in tag name: '{}'").format(tag)
if not ctx.view_repo:
# In some cases the Git repository object is not yet created.
ctx.view_repo = pygit2.Repository(ctx.view_dirs.GIT_DIR)
LOG.debug("process_tags() beginning...")
tags_path = "objects/repos/{repo}/tags/...".format(repo=ctx.config.view_name)
with ctx.p4gf.at_exception_level(P4.P4.RAISE_NONE):
# Raises an exception when there are no files to sync?
ctx.p4gfrun(['sync', '-q', "//{}/{}/...".format(ctx.config.p4client_gf, tags_path)])
# Decide what to do with the tag references.
tags_to_delete = []
tags_to_add = []
tags_to_edit = []
for prt in tags:
tag = prt.ref[10:]
if prt.old_sha1 == p4gf_const.NULL_COMMIT_SHA1:
if prt.new_sha1 == p4gf_const.NULL_COMMIT_SHA1:
# No idea how this happens, but it did, so guard against it.
sys.stderr.write(_('Ignoring double-zero pre-receive-tuple line'))
continue
# Adding a new tag; if it references a commit, check that it
# exists; for other types, it is too costly to verify
# reachability from a known commit, so just ignore them.
obj = _get_tag_target(ctx.view_repo, prt.new_sha1)
is_commit = obj.type == pygit2.GIT_OBJ_COMMIT
if is_commit and not ObjectType.commits_for_sha1(ctx, obj.hex):
return _("Tag '{}' references unknown objects."
" Push commits before tags.").format(tag)
if obj.type == pygit2.GIT_OBJ_TREE:
sys.stderr.write(_("Tag '{}' of tree will not be stored in Perforce\n").format(tag))
continue
if obj.type == pygit2.GIT_OBJ_BLOB:
sys.stderr.write(_("Tag '{}' of blob will not be stored in Perforce\n").format(tag))
continue
_add_tag(ctx, tag, prt.new_sha1, tags_to_edit, tags_to_add)
elif prt.new_sha1 == p4gf_const.NULL_COMMIT_SHA1:
# Removing an existing tag
_remove_tag(ctx, tag, prt.old_sha1, tags_to_edit, tags_to_delete)
else:
# Older versions of Git allowed moving a tag reference, while
# newer ones seemingly do not. We will take the new behavior as
# the correct one and reject such changes.
return _('Updates were rejected because the tag already exists in the remote.')
# Seemingly nothing to do.
if not tags_to_add and not tags_to_edit and not tags_to_delete:
LOG.debug("process_tags() mysteriously came up empty")
return None
# Add and remove tags as appropriate, doing so in batches.
LOG.info("adding {} tags, removing {} tags from Git mirror".format(
len(tags_to_add), len(tags_to_delete)))
desc = _("Git Fusion '{repo}' tag changes").format(repo=ctx.config.view_name)
with p4gf_util.NumberedChangelist(gfctx=ctx, description=desc) as nc:
while len(tags_to_add):
bite = tags_to_add[:_BITE_SIZE]
tags_to_add = tags_to_add[_BITE_SIZE:]
ctx.p4gfrun(["add", "-t", "binary+F", bite])
while len(tags_to_edit):
bite = tags_to_edit[:_BITE_SIZE]
tags_to_edit = tags_to_edit[_BITE_SIZE:]
ctx.p4gfrun(["edit", "-k", bite])
while len(tags_to_delete):
bite = tags_to_delete[:_BITE_SIZE]
tags_to_delete = tags_to_delete[_BITE_SIZE:]
ctx.p4gfrun(["delete", bite])
nc.submit()
if nc.submitted:
_write_last_copied_tag(ctx, nc.change_num)
LOG.debug("process_tags() complete")
return None
def get_last_change_for_commit(commit, ctx, branch_id=None):
"""Given a commit SHA1, find the latest corresponding Perforce change.
Note that a Git commit may correspond to several Perforce changes.
"""
return ObjectType.change_for_sha1(ctx, commit, branch_id)
def main():
"""Do the thing."""
args = _argparse()
_log_config(args)
LOG.debug("args={}".format(args))
ctx = _create_ctx(p4port=args.p4port,
p4user=args.p4user,
repo_name=args.repo_name,
server_id=args.server_id)
LOG.info("P4PORT={}".format(ctx.p4.port))
LOG.info("P4USER={}".format(ctx.p4.user))
di_list = changes_to_desc_info_list(ctx, entire_depot=args.entire_depot)
# Load branch view definitions.
# Gives us Git branch names to go with pushed
# commmit sha1s.
branch_dict = ctx.branch_dict()
# Create index for fast dbid->branch lookups later.
dbid_to_branch = {
b.depot_branch.depot_branch_id: b
for b in branch_dict.values() if b.depot_branch
}
# Use DescInfo and ObjectType backup to
# find branch_id assignments.
for di in di_list:
di.branch_id = None
di.git_branch_name = None
change_num_str = str(di.change_num)
# Use depot branch ID recorded in DescInfo.
if di.depot_branch_id:
branch = dbid_to_branch.get(di.depot_branch_id)
if branch:
di.branch_id = branch.branch_id
di.git_branch_name = branch.git_branch_name
continue
# Fetch from ObjectType.
otl = ObjectType.commits_for_sha1(ctx, di.sha1)
for ot in otl:
if ot.change_num == change_num_str:
di.branch_id = ot.branch_id
if di.branch_id:
branch = branch_dict.get(di.branch_id)
if branch:
di.git_branch_name = branch.git_branch_name
continue
# Report time
attr_list = [
'change_num', 'sha1', 'push_state', 'branch_id', 'git_branch_name'
]
print("# " + "\t".join(attr_list))
for di in di_list:
if args.complete_only and di.push_state != "complete":
continue
v = [str(getattr(di, a)) for a in attr_list]
print("\t".join(v))
def process_tags(ctx, prl):
"""
For each tag reference in the pre-receive-tuple list, add or remove the
object from the Git Fusion mirror.
Arguments:
ctx - P4GF context with initialized pygit2 Repository.
prl - list of PreReceiveTuple objects.
Returns None if successful and an error string otherwise.
"""
tags = [prt for prt in prl if prt.ref.startswith('refs/tags/')]
if not tags:
LOG.debug("process_tags() no incoming tags to process")
return None
# Screen the tags to ensure their names won't cause problems sometime
# in the future (i.e. when we create Perforce labels). Several of these
# characters are not allowed in Git tag names anyway, but better to
# check in case that changes in the future.
# In particular git disallows a leading '-', but we'll check for it anyway
# Otherwise allow internal '-'
regex = re.compile(r'[*@#,]|\.\.\.|%%')
for prt in tags:
tag = prt.ref[10:]
if regex.search(tag) or tag.startswith('-'):
return _(
"illegal characters (@#*,...%%) in tag name: '{}'").format(tag)
if not ctx.view_repo:
# In some cases the Git repository object is not yet created.
ctx.view_repo = pygit2.Repository(ctx.view_dirs.GIT_DIR)
LOG.debug("process_tags() beginning...")
tags_path = "objects/repos/{repo}/tags/...".format(
repo=ctx.config.view_name)
with ctx.p4gf.at_exception_level(P4.P4.RAISE_NONE):
# Raises an exception when there are no files to sync?
ctx.p4gfrun([
'sync', '-q', "//{}/{}/...".format(ctx.config.p4client_gf,
tags_path)
])
# Decide what to do with the tag references.
tags_to_delete = []
tags_to_add = []
tags_to_edit = []
for prt in tags:
tag = prt.ref[10:]
if prt.old_sha1 == p4gf_const.NULL_COMMIT_SHA1:
if prt.new_sha1 == p4gf_const.NULL_COMMIT_SHA1:
# No idea how this happens, but it did, so guard against it.
sys.stderr.write(
_('Ignoring double-zero pre-receive-tuple line'))
continue
# Adding a new tag; if it references a commit, check that it
# exists; for other types, it is too costly to verify
# reachability from a known commit, so just ignore them.
obj = _get_tag_target(ctx.view_repo, prt.new_sha1)
is_commit = obj.type == pygit2.GIT_OBJ_COMMIT
if is_commit and not ObjectType.commits_for_sha1(ctx, obj.hex):
return _("Tag '{}' references unknown objects."
" Push commits before tags.").format(tag)
if obj.type == pygit2.GIT_OBJ_TREE:
sys.stderr.write(
_("Tag '{}' of tree will not be stored in Perforce\n").
format(tag))
continue
if obj.type == pygit2.GIT_OBJ_BLOB:
sys.stderr.write(
_("Tag '{}' of blob will not be stored in Perforce\n").
format(tag))
continue
_add_tag(ctx, tag, prt.new_sha1, tags_to_edit, tags_to_add)
elif prt.new_sha1 == p4gf_const.NULL_COMMIT_SHA1:
# Removing an existing tag
_remove_tag(ctx, tag, prt.old_sha1, tags_to_edit, tags_to_delete)
else:
# Older versions of Git allowed moving a tag reference, while
# newer ones seemingly do not. We will take the new behavior as
# the correct one and reject such changes.
return _(
'Updates were rejected because the tag already exists in the remote.'
)
# Seemingly nothing to do.
if not tags_to_add and not tags_to_edit and not tags_to_delete:
LOG.debug("process_tags() mysteriously came up empty")
return None
# Add and remove tags as appropriate, doing so in batches.
LOG.info("adding {} tags, removing {} tags from Git mirror".format(
len(tags_to_add), len(tags_to_delete)))
desc = _("Git Fusion '{repo}' tag changes").format(
repo=ctx.config.view_name)
with p4gf_util.NumberedChangelist(gfctx=ctx, description=desc) as nc:
while len(tags_to_add):
bite = tags_to_add[:_BITE_SIZE]
tags_to_add = tags_to_add[_BITE_SIZE:]
ctx.p4gfrun(["add", "-t", "binary+F", bite])
while len(tags_to_edit):
bite = tags_to_edit[:_BITE_SIZE]
tags_to_edit = tags_to_edit[_BITE_SIZE:]
ctx.p4gfrun(["edit", "-k", bite])
while len(tags_to_delete):
bite = tags_to_delete[:_BITE_SIZE]
tags_to_delete = tags_to_delete[_BITE_SIZE:]
ctx.p4gfrun(["delete", bite])
nc.submit()
if nc.submitted:
_write_last_copied_tag(ctx, nc.change_num)
LOG.debug("process_tags() complete")
return None