def from_pygit(pg_repo, pg_commit):
"""Factory to convert pygit2 objects to our own."""
message = pg_commit.message
encoding = pg_commit.message_encoding or 'UTF-8'
if '\ufffd' in message:
# Original message failed to decode using UTF-8
_, raw_commit = pg_repo.read(pg_commit.oid)
try:
commit_text = raw_commit.decode('latin_1')
encoding = 'latin_1'
message = commit_text[commit_text.index('\n\n') + 2:]
# convert to UTF-8 for easier comparison
message = message.encode('UTF-8').decode('UTF-8')
except UnicodeDecodeError:
LOG.exception('commit message decoding failed')
subj_end = message.find('\n')
if 0 < subj_end:
subject = message[:subj_end]
else:
subject = message
return Commit(sha1=p4gf_pygit2.object_to_sha1(pg_commit),
tree=p4gf_pygit2.object_to_sha1(pg_commit.tree),
parent_list=[
p4gf_pygit2.object_to_sha1(p)
for p in pg_commit.parents
],
subject=subject,
message=message,
commit_time=pg_commit.commit_time,
encoding=encoding,
author_offset=pg_commit.author.offset,
committer_offset=pg_commit.committer.offset)
def _path_added(self, path, fecommit):
"""Return True if the named path was introduced in the HEAD commit.
:param self: this object
:param path: repo path to be evaluated.
:param fecommit: commit object from fast-export parser.
"""
# Because git-fast-export includes the entire tree in its output,
# regardless of whether the requested commit is the first in the
# branch or not, we need to check the repo itself to be certain if
# this path was truly introduced in this commit, or simply existed
# in the tree prior to the "first" commit.
commit = self.ctx.repo.get(fecommit['sha1'])
if commit is None:
# empty repository?
LOG.debug2("_path_added() commit {} is missing".format(
fecommit['sha1']))
return True
for parent in commit.parents:
if p4gf_git.exists_in_tree(self.ctx.repo, path, parent.tree):
LOG.debug2("_path_added() {} exists in parent tree {}".format(
path,
p4gf_util.abbrev(p4gf_pygit2.object_to_sha1(parent))))
return False
return True
def _count_commits(self, since=None, until=None, count_files=False):
"""Count the number commits from since to until.
:param since: earliest commit SHA1 to visit.
:param until: latest commit SHA1 to visit.
:param count_files: if True, also count the files added.
Returns the number of commits and files introduced within the range of
commits. The number of files will be zero unless count_files is true.
"""
if int(since, 16) == 0:
since = None
if int(until, 16) == 0:
until = None
start = self.repo.get(until) if until else p4gf_pygit2.head_commit(self.repo)
if start is None:
raise RuntimeError(_("Missing starting commit"))
sort = pygit2.GIT_SORT_TOPOLOGICAL | pygit2.GIT_SORT_TIME
commit_count = 0
file_count = 0
for commit in self.repo.walk(start.oid, sort):
if since and p4gf_pygit2.object_to_sha1(commit) == since:
break
commit_count += 1
if count_files:
if commit.parents:
for parent in commit.parents:
file_count += self._compare_commit_with_parent(parent, commit)
else:
file_count += self._count_files_for_commit(commit)
return commit_count, file_count
def _expand_sha1(ctx, partial_sha1):
"""Given partial SHA1 of a git object, return complete SHA1.
If there is no match, returns None.
"""
try:
obj = ctx.repo.git_object_lookup_prefix(partial_sha1)
return p4gf_pygit2.object_to_sha1(obj) if obj else None
except ValueError:
return None
def flatten_tree_1(gwt_path_tree, tree, result_list, tree_q, repo):
"""Process a single pygit2 tree object.
Translate any files to elements appended to result_list.
Translate any directories to (gwt_path, Tree) tuples appended to tree_q.
"""
for tree_entry in tree:
gwt_element_path = os.path.join(gwt_path_tree, tree_entry.name)
git_file = GitFile(gwt_path=gwt_element_path,
mode=tree_entry.filemode,
sha1=p4gf_pygit2.object_to_sha1(tree_entry))
result_list.append(git_file)
if tree_entry.filemode == 0o040000:
tree_q.append(
(gwt_element_path, p4gf_pygit2.tree_object(repo, tree_entry)))
def process_tags(ctx, tags):
"""Add or remove tags objects from the Git Fusion mirror.
:param ctx: P4GF context with initialized pygit2 Repository.
:param tags: list of PreReceiveTuple objects for tags
"""
# pylint:disable=too-many-branches
if not tags:
LOG.debug("process_tags() no incoming tags to process")
return
# Re-sync the tags since preflight_tags() synced with a different temp client.
tags_path = "objects/repos/{repo}/tags".format(repo=ctx.config.repo_name)
with ctx.p4gf.at_exception_level(P4.P4.RAISE_NONE):
# Raises an exception when there are no files to sync?
ctx.p4gfrun('sync', '-q', "//{}/{}/...".format(ctx.p4gf.client,
tags_path))
# Decide what to do with the tag references.
tags_to_delete = []
tags_to_add = []
tags_to_edit = []
for prt in tags:
tag = prt.ref[10:]
if prt.old_sha1 == p4gf_const.NULL_COMMIT_SHA1:
if prt.new_sha1 == p4gf_const.NULL_COMMIT_SHA1:
# No idea how this happens, but it did, so guard against it.
continue
# Adding a new tag; if it references a commit, check that it
# exists; for other types, it is too costly to verify
# reachability from a known commit, so just ignore them.
obj = _get_tag_target(ctx.repo, prt.new_sha1)
is_commit = obj.type == pygit2.GIT_OBJ_COMMIT
if is_commit and not ObjectType.commits_for_sha1(
ctx, p4gf_pygit2.object_to_sha1(obj)):
LOG.debug("Tag '{}' of unknown commit {:7.7} not stored."
" Removing ref from git repo.".format(
tag, prt.new_sha1))
_remove_tag_ref(tag, prt.new_sha1)
continue
if obj.type == pygit2.GIT_OBJ_TREE:
continue
if obj.type == pygit2.GIT_OBJ_BLOB:
continue
_add_tag(ctx, tag, prt.new_sha1, tags_to_edit, tags_to_add)
elif prt.new_sha1 == p4gf_const.NULL_COMMIT_SHA1:
# Removing an existing tag
_remove_tag(ctx, tag, prt.old_sha1, tags_to_edit, tags_to_delete)
# Seemingly nothing to do.
if not tags_to_add and not tags_to_edit and not tags_to_delete:
LOG.debug("process_tags() mysteriously came up empty"
" - probably a tag of a non-existing commit.")
return
# Add and remove tags as appropriate, doing so in batches.
LOG.info(
"adding {} tags, removing {} tags, editing {} tags from Git mirror".
format(len(tags_to_add), len(tags_to_delete), len(tags_to_edit)))
desc = _("Git Fusion '{repo}' tag changes").format(
repo=ctx.config.repo_name)
with p4gf_util.NumberedChangelist(gfctx=ctx, description=desc) as nc:
while len(tags_to_add):
bite = tags_to_add[:_BITE_SIZE]
tags_to_add = tags_to_add[_BITE_SIZE:]
ctx.p4gfrun('add', '-t', 'binary+F', bite)
while len(tags_to_edit):
bite = tags_to_edit[:_BITE_SIZE]
tags_to_edit = tags_to_edit[_BITE_SIZE:]
ctx.p4gfrun('edit', '-k', bite)
while len(tags_to_delete):
bite = tags_to_delete[:_BITE_SIZE]
tags_to_delete = tags_to_delete[_BITE_SIZE:]
ctx.p4gfrun('delete', bite)
nc.submit()
if nc.submitted:
_write_last_copied_tag(ctx, nc.change_num)
LOG.debug("process_tags() complete")
def preflight_tags(ctx, tags, heads):
"""Validate the incoming tags.
:param ctx: P4GF context with initialized pygit2 Repository.
:param tags: list of PreReceiveTuple objects for tags
:param heads: list of commit references for heads (non-tags)
Warnings are printed to stderr so the user knows about them.
Returns None if successful and an error string otherwise.
"""
if not tags:
LOG.debug("preflight_tags() no incoming tags to process")
return None
LOG.debug("preflight_tags() beginning...")
tags_path = "objects/repos/{repo}/tags".format(repo=ctx.config.repo_name)
with ctx.p4gf.at_exception_level(P4.P4.RAISE_NONE):
# Raises an exception when there are no files to sync?
ctx.p4gfrun('sync', '-q', "//{}/{}/...".format(ctx.p4gf.client,
tags_path))
regex = re.compile(r'[*@#,]|\.\.\.|%%')
for prt in tags:
tag = prt.ref[10:]
# Screen the tags to ensure their names won't cause problems
# sometime in the future (i.e. when we create Perforce labels).
# Several of these characters are not allowed in Git tag names
# anyway, but better to check in case that changes in the future.
# In particular git disallows a leading '-', but we'll check for it
# anyway Otherwise allow internal '-'
if regex.search(tag) or tag.startswith('-'):
return _("illegal characters (@#*,...%%) in tag name: '{tag}'"
).format(tag=tag)
if prt.old_sha1 == p4gf_const.NULL_COMMIT_SHA1:
if prt.new_sha1 == p4gf_const.NULL_COMMIT_SHA1:
# No idea how this happens, but it did, so guard against it.
sys.stderr.write(
_('Ignoring double-zero pre-receive-tuple line'))
continue
# Adding a new tag; if it references a commit, check that it
# exists; for other types, it is too costly to verify
# reachability from a known commit, so just ignore them.
obj = _get_tag_target(ctx.repo, prt.new_sha1)
is_commit = obj.type == pygit2.GIT_OBJ_COMMIT
if is_commit and not _is_reachable(p4gf_pygit2.object_to_sha1(obj),
heads):
# Do not fail in preflight but allow the push to proceed.
# Later this tag is ignored without error and not added to the object cache.
# The tag ref however has already been added to the git repo, but
# will be removed later in process_tags.
msg = _(
"Tag '{tag}' of unknown commit {sha1:7.7} not stored in "
"Perforce nor in git.\n"
"You must push a branch containing the target commit either prior to"
" or with the push of the tag.\n").format(
tag=tag, sha1=prt.new_sha1)
LOG.debug(msg)
sys.stderr.write(msg)
if obj.type == pygit2.GIT_OBJ_TREE:
msg = _("Tag '{tag}' of tree will not be stored in Perforce\n"
).format(tag=tag)
sys.stderr.write(msg)
continue
if obj.type == pygit2.GIT_OBJ_BLOB:
msg = _("Tag '{tag}' of blob will not be stored in Perforce\n"
).format(tag=tag)
sys.stderr.write(msg)
continue
fpath = os.path.join(ctx.gitlocalroot,
_client_path(ctx, prt.new_sha1))
if os.path.exists(fpath):
# Overwriting an existing tag? Git prohibits that.
# But, another lightweight tag of the same object is okay.
# Sanity check if this is a lightweight tag of an annotated
# tag and reject with a warning.
with open(fpath, 'rb') as f:
contents = f.read()
try:
zlib.decompress(contents)
msg = _(
"Tag '{tag}' of annotated tag will not be stored in Perforce\n"
)
sys.stderr.write(msg.format(tag=tag))
except zlib.error:
pass
elif prt.new_sha1 != p4gf_const.NULL_COMMIT_SHA1:
# Older versions of Git allowed moving a tag reference, while
# newer ones seemingly do not. We will take the new behavior as
# the correct one and reject such changes.
return _(
'Updates were rejected because the tag already exists in the remote.'
)