def _delete_temp_git_branch_refs(self):
'''
All those temporary Git branch refs whose names we assigned in
_create_branch_id_to_temp_name_dict()? 'git branch -D' them.
'''
for name in self.branch_id_to_temp_name.values():
### unable to use pygit2 to delete reference, no idea why
p4gf_proc.popen_no_throw(['git', 'branch', '-D', name])
def _delete_temp_git_branch_refs(self):
'''
All those temporary Git branch refs whose names we assigned in
_create_branch_id_to_temp_name_dict()? 'git branch -D' them.
'''
for name in self.branch_id_to_temp_name.values():
### unable to use pygit2 to delete reference, no idea why
p4gf_proc.popen_no_throw(['git', 'branch', '-D', name])
def git_prune_and_repack():
"""Invoke git prune and git repack clear any danglers from the object database."""
cmd = ['git', 'prune']
result = p4gf_proc.popen_no_throw(cmd)
LOG.info("git-prune returned: {}".format(result))
cmd = ['git', 'repack']
result = p4gf_proc.popen_no_throw(cmd)
LOG.info("git-repack returned: {}".format(result))
return result
def checkout_detached_head():
"""Detach HEAD so that we have no current branch.
Now we can modify any branch without triggering 'can't ___ current branch' errors.
"""
# no_throw because brand new repos have no commits at all, so
# even HEAD~0 is an invalid reference.
if not p4gf_util.git_empty():
with non_bare_git():
p4gf_proc.popen_no_throw(['git', 'checkout', 'HEAD~0'])
def git_checkout(sha1, force=False):
"""Switch to the given sha1.
Returns True if the checkout was successful (exit status of 0),
and False otherwise.
"""
with non_bare_git():
if force:
result = p4gf_proc.popen_no_throw(['git', 'checkout', '-f', sha1])
else:
result = p4gf_proc.popen_no_throw(['git', 'checkout', sha1])
return result['ec'] == 0
def _dump_instrumentation(self):
'''
Debugging dump of timing and other info.
'''
if _DUMP_LOG and LOG_GRAPH.isEnabledFor(logging.DEBUG3):
cmd = list(_DUMP_LOG) + [prt.new_sha1 for prt in self.pre_receive_list]
p = p4gf_proc.popen_no_throw(cmd)
l = self.annotate_lines(p['out'].splitlines())
LOG_GRAPH.debug3('Log: {}\n{}'.format(' '.join(cmd), '\n'.join(l)))
total_seconds = Timer(TIMER_OVERALL).time
total_rev_ct = len(self.assign_dict)
LOG_TIME.debug("branches : {}".format(len(self.branch_dict)))
LOG_TIME.debug("commits : {}".format(total_rev_ct))
LOG_TIME.debug("seconds : {}".format(int(total_seconds + 0.5)))
if 1.0 <= total_seconds:
# Commits per second math becomes unreliable for short runs.
rev_per_second = total_rev_ct / total_seconds
LOG_TIME.debug("commits/second: {}".format(int(rev_per_second + 0.5)))
if self.branch_len:
histo = p4gf_histogram.to_histogram(self.branch_len)
histo_lines = p4gf_histogram.to_lines(histo)
LOG_TIME.debug('Branch length histogram: how many branches have N commits?\n'
+ '\n'.join(histo_lines))
def git_gc_auto(git_dir=None):
"""Invoke git gc --auto to pack objects after copy_p4_to_git only if poll_only."""
if git_dir:
cmd = ['git', '--git-dir=' + git_dir, 'gc', '--auto']
else:
cmd = ['git', 'gc', '--auto']
result = p4gf_proc.popen_no_throw(cmd)
LOG.info("poll_only: git gc --auto returns: {0}".format(result))
def git_fsck(obj=None):
"""Invoke git fsck to verify the object database."""
cmd = ['git', 'fsck']
if obj:
cmd.append(str(obj))
result = p4gf_proc.popen_no_throw(cmd)
LOG.info("git-fsck returned: {}".format(result))
return result
def delete_empty_repo_branch(_ctx, git_dir):
'''
Delete branch empty_repo. If we are currently on that branch,
detach head before switching.
Only do this if our HEAD points to an actual sha1: we have to have
at least one commit.
'''
p4gf_proc.popen_no_throw(
['git', '--git-dir=' + git_dir, 'checkout', 'HEAD~0'])
p = p4gf_proc.popen_no_throw([
'git', '--git-dir=' + git_dir, 'branch', '--list',
p4gf_const.P4GF_BRANCH_EMPTY_REPO
])
if p['out']:
p = p4gf_proc.popen_no_throw([
'git', '--git-dir=' + git_dir, 'branch', '-D',
p4gf_const.P4GF_BRANCH_EMPTY_REPO
])
def is_bare_git_repo(git_dir):
'''
Is this Git repo already loaded for --bare?
'''
### this explodes the Python process, in p4api.so, oddly enough
# repo = pygit2.Repository(git_dir)
# return 'core.bare' in repo.config and repo.config['core.bare'] == 'true'
cmd = ['git', '--git-dir=' + git_dir, 'config', '--get', NTR('core.bare')]
result = p4gf_proc.popen_no_throw(cmd)
return 'true' in result['out']
def __call__( self
, ctx
, fe_commit
, branch_id
, jobs ):
'''
If preflight hook configured, invoke it (or PASS/FAIL it).
If fail, raise exception detailing why.
Route hook's stdout and stderr to our stderr.
'''
_debug3('call() {} {} {}'
, p4gf_util.abbrev(fe_commit['sha1'])
, p4gf_util.abbrev(branch_id)
, self )
if self.action is ACTION_NONE:
return
elif self.action is ACTION_PASS:
if self.msg:
sys.stderr.write(self.msg + '\n')
return
elif self.action is ACTION_FAIL:
self.raise_rejection(fe_commit['sha1'], self.msg)
else: # self.action is ACTION_RUN:
cmd_line_vars = self.calc_cmd_line_vars(
ctx = ctx
, fe_commit = fe_commit
, branch_id = branch_id
, jobs = jobs
, spec_file_path = self.spec_file_path(ctx)
)
d = (ctx.gwt_to_depot_path(fe_file['path'])
for fe_file in fe_commit['files'])
depot_file_list = (dd for dd in d if dd)
self._write_spec_file(
ctx = ctx
, fe_commit = fe_commit
, depot_file_list = depot_file_list
, jobs = jobs
, spec_file_path = self.spec_file_path(ctx)
, cmd_line_vars = cmd_line_vars )
cmd = [self.substitute_cmd_line_vars(cmd_line_vars, word)
for word in self.cmd]
_debug3('cmd {}', cmd)
d = p4gf_proc.popen_no_throw(cmd)
_debug3('{}', d)
msg = p4gf_util.join_non_empty('\n', d['out'], d['err'])
if d['ec']:
self.raise_rejection(fe_commit['sha1'], msg)
sys.stderr.write(msg)
def _git_is_ancestor(parent_sha1, child_sha1):
"""Ask Git if one already-copied-to-Git commit is an ancestor of
another.
"""
# Make sure these arguments are strings, or p4gf_proc will have problems (GF-2800).
cmd = [
'git', 'merge-base', '--is-ancestor',
str(parent_sha1),
str(child_sha1)
]
p = p4gf_proc.popen_no_throw(cmd)
return p['ec'] == 0
def run(self, fe_commit, cmd_line_vars):
"""Run the command for the given commit."""
cmd = [
substitute_cmd_line_vars(cmd_line_vars, word) for word in self.cmd
]
_debug3('cmd {}', cmd)
d = p4gf_proc.popen_no_throw(cmd)
_debug3('{}', d)
msg = p4gf_path.join_non_empty('\n', d['out'], d['err'])
if d['ec']:
raise_rejection(fe_commit['sha1'], msg)
sys.stderr.write(msg)
def delete_refs_for_closed_reviews(ctx):
"""Remove Git branch references to any Git Swarm reviews that are no longer pending.
Keeps the branch list from expanding unbounded and swamping some Git
user's 'git branch -a' report.
"""
# Just in case pygit2.listall_references() behaves
# poorly if we modify references out from under it
# during an iteration, collect all the doomed refs
# before deleting them.
#
closed_ref_list = [
ref_review_id.ref for ref_review_id in _ref_review_id_list(ctx) if
not GSReviewCollection.is_review_open(ctx, ref_review_id.review_id)
]
LOG.debug(
'delete_refs_for_closed_reviews() {}'.format(closed_ref_list))
if not closed_ref_list:
return
p4gf_proc.popen_no_throw(['git', 'branch', '-D'] + closed_ref_list)
def gitrun(self, cmd):
"""
Preview or run a git command that changes things.
Use this only for mutating calls that we must preview.
"""
if self.is_preview:
LOG.info(" ".join(cmd))
return
LOG.info(" ".join(cmd))
result = p4gf_proc.popen_no_throw(cmd)
LOG.debug("git returns: {0}".format(result))
return result
def delete_refs_for_closed_reviews(ctx):
'''
Remove Git branch references to any Git Swarm reviews that are no longer
pending.
Keeps the branch list from expanding unbounded and swamping some Git
user's 'git branch -a' report.
'''
# Just in case pygit2.listall_references() behaves
# poorly if we modify references out from under it
# during an iteration, collect all the doomed refs
# before deleting them.
#
closed_ref_list = [
ref_review_id.ref
for ref_review_id in _ref_review_id_list(ctx)
if not GSReviewCollection.is_review_open(ctx, ref_review_id.review_id)]
LOG.debug('delete_refs_for_closed_reviews() {}'.format(closed_ref_list))
if not closed_ref_list:
return
p4gf_proc.popen_no_throw(['git', 'branch', '-D'] + closed_ref_list)
def __call__(self, ctx, fe_commit, branch_id, jobs):
'''
If preflight hook configured, invoke it (or PASS/FAIL it).
If fail, raise exception detailing why.
Route hook's stdout and stderr to our stderr.
'''
_debug3('call() {} {} {}', p4gf_util.abbrev(fe_commit['sha1']),
p4gf_util.abbrev(branch_id), self)
if self.action is ACTION_NONE:
return
elif self.action is ACTION_PASS:
if self.msg:
sys.stderr.write(self.msg + '\n')
return
elif self.action is ACTION_FAIL:
self.raise_rejection(fe_commit['sha1'], self.msg)
else: # self.action is ACTION_RUN:
cmd_line_vars = self.calc_cmd_line_vars(
ctx=ctx,
fe_commit=fe_commit,
branch_id=branch_id,
jobs=jobs,
spec_file_path=self.spec_file_path(ctx))
d = (ctx.gwt_to_depot_path(fe_file['path'])
for fe_file in fe_commit['files'])
depot_file_list = (dd for dd in d if dd)
self._write_spec_file(ctx=ctx,
fe_commit=fe_commit,
depot_file_list=depot_file_list,
jobs=jobs,
spec_file_path=self.spec_file_path(ctx),
cmd_line_vars=cmd_line_vars)
cmd = [
self.substitute_cmd_line_vars(cmd_line_vars, word)
for word in self.cmd
]
_debug3('cmd {}', cmd)
d = p4gf_proc.popen_no_throw(cmd)
_debug3('{}', d)
msg = p4gf_util.join_non_empty('\n', d['out'], d['err'])
if d['ec']:
self.raise_rejection(fe_commit['sha1'], msg)
sys.stderr.write(msg)
def __get_snapshot_trees(commit, trees):
"""get all tree objects for a given commit
commit: SHA1 of commit
each tree is added to the list to be mirrored
"""
#ls-tree doesn't return the top level tree, so add it here
commit_tree = __get_commit_tree(commit, trees)
po = p4gf_proc.popen_no_throw(['git', 'ls-tree', '-rt', commit_tree])['out']
for line in po.splitlines():
m = TREE_REGEX.match(line)
if m:
LOG.debug("adding subtree {}".format(m.group(1)))
trees.add(m.group(1))
return commit_tree
def space_total(self):
"""Return the disk usage in megabytes of the repository as a float."""
# Allow _space_total to be 0, that is perfectly legal.
if self._space_total is None:
# self._space_total = self._get_disk_usage()
# $ /usr/bin/du -sk .git
# 337524 .git
result = p4gf_proc.popen_no_throw(['/usr/bin/du', '-sk', self.repo.path])
if result['ec'] == 0:
self._space_total = int(result['out'].split()[0]) / 1024
else:
LOG.error("Push limits not enforced: unable to get disk usage for {}".format(
self.repo.path))
self._space_total = 0
return self._space_total
def __get_snapshot_trees(commit, trees):
"""get all tree objects for a given commit
commit: SHA1 of commit
each tree is added to the list to be mirrored
"""
#ls-tree doesn't return the top level tree, so add it here
commit_tree = __get_commit_tree(commit, trees)
po = p4gf_proc.popen_no_throw(['git', 'ls-tree', '-rt',
commit_tree])['out']
for line in po.splitlines():
m = TREE_REGEX.match(line)
if m:
LOG.debug("adding subtree {}".format(m.group(1)))
trees.add(m.group(1))
return commit_tree
def __get_delta_trees(commit_tree1, commit2, trees):
"""get all tree objects new in one commit vs another commit
commit1: SHA1 of first commit
commit2: SHA1 of second commit
each tree is added to the list to be mirrored
"""
# diff-tree doesn't return the top level tree, so add it here
commit_tree2 = __get_commit_tree(commit2, trees)
#po = p4gf_proc.popen_no_throw(['git', 'diff-tree', '-t', commit1, commit2])['out']
po = p4gf_proc.popen_no_throw(['git', 'diff-tree', '-t', commit_tree1, commit_tree2])['out']
for line in po.splitlines():
m = TREE_ENT_RE.match(line)
if m:
LOG.debug("adding subtree {}".format(m.group(1)))
trees.add(m.group(1))
return commit_tree2
def __get_delta_trees(commit_tree1, commit2, trees):
"""get all tree objects new in one commit vs another commit
commit1: SHA1 of first commit
commit2: SHA1 of second commit
each tree is added to the list to be mirrored
"""
# diff-tree doesn't return the top level tree, so add it here
commit_tree2 = __get_commit_tree(commit2, trees)
#po = p4gf_proc.popen_no_throw(['git', 'diff-tree', '-t', commit1, commit2])['out']
po = p4gf_proc.popen_no_throw(
['git', 'diff-tree', '-t', commit_tree1, commit_tree2])['out']
for line in po.splitlines():
m = TREE_ENT_RE.match(line)
if m:
LOG.debug("adding subtree {}".format(m.group(1)))
trees.add(m.group(1))
return commit_tree2
def fast_reclone(ctx):
"""Try to do fast reclone from mirror."""
# don't try this on a non-empty GF repo
if not ctx.repo.is_empty:
LOG.debug("fast_reclone: repo not empty")
return None, None
# write empty tree into repo, just in case it's referenced by any
# commits we'll be copying
result = p4gf_proc.popen_no_throw(['git', 'write-tree'])
if result['ec']:
LOG.debug(
'fast_reclone: failed to write empty tree: {}'.format(result))
return None, None
# copy commits and trees into the repo
branch_heads, commits, dirs = _get_commits(ctx)
trees = _get_trees(ctx, dirs)
return branch_heads, commits + trees
def _is_reachable(sha1, heads):
"""Return True if the commit is reachable from one of the heads.
:param sha1: SHA1 of the commit to find.
:param heads: list of commit references for heads (non-tags)
:return: True if commit is reachable, False otherwise.
"""
LOG.debug2('_is_reachable() checking for {}'.format(sha1))
for head_sha1 in heads:
# ### newer pygit2.Repository.merge_base(oid, oid) would do this for us,
# ### too bad we're not updating any time soon...
cmd = ['git', 'merge-base', '--is-ancestor', sha1, head_sha1]
result = p4gf_proc.popen_no_throw(cmd)
if LOG.isEnabledFor(logging.DEBUG2):
reachable = result['ec'] == 0
LOG.debug2('_is_reachable() {} is reachable {}'.format(
sha1, reachable))
if result['ec'] == 0:
return True
return False
def update_git_config_and_repack(git_dir, git_autopack, git_gc_auto):
"""Unset git config settings which disable packing.
Repos created prior to 14.1 disabled packing.
Detect these old settings, remove them and call git repack.
"""
# See if there is anything to do before doing a bunch of work.
repo = pygit2.Repository(git_dir)
if 'receive.autogc' not in repo.config:
return
cwd = os.getcwd()
os.chdir(os.path.dirname(git_dir))
remove_pre_14_settings = {
'receive.autogc': 'unset',
'receive.unpacklimit': 'unset',
'gc.auto': 'unset',
'gc.autopacklimit': 'unset',
'pack.compression': 'unset',
'transfer.unpacklimit': 'unset',
}
set_git_config(git_dir, remove_pre_14_settings)
gc_settings = get_autopack_settings(git_autopack, git_gc_auto)
set_git_config(git_dir, gc_settings)
cmd = ['git', 'repack', '-ad']
result = p4gf_proc.popen_no_throw(cmd)
if result['ec']:
LOG.error("update_git_config_and_repack: git repack failed with: %s",
result['err'])
sys.stderr.write(
_("Perforce: error: git repack failed with '{error}'\n").format(
error=result['ec']))
else:
LOG.debug3("update_git_config_and_repack: %s", result)
os.chdir(cwd)
def force_tag_ref(ref_name, sha1):
"""Create or change one Git tag reference."""
p4gf_proc.popen_no_throw(['git', 'tag', '-f', ref_name, sha1])
def create_git_repo(ctx, git_dir):
"""Create the git repository in the given root directory."""
# Test if the Git repository has already been created.
if os.path.exists(os.path.join(git_dir, 'HEAD')):
return
# Prepare the Git repository directory, cleaning up if necessary.
work_tree = os.path.dirname(git_dir)
if not os.path.exists(git_dir):
if os.path.exists(work_tree):
# weird case where git view dir exists but repo was deleted
LOG.warn(
"mirror Git repository {} in bad state, repairing...".format(
git_dir))
shutil.rmtree(work_tree)
LOG.debug("creating directory %s for Git repo", git_dir)
os.makedirs(git_dir)
# Initialize the Git repository for that directory.
LOG.debug("creating Git repository in %s", git_dir)
pygit2.init_repository(git_dir)
# Configure the Git repository to avoid creating pack objects, so we can
# read them directly ourselves and avoid the overhead of git-cat-file.
cwd = os.getcwd()
os.chdir(work_tree)
settings = {
# Prevent conflicting change history by disallowing rewinds.
'receive.denyNonFastForwards': 'true',
# # Turn off default compression (suspenders)
# 'core.compression': '0',
# # Turn off compression for loose objects (belt)
# 'core.loosecompression': '0',
# # threshold beyond which deltaCompression is compression is disabled
# 'core.bigFileThreshold': '0',
# # Packs smaller than this are unpacked into loose object files (belt)
# 'fetch.unpackLimit': '1000000000',
# Turn off garbage collection
'gc.auto': '0',
# Should never be used
'gc.autopacklimit': '0',
# Should never be used
'pack.compression': '0',
# git-receive-pack will not run gc
'receive.autogc': 'false',
# Packs smaller than this are unpacked into loose object files (belt)
'receive.unpackLimit': '1000000000',
# Packs smaller than this are unpacked into loose object files (suspenders)
'transfer.unpackLimit': '1000000000'
}
# repo = pygit2.Repository(git_dir)
for k, v in settings.items():
### this explodes the Python process, in p4api.so, oddly enough
# repo.config[k] = v
cmd = ['git', 'config', '--local', '--replace-all', k, v]
result = p4gf_proc.popen_no_throw(cmd)
if result['ec']:
LOG.error("configuring git repo failed for {}={} => {}".format(
k, v, result['err']))
sys.stderr.write(
_("error: git init failed with '{}' for '{}'\n").format(
result['ec'], work_tree))
os.chdir(cwd)
install_hook(git_dir)
# Don't bother changing branches in a --bare repo.
if is_bare_git_repo(git_dir):
return
create_empty_repo_branch(ctx, git_dir)
def delete_branch_ref(ref_name):
"""Delete one Git branch reference."""
p4gf_proc.popen_no_throw(['git', 'branch', '-D', ref_name])
def delete_branch_refs(ref_names):
"""Delete several Git branch references at once."""
for chunk in p4gf_util.iter_chunks(ref_names, 300):
p4gf_proc.popen_no_throw(['git', 'branch', '-D'] + chunk)
def is_valid_git_branch_name(git_branch_name):
"""Determine if the given name refers to a valid git branch."""
cmd = ['git', 'check-ref-format', '--allow-onelevel', git_branch_name]
result = p4gf_proc.popen_no_throw(cmd)
return not result['ec']
def force_branch_ref(ref_name, sha1):
"""Create or change one Git branch reference."""
p4gf_proc.popen_no_throw(['git', 'branch', '-f', ref_name, sha1])
def delete_tag_ref(ref_name):
"""Delete one Git tag reference."""
p4gf_proc.popen_no_throw(['git', 'tag', '-d', ref_name])
