def _delete_temp_git_branch_refs(self): ''' All those temporary Git branch refs whose names we assigned in _create_branch_id_to_temp_name_dict()? 'git branch -D' them. ''' for name in self.branch_id_to_temp_name.values(): ### unable to use pygit2 to delete reference, no idea why p4gf_proc.popen_no_throw(['git', 'branch', '-D', name])
def git_prune_and_repack(): """Invoke git prune and git repack clear any danglers from the object database.""" cmd = ['git', 'prune'] result = p4gf_proc.popen_no_throw(cmd) LOG.info("git-prune returned: {}".format(result)) cmd = ['git', 'repack'] result = p4gf_proc.popen_no_throw(cmd) LOG.info("git-repack returned: {}".format(result)) return result
def checkout_detached_head(): """Detach HEAD so that we have no current branch. Now we can modify any branch without triggering 'can't ___ current branch' errors. """ # no_throw because brand new repos have no commits at all, so # even HEAD~0 is an invalid reference. if not p4gf_util.git_empty(): with non_bare_git(): p4gf_proc.popen_no_throw(['git', 'checkout', 'HEAD~0'])
def git_checkout(sha1, force=False): """Switch to the given sha1. Returns True if the checkout was successful (exit status of 0), and False otherwise. """ with non_bare_git(): if force: result = p4gf_proc.popen_no_throw(['git', 'checkout', '-f', sha1]) else: result = p4gf_proc.popen_no_throw(['git', 'checkout', sha1]) return result['ec'] == 0
def _dump_instrumentation(self): ''' Debugging dump of timing and other info. ''' if _DUMP_LOG and LOG_GRAPH.isEnabledFor(logging.DEBUG3): cmd = list(_DUMP_LOG) + [prt.new_sha1 for prt in self.pre_receive_list] p = p4gf_proc.popen_no_throw(cmd) l = self.annotate_lines(p['out'].splitlines()) LOG_GRAPH.debug3('Log: {}\n{}'.format(' '.join(cmd), '\n'.join(l))) total_seconds = Timer(TIMER_OVERALL).time total_rev_ct = len(self.assign_dict) LOG_TIME.debug("branches : {}".format(len(self.branch_dict))) LOG_TIME.debug("commits : {}".format(total_rev_ct)) LOG_TIME.debug("seconds : {}".format(int(total_seconds + 0.5))) if 1.0 <= total_seconds: # Commits per second math becomes unreliable for short runs. rev_per_second = total_rev_ct / total_seconds LOG_TIME.debug("commits/second: {}".format(int(rev_per_second + 0.5))) if self.branch_len: histo = p4gf_histogram.to_histogram(self.branch_len) histo_lines = p4gf_histogram.to_lines(histo) LOG_TIME.debug('Branch length histogram: how many branches have N commits?\n' + '\n'.join(histo_lines))
def git_gc_auto(git_dir=None): """Invoke git gc --auto to pack objects after copy_p4_to_git only if poll_only.""" if git_dir: cmd = ['git', '--git-dir=' + git_dir, 'gc', '--auto'] else: cmd = ['git', 'gc', '--auto'] result = p4gf_proc.popen_no_throw(cmd) LOG.info("poll_only: git gc --auto returns: {0}".format(result))
def git_fsck(obj=None): """Invoke git fsck to verify the object database.""" cmd = ['git', 'fsck'] if obj: cmd.append(str(obj)) result = p4gf_proc.popen_no_throw(cmd) LOG.info("git-fsck returned: {}".format(result)) return result
def delete_empty_repo_branch(_ctx, git_dir): ''' Delete branch empty_repo. If we are currently on that branch, detach head before switching. Only do this if our HEAD points to an actual sha1: we have to have at least one commit. ''' p4gf_proc.popen_no_throw( ['git', '--git-dir=' + git_dir, 'checkout', 'HEAD~0']) p = p4gf_proc.popen_no_throw([ 'git', '--git-dir=' + git_dir, 'branch', '--list', p4gf_const.P4GF_BRANCH_EMPTY_REPO ]) if p['out']: p = p4gf_proc.popen_no_throw([ 'git', '--git-dir=' + git_dir, 'branch', '-D', p4gf_const.P4GF_BRANCH_EMPTY_REPO ])
def is_bare_git_repo(git_dir): ''' Is this Git repo already loaded for --bare? ''' ### this explodes the Python process, in p4api.so, oddly enough # repo = pygit2.Repository(git_dir) # return 'core.bare' in repo.config and repo.config['core.bare'] == 'true' cmd = ['git', '--git-dir=' + git_dir, 'config', '--get', NTR('core.bare')] result = p4gf_proc.popen_no_throw(cmd) return 'true' in result['out']
def __call__( self , ctx , fe_commit , branch_id , jobs ): ''' If preflight hook configured, invoke it (or PASS/FAIL it). If fail, raise exception detailing why. Route hook's stdout and stderr to our stderr. ''' _debug3('call() {} {} {}' , p4gf_util.abbrev(fe_commit['sha1']) , p4gf_util.abbrev(branch_id) , self ) if self.action is ACTION_NONE: return elif self.action is ACTION_PASS: if self.msg: sys.stderr.write(self.msg + '\n') return elif self.action is ACTION_FAIL: self.raise_rejection(fe_commit['sha1'], self.msg) else: # self.action is ACTION_RUN: cmd_line_vars = self.calc_cmd_line_vars( ctx = ctx , fe_commit = fe_commit , branch_id = branch_id , jobs = jobs , spec_file_path = self.spec_file_path(ctx) ) d = (ctx.gwt_to_depot_path(fe_file['path']) for fe_file in fe_commit['files']) depot_file_list = (dd for dd in d if dd) self._write_spec_file( ctx = ctx , fe_commit = fe_commit , depot_file_list = depot_file_list , jobs = jobs , spec_file_path = self.spec_file_path(ctx) , cmd_line_vars = cmd_line_vars ) cmd = [self.substitute_cmd_line_vars(cmd_line_vars, word) for word in self.cmd] _debug3('cmd {}', cmd) d = p4gf_proc.popen_no_throw(cmd) _debug3('{}', d) msg = p4gf_util.join_non_empty('\n', d['out'], d['err']) if d['ec']: self.raise_rejection(fe_commit['sha1'], msg) sys.stderr.write(msg)
def _git_is_ancestor(parent_sha1, child_sha1): """Ask Git if one already-copied-to-Git commit is an ancestor of another. """ # Make sure these arguments are strings, or p4gf_proc will have problems (GF-2800). cmd = [ 'git', 'merge-base', '--is-ancestor', str(parent_sha1), str(child_sha1) ] p = p4gf_proc.popen_no_throw(cmd) return p['ec'] == 0
def run(self, fe_commit, cmd_line_vars): """Run the command for the given commit.""" cmd = [ substitute_cmd_line_vars(cmd_line_vars, word) for word in self.cmd ] _debug3('cmd {}', cmd) d = p4gf_proc.popen_no_throw(cmd) _debug3('{}', d) msg = p4gf_path.join_non_empty('\n', d['out'], d['err']) if d['ec']: raise_rejection(fe_commit['sha1'], msg) sys.stderr.write(msg)
def delete_refs_for_closed_reviews(ctx): """Remove Git branch references to any Git Swarm reviews that are no longer pending. Keeps the branch list from expanding unbounded and swamping some Git user's 'git branch -a' report. """ # Just in case pygit2.listall_references() behaves # poorly if we modify references out from under it # during an iteration, collect all the doomed refs # before deleting them. # closed_ref_list = [ ref_review_id.ref for ref_review_id in _ref_review_id_list(ctx) if not GSReviewCollection.is_review_open(ctx, ref_review_id.review_id) ] LOG.debug( 'delete_refs_for_closed_reviews() {}'.format(closed_ref_list)) if not closed_ref_list: return p4gf_proc.popen_no_throw(['git', 'branch', '-D'] + closed_ref_list)
def gitrun(self, cmd): """ Preview or run a git command that changes things. Use this only for mutating calls that we must preview. """ if self.is_preview: LOG.info(" ".join(cmd)) return LOG.info(" ".join(cmd)) result = p4gf_proc.popen_no_throw(cmd) LOG.debug("git returns: {0}".format(result)) return result
def delete_refs_for_closed_reviews(ctx): ''' Remove Git branch references to any Git Swarm reviews that are no longer pending. Keeps the branch list from expanding unbounded and swamping some Git user's 'git branch -a' report. ''' # Just in case pygit2.listall_references() behaves # poorly if we modify references out from under it # during an iteration, collect all the doomed refs # before deleting them. # closed_ref_list = [ ref_review_id.ref for ref_review_id in _ref_review_id_list(ctx) if not GSReviewCollection.is_review_open(ctx, ref_review_id.review_id)] LOG.debug('delete_refs_for_closed_reviews() {}'.format(closed_ref_list)) if not closed_ref_list: return p4gf_proc.popen_no_throw(['git', 'branch', '-D'] + closed_ref_list)
def __call__(self, ctx, fe_commit, branch_id, jobs): ''' If preflight hook configured, invoke it (or PASS/FAIL it). If fail, raise exception detailing why. Route hook's stdout and stderr to our stderr. ''' _debug3('call() {} {} {}', p4gf_util.abbrev(fe_commit['sha1']), p4gf_util.abbrev(branch_id), self) if self.action is ACTION_NONE: return elif self.action is ACTION_PASS: if self.msg: sys.stderr.write(self.msg + '\n') return elif self.action is ACTION_FAIL: self.raise_rejection(fe_commit['sha1'], self.msg) else: # self.action is ACTION_RUN: cmd_line_vars = self.calc_cmd_line_vars( ctx=ctx, fe_commit=fe_commit, branch_id=branch_id, jobs=jobs, spec_file_path=self.spec_file_path(ctx)) d = (ctx.gwt_to_depot_path(fe_file['path']) for fe_file in fe_commit['files']) depot_file_list = (dd for dd in d if dd) self._write_spec_file(ctx=ctx, fe_commit=fe_commit, depot_file_list=depot_file_list, jobs=jobs, spec_file_path=self.spec_file_path(ctx), cmd_line_vars=cmd_line_vars) cmd = [ self.substitute_cmd_line_vars(cmd_line_vars, word) for word in self.cmd ] _debug3('cmd {}', cmd) d = p4gf_proc.popen_no_throw(cmd) _debug3('{}', d) msg = p4gf_util.join_non_empty('\n', d['out'], d['err']) if d['ec']: self.raise_rejection(fe_commit['sha1'], msg) sys.stderr.write(msg)
def __get_snapshot_trees(commit, trees): """get all tree objects for a given commit commit: SHA1 of commit each tree is added to the list to be mirrored """ #ls-tree doesn't return the top level tree, so add it here commit_tree = __get_commit_tree(commit, trees) po = p4gf_proc.popen_no_throw(['git', 'ls-tree', '-rt', commit_tree])['out'] for line in po.splitlines(): m = TREE_REGEX.match(line) if m: LOG.debug("adding subtree {}".format(m.group(1))) trees.add(m.group(1)) return commit_tree
def space_total(self): """Return the disk usage in megabytes of the repository as a float.""" # Allow _space_total to be 0, that is perfectly legal. if self._space_total is None: # self._space_total = self._get_disk_usage() # $ /usr/bin/du -sk .git # 337524 .git result = p4gf_proc.popen_no_throw(['/usr/bin/du', '-sk', self.repo.path]) if result['ec'] == 0: self._space_total = int(result['out'].split()[0]) / 1024 else: LOG.error("Push limits not enforced: unable to get disk usage for {}".format( self.repo.path)) self._space_total = 0 return self._space_total
def __get_delta_trees(commit_tree1, commit2, trees): """get all tree objects new in one commit vs another commit commit1: SHA1 of first commit commit2: SHA1 of second commit each tree is added to the list to be mirrored """ # diff-tree doesn't return the top level tree, so add it here commit_tree2 = __get_commit_tree(commit2, trees) #po = p4gf_proc.popen_no_throw(['git', 'diff-tree', '-t', commit1, commit2])['out'] po = p4gf_proc.popen_no_throw(['git', 'diff-tree', '-t', commit_tree1, commit_tree2])['out'] for line in po.splitlines(): m = TREE_ENT_RE.match(line) if m: LOG.debug("adding subtree {}".format(m.group(1))) trees.add(m.group(1)) return commit_tree2
def __get_delta_trees(commit_tree1, commit2, trees): """get all tree objects new in one commit vs another commit commit1: SHA1 of first commit commit2: SHA1 of second commit each tree is added to the list to be mirrored """ # diff-tree doesn't return the top level tree, so add it here commit_tree2 = __get_commit_tree(commit2, trees) #po = p4gf_proc.popen_no_throw(['git', 'diff-tree', '-t', commit1, commit2])['out'] po = p4gf_proc.popen_no_throw( ['git', 'diff-tree', '-t', commit_tree1, commit_tree2])['out'] for line in po.splitlines(): m = TREE_ENT_RE.match(line) if m: LOG.debug("adding subtree {}".format(m.group(1))) trees.add(m.group(1)) return commit_tree2
def fast_reclone(ctx): """Try to do fast reclone from mirror.""" # don't try this on a non-empty GF repo if not ctx.repo.is_empty: LOG.debug("fast_reclone: repo not empty") return None, None # write empty tree into repo, just in case it's referenced by any # commits we'll be copying result = p4gf_proc.popen_no_throw(['git', 'write-tree']) if result['ec']: LOG.debug( 'fast_reclone: failed to write empty tree: {}'.format(result)) return None, None # copy commits and trees into the repo branch_heads, commits, dirs = _get_commits(ctx) trees = _get_trees(ctx, dirs) return branch_heads, commits + trees
def _is_reachable(sha1, heads): """Return True if the commit is reachable from one of the heads. :param sha1: SHA1 of the commit to find. :param heads: list of commit references for heads (non-tags) :return: True if commit is reachable, False otherwise. """ LOG.debug2('_is_reachable() checking for {}'.format(sha1)) for head_sha1 in heads: # ### newer pygit2.Repository.merge_base(oid, oid) would do this for us, # ### too bad we're not updating any time soon... cmd = ['git', 'merge-base', '--is-ancestor', sha1, head_sha1] result = p4gf_proc.popen_no_throw(cmd) if LOG.isEnabledFor(logging.DEBUG2): reachable = result['ec'] == 0 LOG.debug2('_is_reachable() {} is reachable {}'.format( sha1, reachable)) if result['ec'] == 0: return True return False
def update_git_config_and_repack(git_dir, git_autopack, git_gc_auto): """Unset git config settings which disable packing. Repos created prior to 14.1 disabled packing. Detect these old settings, remove them and call git repack. """ # See if there is anything to do before doing a bunch of work. repo = pygit2.Repository(git_dir) if 'receive.autogc' not in repo.config: return cwd = os.getcwd() os.chdir(os.path.dirname(git_dir)) remove_pre_14_settings = { 'receive.autogc': 'unset', 'receive.unpacklimit': 'unset', 'gc.auto': 'unset', 'gc.autopacklimit': 'unset', 'pack.compression': 'unset', 'transfer.unpacklimit': 'unset', } set_git_config(git_dir, remove_pre_14_settings) gc_settings = get_autopack_settings(git_autopack, git_gc_auto) set_git_config(git_dir, gc_settings) cmd = ['git', 'repack', '-ad'] result = p4gf_proc.popen_no_throw(cmd) if result['ec']: LOG.error("update_git_config_and_repack: git repack failed with: %s", result['err']) sys.stderr.write( _("Perforce: error: git repack failed with '{error}'\n").format( error=result['ec'])) else: LOG.debug3("update_git_config_and_repack: %s", result) os.chdir(cwd)
def force_tag_ref(ref_name, sha1): """Create or change one Git tag reference.""" p4gf_proc.popen_no_throw(['git', 'tag', '-f', ref_name, sha1])
def create_git_repo(ctx, git_dir): """Create the git repository in the given root directory.""" # Test if the Git repository has already been created. if os.path.exists(os.path.join(git_dir, 'HEAD')): return # Prepare the Git repository directory, cleaning up if necessary. work_tree = os.path.dirname(git_dir) if not os.path.exists(git_dir): if os.path.exists(work_tree): # weird case where git view dir exists but repo was deleted LOG.warn( "mirror Git repository {} in bad state, repairing...".format( git_dir)) shutil.rmtree(work_tree) LOG.debug("creating directory %s for Git repo", git_dir) os.makedirs(git_dir) # Initialize the Git repository for that directory. LOG.debug("creating Git repository in %s", git_dir) pygit2.init_repository(git_dir) # Configure the Git repository to avoid creating pack objects, so we can # read them directly ourselves and avoid the overhead of git-cat-file. cwd = os.getcwd() os.chdir(work_tree) settings = { # Prevent conflicting change history by disallowing rewinds. 'receive.denyNonFastForwards': 'true', # # Turn off default compression (suspenders) # 'core.compression': '0', # # Turn off compression for loose objects (belt) # 'core.loosecompression': '0', # # threshold beyond which deltaCompression is compression is disabled # 'core.bigFileThreshold': '0', # # Packs smaller than this are unpacked into loose object files (belt) # 'fetch.unpackLimit': '1000000000', # Turn off garbage collection 'gc.auto': '0', # Should never be used 'gc.autopacklimit': '0', # Should never be used 'pack.compression': '0', # git-receive-pack will not run gc 'receive.autogc': 'false', # Packs smaller than this are unpacked into loose object files (belt) 'receive.unpackLimit': '1000000000', # Packs smaller than this are unpacked into loose object files (suspenders) 'transfer.unpackLimit': '1000000000' } # repo = pygit2.Repository(git_dir) for k, v in settings.items(): ### this explodes the Python process, in p4api.so, oddly enough # repo.config[k] = v cmd = ['git', 'config', '--local', '--replace-all', k, v] result = p4gf_proc.popen_no_throw(cmd) if result['ec']: LOG.error("configuring git repo failed for {}={} => {}".format( k, v, result['err'])) sys.stderr.write( _("error: git init failed with '{}' for '{}'\n").format( result['ec'], work_tree)) os.chdir(cwd) install_hook(git_dir) # Don't bother changing branches in a --bare repo. if is_bare_git_repo(git_dir): return create_empty_repo_branch(ctx, git_dir)
def delete_branch_ref(ref_name): """Delete one Git branch reference.""" p4gf_proc.popen_no_throw(['git', 'branch', '-D', ref_name])
def delete_branch_refs(ref_names): """Delete several Git branch references at once.""" for chunk in p4gf_util.iter_chunks(ref_names, 300): p4gf_proc.popen_no_throw(['git', 'branch', '-D'] + chunk)
def is_valid_git_branch_name(git_branch_name): """Determine if the given name refers to a valid git branch.""" cmd = ['git', 'check-ref-format', '--allow-onelevel', git_branch_name] result = p4gf_proc.popen_no_throw(cmd) return not result['ec']
def force_branch_ref(ref_name, sha1): """Create or change one Git branch reference.""" p4gf_proc.popen_no_throw(['git', 'branch', '-f', ref_name, sha1])
def delete_tag_ref(ref_name): """Delete one Git tag reference.""" p4gf_proc.popen_no_throw(['git', 'tag', '-d', ref_name])