def exec_out(cmd): """ Return the stdout output of command 'cmd'. Raise exception on failure. By failure, we mean when the command's return code is non-zero. """ return execute(cmd, must_succeed=True)
def exec_all(cmd): """ Return the return code, stdout and stderr of the command. """ logger.debug("Running command: %s" % cmd) r_code, stdout, stderr = execute(cmd, return_stdout=True, return_stderr=True) return r_code, stdout, stderr
def get_root_commits(): """ Return the list of root commits in the repository. If unable to find any or if an error occurs, an empty list is returned. Note that a repository may have multiple root commits (this is why the returned value is a list). """ rcode, stdout = execute('git rev-list --max-parents=0 HEAD', return_stdout=True) if rcode != 0: return [] return [s.strip() for s in stdout]
def exec_code(cmd): """ Return the return code only of the command. The stdout and stderr outputs are ignored (only logged in debug mode). """ logger.debug("Running command: %s" % cmd) r_code, stdout, stderr = execute(cmd, return_stdout=True, return_stderr=True) if stdout: logger.debug("Command stdout: %s" % stdout) if stderr: logger.debug("Command stderr: %s" % stderr) return r_code
def exec_out(cmd): """ Return the stdout output of command 'cmd'. Raise exception on failure. By failure, we mean when the command's return code is non-zero. The stderr output is ignored (only logged in debug mode). """ logger.debug("Running command: %s" % cmd) r_code, stdout, stderr = execute(cmd, return_stdout=True, return_stderr=True) if stdout: logger.debug("Command stdout: %s" % stdout) if stderr: logger.debug("Command stderr: %s" % stderr) if r_code != 0: raise RuntimeError("Non-zero return code in command: %s\nstdout: %s\nstderr: %s" % (cmd, stdout, stderr)) return stdout
def _flatten(start, end, state): """ Actual implementation of `flatten`. """ logger = state.logger # Helper function to create a new branch with a unique name. def make_new_branch(commit): logger.debug('Creating new branch #%s' % state.branch_idx) name = 'flatten_tmp_branch_%s' % state.branch_idx exec_out('git checkout -b %s %s' % (name, commit)) state.branch_idx += 1 return name # Work in a new temporary branch. base_branch = make_new_branch(end) # First attempt: simple rebase. If it works then we are good to go. logger.debug('Attempting a rebase on top of %s' % start) r_code, stdout, stderr = execute('git rebase %s' % start, return_stdout=True, return_stderr=True) if r_code == 0: assert execute('git diff --quiet %s' % end) == 0 logger.debug('Rebase successful!') return base_branch logger.debug('Rebase failure -- rolling back') exec_out('git rebase --abort') # Get parent/child relationships. # The `data` dictionary maps a commit hash to a `Storage` instance holding: # - the commit hash # - its parents # - its children logger.debug('Analyzing parent/child relationships') data = dict() commits = exec_out('git rev-list --parents --reverse --topo-order %s' % end) for commit_info in commits: tokens = commit_info.split(' ') commit = tokens[0] parents = tokens[1:] commit_info = util.Storage( hash=commit, parents=[data[h] for h in parents], children=[]) for p in commit_info.parents: assert commit_info not in p.children p.children.append(commit_info) assert commit not in data data[commit] = commit_info # TODO Explain somewhere why not recursive. # In this branch we will perform temporary rebases. rebase_branch = make_new_branch(end) # Need another work branch for the final result. work_branch = make_new_branch(start) # The algorithm works as follows: # * We apply commits that can be applied on top of each other, until # we find a merge commit (>= 2 parents). # * When a merge commit is found, we attempt to rebase the unprocessed # child on top of current head: # - If this works, the resulting commits are cherry-picked into # the work branch. # - If this fails, then we manually create a patch that represents # the diff from the merge commit, and apply this patch instead. head = data[start] while head.children: child = head.children[0] assert child.parents if len(child.parents) == 1: # This is not a merge commit: apply it. logger.debug('Cherry-picking %s' % child.hash) exec_out('git cherry-pick --allow-empty %s' % child.hash) # Ensure end result is as expected. assert execute('git diff --quiet %s' % child.hash) == 0 else: if len(child.parents) > 2: raise NotImplementedError( 'Found a merge commit with %s parents, but the ' 'current implementation only supports two parents.' % len(child.parents)) # Get the other parent of this child. if head is child.parents[0]: other = child.parents[1] else: assert head is child.parents[1] other = child.parents[0] # Attempt to rebase the other parent on top of the current head. # We do this in a new branch whose head is the other parent. exec_out('git branch -D %s' % rebase_branch) logger.debug('Attempting to rebase %s on top of %s to yield %s' % (other.hash, head.hash, child.hash)) exec_out('git checkout -b %s %s' % (rebase_branch, other.hash)) r_code, stdout, stderr = execute( 'git rebase %s' % head.hash, return_stdout=True, return_stderr=True) if r_code == 0: logger.debug('Rebase successful') # Apply resulting commits on top of current head of the work # branch. # First we obtain these commits. rebase_head = get_current_head() to_apply = exec_out('git rev-list --reverse HEAD ^%s' % head.hash) exec_out('git checkout %s' % work_branch) for commit in to_apply: exec_out('git cherry-pick %s' % commit) # It can happen that the end result is not as expected. This # is the case when 'rebase' and 'merge' both succeed without # conflict and yet give different results. Another situation is # when the merge commit contains some manual changes. has_diff = execute('git diff --quiet %s' % child.hash) if has_diff != 0: cherry_head = get_current_head() logger.debug( 'Rebase + cherry-pick did not yield the expected ' 'result: it yielded %s while the expected result was ' '%s (the rebased branch can be found at %s)' % (cherry_head, child.hash, rebase_head)) logger.debug('Adding a new commit to fix this situation') # In such a situation, we add a new commit to fix it. # First set working directory to its expected state. This # is a bit tricy because 'git checkout X -- .' does not # always work with files being moved around. Thus achieve fix_branch = make_new_branch(child.hash) exec_out('git reset %s' % cherry_head) exec_out('git add -A') exec_out(['git', 'commit', '-m', 'Automated recovery from ' 'unexpected rebase result']) exec_out('git checkout %s' % work_branch) exec_out('git reset --hard %s' % fix_branch) exec_out('git branch -D %s' % fix_branch) # Repository should be clean. assert not exec_out('git status --porcelain') # Now there should be no more diff. if execute('git diff --quiet %s' % child.hash) != 0: raise RuntimeError( 'There remain changes after attempted recovery. ' 'The current head is %s, and differs from %s' % (get_current_head(), child.hash)) else: logger.debug('Rebase failed -- rolling back') exec_out('git rebase --abort') # We use the so-called "symmetric difference" to figure out # which commits are being included in the merge, in order to # build an informative log message. cmd = 'git rev-list --left-only --graph ' c_range = ' %s...%s' % (other.hash, head.hash) short_info = exec_out(cmd + '--oneline' + c_range) full_info = exec_out(cmd + '--pretty=fuller' + c_range) patch_info = [ 'Automatic patch built from combined commits', '', 'This patch is made of the following commits (the full ' 'description\nfollows the short summary):', ''] + short_info + [ '', '', 'Full information about combined commits:', ''] + full_info # Restore work branch. exec_out('git checkout %s' % work_branch) # Build a patch with the diff. logger.debug('Building patch') diff = exec_out('git diff --full-index --binary %s..%s' % (head.hash, child.hash)) diff_f_name = '.tmp.flatten_patch' diff_file = open(diff_f_name, 'w') try: # Note the last empty line to ensure binary diffs are not # corrupted. diff_file.write('\n'.join(diff) + '\n\n') finally: diff_file.close() patch_success = False try: # Apply the patch. if diff: logger.debug('Applying patch') r_code = execute('git apply --check %s' % diff_f_name) else: logger.debug('Patch is empty: skipping it') r_code = 0 head_before = get_current_head() if r_code != 0: raise RuntimeError( 'Merge patch cannot be applied on top of %s' % head_before) if diff: exec_out('git apply --index %s' % diff_f_name) logger.debug( 'Patch successfully applied, committing it') else: logger.debug('Committing an empty commit') commit_f_name = '.tmp.flatten_msg' commit_file = open(commit_f_name, 'w') try: commit_file.write('\n'.join(patch_info)) finally: commit_file.close() try: exec_out('git commit --allow-empty -F %s' % commit_f_name) finally: os.remove(commit_f_name) # Ensure the end result is the one expected. has_diff = execute('git diff --quiet %s' % child.hash) if has_diff != 0: # The patch did not yield the expected result. raise RuntimeError( 'Patch was applied on %s but did not work as ' 'expected.\n' 'It yielded %s while the expected result was %s' % (head_before, get_current_head(), child.hash)) patch_success = True finally: if patch_success: os.remove(diff_f_name) # Update current head. head = child return work_branch