def exec_out(cmd):
"""
Return the stdout output of command 'cmd'. Raise exception on failure.
By failure, we mean when the command's return code is non-zero.
"""
return execute(cmd, must_succeed=True)
def exec_all(cmd):
"""
Return the return code, stdout and stderr of the command.
"""
logger.debug("Running command: %s" % cmd)
r_code, stdout, stderr = execute(cmd, return_stdout=True, return_stderr=True)
return r_code, stdout, stderr
def get_root_commits():
"""
Return the list of root commits in the repository.
If unable to find any or if an error occurs, an empty list is returned.
Note that a repository may have multiple root commits (this is why the
returned value is a list).
"""
rcode, stdout = execute('git rev-list --max-parents=0 HEAD',
return_stdout=True)
if rcode != 0:
return []
return [s.strip() for s in stdout]
def exec_code(cmd):
"""
Return the return code only of the command.
The stdout and stderr outputs are ignored (only logged in debug mode).
"""
logger.debug("Running command: %s" % cmd)
r_code, stdout, stderr = execute(cmd, return_stdout=True, return_stderr=True)
if stdout:
logger.debug("Command stdout: %s" % stdout)
if stderr:
logger.debug("Command stderr: %s" % stderr)
return r_code
def exec_out(cmd):
"""
Return the stdout output of command 'cmd'. Raise exception on failure.
By failure, we mean when the command's return code is non-zero.
The stderr output is ignored (only logged in debug mode).
"""
logger.debug("Running command: %s" % cmd)
r_code, stdout, stderr = execute(cmd, return_stdout=True, return_stderr=True)
if stdout:
logger.debug("Command stdout: %s" % stdout)
if stderr:
logger.debug("Command stderr: %s" % stderr)
if r_code != 0:
raise RuntimeError("Non-zero return code in command: %s\nstdout: %s\nstderr: %s" % (cmd, stdout, stderr))
return stdout
def _flatten(start, end, state):
"""
Actual implementation of `flatten`.
"""
logger = state.logger
# Helper function to create a new branch with a unique name.
def make_new_branch(commit):
logger.debug('Creating new branch #%s' % state.branch_idx)
name = 'flatten_tmp_branch_%s' % state.branch_idx
exec_out('git checkout -b %s %s' % (name, commit))
state.branch_idx += 1
return name
# Work in a new temporary branch.
base_branch = make_new_branch(end)
# First attempt: simple rebase. If it works then we are good to go.
logger.debug('Attempting a rebase on top of %s' % start)
r_code, stdout, stderr = execute('git rebase %s' % start,
return_stdout=True,
return_stderr=True)
if r_code == 0:
assert execute('git diff --quiet %s' % end) == 0
logger.debug('Rebase successful!')
return base_branch
logger.debug('Rebase failure -- rolling back')
exec_out('git rebase --abort')
# Get parent/child relationships.
# The `data` dictionary maps a commit hash to a `Storage` instance holding:
# - the commit hash
# - its parents
# - its children
logger.debug('Analyzing parent/child relationships')
data = dict()
commits = exec_out('git rev-list --parents --reverse --topo-order %s' %
end)
for commit_info in commits:
tokens = commit_info.split(' ')
commit = tokens[0]
parents = tokens[1:]
commit_info = util.Storage(
hash=commit,
parents=[data[h] for h in parents],
children=[])
for p in commit_info.parents:
assert commit_info not in p.children
p.children.append(commit_info)
assert commit not in data
data[commit] = commit_info
# TODO Explain somewhere why not recursive.
# In this branch we will perform temporary rebases.
rebase_branch = make_new_branch(end)
# Need another work branch for the final result.
work_branch = make_new_branch(start)
# The algorithm works as follows:
# * We apply commits that can be applied on top of each other, until
# we find a merge commit (>= 2 parents).
# * When a merge commit is found, we attempt to rebase the unprocessed
# child on top of current head:
# - If this works, the resulting commits are cherry-picked into
# the work branch.
# - If this fails, then we manually create a patch that represents
# the diff from the merge commit, and apply this patch instead.
head = data[start]
while head.children:
child = head.children[0]
assert child.parents
if len(child.parents) == 1:
# This is not a merge commit: apply it.
logger.debug('Cherry-picking %s' % child.hash)
exec_out('git cherry-pick --allow-empty %s' % child.hash)
# Ensure end result is as expected.
assert execute('git diff --quiet %s' % child.hash) == 0
else:
if len(child.parents) > 2:
raise NotImplementedError(
'Found a merge commit with %s parents, but the '
'current implementation only supports two parents.' %
len(child.parents))
# Get the other parent of this child.
if head is child.parents[0]:
other = child.parents[1]
else:
assert head is child.parents[1]
other = child.parents[0]
# Attempt to rebase the other parent on top of the current head.
# We do this in a new branch whose head is the other parent.
exec_out('git branch -D %s' % rebase_branch)
logger.debug('Attempting to rebase %s on top of %s to yield %s' %
(other.hash, head.hash, child.hash))
exec_out('git checkout -b %s %s' % (rebase_branch, other.hash))
r_code, stdout, stderr = execute(
'git rebase %s' % head.hash, return_stdout=True,
return_stderr=True)
if r_code == 0:
logger.debug('Rebase successful')
# Apply resulting commits on top of current head of the work
# branch.
# First we obtain these commits.
rebase_head = get_current_head()
to_apply = exec_out('git rev-list --reverse HEAD ^%s' %
head.hash)
exec_out('git checkout %s' % work_branch)
for commit in to_apply:
exec_out('git cherry-pick %s' % commit)
# It can happen that the end result is not as expected. This
# is the case when 'rebase' and 'merge' both succeed without
# conflict and yet give different results. Another situation is
# when the merge commit contains some manual changes.
has_diff = execute('git diff --quiet %s' % child.hash)
if has_diff != 0:
cherry_head = get_current_head()
logger.debug(
'Rebase + cherry-pick did not yield the expected '
'result: it yielded %s while the expected result was '
'%s (the rebased branch can be found at %s)' %
(cherry_head, child.hash, rebase_head))
logger.debug('Adding a new commit to fix this situation')
# In such a situation, we add a new commit to fix it.
# First set working directory to its expected state. This
# is a bit tricy because 'git checkout X -- .' does not
# always work with files being moved around. Thus achieve
fix_branch = make_new_branch(child.hash)
exec_out('git reset %s' % cherry_head)
exec_out('git add -A')
exec_out(['git', 'commit', '-m', 'Automated recovery from '
'unexpected rebase result'])
exec_out('git checkout %s' % work_branch)
exec_out('git reset --hard %s' % fix_branch)
exec_out('git branch -D %s' % fix_branch)
# Repository should be clean.
assert not exec_out('git status --porcelain')
# Now there should be no more diff.
if execute('git diff --quiet %s' % child.hash) != 0:
raise RuntimeError(
'There remain changes after attempted recovery. '
'The current head is %s, and differs from %s' %
(get_current_head(), child.hash))
else:
logger.debug('Rebase failed -- rolling back')
exec_out('git rebase --abort')
# We use the so-called "symmetric difference" to figure out
# which commits are being included in the merge, in order to
# build an informative log message.
cmd = 'git rev-list --left-only --graph '
c_range = ' %s...%s' % (other.hash, head.hash)
short_info = exec_out(cmd + '--oneline' + c_range)
full_info = exec_out(cmd + '--pretty=fuller' + c_range)
patch_info = [
'Automatic patch built from combined commits',
'',
'This patch is made of the following commits (the full '
'description\nfollows the short summary):',
''] + short_info + [
'',
'',
'Full information about combined commits:',
''] + full_info
# Restore work branch.
exec_out('git checkout %s' % work_branch)
# Build a patch with the diff.
logger.debug('Building patch')
diff = exec_out('git diff --full-index --binary %s..%s' %
(head.hash, child.hash))
diff_f_name = '.tmp.flatten_patch'
diff_file = open(diff_f_name, 'w')
try:
# Note the last empty line to ensure binary diffs are not
# corrupted.
diff_file.write('\n'.join(diff) + '\n\n')
finally:
diff_file.close()
patch_success = False
try:
# Apply the patch.
if diff:
logger.debug('Applying patch')
r_code = execute('git apply --check %s' % diff_f_name)
else:
logger.debug('Patch is empty: skipping it')
r_code = 0
head_before = get_current_head()
if r_code != 0:
raise RuntimeError(
'Merge patch cannot be applied on top of %s' %
head_before)
if diff:
exec_out('git apply --index %s' % diff_f_name)
logger.debug(
'Patch successfully applied, committing it')
else:
logger.debug('Committing an empty commit')
commit_f_name = '.tmp.flatten_msg'
commit_file = open(commit_f_name, 'w')
try:
commit_file.write('\n'.join(patch_info))
finally:
commit_file.close()
try:
exec_out('git commit --allow-empty -F %s' %
commit_f_name)
finally:
os.remove(commit_f_name)
# Ensure the end result is the one expected.
has_diff = execute('git diff --quiet %s' % child.hash)
if has_diff != 0:
# The patch did not yield the expected result.
raise RuntimeError(
'Patch was applied on %s but did not work as '
'expected.\n'
'It yielded %s while the expected result was %s' %
(head_before, get_current_head(), child.hash))
patch_success = True
finally:
if patch_success:
os.remove(diff_f_name)
# Update current head.
head = child
return work_branch