def test_push_root_commit(cd_to_monorepo_clone,
monorepo_simple_root_remote_git_dir, capfd):
current_root_top = git_output('rev-parse',
'internal/master',
git_dir=monorepo_simple_root_remote_git_dir)
file_contents = 'internal: cool file'
commit_file('root-file', file_contents)
result = CliRunner().invoke(
git_apple_llvm_push,
['HEAD:internal/master', '--merge-strategy=ff-only'])
if 'Pushing to monorepo root' not in result.output:
raise AssertionError
if result.exit_code != 0:
raise AssertionError
captured = capfd.readouterr()
new_root_top = git_output('rev-parse',
'internal/master',
git_dir=monorepo_simple_root_remote_git_dir)
# Verify that the `git push` output is printed.
if f'{new_root_top} -> internal/master' not in captured.err:
raise AssertionError
if new_root_top == current_root_top:
raise AssertionError
if git_output(
'rev-parse',
'internal/master~1',
git_dir=monorepo_simple_root_remote_git_dir) != current_root_top:
raise AssertionError
if git_output(
'show',
'internal/master:root-file',
git_dir=monorepo_simple_root_remote_git_dir) != file_contents:
raise AssertionError
def test_push_root_commit(cd_to_monorepo_clone,
monorepo_simple_root_remote_git_dir, capfd):
current_root_top = git_output('rev-parse',
'internal/master',
git_dir=monorepo_simple_root_remote_git_dir)
file_contents = 'internal: cool file'
commit_file('root-file', file_contents)
result = CliRunner().invoke(
git_apple_llvm_push,
['HEAD:internal/master', '--merge-strategy=ff-only'])
assert 'Pushing to monorepo root' in result.output
assert result.exit_code == 0
captured = capfd.readouterr()
new_root_top = git_output('rev-parse',
'internal/master',
git_dir=monorepo_simple_root_remote_git_dir)
# Verify that the `git push` output is printed.
assert f'{new_root_top} -> internal/master' in captured.err
assert new_root_top != current_root_top
assert git_output(
'rev-parse',
'internal/master~1',
git_dir=monorepo_simple_root_remote_git_dir) == current_root_top
assert git_output(
'show',
'internal/master:root-file',
git_dir=monorepo_simple_root_remote_git_dir) == file_contents
def get_tracked_branch_ref(branch_name: str) -> Optional[TrackedBranchRef]:
""" Returns a valid TrackedBranchRef if the given branch resolves to one """
tracking_branch_name: Optional[str] = git_output('rev-parse',
'--abbrev-ref',
'--symbolic-full-name',
branch_name + '@{u}',
ignore_error=True)
if not tracking_branch_name:
if '/' in branch_name:
tracking_branch_name = branch_name
else:
return None
assert '/' in tracking_branch_name
rb = tracking_branch_name.split('/', 1)
remote_branch = rb[1]
output: Optional[str] = git_output('ls-remote',
'--exit-code',
rb[0],
remote_branch,
ignore_error=True)
if not output:
return None
hash_refname: List[str] = output.split()
if len(hash_refname
) >= 2 and hash_refname[1] == f'refs/heads/{remote_branch}':
return TrackedBranchRef(rb[0], git_output('remote', 'get-url', rb[0]),
remote_branch, hash_refname[0])
return None
def test_push_many_llvm_commits(cd_to_monorepo_clone,
monorepo_simple_llvm_remote_git_dir):
current_llvm_top = git_output('rev-parse',
'master',
git_dir=monorepo_simple_llvm_remote_git_dir)
for i in range(0, 50):
commit_file(f'llvm/a-new-file{i}', 'internal: cool file')
result = CliRunner().invoke(
git_apple_llvm_push,
['HEAD:internal/master', '--merge-strategy=ff-only'])
if 'pushing 50 commits, are you really sure' not in result.output:
raise AssertionError
if result.exit_code != 1:
raise AssertionError
result = CliRunner().invoke(git_apple_llvm_push, [
'HEAD:internal/master', '--merge-strategy=ff-only', '--push-limit=51'
])
if 'Pushing to llvm' not in result.output:
raise AssertionError
if result.exit_code != 0:
raise AssertionError
if git_output(
'rev-parse', 'master~50',
git_dir=monorepo_simple_llvm_remote_git_dir) != current_llvm_top:
raise AssertionError
def test_monorepo_simple_test_harness(cd_to_monorepo):
internal_commits = git_output('rev-list', 'internal/master').splitlines()
if len(internal_commits) != 16:
raise AssertionError
trailers = git_output('show', '--format=%B', internal_commits[0])
if 'apple-llvm-split-commit:' not in trailers:
raise AssertionError
if 'apple-llvm-split-dir: -/' not in trailers:
raise AssertionError
if not PosixPath('clang/dir/file2').is_file():
raise AssertionError
upstream_commits = git_output('rev-list', 'llvm/master').splitlines()
if len(upstream_commits) != 9:
raise AssertionError
if internal_commits[-1] != upstream_commits[-1]:
raise AssertionError
# Verify that each upstream commit is in downstream.
for commit in upstream_commits:
git('merge-base', '--is-ancestor', commit, 'internal/master')
internal_clang_commits = git_output('rev-list', 'split/clang/internal/master').splitlines()
if len(internal_clang_commits) != 7:
raise AssertionError
upstream_clang_commits = git_output('rev-list', 'split/clang/upstream/master').splitlines()
if len(upstream_clang_commits) != 4:
raise AssertionError
def test_git_invocation(tmp_path):
""" Tests for the git/git_output functions. """
repo_path = tmp_path / 'repo'
repo_path.mkdir()
assert repo_path.is_dir() # Ensure the dir is there for us to work with.
repo_dir = str(repo_path)
git('init', git_dir=repo_dir)
(repo_path / 'initial').write_text(u'initial')
git('add', 'initial', git_dir=repo_dir)
# Check that we can report an error on failure.
with pytest.raises(GitError) as err:
git('add', 'foo', git_dir=repo_dir)
assert err.value.stderr.startswith('fatal')
assert repr(err.value).startswith('GitError')
# Check that errors can be ignored.
git('add', 'foo', git_dir=repo_dir, ignore_error=True)
output = git_output('commit', '-m', 'initial', git_dir=repo_dir)
assert len(output) > 0
# Ensure that the output is stripped.
output = git_output('rev-list', 'HEAD', git_dir=repo_dir)
assert '\n' not in output
output = git_output('rev-list', 'HEAD', git_dir=repo_dir, strip=False)
assert '\n' in output
# Ensure that commit exists works only for commit hashes.
hash = output.strip()
assert commit_exists(hash)
assert not commit_exists('HEAD')
assert not commit_exists(hash + 'abc')
assert not commit_exists('000000')
# Ensure that we can get the directory of the checkout even when the
# working directory is a subdirectory.
os.chdir(repo_dir)
dir_a = get_current_checkout_directory()
(repo_path / 'subdir').mkdir()
cwd = os.getcwd()
os.chdir(os.path.join(repo_dir, 'subdir'))
dir_b = get_current_checkout_directory()
os.chdir(cwd)
assert dir_a == dir_b
assert read_file_or_none('HEAD', 'initial') == 'initial'
assert read_file_or_none(hash, 'initial') == 'initial'
assert read_file_or_none('HEAD', 'path/does-not-exist') is None
assert read_file_or_none('foo', 'initial') is None
def find_base_split_commit(split_dir, base_commit) -> Optional[str]:
""" Return the hash of the base commit in the specified
split repository derived from the specified monorepo base commit. """
mono_base_commit = git_output('rev-list', '--first-parent', '-n', '1', '--grep',
f'^apple-llvm-split-dir: {split_dir}/*$', base_commit,
ignore_error=True)
if not mono_base_commit:
return None
SPLIT_COMMIT_TRAILER = 'apple-llvm-split-commit:'
for line in git_output('rev-list', '-n', '1', '--format=%B',
mono_base_commit).splitlines():
if line.startswith(SPLIT_COMMIT_TRAILER):
return line[len(SPLIT_COMMIT_TRAILER):].strip()
return None
def create_monorepo(commits: List[CommitBlueprint]):
# Create the upstream monorepo.
start_new_orphan_branch('llvm/master')
for commit in commits:
if isinstance(commit, BlobCommitBlueprint) and not commit.is_internal:
commit.monorepo_commit_hash = commit_file(commit.monorepo_filename,
commit.text)
# Create the downstream monorepo.
is_first = True
for commit in commits:
hs = commit.split_commit_hash
trailers = f'\n---\napple-llvm-split-commit: {hs}\napple-llvm-split-dir: {commit.split_dir}/'
if isinstance(commit, BlobCommitBlueprint) and commit.is_internal:
assert commit.parent and commit.parent.monorepo_commit_hash is not None
if is_first:
git('checkout', '-b', 'internal/master',
commit.parent.monorepo_commit_hash)
git('clean', '-d', '-f')
is_first = False
commit.monorepo_commit_hash = commit_file(commit.monorepo_filename,
commit.text,
trailers=trailers)
elif isinstance(commit, MergeCommitBlueprint):
assert not is_first
# Verify the the expected split merge is already there.
git('merge-base', '--is-ancestor',
commit.downstream.monorepo_commit_hash, 'internal/master')
# Recreate the merge.
assert commit.upstream.monorepo_commit_hash is not None
git('merge', '--no-commit', commit.upstream.monorepo_commit_hash)
msg = f'Merge {commit.upstream.monorepo_commit_hash} into internal/master\n{trailers}'
git('commit', '-m', msg)
commit.monorepo_commit_hash = git_output('rev-parse', 'HEAD')
def test_rebase_commit_graph(split_clang_head_in_monorepo: str,
merge_strategy: MergeStrategy):
work_head = commit_file('file1', 'top of branch is here')
branch_name = f'new-rebase/split/clang/{merge_strategy}'
git('checkout', '-b', branch_name, split_clang_head_in_monorepo)
new_clang_head = commit_file('file2', 'top of clang/master is here')
def doit() -> str:
return merge_commit_graph_with_top_of_branch(CommitGraph([work_head], [split_clang_head_in_monorepo]),
'clang',
branch_name,
merge_strategy)
# Fast forward only fails with disjoint graph.
if merge_strategy == MergeStrategy.FastForwardOnly:
with pytest.raises(ImpossibleMergeError) as err:
doit()
if 'Not possible to fast-forward' not in err.value.git_error.stderr:
raise AssertionError
else:
# However, other strategies rebase the commit graph on top of the
# destination branch.
commit = doit()
if work_head == commit:
raise AssertionError
if new_clang_head == commit:
raise AssertionError
if new_clang_head != git_output('rev-parse', f'{commit}~1'):
raise AssertionError
def test_regraft_no_split_root(cd_to_monorepo):
internal_head = git_output('rev-parse', 'internal/master')
clang_change_hash = commit_file('clang/new-file', 'internal: new file')
# Try to regraft something that has no `apple-llvm-split-*` history.
with pytest.raises(RegraftNoSplitRootError) as err:
regraft_commit_graph_onto_split_repo(CommitGraph([clang_change_hash], [internal_head]), 'polly')
assert err.value.root_commit_hash == internal_head
def test_regraft_missing_split_root(cd_to_monorepo):
# Test the scenario with a missing split root.
git('checkout', '-b', 'fake-internal-master', 'internal/master')
git('commit', '--allow-empty', '-m', 'error\n\napple-llvm-split-commit: 00000000000\napple-llvm-split-dir: clang/')
bad_master_head = git_output('rev-parse', 'HEAD')
bad_clang_hash = commit_file('clang/another-file', 'internal: new file')
with pytest.raises(RegraftMissingSplitRootError) as err:
regraft_commit_graph_onto_split_repo(CommitGraph([bad_clang_hash], [bad_master_head]), 'clang')
assert err.value.root_commit_hash == bad_master_head
def has_existing_remote(remote_name: str, remote_url: str, **kwargs):
try:
url = git_output('remote', 'get-url', remote_name, **kwargs)
if url != remote_url:
git('remote', 'remove', remote_name, **kwargs)
return False
return True
except GitError:
return False
def __init__(self, split_dir: str, remote_url: str,
destination_branch: str):
self.split_dir = split_dir
self.remote_url = remote_url
self.destination_branch = destination_branch
# Setup the directory into which the split repo is actually
# cloned.
assert os.path.isdir('.git')
self.remote_clone_dir = os.path.abspath(
os.path.join('.git', f'apple-llvm-split-{split_dir}.git'))
if not os.path.isdir(self.remote_clone_dir):
os.mkdir(self.remote_clone_dir)
git_output('init', '--bare', git_dir=self.remote_clone_dir)
self.monorepo_remote_url = os.path.abspath(os.getcwd())
# The final commit hash that should be pushed.
self.commit_hash = None
def test_merge_commit_graph(split_clang_head_in_monorepo: str,
merge_strategy: MergeStrategy):
# Create the test scenario:
# * merge <-- top of commit graph.
# |\
# | * [new-merge/split/clang] <-- destination.
# | \/
# | * file2 <-- root of commit graph.
# * | file1
# |/
# * [split/clang/internal/master] <-- root of commit graph.
push_clang_commit = commit_file('file1', 'top of branch is here')
branch_name = f'new-merge/split/clang/{merge_strategy}'
git('checkout', '-b', branch_name, split_clang_head_in_monorepo)
up_clang_head = commit_file('file2', 'merging this in')
new_clang_head = commit_file('file3', 'top of clang/master is here')
git('checkout', '--detach', up_clang_head)
git('clean', '-d', '-f')
git('merge', push_clang_commit)
merge_commit = git_output('rev-parse', 'HEAD')
def doit() -> str:
graph = CommitGraph([merge_commit, push_clang_commit], [up_clang_head, split_clang_head_in_monorepo])
return merge_commit_graph_with_top_of_branch(graph,
'clang',
branch_name,
merge_strategy)
# Graph with merges can only be merged.
if merge_strategy == MergeStrategy.RebaseOrMerge:
commit = doit()
if merge_commit == commit:
raise AssertionError
parents = git_output('show', '--format=%P', commit).split()
if 2 != len(parents):
raise AssertionError
if new_clang_head != parents[0]:
raise AssertionError
if merge_commit != parents[1]:
raise AssertionError
else:
with pytest.raises(ImpossibleMergeError):
doit()
def load_push_config(source_ref: str, dest_branch: str) -> Optional[GitPushConfiguration]:
config_name = dest_branch.replace('/', '-')
config = git_output(
'show', f'{source_ref}:apple-llvm-config/push/{config_name}.json', ignore_error=True)
if not config:
return None
value = json.loads(config)
# FIXME: Validate json.
return GitPushConfiguration(name=config_name,
branch_to_dest_branch_mapping=value['branch_to_dest_branch_mapping'],
repo_mapping=value['repo_mapping'])
def dispatch(self, params):
ci_job_config_filename = f'apple-llvm-config/ci-jobs/{self.ci_jobs}.json'
log.debug('Test plan %s: loading ci config %s', self.name,
ci_job_config_filename)
ci_job_config_json = json.loads(
git_output('show', f'HEAD:{ci_job_config_filename}'))
ci_job_config = JenkinsCIConfig(ci_job_config_json)
params.update(self.params)
log.debug('Test plan %s: dispatching ci job requests for params: %s',
self.name, params)
ci_job_config.dispatch(params, self.name)
def commit_merge(self, downstream: CommitBlueprint,
upstream: CommitBlueprint) -> MergeCommitBlueprint:
checkout_and_clean(downstream.split_commit_hash)
git('merge', upstream.split_commit_hash)
commit_hash = git_output('rev-parse', 'HEAD')
git('branch', '-f', f'split/{self.split_dir}/internal/master',
commit_hash)
return MergeCommitBlueprint(split_dir=self.split_dir,
split_commit_hash=commit_hash,
downstream=downstream,
upstream=upstream)
def find_inflight_merges(remote: str = 'origin') -> Dict[str, List[str]]:
"""
This function fetches the refs created by the automerger to find
the inflight merges that are currently being processed.
"""
# Delete the previously fetched refs to avoid fetch failures
# where there were force pushes.
existing_refs = git_output('for-each-ref', AM_STATUS_PREFIX,
'--format=%(refname)').split('\n')
for ref in existing_refs:
if not ref:
continue
log.debug(f'Deleting local ref "{ref}" before fetching')
git('update-ref', '-d', ref)
git('fetch', remote,
f'{AM_PREFIX}*:{AM_STATUS_PREFIX}*') # FIXME: handle fetch failures.
refs = git_output('for-each-ref', AM_STATUS_PREFIX,
'--format=%(refname)').split('\n')
inflight_merges: Dict[str, List[str]] = {}
for ref in refs:
if not ref:
continue
if not ref.startswith(AM_STATUS_PREFIX):
raise AssertionError
merge_name = ref[len(AM_STATUS_PREFIX):]
underscore_idx = merge_name.find('_')
if underscore_idx == -1:
raise AssertionError
commit_hash = merge_name[:underscore_idx]
dest_branch = merge_name[underscore_idx + 1:]
if dest_branch in inflight_merges:
inflight_merges[dest_branch].append(commit_hash)
else:
inflight_merges[dest_branch] = [commit_hash]
for (m, k) in inflight_merges.items():
log.debug(f'in-flight {m}: {k}')
return inflight_merges
def __init__(self, path: str, clone_path: str,
clang_split_remote_path: str, llvm_split_remote_path: str,
root_split_remote_path: str):
self.path = path
self.clone_path = clone_path
self.clang_split_remote_path = clang_split_remote_path
self.llvm_split_remote_path = llvm_split_remote_path
self.root_split_remote_path = root_split_remote_path
# Configure the push configuration.
push_config = {
'branch_to_dest_branch_mapping': {
'internal/master:-': 'internal/master',
'internal/master:*': 'master'
},
'repo_mapping': {
'clang': self.clang_split_remote_path,
'llvm': self.llvm_split_remote_path,
'-': self.root_split_remote_path
}
}
push_config_json = json.dumps(push_config)
# Create the repo.
cwd = os.getcwd()
os.chdir(self.path)
git('init')
monorepo_test_harness.create_simple_test_harness(
push_config_json=push_config_json)
self.internal_head = git_output('rev-parse', 'internal/master')
# Create the clone.
os.chdir(self.clone_path)
git('init')
git('remote', 'add', 'origin', self.path)
git('fetch', 'origin')
# Create the split clones.
def create_split_remote(path: str, split_dir: str, branch_name: str):
os.chdir(path)
git('init', '--bare')
git('remote', 'add', 'origin', self.path)
git('fetch', 'origin')
git('branch', '-f', branch_name,
f'origin/split/{split_dir}/internal/master')
create_split_remote(self.clang_split_remote_path, 'clang', 'master')
create_split_remote(self.llvm_split_remote_path, 'llvm', 'master')
create_split_remote(self.root_split_remote_path, '-',
'internal/master')
# Back to the original CWD.
os.chdir(cwd)
def test_monorepo_simple_test_harness(cd_to_monorepo):
internal_commits = git_output('rev-list', 'internal/master').splitlines()
assert len(internal_commits) == 16
trailers = git_output('show', '--format=%B', internal_commits[0])
assert 'apple-llvm-split-commit:' in trailers
assert 'apple-llvm-split-dir: -/' in trailers
assert PosixPath('clang/dir/file2').is_file()
upstream_commits = git_output('rev-list', 'llvm/master').splitlines()
assert len(upstream_commits) == 9
assert internal_commits[-1] == upstream_commits[-1]
# Verify that each upstream commit is in downstream.
for commit in upstream_commits:
git('merge-base', '--is-ancestor', commit, 'internal/master')
internal_clang_commits = git_output(
'rev-list', 'split/clang/internal/master').splitlines()
assert len(internal_clang_commits) == 7
upstream_clang_commits = git_output(
'rev-list', 'split/clang/upstream/master').splitlines()
assert len(upstream_clang_commits) == 4
def compute_unmerged_commits(remote: str,
target_branch: str,
upstream_branch: str,
format: str = '%H') -> Optional[List[str]]:
""" Returns the list of commits that are not yet merged from upstream to the target branch. """
commit_log_output = git_output(
'log',
'--first-parent',
f'--pretty=format:{format}',
'--no-patch',
f'{remote}/{target_branch}..{remote}/{upstream_branch}',
)
if not commit_log_output:
return None
return commit_log_output.split('\n')
def dispatch_test_plan_for_pull_request(self, name: str, pr_number: int):
""" Loads a test plan and dispatches it for a given pull request. """
test_plans_filename = 'apple-llvm-config/ci-test-plans.json'
log.debug('Test plan dispatcher: loading test plans %s',
test_plans_filename)
test_plans: Dict[str, TestPlan] = {}
tp = json.loads(git_output(
'show', f'HEAD:{test_plans_filename}'))['test-plans']
for k in tp:
test_plans[k] = TestPlan(k, tp[k])
if name not in test_plans:
raise TestPlanNotFoundError(name)
log.debug('Test plan dispatcher: invoking %s for pull request #%s',
name, str(pr_number))
test_plans[name].dispatch({'pullRequestID': str(pr_number)})
def commit_file(filename: str,
text: str,
trailers: Optional[str] = None) -> str:
""" Commits the contents to the given filename and returns the commit hash. """
update_path = PosixPath(filename)
if len(update_path.parent.name) > 0:
(update_path.parent).mkdir(parents=True, exist_ok=True)
update_path.resolve().write_text(text)
git('add', filename)
msg = f'Updated {filename}'
if 'internal' in text:
msg = f'[internal] {msg}'
if trailers is not None:
msg = f'{msg}\n{trailers}'
git('commit', '-m', msg)
head_commit = git_output('rev-parse', 'HEAD')
return head_commit
def load_pr_config() -> Optional[PRToolConfiguration]:
config = git_output('show',
f'HEAD:apple-llvm-config/pr.json',
ignore_error=True)
if not config:
return None
value = json.loads(config)
type = pr_tool_type_from_string(value['type'])
if not type:
return None
# FIXME: Validate json.
test_type: Optional[CISystemType] = None
if 'test' in value:
test_type = pr_test_type_from_string(value['test']['type'])
return PRToolConfiguration(type=type,
domain=value['domain'],
user=value['user'],
repo=value['repo'],
test_type=test_type)
def test_regraft_commit_graph(cd_to_monorepo):
# Create the test scenario:
# * new-dir/root-file [internal/master] <-- top of commit graph.
# * llvm/file1
# * clang/new-file
# * [origin/internal/master] <-- root of commit graph.
internal_head = git_output('rev-parse', 'internal/master')
clang_change_hash = commit_file('clang/new-file', 'internal: new file')
llvm_change_hash = commit_file('llvm/file1', 'internal: rewrite file1')
root_change_hash = commit_file('new-dir/root-file', 'internal: new file in new dir in -')
commit_graph = CommitGraph([root_change_hash, llvm_change_hash,
clang_change_hash], [internal_head])
# Verify that the expected split commit graphs have been produced.
clang_commit_graph = regraft_commit_graph_onto_split_repo(commit_graph, 'clang')
if 1 != len(clang_commit_graph.roots):
raise AssertionError
if git_output('rev-parse', 'split/clang/internal/master') != clang_commit_graph.roots[0]:
raise AssertionError
if 1 != len(clang_commit_graph.commits):
raise AssertionError
if 'new-file' != git_output('show', clang_commit_graph.commits[0], '--name-only', '--format='):
raise AssertionError
llvm_commit_graph = regraft_commit_graph_onto_split_repo(commit_graph, 'llvm')
if 1 != len(llvm_commit_graph.roots):
raise AssertionError
if git_output('rev-parse', 'split/llvm/internal/master') != llvm_commit_graph.roots[0]:
raise AssertionError
if 1 != len(llvm_commit_graph.commits):
raise AssertionError
if 'file1' != git_output('show', llvm_commit_graph.commits[0], '--name-only', '--format='):
raise AssertionError
root_commit_graph = regraft_commit_graph_onto_split_repo(commit_graph, '-')
if 1 != len(root_commit_graph.roots):
raise AssertionError
if git_output('rev-parse', 'split/-/internal/master') != root_commit_graph.roots[0]:
raise AssertionError
if 1 != len(root_commit_graph.commits):
raise AssertionError
if 'new-dir/root-file' != git_output('show', root_commit_graph.commits[0], '--name-only', '--format='):
raise AssertionError
# "Regenerate" the clang change from the test scenario with appropriate
# monorepo metadata.
# * clang/new-file <-- [internal/master-regenerated]
# * [origin/internal/master] <-- root of commit graph.
git('checkout', '-b', 'internal/master-regenerated', internal_head)
git('clean', '-d', '-f')
trailer = f'\napple-llvm-split-commit: {clang_commit_graph.commits[0]}\napple-llvm-split-dir: clang/'
regenerated_clang_change_hash = commit_file('clang/new-file', 'internal: new file',
trailers=trailer)
# Create the test scenario:
# * clang/dir/subchange <-- top of commit graph.
# * merge
# |\
# | * [internal/master-regenerated] <-- root of commit graph.
# * | clang/dir/subchange
# |/
# * [origin/internal/master] <-- root of commit graph.
git('checkout', '-b', 'internal/feature', internal_head)
git('clean', '-d', '-f')
first_clang_change = commit_file('clang/dir/subchange', 'internal: feature file')
git('merge', regenerated_clang_change_hash)
clang_merge = git_output('rev-parse', 'HEAD')
clang_change_after_merge = commit_file('clang/dir/subchange', 'internal: feature file\nnewline')
merge_graph = CommitGraph([clang_change_after_merge, clang_merge,
first_clang_change], [internal_head,
regenerated_clang_change_hash])
merged_clang_commit_graph = regraft_commit_graph_onto_split_repo(merge_graph, 'clang')
if 2 != len(merged_clang_commit_graph.roots):
raise AssertionError
if clang_commit_graph.roots[0] != merged_clang_commit_graph.roots[1]:
raise AssertionError
if clang_commit_graph.commits[0] != merged_clang_commit_graph.roots[0]:
raise AssertionError
if 3 != len(merged_clang_commit_graph.commits):
raise AssertionError
if 'dir/subchange' != git_output('show', merged_clang_commit_graph.commits[0],
'--name-only', '--format='):
raise AssertionError
if 'dir/subchange' != git_output('show', merged_clang_commit_graph.commits[2],
'--name-only', '--format='):
raise AssertionError
# Try to regraft something unmodified in the commit graph.
clang_only_commit_graph = CommitGraph([clang_change_hash], [internal_head])
no_llvm_commit_graph = regraft_commit_graph_onto_split_repo(clang_only_commit_graph, 'llvm')
if None != no_llvm_commit_graph:
raise AssertionError
no_root_commit_graph = regraft_commit_graph_onto_split_repo(clang_only_commit_graph, '-')
if None != no_root_commit_graph:
raise AssertionError
def clone_url(self):
return git_output('remote', 'get-url', 'origin')
def git_apple_llvm_push(refspec, dry_run, verbose, merge_strategy, push_limit):
""" Push changes back to the split Git repositories. """
logging.basicConfig(level=logging.DEBUG if verbose else logging.WARNING,
format='%(levelname)s: %(message)s')
# Verify that we're in a git checkout.
git_path = get_current_checkout_directory()
if git_path is None:
fatal('not a git repository')
os.chdir(git_path)
# Figure out the set of remote branches we care about.
remote = 'origin'
remote_monorepo_branches = [
x.strip() for x in git_output('branch', '-r', '-l').splitlines()
]
remote_monorepo_branches = list(
filter(lambda x: isKnownTrackingBranch(remote, x),
remote_monorepo_branches))
log.info('Branches we care about %s', remote_monorepo_branches)
refs = refspec.split(':')
if len(refs) < 2:
fatal(f'Git refspec "{refspec}" is invalid')
source_ref = refs[0]
dest_ref = refs[1]
remote_dest_ref = f'{remote}/{dest_ref}'
# Verify that the source ref is valid and get its commit hash.
source_commit_hash = git_output('rev-parse', source_ref, ignore_error=True)
if source_commit_hash is None:
fatal(f'source Git refspec "{source_ref}" is invalid')
# Ensure that the source ref is associated with a ref that can be fetched.
git('branch', '-f', MONOREPO_SRC_REF_NAME, source_commit_hash)
# Verify that the destination ref is valid and load its push config.
dest_commit_hash = git_output('rev-parse',
remote_dest_ref,
ignore_error=True)
if dest_commit_hash is None:
fatal(f'destination Git refspec "{dest_ref}" is invalid')
push_config = load_push_config(source_commit_hash, dest_ref)
if push_config is None:
fatal(f'destination Git refspec "{dest_ref}" cannot be pushed to.')
# The rev-list command is used to compute the graph we would like to
# commit.
rev_list = git_output('rev-list',
'--boundary',
source_commit_hash,
'--not',
*remote_monorepo_branches,
ignore_error=True)
if rev_list is None:
fatal('unable to determine the commit graph to push')
commit_graph = compute_commit_graph(rev_list)
if commit_graph is None:
print('No commits to commit: everything up-to-date.')
return
# Prohibit pushing more than 50 commits by default in a bid to avoid
# inadvertent mistakes.
if push_limit != 0 and len(commit_graph.commits) >= push_limit:
fatal(
f'pushing {len(commit_graph.commits)} commits, are you really sure?'
f'\nPass --push-limit={len(commit_graph.commits)+1} if yes.')
click.echo(
click.style(
f'Preparing to push to {len(commit_graph.commits)} commits:',
bold=True))
git('log', '--format=%h %s', '--graph', commit_graph.source_commit_hash,
'--not', *commit_graph.roots)
message_bodies = git_output('log', '--format=%b',
commit_graph.source_commit_hash, '--not',
*commit_graph.roots)
if 'apple-llvm-split-commit:' in message_bodies:
fatal('one or more commits is already present in the split repo')
# Prepare the split remotes.
split_repos_of_interest = commit_graph.compute_changed_split_repos()
click.echo(
f'Split repos that should be updated: {", ".join(map(split_dir_to_str, split_repos_of_interest))}\n'
)
split_remotes = {}
for split_dir in split_repos_of_interest:
if not push_config.can_push_to_split_dir(split_dir):
fatal(
f'push configuration "{push_config.name}" prohibits pushing to "{split_dir}"'
)
remote = SplitRemote(
split_dir, push_config.repo_mapping[split_dir],
push_config.get_split_repo_branch(split_dir, dest_ref))
click.echo(
click.style(
f'Fetching "{remote.destination_branch}" for {split_dir_to_str(split_dir)}...',
bold=True))
try:
remote.update_remote()
except GitError:
fatal(
f'failed to fetch from the remote for {split_dir_to_str(split_dir)}.'
)
click.echo(
'Fetching monorepo commits from monorepo to the split clone (takes time on first push)...\n'
)
remote.update_mono_remote()
split_remotes[split_dir] = remote
# Regraft the commit history.
for split_dir in split_repos_of_interest:
click.echo(
click.style(
f'Regrafting the commits from monorepo to {split_dir_to_str(split_dir)}...',
bold=True))
split_remotes[split_dir].begin()
split_remotes[
split_dir].regrafted_graph = regraft_commit_graph_onto_split_repo(
commit_graph, split_dir)
# Merge/rebase the commit history.
for split_dir in split_repos_of_interest:
click.echo(
click.style(
f'\nRebasing/merging the {split_dir_to_str(split_dir)} commits...',
bold=True))
remote = split_remotes[split_dir]
remote.begin()
try:
remote.commit_hash = merge_commit_graph_with_top_of_branch(
remote.regrafted_graph, split_dir,
'origin/' + remote.destination_branch, merge_strategy)
except ImpossibleMergeError as err:
fatal(
f'unable to {err.operation} commits in {split_dir_to_str(split_dir)}.'
f'Please rebase your monorepo commits first.')
# Once everything is ready, push!
for split_dir in split_repos_of_interest:
split_remotes[split_dir].push(dry_run)
def merge_commit_graph_with_top_of_branch(commit_graph: CommitGraph,
split_dir: str,
destination_branch: str,
strategy: MergeStrategy) -> str:
""" Merge/rebase a regrafted split commit graph on top of the
destination split branch. """
# The relative path (from the split .git directory) to the temporary
# working checkout.
split_worktree_path = f'.git/apple-llvm-push-checkout-{split_dir}'
branch_name = f'temp-apple-llvm-push-merged-{split_dir}'
git('worktree',
'remove',
'--force',
split_worktree_path,
ignore_error=True,
stdout=get_dev_null(),
stderr=get_dev_null())
git('branch',
'-f',
'-D',
branch_name,
ignore_error=True,
stdout=get_dev_null(),
stderr=get_dev_null())
git('worktree', 'add', '-f', '-b', branch_name, split_worktree_path,
destination_branch)
with ExitStack() as cleanups:
# Remove the worktree once we're done.
cleanups.callback(lambda: git(
'worktree', 'remove', '-f', split_worktree_path, ignore_error=True)
)
# Try the fast-forward only first.
try:
git('merge',
'--ff-only',
commit_graph.source_commit_hash,
git_dir=split_worktree_path)
except GitError as err:
if strategy == MergeStrategy.FastForwardOnly:
raise ImpossibleMergeError('fast forward', err)
if strategy != MergeStrategy.FastForwardOnly:
# Try rebasing.
if strategy != MergeStrategy.MergeOnly and not commit_graph.has_merges:
try:
git('rebase',
'--onto',
branch_name,
branch_name,
commit_graph.source_commit_hash,
git_dir=split_worktree_path)
except GitError as err:
if strategy == MergeStrategy.Rebase:
raise ImpossibleMergeError('rebase', err)
click.echo('rebase failed, trying merge...')
git('rebase', '--abort', git_dir=split_worktree_path)
git('worktree',
'remove',
'-f',
split_worktree_path,
ignore_error=True)
cleanups.pop_all()
return merge_commit_graph_with_top_of_branch(
commit_graph, split_dir, destination_branch,
MergeStrategy.MergeOnly)
elif strategy == MergeStrategy.Rebase:
raise ImpossibleMergeError(
'rebase',
GitError(['rebase'],
1,
stdout='',
stderr='unable to rebase history with merges'))
else:
assert strategy == MergeStrategy.RebaseOrMerge or strategy == MergeStrategy.MergeOnly
# Fallback to merge.
try:
git('merge',
commit_graph.source_commit_hash,
git_dir=split_worktree_path)
except GitError as err:
raise ImpossibleMergeError('merge', err)
result = git_output('rev-parse', 'HEAD', git_dir=split_worktree_path)
return result
def regraft_commit_graph_onto_split_repo(
commit_graph: CommitGraph, split_dir: str) -> Optional[CommitGraph]:
""" This function takes a monorepo commit graph and regrafts it on top of
the appropriate split repo commit graph. The resulting commit graph
is returned, or nothing if the rewrite lead to empty commits.
This is done using the git `filter-branch` command to recreate commits.
It operates on trees instead of diffs, so the commits have to be
re-parented appropriately to preserve the intended diff between the two
trees.
The roots on which the regraft is performed are found by looking for
the matching `apple-llvm-split-*` trailers to determine the approriate
split repo root. Right now regrafting from the upstream LLVM.org monorepo
roots isn't unsupported (FIXME).
Raises `RegraftMissingSplitRootError` or `RegraftMissingSplitRootError`
when the commit graph roots can't be remapped.
"""
assert split_dir == '-' or split_dir in all_monorepo_split_dirs
base_split_commits = {}
for root in commit_graph.roots:
base_split_commit = find_base_split_commit(split_dir, root)
if not base_split_commit:
raise RegraftNoSplitRootError(root)
if not commit_exists(base_split_commit):
raise RegraftMissingSplitRootError(root)
base_split_commits[root] = base_split_commit
# Construct a filter the rewrites the history to the split directory.
dir_filter_args: List[str] = []
if split_dir == '-':
split_dirs = ' '.join(list(all_monorepo_split_dirs))
dir_filter_args = [
'--index-filter',
f'git rm -r --cached --ignore-unmatch {split_dirs}'
]
else:
dir_filter_args = [
'--index-filter',
f'git read-tree $(git rev-parse $GIT_COMMIT:{split_dir})'
]
# Construct a parent filter to replace roots with split base commits.
parent_filter_cmd = 'cat | sed ' + ' '.join([
f'-e s,{mono},{split},'
for (mono, split) in base_split_commits.items()
])
# Setup a work branch that should be rewritten.
branch_name = f'temp-apple-llvm-push-{split_dir}'
git('branch', '-f', branch_name, commit_graph.source_commit_hash)
# Rewrite the branch.
try:
# We need the `-f` filter-branch argument to force overwrite of the
# backup created by git.
# FIXME: Investigate if we can remove it.
git('filter-branch',
'-f',
'--prune-empty',
'--parent-filter',
parent_filter_cmd,
*dir_filter_args,
branch_name,
'--not',
*commit_graph.roots,
stdout=get_dev_null())
except GitError as err:
# Nothing was rewritten!
if err.stderr.find('nothing to rewrite') != -1:
return None
raise err
# Compute the updated commit graph.
rev_list = git_output(
'rev-list', '--boundary', branch_name, '--not',
*[split for (mono, split) in base_split_commits.items()])
split_commit_graph = compute_commit_graph(rev_list)
if split_commit_graph is None:
return None
result = CommitGraph(split_commit_graph.commits, split_commit_graph.roots)
# Verify the integrity of the regraft by checking changed files.
original_changed_files = commit_graph.compute_changed_files(
split_dir=split_dir)
regrafted_changed_files = result.compute_changed_files()
assert original_changed_files == regrafted_changed_files
return result
def __get_changed_filenames(self) -> str:
return git_output('log', '--format=', '--name-only',
self.source_commit_hash, '--not', *self.roots)