def update_mono_remote(self):
if not has_existing_remote('mono', self.monorepo_remote_url,
git_dir=self.remote_clone_dir):
git('remote', 'add', 'mono', self.monorepo_remote_url,
git_dir=self.remote_clone_dir)
git('fetch', 'mono', MONOREPO_SRC_REF_NAME,
git_dir=self.remote_clone_dir)
def update_remote(self):
if not has_existing_remote('origin', self.remote_url, git_dir=self.remote_clone_dir):
git('remote', 'add', 'origin', self.remote_url,
git_dir=self.remote_clone_dir)
log.debug('fetching the remote for %s', self.split_dir)
git('fetch', '--no-tags', 'origin', self.destination_branch,
git_dir=self.remote_clone_dir, stderr=None)
def status(target: Optional[str], all_commits: bool, no_fetch: bool, ci_status: bool):
remote = 'origin'
if not no_fetch:
click.echo(f'❕ Fetching "{remote}" to provide the latest status...')
git('fetch', remote, stderr=None)
click.echo('✅ Fetch succeeded!\n')
print_status(remote=remote, target_branch=target, list_commits=all_commits, query_ci_status=ci_status)
def test_rebase_commit_graph(split_clang_head_in_monorepo: str,
merge_strategy: MergeStrategy):
work_head = commit_file('file1', 'top of branch is here')
branch_name = f'new-rebase/split/clang/{merge_strategy}'
git('checkout', '-b', branch_name, split_clang_head_in_monorepo)
new_clang_head = commit_file('file2', 'top of clang/master is here')
def doit() -> str:
return merge_commit_graph_with_top_of_branch(CommitGraph([work_head], [split_clang_head_in_monorepo]),
'clang',
branch_name,
merge_strategy)
# Fast forward only fails with disjoint graph.
if merge_strategy == MergeStrategy.FastForwardOnly:
with pytest.raises(ImpossibleMergeError) as err:
doit()
if 'Not possible to fast-forward' not in err.value.git_error.stderr:
raise AssertionError
else:
# However, other strategies rebase the commit graph on top of the
# destination branch.
commit = doit()
if work_head == commit:
raise AssertionError
if new_clang_head == commit:
raise AssertionError
if new_clang_head != git_output('rev-parse', f'{commit}~1'):
raise AssertionError
def create_monorepo(commits: List[CommitBlueprint]):
# Create the upstream monorepo.
start_new_orphan_branch('llvm/master')
for commit in commits:
if isinstance(commit, BlobCommitBlueprint) and not commit.is_internal:
commit.monorepo_commit_hash = commit_file(commit.monorepo_filename,
commit.text)
# Create the downstream monorepo.
is_first = True
for commit in commits:
hs = commit.split_commit_hash
trailers = f'\n---\napple-llvm-split-commit: {hs}\napple-llvm-split-dir: {commit.split_dir}/'
if isinstance(commit, BlobCommitBlueprint) and commit.is_internal:
assert commit.parent and commit.parent.monorepo_commit_hash is not None
if is_first:
git('checkout', '-b', 'internal/master',
commit.parent.monorepo_commit_hash)
git('clean', '-d', '-f')
is_first = False
commit.monorepo_commit_hash = commit_file(commit.monorepo_filename,
commit.text,
trailers=trailers)
elif isinstance(commit, MergeCommitBlueprint):
assert not is_first
# Verify the the expected split merge is already there.
git('merge-base', '--is-ancestor',
commit.downstream.monorepo_commit_hash, 'internal/master')
# Recreate the merge.
assert commit.upstream.monorepo_commit_hash is not None
git('merge', '--no-commit', commit.upstream.monorepo_commit_hash)
msg = f'Merge {commit.upstream.monorepo_commit_hash} into internal/master\n{trailers}'
git('commit', '-m', msg)
commit.monorepo_commit_hash = git_output('rev-parse', 'HEAD')
def graph(format: str, remote: List[str], no_fetch: bool, ci_status: bool):
remotes = remote if remote else ['origin']
if not no_fetch:
for r in remotes:
click.echo(f'❕ Fetching "{r}" to provide the latest status...')
git('fetch', r, stderr=None)
print_graph(remotes=remotes, query_ci_status=ci_status, fmt=format)
def test_monorepo_simple_test_harness(cd_to_monorepo):
internal_commits = git_output('rev-list', 'internal/master').splitlines()
if len(internal_commits) != 16:
raise AssertionError
trailers = git_output('show', '--format=%B', internal_commits[0])
if 'apple-llvm-split-commit:' not in trailers:
raise AssertionError
if 'apple-llvm-split-dir: -/' not in trailers:
raise AssertionError
if not PosixPath('clang/dir/file2').is_file():
raise AssertionError
upstream_commits = git_output('rev-list', 'llvm/master').splitlines()
if len(upstream_commits) != 9:
raise AssertionError
if internal_commits[-1] != upstream_commits[-1]:
raise AssertionError
# Verify that each upstream commit is in downstream.
for commit in upstream_commits:
git('merge-base', '--is-ancestor', commit, 'internal/master')
internal_clang_commits = git_output('rev-list', 'split/clang/internal/master').splitlines()
if len(internal_clang_commits) != 7:
raise AssertionError
upstream_clang_commits = git_output('rev-list', 'split/clang/upstream/master').splitlines()
if len(upstream_clang_commits) != 4:
raise AssertionError
def test_reject_mapped_commit(cd_to_monorepo_clone):
git('commit', '--allow-empty', '-m', '''This commit is already mapped!
apple-llvm-split-commit: f0931a1b36c88157ffc25a9ed1295f3addff85b9\n
apple-llvm-split-dir: llvm/''')
result = CliRunner().invoke(git_apple_llvm_push, ['HEAD:internal/master'])
assert 'one or more commits is already present in the split repo' in result.output
assert result.exit_code == 1
def test_cli_tool_no_pr_config(tmp_path):
prev = os.getcwd()
os.chdir(str(tmp_path))
git('init')
result = CliRunner().invoke(pr, ['list'], mix_stderr=True)
assert result.exit_code == 1
assert 'missing `git apple-llvm pr` configuration file' in result.output
os.chdir(prev)
def test_regraft_missing_split_root(cd_to_monorepo):
# Test the scenario with a missing split root.
git('checkout', '-b', 'fake-internal-master', 'internal/master')
git('commit', '--allow-empty', '-m', 'error\n\napple-llvm-split-commit: 00000000000\napple-llvm-split-dir: clang/')
bad_master_head = git_output('rev-parse', 'HEAD')
bad_clang_hash = commit_file('clang/another-file', 'internal: new file')
with pytest.raises(RegraftMissingSplitRootError) as err:
regraft_commit_graph_onto_split_repo(CommitGraph([bad_clang_hash], [bad_master_head]), 'clang')
assert err.value.root_commit_hash == bad_master_head
def has_existing_remote(remote_name: str, remote_url: str, **kwargs):
try:
url = git_output('remote', 'get-url', remote_name, **kwargs)
if url != remote_url:
git('remote', 'remove', remote_name, **kwargs)
return False
return True
except GitError:
return False
def test_cli_tool_no_pr_config(tmp_path):
prev = os.getcwd()
os.chdir(str(tmp_path))
git('init')
result = CliRunner().invoke(pr, ['list'])
if result.exit_code != 1:
raise AssertionError
if 'missing `git apple-llvm pr` configuration file' not in result.output:
raise AssertionError
os.chdir(prev)
def push(self, dry_run: bool = False):
click.echo(click.style(
f'\nPushing to {split_dir_to_str(self.split_dir)}:', bold=True))
if dry_run:
click.echo('🛑 dry run, stopping before pushing.')
return
git('push', 'origin',
f'{self.commit_hash}:{self.destination_branch}',
git_dir=self.remote_clone_dir,
stderr=None)
def commit_merge(self, downstream: CommitBlueprint,
upstream: CommitBlueprint) -> MergeCommitBlueprint:
checkout_and_clean(downstream.split_commit_hash)
git('merge', upstream.split_commit_hash)
commit_hash = git_output('rev-parse', 'HEAD')
git('branch', '-f', f'split/{self.split_dir}/internal/master',
commit_hash)
return MergeCommitBlueprint(split_dir=self.split_dir,
split_commit_hash=commit_hash,
downstream=downstream,
upstream=upstream)
def create_split_remote(path: str, split_dir: str, branch_name: str):
os.chdir(path)
git('init', '--bare')
git('remote', 'add', 'origin', self.path)
git('fetch', 'origin')
git('branch', '-f', branch_name,
f'origin/split/{split_dir}/internal/master')
def am_tool_git_repo_clone(tmp_path_factory, am_tool_git_repo: str) -> str:
path = str(tmp_path_factory.mktemp('simple-am-tool-dir-clone'))
git('init', git_dir=path)
git('remote', 'add', 'origin', am_tool_git_repo, git_dir=path)
git('fetch', 'origin', git_dir=path)
git('checkout', 'master', git_dir=path)
return path
def status(target: Optional[str], all_commits: bool, remote: List[str],
no_fetch: bool, ci_status: bool, graph: bool, graph_format: str):
remotes = remote if remote else ['origin']
if not no_fetch:
for r in remotes:
click.echo(f'❕ Fetching "{r}" to provide the latest status...')
git('fetch', r, stderr=None)
click.echo('✅ Fetch succeeded!\n')
if graph and not graph_format:
graph_format = 'pdf'
print_status(remotes=remotes,
target_branch=target,
list_commits=all_commits,
query_ci_status=ci_status,
graph_format=graph_format)
def test_merge_conflict(split_clang_head_in_monorepo: str,
merge_strategy: MergeStrategy):
push_clang_commit = commit_file('file1', 'top of branch is here')
branch_name = f'rebase-fail/split/clang/{merge_strategy}'
git('checkout', '-b', branch_name, split_clang_head_in_monorepo)
commit_file('file1', 'rebase this without conflict')
graph = CommitGraph([push_clang_commit], [split_clang_head_in_monorepo])
with pytest.raises(ImpossibleMergeError) as err:
merge_commit_graph_with_top_of_branch(graph, 'clang', branch_name,
merge_strategy)
if merge_strategy == MergeStrategy.Rebase:
assert err.value.operation == 'rebase'
elif merge_strategy == MergeStrategy.RebaseOrMerge:
assert err.value.operation == 'merge'
def commit_file(filename: str,
text: str,
trailers: Optional[str] = None) -> str:
""" Commits the contents to the given filename and returns the commit hash. """
update_path = PosixPath(filename)
if len(update_path.parent.name) > 0:
(update_path.parent).mkdir(parents=True, exist_ok=True)
update_path.resolve().write_text(text)
git('add', filename)
msg = f'Updated {filename}'
if 'internal' in text:
msg = f'[internal] {msg}'
if trailers is not None:
msg = f'{msg}\n{trailers}'
git('commit', '-m', msg)
head_commit = git_output('rev-parse', 'HEAD')
return head_commit
def __init__(self, path: str, clone_path: str,
clang_split_remote_path: str, llvm_split_remote_path: str,
root_split_remote_path: str):
self.path = path
self.clone_path = clone_path
self.clang_split_remote_path = clang_split_remote_path
self.llvm_split_remote_path = llvm_split_remote_path
self.root_split_remote_path = root_split_remote_path
# Configure the push configuration.
push_config = {
'branch_to_dest_branch_mapping': {
'internal/master:-': 'internal/master',
'internal/master:*': 'master'
},
'repo_mapping': {
'clang': self.clang_split_remote_path,
'llvm': self.llvm_split_remote_path,
'-': self.root_split_remote_path
}
}
push_config_json = json.dumps(push_config)
# Create the repo.
cwd = os.getcwd()
os.chdir(self.path)
git('init')
monorepo_test_harness.create_simple_test_harness(
push_config_json=push_config_json)
self.internal_head = git_output('rev-parse', 'internal/master')
# Create the clone.
os.chdir(self.clone_path)
git('init')
git('remote', 'add', 'origin', self.path)
git('fetch', 'origin')
# Create the split clones.
def create_split_remote(path: str, split_dir: str, branch_name: str):
os.chdir(path)
git('init', '--bare')
git('remote', 'add', 'origin', self.path)
git('fetch', 'origin')
git('branch', '-f', branch_name,
f'origin/split/{split_dir}/internal/master')
create_split_remote(self.clang_split_remote_path, 'clang', 'master')
create_split_remote(self.llvm_split_remote_path, 'llvm', 'master')
create_split_remote(self.root_split_remote_path, '-',
'internal/master')
# Back to the original CWD.
os.chdir(cwd)
def test_cli_tool_create_pr_invalid_base(cd_to_pr_tool_repo_clone, pr_tool_type):
git('checkout', 'master')
git('branch', '-D', 'pr_branch2', ignore_error=True)
git('checkout', '-b', 'pr_branch2')
git('push', 'origin', '-u', '-f', 'pr_branch2')
mock_tool = MockPRTool()
git_apple_llvm.pr.main.pr_tool = create_pr_tool(mock_tool, pr_tool_type)
# PR creation fails when the branch is not pushed.
result = CliRunner().invoke(pr, ['create', '-m', 'test pr', '-b', 'mastar', '-h', 'pr_branch2'])
assert result.exit_code == 1
assert 'base branch "mastar" is not a valid remote tracking branch' in result.output
def test_git_invocation(tmp_path):
""" Tests for the git/git_output functions. """
repo_path = tmp_path / 'repo'
repo_path.mkdir()
assert repo_path.is_dir() # Ensure the dir is there for us to work with.
repo_dir = str(repo_path)
git('init', git_dir=repo_dir)
(repo_path / 'initial').write_text(u'initial')
git('add', 'initial', git_dir=repo_dir)
# Check that we can report an error on failure.
with pytest.raises(GitError) as err:
git('add', 'foo', git_dir=repo_dir)
assert err.value.stderr.startswith('fatal')
assert repr(err.value).startswith('GitError')
# Check that errors can be ignored.
git('add', 'foo', git_dir=repo_dir, ignore_error=True)
output = git_output('commit', '-m', 'initial', git_dir=repo_dir)
assert len(output) > 0
# Ensure that the output is stripped.
output = git_output('rev-list', 'HEAD', git_dir=repo_dir)
assert '\n' not in output
output = git_output('rev-list', 'HEAD', git_dir=repo_dir, strip=False)
assert '\n' in output
# Ensure that commit exists works only for commit hashes.
hash = output.strip()
assert commit_exists(hash)
assert not commit_exists('HEAD')
assert not commit_exists(hash + 'abc')
assert not commit_exists('000000')
# Ensure that we can get the directory of the checkout even when the
# working directory is a subdirectory.
os.chdir(repo_dir)
dir_a = get_current_checkout_directory()
(repo_path / 'subdir').mkdir()
cwd = os.getcwd()
os.chdir(os.path.join(repo_dir, 'subdir'))
dir_b = get_current_checkout_directory()
os.chdir(cwd)
assert dir_a == dir_b
assert read_file_or_none('HEAD', 'initial') == 'initial'
assert read_file_or_none(hash, 'initial') == 'initial'
assert read_file_or_none('HEAD', 'path/does-not-exist') is None
assert read_file_or_none('foo', 'initial') is None
def commit_file(self,
filename: str,
text: str,
parent: Optional[BlobCommitBlueprint] = None,
internal: bool = False) -> BlobCommitBlueprint:
if parent is not None:
checkout_and_clean(parent.split_commit_hash)
commit_hash = commit_file(filename, text)
result = BlobCommitBlueprint(
split_dir=self.split_dir,
filename=filename,
text=text,
split_commit_hash=commit_hash,
parent=parent if parent else self.prev_parent,
is_internal=internal)
kind = 'internal' if internal else 'upstream'
git('branch', '-f', f'split/{self.split_dir}/{kind}/master',
commit_hash)
self.prev_parent = result
return result
def find_inflight_merges(remote: str = 'origin') -> Dict[str, List[str]]:
"""
This function fetches the refs created by the automerger to find
the inflight merges that are currently being processed.
"""
# Delete the previously fetched refs to avoid fetch failures
# where there were force pushes.
existing_refs = git_output('for-each-ref', AM_STATUS_PREFIX,
'--format=%(refname)').split('\n')
for ref in existing_refs:
if not ref:
continue
log.debug(f'Deleting local ref "{ref}" before fetching')
git('update-ref', '-d', ref)
git('fetch', remote,
f'{AM_PREFIX}*:{AM_STATUS_PREFIX}*') # FIXME: handle fetch failures.
refs = git_output('for-each-ref', AM_STATUS_PREFIX,
'--format=%(refname)').split('\n')
inflight_merges: Dict[str, List[str]] = {}
for ref in refs:
if not ref:
continue
if not ref.startswith(AM_STATUS_PREFIX):
raise AssertionError
merge_name = ref[len(AM_STATUS_PREFIX):]
underscore_idx = merge_name.find('_')
if underscore_idx == -1:
raise AssertionError
commit_hash = merge_name[:underscore_idx]
dest_branch = merge_name[underscore_idx + 1:]
if dest_branch in inflight_merges:
inflight_merges[dest_branch].append(commit_hash)
else:
inflight_merges[dest_branch] = [commit_hash]
for (m, k) in inflight_merges.items():
log.debug(f'in-flight {m}: {k}')
return inflight_merges
def ci_tool_git_repo(tmp_path_factory) -> str:
path = str(tmp_path_factory.mktemp('simple-ci-tool-dir'))
test_plans = {
"test-plans": {
"check-llvm": {
"description": "Runs lit and unit tests for LLVM",
"infer-from-changes": ["llvm"],
"ci-jobs": "pull-request-RA",
"params": {
"monorepo_projects": "",
"test_targets": "check-llvm"
}
}
}
}
ci_jobs = {
"type":
"jenkins",
"url":
TEST_API_URL,
"jobs": [{
"name": "a-RA",
"url": TEST_API_URL + "/view/monorepo/job/pr-build-test",
"params": {
"build_variant": "a"
}
}, {
"name": "b-RA",
"url": TEST_API_URL + "/view/monorepo/job/pr-build-test",
"params": {
"build_variant": "b"
}
}]
}
# Create the repo with the CI and test plan configs.
git('init', git_dir=path)
os.mkdir(os.path.join(path, 'apple-llvm-config'))
with open(os.path.join(path, 'apple-llvm-config', 'ci-test-plans.json'),
'w') as f:
f.write(json.dumps(test_plans))
os.mkdir(os.path.join(path, 'apple-llvm-config/ci-jobs'))
with open(
os.path.join(path, 'apple-llvm-config/ci-jobs',
'pull-request-RA.json'), 'w') as f:
f.write(json.dumps(ci_jobs))
git('add', 'apple-llvm-config', git_dir=path)
git('commit', '-m', 'ci config', git_dir=path)
git('checkout', '-b', 'repo/apple-llvm-config/pr', git_dir=path)
return path
def test_monorepo_simple_test_harness(cd_to_monorepo):
internal_commits = git_output('rev-list', 'internal/master').splitlines()
assert len(internal_commits) == 16
trailers = git_output('show', '--format=%B', internal_commits[0])
assert 'apple-llvm-split-commit:' in trailers
assert 'apple-llvm-split-dir: -/' in trailers
assert PosixPath('clang/dir/file2').is_file()
upstream_commits = git_output('rev-list', 'llvm/master').splitlines()
assert len(upstream_commits) == 9
assert internal_commits[-1] == upstream_commits[-1]
# Verify that each upstream commit is in downstream.
for commit in upstream_commits:
git('merge-base', '--is-ancestor', commit, 'internal/master')
internal_clang_commits = git_output(
'rev-list', 'split/clang/internal/master').splitlines()
assert len(internal_clang_commits) == 7
upstream_clang_commits = git_output(
'rev-list', 'split/clang/upstream/master').splitlines()
assert len(upstream_clang_commits) == 4
def test_merge_commit_graph(split_clang_head_in_monorepo: str,
merge_strategy: MergeStrategy):
# Create the test scenario:
# * merge <-- top of commit graph.
# |\
# | * [new-merge/split/clang] <-- destination.
# | \/
# | * file2 <-- root of commit graph.
# * | file1
# |/
# * [split/clang/internal/master] <-- root of commit graph.
push_clang_commit = commit_file('file1', 'top of branch is here')
branch_name = f'new-merge/split/clang/{merge_strategy}'
git('checkout', '-b', branch_name, split_clang_head_in_monorepo)
up_clang_head = commit_file('file2', 'merging this in')
new_clang_head = commit_file('file3', 'top of clang/master is here')
git('checkout', '--detach', up_clang_head)
git('clean', '-d', '-f')
git('merge', push_clang_commit)
merge_commit = git_output('rev-parse', 'HEAD')
def doit() -> str:
graph = CommitGraph([merge_commit, push_clang_commit], [up_clang_head, split_clang_head_in_monorepo])
return merge_commit_graph_with_top_of_branch(graph,
'clang',
branch_name,
merge_strategy)
# Graph with merges can only be merged.
if merge_strategy == MergeStrategy.RebaseOrMerge:
commit = doit()
if merge_commit == commit:
raise AssertionError
parents = git_output('show', '--format=%P', commit).split()
if 2 != len(parents):
raise AssertionError
if new_clang_head != parents[0]:
raise AssertionError
if merge_commit != parents[1]:
raise AssertionError
else:
with pytest.raises(ImpossibleMergeError):
doit()
def has_merge_conflict(commit: str,
target_branch: str,
remote: str = 'origin') -> bool:
""" Returns true if the given commit hash has a merge conflict with the given target branch.
"""
try:
# Always remove the temporary worktree. It's possible that we got
# interrupted and left it around. This will raise an exception if the
# worktree doesn't exist, which can be safely ignored.
git('worktree',
'remove',
'--force',
'.git/temp-worktree',
stdout=get_dev_null(),
stderr=get_dev_null())
except GitError:
pass
git('worktree',
'add',
'.git/temp-worktree',
f'{remote}/{target_branch}',
'--detach',
stdout=get_dev_null(),
stderr=get_dev_null())
try:
git('merge',
'--no-commit',
commit,
git_dir='.git/temp-worktree',
stdout=get_dev_null(),
stderr=get_dev_null())
return False
except GitError:
return True
finally:
git('worktree',
'remove',
'--force',
'.git/temp-worktree',
stdout=get_dev_null(),
stderr=get_dev_null())
def test_different_tracked_branch_ref(cd_to_monorepo_clone, monorepo_test_fixture, optional_remote_prefix: str):
branch_name = 'test-tracked-branch'
git('branch', '-D', branch_name, ignore_error=True)
git('checkout', '-b', branch_name)
git('branch', '-u', 'origin/internal/master', branch_name)
ref: Optional[TrackedBranchRef] = get_tracked_branch_ref(optional_remote_prefix + branch_name)
if len(optional_remote_prefix) > 0:
# The ref doesn't exist on the remote
assert ref is None
return
assert ref is not None
assert ref.remote_name == 'origin'
assert ref.remote_url == monorepo_test_fixture.path
assert ref.branch_name == 'internal/master'
def cd_to_pr_tool_repo_clone_adjust_jenkins_ci(cd_to_pr_tool_repo):
pr_config = {
'type': 'github',
'domain': 'github.com',
'user': '******',
'repo': 'apple-llvm-infrastructure-tools',
'test': {
'type': 'jenkins-test-plans'
}
}
with open('apple-llvm-config/pr.json', 'w') as f:
f.write(json.dumps(pr_config))
test_plans = {
"test-plans": {
"pr": {
"description": "",
"ci-jobs": "pull-request-RA",
"params": {
"test_targets": "check-llvm"
}
}
}
}
ci_jobs = {
"type":
"jenkins",
"url":
JENKINS_TEST_API_URL,
"jobs": [{
"name": "a-RA",
"url": JENKINS_TEST_API_URL + "/view/monorepo/job/pr-build-test",
"params": {}
}]
}
# Create the repo with the CI and test plan configs.
with open(os.path.join('apple-llvm-config', 'ci-test-plans.json'),
'w') as f:
f.write(json.dumps(test_plans))
os.mkdir(os.path.join('apple-llvm-config/ci-jobs'))
with open(
os.path.join('apple-llvm-config/ci-jobs', 'pull-request-RA.json'),
'w') as f:
f.write(json.dumps(ci_jobs))
git('add', 'apple-llvm-config')
git('commit', '-m', 'use jenkins now')
yield
git('reset', '--hard', 'HEAD~1')
