def update_mono_remote(self): if not has_existing_remote('mono', self.monorepo_remote_url, git_dir=self.remote_clone_dir): git('remote', 'add', 'mono', self.monorepo_remote_url, git_dir=self.remote_clone_dir) git('fetch', 'mono', MONOREPO_SRC_REF_NAME, git_dir=self.remote_clone_dir)
def update_remote(self): if not has_existing_remote('origin', self.remote_url, git_dir=self.remote_clone_dir): git('remote', 'add', 'origin', self.remote_url, git_dir=self.remote_clone_dir) log.debug('fetching the remote for %s', self.split_dir) git('fetch', '--no-tags', 'origin', self.destination_branch, git_dir=self.remote_clone_dir, stderr=None)
def status(target: Optional[str], all_commits: bool, no_fetch: bool, ci_status: bool): remote = 'origin' if not no_fetch: click.echo(f'❕ Fetching "{remote}" to provide the latest status...') git('fetch', remote, stderr=None) click.echo('✅ Fetch succeeded!\n') print_status(remote=remote, target_branch=target, list_commits=all_commits, query_ci_status=ci_status)
def test_rebase_commit_graph(split_clang_head_in_monorepo: str, merge_strategy: MergeStrategy): work_head = commit_file('file1', 'top of branch is here') branch_name = f'new-rebase/split/clang/{merge_strategy}' git('checkout', '-b', branch_name, split_clang_head_in_monorepo) new_clang_head = commit_file('file2', 'top of clang/master is here') def doit() -> str: return merge_commit_graph_with_top_of_branch(CommitGraph([work_head], [split_clang_head_in_monorepo]), 'clang', branch_name, merge_strategy) # Fast forward only fails with disjoint graph. if merge_strategy == MergeStrategy.FastForwardOnly: with pytest.raises(ImpossibleMergeError) as err: doit() if 'Not possible to fast-forward' not in err.value.git_error.stderr: raise AssertionError else: # However, other strategies rebase the commit graph on top of the # destination branch. commit = doit() if work_head == commit: raise AssertionError if new_clang_head == commit: raise AssertionError if new_clang_head != git_output('rev-parse', f'{commit}~1'): raise AssertionError
def create_monorepo(commits: List[CommitBlueprint]): # Create the upstream monorepo. start_new_orphan_branch('llvm/master') for commit in commits: if isinstance(commit, BlobCommitBlueprint) and not commit.is_internal: commit.monorepo_commit_hash = commit_file(commit.monorepo_filename, commit.text) # Create the downstream monorepo. is_first = True for commit in commits: hs = commit.split_commit_hash trailers = f'\n---\napple-llvm-split-commit: {hs}\napple-llvm-split-dir: {commit.split_dir}/' if isinstance(commit, BlobCommitBlueprint) and commit.is_internal: assert commit.parent and commit.parent.monorepo_commit_hash is not None if is_first: git('checkout', '-b', 'internal/master', commit.parent.monorepo_commit_hash) git('clean', '-d', '-f') is_first = False commit.monorepo_commit_hash = commit_file(commit.monorepo_filename, commit.text, trailers=trailers) elif isinstance(commit, MergeCommitBlueprint): assert not is_first # Verify the the expected split merge is already there. git('merge-base', '--is-ancestor', commit.downstream.monorepo_commit_hash, 'internal/master') # Recreate the merge. assert commit.upstream.monorepo_commit_hash is not None git('merge', '--no-commit', commit.upstream.monorepo_commit_hash) msg = f'Merge {commit.upstream.monorepo_commit_hash} into internal/master\n{trailers}' git('commit', '-m', msg) commit.monorepo_commit_hash = git_output('rev-parse', 'HEAD')
def graph(format: str, remote: List[str], no_fetch: bool, ci_status: bool): remotes = remote if remote else ['origin'] if not no_fetch: for r in remotes: click.echo(f'❕ Fetching "{r}" to provide the latest status...') git('fetch', r, stderr=None) print_graph(remotes=remotes, query_ci_status=ci_status, fmt=format)
def test_monorepo_simple_test_harness(cd_to_monorepo): internal_commits = git_output('rev-list', 'internal/master').splitlines() if len(internal_commits) != 16: raise AssertionError trailers = git_output('show', '--format=%B', internal_commits[0]) if 'apple-llvm-split-commit:' not in trailers: raise AssertionError if 'apple-llvm-split-dir: -/' not in trailers: raise AssertionError if not PosixPath('clang/dir/file2').is_file(): raise AssertionError upstream_commits = git_output('rev-list', 'llvm/master').splitlines() if len(upstream_commits) != 9: raise AssertionError if internal_commits[-1] != upstream_commits[-1]: raise AssertionError # Verify that each upstream commit is in downstream. for commit in upstream_commits: git('merge-base', '--is-ancestor', commit, 'internal/master') internal_clang_commits = git_output('rev-list', 'split/clang/internal/master').splitlines() if len(internal_clang_commits) != 7: raise AssertionError upstream_clang_commits = git_output('rev-list', 'split/clang/upstream/master').splitlines() if len(upstream_clang_commits) != 4: raise AssertionError
def test_reject_mapped_commit(cd_to_monorepo_clone): git('commit', '--allow-empty', '-m', '''This commit is already mapped! apple-llvm-split-commit: f0931a1b36c88157ffc25a9ed1295f3addff85b9\n apple-llvm-split-dir: llvm/''') result = CliRunner().invoke(git_apple_llvm_push, ['HEAD:internal/master']) assert 'one or more commits is already present in the split repo' in result.output assert result.exit_code == 1
def test_cli_tool_no_pr_config(tmp_path): prev = os.getcwd() os.chdir(str(tmp_path)) git('init') result = CliRunner().invoke(pr, ['list'], mix_stderr=True) assert result.exit_code == 1 assert 'missing `git apple-llvm pr` configuration file' in result.output os.chdir(prev)
def test_regraft_missing_split_root(cd_to_monorepo): # Test the scenario with a missing split root. git('checkout', '-b', 'fake-internal-master', 'internal/master') git('commit', '--allow-empty', '-m', 'error\n\napple-llvm-split-commit: 00000000000\napple-llvm-split-dir: clang/') bad_master_head = git_output('rev-parse', 'HEAD') bad_clang_hash = commit_file('clang/another-file', 'internal: new file') with pytest.raises(RegraftMissingSplitRootError) as err: regraft_commit_graph_onto_split_repo(CommitGraph([bad_clang_hash], [bad_master_head]), 'clang') assert err.value.root_commit_hash == bad_master_head
def has_existing_remote(remote_name: str, remote_url: str, **kwargs): try: url = git_output('remote', 'get-url', remote_name, **kwargs) if url != remote_url: git('remote', 'remove', remote_name, **kwargs) return False return True except GitError: return False
def test_cli_tool_no_pr_config(tmp_path): prev = os.getcwd() os.chdir(str(tmp_path)) git('init') result = CliRunner().invoke(pr, ['list']) if result.exit_code != 1: raise AssertionError if 'missing `git apple-llvm pr` configuration file' not in result.output: raise AssertionError os.chdir(prev)
def push(self, dry_run: bool = False): click.echo(click.style( f'\nPushing to {split_dir_to_str(self.split_dir)}:', bold=True)) if dry_run: click.echo('🛑 dry run, stopping before pushing.') return git('push', 'origin', f'{self.commit_hash}:{self.destination_branch}', git_dir=self.remote_clone_dir, stderr=None)
def commit_merge(self, downstream: CommitBlueprint, upstream: CommitBlueprint) -> MergeCommitBlueprint: checkout_and_clean(downstream.split_commit_hash) git('merge', upstream.split_commit_hash) commit_hash = git_output('rev-parse', 'HEAD') git('branch', '-f', f'split/{self.split_dir}/internal/master', commit_hash) return MergeCommitBlueprint(split_dir=self.split_dir, split_commit_hash=commit_hash, downstream=downstream, upstream=upstream)
def create_split_remote(path: str, split_dir: str, branch_name: str): os.chdir(path) git('init', '--bare') git('remote', 'add', 'origin', self.path) git('fetch', 'origin') git('branch', '-f', branch_name, f'origin/split/{split_dir}/internal/master')
def am_tool_git_repo_clone(tmp_path_factory, am_tool_git_repo: str) -> str: path = str(tmp_path_factory.mktemp('simple-am-tool-dir-clone')) git('init', git_dir=path) git('remote', 'add', 'origin', am_tool_git_repo, git_dir=path) git('fetch', 'origin', git_dir=path) git('checkout', 'master', git_dir=path) return path
def status(target: Optional[str], all_commits: bool, remote: List[str], no_fetch: bool, ci_status: bool, graph: bool, graph_format: str): remotes = remote if remote else ['origin'] if not no_fetch: for r in remotes: click.echo(f'❕ Fetching "{r}" to provide the latest status...') git('fetch', r, stderr=None) click.echo('✅ Fetch succeeded!\n') if graph and not graph_format: graph_format = 'pdf' print_status(remotes=remotes, target_branch=target, list_commits=all_commits, query_ci_status=ci_status, graph_format=graph_format)
def test_merge_conflict(split_clang_head_in_monorepo: str, merge_strategy: MergeStrategy): push_clang_commit = commit_file('file1', 'top of branch is here') branch_name = f'rebase-fail/split/clang/{merge_strategy}' git('checkout', '-b', branch_name, split_clang_head_in_monorepo) commit_file('file1', 'rebase this without conflict') graph = CommitGraph([push_clang_commit], [split_clang_head_in_monorepo]) with pytest.raises(ImpossibleMergeError) as err: merge_commit_graph_with_top_of_branch(graph, 'clang', branch_name, merge_strategy) if merge_strategy == MergeStrategy.Rebase: assert err.value.operation == 'rebase' elif merge_strategy == MergeStrategy.RebaseOrMerge: assert err.value.operation == 'merge'
def commit_file(filename: str, text: str, trailers: Optional[str] = None) -> str: """ Commits the contents to the given filename and returns the commit hash. """ update_path = PosixPath(filename) if len(update_path.parent.name) > 0: (update_path.parent).mkdir(parents=True, exist_ok=True) update_path.resolve().write_text(text) git('add', filename) msg = f'Updated {filename}' if 'internal' in text: msg = f'[internal] {msg}' if trailers is not None: msg = f'{msg}\n{trailers}' git('commit', '-m', msg) head_commit = git_output('rev-parse', 'HEAD') return head_commit
def __init__(self, path: str, clone_path: str, clang_split_remote_path: str, llvm_split_remote_path: str, root_split_remote_path: str): self.path = path self.clone_path = clone_path self.clang_split_remote_path = clang_split_remote_path self.llvm_split_remote_path = llvm_split_remote_path self.root_split_remote_path = root_split_remote_path # Configure the push configuration. push_config = { 'branch_to_dest_branch_mapping': { 'internal/master:-': 'internal/master', 'internal/master:*': 'master' }, 'repo_mapping': { 'clang': self.clang_split_remote_path, 'llvm': self.llvm_split_remote_path, '-': self.root_split_remote_path } } push_config_json = json.dumps(push_config) # Create the repo. cwd = os.getcwd() os.chdir(self.path) git('init') monorepo_test_harness.create_simple_test_harness( push_config_json=push_config_json) self.internal_head = git_output('rev-parse', 'internal/master') # Create the clone. os.chdir(self.clone_path) git('init') git('remote', 'add', 'origin', self.path) git('fetch', 'origin') # Create the split clones. def create_split_remote(path: str, split_dir: str, branch_name: str): os.chdir(path) git('init', '--bare') git('remote', 'add', 'origin', self.path) git('fetch', 'origin') git('branch', '-f', branch_name, f'origin/split/{split_dir}/internal/master') create_split_remote(self.clang_split_remote_path, 'clang', 'master') create_split_remote(self.llvm_split_remote_path, 'llvm', 'master') create_split_remote(self.root_split_remote_path, '-', 'internal/master') # Back to the original CWD. os.chdir(cwd)
def test_cli_tool_create_pr_invalid_base(cd_to_pr_tool_repo_clone, pr_tool_type): git('checkout', 'master') git('branch', '-D', 'pr_branch2', ignore_error=True) git('checkout', '-b', 'pr_branch2') git('push', 'origin', '-u', '-f', 'pr_branch2') mock_tool = MockPRTool() git_apple_llvm.pr.main.pr_tool = create_pr_tool(mock_tool, pr_tool_type) # PR creation fails when the branch is not pushed. result = CliRunner().invoke(pr, ['create', '-m', 'test pr', '-b', 'mastar', '-h', 'pr_branch2']) assert result.exit_code == 1 assert 'base branch "mastar" is not a valid remote tracking branch' in result.output
def test_git_invocation(tmp_path): """ Tests for the git/git_output functions. """ repo_path = tmp_path / 'repo' repo_path.mkdir() assert repo_path.is_dir() # Ensure the dir is there for us to work with. repo_dir = str(repo_path) git('init', git_dir=repo_dir) (repo_path / 'initial').write_text(u'initial') git('add', 'initial', git_dir=repo_dir) # Check that we can report an error on failure. with pytest.raises(GitError) as err: git('add', 'foo', git_dir=repo_dir) assert err.value.stderr.startswith('fatal') assert repr(err.value).startswith('GitError') # Check that errors can be ignored. git('add', 'foo', git_dir=repo_dir, ignore_error=True) output = git_output('commit', '-m', 'initial', git_dir=repo_dir) assert len(output) > 0 # Ensure that the output is stripped. output = git_output('rev-list', 'HEAD', git_dir=repo_dir) assert '\n' not in output output = git_output('rev-list', 'HEAD', git_dir=repo_dir, strip=False) assert '\n' in output # Ensure that commit exists works only for commit hashes. hash = output.strip() assert commit_exists(hash) assert not commit_exists('HEAD') assert not commit_exists(hash + 'abc') assert not commit_exists('000000') # Ensure that we can get the directory of the checkout even when the # working directory is a subdirectory. os.chdir(repo_dir) dir_a = get_current_checkout_directory() (repo_path / 'subdir').mkdir() cwd = os.getcwd() os.chdir(os.path.join(repo_dir, 'subdir')) dir_b = get_current_checkout_directory() os.chdir(cwd) assert dir_a == dir_b assert read_file_or_none('HEAD', 'initial') == 'initial' assert read_file_or_none(hash, 'initial') == 'initial' assert read_file_or_none('HEAD', 'path/does-not-exist') is None assert read_file_or_none('foo', 'initial') is None
def commit_file(self, filename: str, text: str, parent: Optional[BlobCommitBlueprint] = None, internal: bool = False) -> BlobCommitBlueprint: if parent is not None: checkout_and_clean(parent.split_commit_hash) commit_hash = commit_file(filename, text) result = BlobCommitBlueprint( split_dir=self.split_dir, filename=filename, text=text, split_commit_hash=commit_hash, parent=parent if parent else self.prev_parent, is_internal=internal) kind = 'internal' if internal else 'upstream' git('branch', '-f', f'split/{self.split_dir}/{kind}/master', commit_hash) self.prev_parent = result return result
def find_inflight_merges(remote: str = 'origin') -> Dict[str, List[str]]: """ This function fetches the refs created by the automerger to find the inflight merges that are currently being processed. """ # Delete the previously fetched refs to avoid fetch failures # where there were force pushes. existing_refs = git_output('for-each-ref', AM_STATUS_PREFIX, '--format=%(refname)').split('\n') for ref in existing_refs: if not ref: continue log.debug(f'Deleting local ref "{ref}" before fetching') git('update-ref', '-d', ref) git('fetch', remote, f'{AM_PREFIX}*:{AM_STATUS_PREFIX}*') # FIXME: handle fetch failures. refs = git_output('for-each-ref', AM_STATUS_PREFIX, '--format=%(refname)').split('\n') inflight_merges: Dict[str, List[str]] = {} for ref in refs: if not ref: continue if not ref.startswith(AM_STATUS_PREFIX): raise AssertionError merge_name = ref[len(AM_STATUS_PREFIX):] underscore_idx = merge_name.find('_') if underscore_idx == -1: raise AssertionError commit_hash = merge_name[:underscore_idx] dest_branch = merge_name[underscore_idx + 1:] if dest_branch in inflight_merges: inflight_merges[dest_branch].append(commit_hash) else: inflight_merges[dest_branch] = [commit_hash] for (m, k) in inflight_merges.items(): log.debug(f'in-flight {m}: {k}') return inflight_merges
def ci_tool_git_repo(tmp_path_factory) -> str: path = str(tmp_path_factory.mktemp('simple-ci-tool-dir')) test_plans = { "test-plans": { "check-llvm": { "description": "Runs lit and unit tests for LLVM", "infer-from-changes": ["llvm"], "ci-jobs": "pull-request-RA", "params": { "monorepo_projects": "", "test_targets": "check-llvm" } } } } ci_jobs = { "type": "jenkins", "url": TEST_API_URL, "jobs": [{ "name": "a-RA", "url": TEST_API_URL + "/view/monorepo/job/pr-build-test", "params": { "build_variant": "a" } }, { "name": "b-RA", "url": TEST_API_URL + "/view/monorepo/job/pr-build-test", "params": { "build_variant": "b" } }] } # Create the repo with the CI and test plan configs. git('init', git_dir=path) os.mkdir(os.path.join(path, 'apple-llvm-config')) with open(os.path.join(path, 'apple-llvm-config', 'ci-test-plans.json'), 'w') as f: f.write(json.dumps(test_plans)) os.mkdir(os.path.join(path, 'apple-llvm-config/ci-jobs')) with open( os.path.join(path, 'apple-llvm-config/ci-jobs', 'pull-request-RA.json'), 'w') as f: f.write(json.dumps(ci_jobs)) git('add', 'apple-llvm-config', git_dir=path) git('commit', '-m', 'ci config', git_dir=path) git('checkout', '-b', 'repo/apple-llvm-config/pr', git_dir=path) return path
def test_monorepo_simple_test_harness(cd_to_monorepo): internal_commits = git_output('rev-list', 'internal/master').splitlines() assert len(internal_commits) == 16 trailers = git_output('show', '--format=%B', internal_commits[0]) assert 'apple-llvm-split-commit:' in trailers assert 'apple-llvm-split-dir: -/' in trailers assert PosixPath('clang/dir/file2').is_file() upstream_commits = git_output('rev-list', 'llvm/master').splitlines() assert len(upstream_commits) == 9 assert internal_commits[-1] == upstream_commits[-1] # Verify that each upstream commit is in downstream. for commit in upstream_commits: git('merge-base', '--is-ancestor', commit, 'internal/master') internal_clang_commits = git_output( 'rev-list', 'split/clang/internal/master').splitlines() assert len(internal_clang_commits) == 7 upstream_clang_commits = git_output( 'rev-list', 'split/clang/upstream/master').splitlines() assert len(upstream_clang_commits) == 4
def test_merge_commit_graph(split_clang_head_in_monorepo: str, merge_strategy: MergeStrategy): # Create the test scenario: # * merge <-- top of commit graph. # |\ # | * [new-merge/split/clang] <-- destination. # | \/ # | * file2 <-- root of commit graph. # * | file1 # |/ # * [split/clang/internal/master] <-- root of commit graph. push_clang_commit = commit_file('file1', 'top of branch is here') branch_name = f'new-merge/split/clang/{merge_strategy}' git('checkout', '-b', branch_name, split_clang_head_in_monorepo) up_clang_head = commit_file('file2', 'merging this in') new_clang_head = commit_file('file3', 'top of clang/master is here') git('checkout', '--detach', up_clang_head) git('clean', '-d', '-f') git('merge', push_clang_commit) merge_commit = git_output('rev-parse', 'HEAD') def doit() -> str: graph = CommitGraph([merge_commit, push_clang_commit], [up_clang_head, split_clang_head_in_monorepo]) return merge_commit_graph_with_top_of_branch(graph, 'clang', branch_name, merge_strategy) # Graph with merges can only be merged. if merge_strategy == MergeStrategy.RebaseOrMerge: commit = doit() if merge_commit == commit: raise AssertionError parents = git_output('show', '--format=%P', commit).split() if 2 != len(parents): raise AssertionError if new_clang_head != parents[0]: raise AssertionError if merge_commit != parents[1]: raise AssertionError else: with pytest.raises(ImpossibleMergeError): doit()
def has_merge_conflict(commit: str, target_branch: str, remote: str = 'origin') -> bool: """ Returns true if the given commit hash has a merge conflict with the given target branch. """ try: # Always remove the temporary worktree. It's possible that we got # interrupted and left it around. This will raise an exception if the # worktree doesn't exist, which can be safely ignored. git('worktree', 'remove', '--force', '.git/temp-worktree', stdout=get_dev_null(), stderr=get_dev_null()) except GitError: pass git('worktree', 'add', '.git/temp-worktree', f'{remote}/{target_branch}', '--detach', stdout=get_dev_null(), stderr=get_dev_null()) try: git('merge', '--no-commit', commit, git_dir='.git/temp-worktree', stdout=get_dev_null(), stderr=get_dev_null()) return False except GitError: return True finally: git('worktree', 'remove', '--force', '.git/temp-worktree', stdout=get_dev_null(), stderr=get_dev_null())
def test_different_tracked_branch_ref(cd_to_monorepo_clone, monorepo_test_fixture, optional_remote_prefix: str): branch_name = 'test-tracked-branch' git('branch', '-D', branch_name, ignore_error=True) git('checkout', '-b', branch_name) git('branch', '-u', 'origin/internal/master', branch_name) ref: Optional[TrackedBranchRef] = get_tracked_branch_ref(optional_remote_prefix + branch_name) if len(optional_remote_prefix) > 0: # The ref doesn't exist on the remote assert ref is None return assert ref is not None assert ref.remote_name == 'origin' assert ref.remote_url == monorepo_test_fixture.path assert ref.branch_name == 'internal/master'
def cd_to_pr_tool_repo_clone_adjust_jenkins_ci(cd_to_pr_tool_repo): pr_config = { 'type': 'github', 'domain': 'github.com', 'user': '******', 'repo': 'apple-llvm-infrastructure-tools', 'test': { 'type': 'jenkins-test-plans' } } with open('apple-llvm-config/pr.json', 'w') as f: f.write(json.dumps(pr_config)) test_plans = { "test-plans": { "pr": { "description": "", "ci-jobs": "pull-request-RA", "params": { "test_targets": "check-llvm" } } } } ci_jobs = { "type": "jenkins", "url": JENKINS_TEST_API_URL, "jobs": [{ "name": "a-RA", "url": JENKINS_TEST_API_URL + "/view/monorepo/job/pr-build-test", "params": {} }] } # Create the repo with the CI and test plan configs. with open(os.path.join('apple-llvm-config', 'ci-test-plans.json'), 'w') as f: f.write(json.dumps(test_plans)) os.mkdir(os.path.join('apple-llvm-config/ci-jobs')) with open( os.path.join('apple-llvm-config/ci-jobs', 'pull-request-RA.json'), 'w') as f: f.write(json.dumps(ci_jobs)) git('add', 'apple-llvm-config') git('commit', '-m', 'use jenkins now') yield git('reset', '--hard', 'HEAD~1')