def test_dry_run_stops_before_creating_pr(self):
host = MockHost()
host.executive = mock_git_commands({
'crrev-parse':
'c2087acb00eee7960339a0be34ea27d6b20e1131',
})
test_exporter = TestExporter(host,
'gh-username',
'gh-token',
gerrit_user=None,
gerrit_token=None,
dry_run=True)
test_exporter.wpt_github = MockWPTGitHub(pull_requests=[
PullRequest(title='title1', number=1234, body='', state='open'),
])
test_exporter.gerrit = MockGerritAPI(host, 'gerrit-username',
'gerrit-token')
test_exporter.get_exportable_commits = lambda limit: [
ChromiumCommit(host, position='refs/heads/master@{#458475}'),
ChromiumCommit(host, position='refs/heads/master@{#458476}'),
ChromiumCommit(host, position='refs/heads/master@{#458477}'),
]
test_exporter.run()
self.assertEqual(test_exporter.wpt_github.calls, [
'pr_with_position',
'pr_with_position',
'pr_with_position',
])
def test_dry_run_stops_before_creating_pr(self):
self.host.executive = mock_git_commands({
'crrev-parse':
'c2087acb00eee7960339a0be34ea27d6b20e1131',
})
test_exporter = TestExporter(self.host)
test_exporter.wpt_github = MockWPTGitHub(pull_requests=[
PullRequest(
title='title1', number=1234, body='', state='open', labels=[]),
])
test_exporter.gerrit = MockGerritAPI(self.host, 'gerrit-username',
'gerrit-token')
test_exporter.get_exportable_commits = lambda: ([
ChromiumCommit(self.host, position='refs/heads/master@{#458475}'),
ChromiumCommit(self.host, position='refs/heads/master@{#458476}'),
ChromiumCommit(self.host, position='refs/heads/master@{#458477}'),
], [])
success = test_exporter.main(
['--credentials-json', '/tmp/credentials.json', '--dry-run'])
self.assertTrue(success)
self.assertEqual(test_exporter.wpt_github.calls, [
'pr_for_chromium_commit',
'pr_for_chromium_commit',
'pr_for_chromium_commit',
])
def test_accepts_sha(self):
chromium_commit = ChromiumCommit(
MockHost(), sha='c881563d734a86f7d9cd57ac509653a61c45c240')
self.assertEqual(chromium_commit.sha,
'c881563d734a86f7d9cd57ac509653a61c45c240')
self.assertIsNone(chromium_commit.position)
def exportable_commits_since(chromium_commit_hash, host, local_wpt):
"""Lists exportable commits after a certain point.
Args:
chromium_commit_hash: The SHA of the Chromium commit from which this
method will look. This commit is not included in the commits searched.
host: A Host object.
local_wpt: A LocalWPT instance, used to see whether a Chromium commit
can be applied cleanly in the upstream repo.
Returns:
A list of ChromiumCommit objects for commits that are exportable after
the given commit, in chronological order.
"""
chromium_repo_root = host.executive.run_command(
['git', 'rev-parse', '--show-toplevel']).strip()
wpt_path = chromium_repo_root + '/' + CHROMIUM_WPT_DIR
commit_range = '{}..HEAD'.format(chromium_commit_hash)
commit_hashes = host.executive.run_command(
['git', 'rev-list', commit_range, '--reverse', '--',
wpt_path]).splitlines()
chromium_commits = [ChromiumCommit(host, sha=sha) for sha in commit_hashes]
return [
commit for commit in chromium_commits
if is_exportable(commit, local_wpt)
]
def test_attempts_to_merge_landed_gerrit_cl(self):
host = MockHost()
host.executive = mock_git_commands({
'footers': 'decafbad',
})
test_exporter = TestExporter(host,
'gh-username',
'gh-token',
gerrit_user=None,
gerrit_token=None,
dry_run=False)
test_exporter.wpt_github = MockWPTGitHub(pull_requests=[
PullRequest(
title='title1',
number=1234,
body='description\nWPT-Export-Revision: 9\nChange-Id: decafbad',
state='open'),
])
test_exporter.get_exportable_commits = lambda limit: [
ChromiumCommit(host,
sha='c881563d734a86f7d9cd57ac509653a61c45c240'),
]
test_exporter.gerrit = MockGerritAPI(host, 'gerrit-username',
'gerrit-token')
test_exporter.gerrit.query_exportable_open_cls = lambda: []
test_exporter.run()
self.assertEqual(test_exporter.wpt_github.calls, [
'pr_with_position',
'get_pr_branch',
'merge_pull_request',
'delete_remote_branch',
])
self.assertEqual(test_exporter.wpt_github.pull_requests_created, [])
self.assertEqual(test_exporter.wpt_github.pull_requests_merged, [1234])
def _exportable_commits_since(chromium_commit_hash, host, local_wpt, wpt_github,
require_clean=True, verify_merged_pr=False):
"""Lists exportable commits after the given commit.
Args:
chromium_commit_hash: The SHA of the Chromium commit from which this
method will look. This commit is not included in the commits searched.
Return values and remaining arguments are the same as exportable_commits_over_last_n_commits.
"""
chromium_repo_root = host.executive.run_command([
'git', 'rev-parse', '--show-toplevel'
], cwd=absolute_chromium_dir(host)).strip()
wpt_path = chromium_repo_root + '/' + CHROMIUM_WPT_DIR
commit_range = '{}..HEAD'.format(chromium_commit_hash)
commit_hashes = host.executive.run_command([
'git', 'rev-list', commit_range, '--reverse', '--', wpt_path
], cwd=absolute_chromium_dir(host)).splitlines()
chromium_commits = [ChromiumCommit(host, sha=sha) for sha in commit_hashes]
exportable_commits = []
errors = []
for commit in chromium_commits:
state, error = get_commit_export_state(commit, local_wpt, wpt_github, verify_merged_pr)
if require_clean:
success = state == CommitExportState.EXPORTABLE_CLEAN
else:
success = state in (CommitExportState.EXPORTABLE_CLEAN, CommitExportState.EXPORTABLE_DIRTY)
if success:
exportable_commits.append(commit)
elif error != '':
errors.append('The following commit did not apply cleanly:\nSubject: %s (%s)\n%s' %
(commit.subject(), commit.url(), error))
return exportable_commits, errors
def _exportable_commits_since(chromium_commit_hash, host, local_wpt,
wpt_github):
"""Lists exportable commits after a certain point.
Args:
chromium_commit_hash: The SHA of the Chromium commit from which this
method will look. This commit is not included in the commits searched.
host: A Host object.
local_wpt: A LocalWPT instance.
wpt_github: A WPTGitHub instance.
Returns:
A list of ChromiumCommit objects for commits that are exportable after
the given commit, in chronological order.
"""
chromium_repo_root = host.executive.run_command(
['git', 'rev-parse', '--show-toplevel'],
cwd=absolute_chromium_dir(host)).strip()
wpt_path = chromium_repo_root + '/' + CHROMIUM_WPT_DIR
commit_range = '{}..HEAD'.format(chromium_commit_hash)
commit_hashes = host.executive.run_command(
['git', 'rev-list', commit_range, '--reverse', '--', wpt_path],
cwd=absolute_chromium_dir(host)).splitlines()
chromium_commits = [ChromiumCommit(host, sha=sha) for sha in commit_hashes]
exportable_commits = []
for commit in chromium_commits:
if is_exportable(commit, local_wpt, wpt_github):
exportable_commits.append(commit)
return exportable_commits
def test_attempts_to_merge_landed_gerrit_cl(self):
host = MockHost()
def mock_command(args):
if args[1] != 'footers':
return ''
if args[2] == '--key':
return 'decafbad\n'
elif args[2] == '--position':
return 'refs/heads/master@{#475994}'
host.executive = MockExecutive(run_command_fn=mock_command)
test_exporter = TestExporter(host, 'gh-username', 'gh-token', gerrit_user=None,
gerrit_token=None, dry_run=False)
test_exporter.wpt_github = MockWPTGitHub(pull_requests=[
PullRequest(title='title1', number=1234,
body='description\nWPT-Export-Revision: 9\nChange-Id: decafbad',
state='open', labels=['do not merge yet']),
])
test_exporter.get_exportable_commits = lambda: [
ChromiumCommit(host, sha='c881563d734a86f7d9cd57ac509653a61c45c240'),
]
test_exporter.gerrit = MockGerritAPI(host, 'gerrit-username', 'gerrit-token')
test_exporter.gerrit.query_exportable_open_cls = lambda: []
test_exporter.run()
self.assertEqual(test_exporter.wpt_github.calls, [
'pr_for_chromium_commit',
'remove_label "do not merge yet"',
'get_pr_branch',
'merge_pull_request',
'delete_remote_branch',
])
self.assertEqual(test_exporter.wpt_github.pull_requests_created, [])
self.assertEqual(test_exporter.wpt_github.pull_requests_merged, [1234])
def test_derives_sha_from_position(self):
host = MockHost()
host.executive = MockExecutive(output='deadbeefcafe')
pos = 'Cr-Commit-Position: refs/heads/master@{#789}'
chromium_commit = ChromiumCommit(host, position=pos)
self.assertEqual(chromium_commit.position, 'refs/heads/master@{#789}')
self.assertEqual(chromium_commit.sha, 'deadbeefcafe')
def test_derives_sha_from_position(self):
host = MockHost()
host.executive = MockExecutive(
output='c881563d734a86f7d9cd57ac509653a61c45c240')
pos = 'Cr-Commit-Position: refs/heads/master@{#789}'
chromium_commit = ChromiumCommit(host, position=pos)
self.assertEqual(chromium_commit.position, 'refs/heads/master@{#789}')
self.assertEqual(chromium_commit.sha,
'c881563d734a86f7d9cd57ac509653a61c45c240')
def test_derives_position_from_sha(self):
host = MockHost()
host.executive = mock_git_commands(
{'footers': 'refs/heads/master@{#789}'})
chromium_commit = ChromiumCommit(
host, sha='c881563d734a86f7d9cd57ac509653a61c45c240')
self.assertEqual(chromium_commit.position, 'refs/heads/master@{#789}')
self.assertEqual(chromium_commit.sha,
'c881563d734a86f7d9cd57ac509653a61c45c240')
def test_filtered_changed_files_blacklist(self):
host = MockHost()
fake_files = ['file1', 'MANIFEST.json', 'file3']
qualified_fake_files = [CHROMIUM_WPT_DIR + f for f in fake_files]
host.executive = mock_command_exec({
'diff-tree': '\n'.join(qualified_fake_files),
'crrev-parse': 'fake rev',
})
position_footer = 'Cr-Commit-Position: refs/heads/master@{#789}'
chromium_commit = ChromiumCommit(host, position=position_footer)
files = chromium_commit.filtered_changed_files()
expected_files = ['file1', 'file3']
qualified_expected_files = [CHROMIUM_WPT_DIR + f for f in expected_files]
self.assertEqual(files, qualified_expected_files)
def main():
configure_logging()
options = parse_args()
host = Host()
github = GitHub(host)
local_wpt = LocalWPT(host, no_fetch=options.no_fetch, use_github=True)
chromium_wpt = ChromiumWPT(host)
wpt_commit, chromium_commit = local_wpt.most_recent_chromium_commit()
assert chromium_commit, 'No Chromium commit found, this is impossible'
_log.info('web-platform-tests@%s (%d behind origin/master)', wpt_commit,
local_wpt.commits_behind_master(wpt_commit))
_log.info('chromium@%s (%d behind origin/master)', chromium_commit.sha,
chromium_commit.num_behind_master())
exportable_commits = chromium_wpt.exportable_commits_since(
chromium_commit.sha)
if exportable_commits:
_log.info('Found %s exportable commits in chromium:',
len(exportable_commits))
for commit in exportable_commits:
_log.info('- %s %s', commit, chromium_wpt.subject(commit))
else:
_log.info('No exportable commits found in Chromium, stopping.')
return
for commit in exportable_commits:
_log.info('Uploading %s', chromium_wpt.subject(commit))
chromium_commit = ChromiumCommit(host, sha=commit)
patch = chromium_wpt.format_patch(commit)
message = chromium_wpt.message(commit)
local_wpt.create_branch_with_patch(branch_name, message, patch)
github.create_pr(local_branch_name='chromium-try-{}'.format(commit),
desc_title=chromium_commit.subject(),
body=chromium_commit.body())
def most_recent_chromium_commit(self):
"""Finds the most recent commit in WPT with a Chromium commit position."""
wpt_commit_hash = self.run(['git', 'rev-list', 'HEAD', '-n', '1', '--grep=Cr-Commit-Position'])
if not wpt_commit_hash:
return None, None
wpt_commit_hash = wpt_commit_hash.strip()
position = self.run(['git', 'footers', '--position', wpt_commit_hash])
position = position.strip()
assert position
chromium_commit = ChromiumCommit(self.host, position=position)
return wpt_commit_hash, chromium_commit
def test_when_commit_has_no_position(self):
host = MockHost()
def run_command(_):
raise ScriptError(
'Unable to infer commit position from footers rutabaga')
host.executive = MockExecutive(run_command_fn=run_command)
chromium_commit = ChromiumCommit(
host, sha='c881563d734a86f7d9cd57ac509653a61c45c240')
self.assertEqual(chromium_commit.position, 'no-commit-position-yet')
self.assertEqual(chromium_commit.sha,
'c881563d734a86f7d9cd57ac509653a61c45c240')
def most_recent_chromium_commit(self):
"""Finds the most recent commit in WPT with a Chromium commit position."""
sha = self.run([
'git', 'rev-list', 'HEAD', '-n', '1', '--grep=Cr-Commit-Position'
])
if not sha:
return None, None
sha = sha.strip()
position = self.run(['git', 'footers', '--position', sha])
position = position.strip()
assert position
chromium_commit = ChromiumCommit(self.host, position=position)
return sha, chromium_commit
def most_recent_chromium_commit(self):
"""Goes back in WPT commit history and gets the most recent commit
that contains 'Cr-Commit-Position:'
"""
sha = self.run(['git', 'rev-list', 'HEAD', '-n', '1', '--grep=Cr-Commit-Position'])
if not sha:
return None, None
sha = sha.strip()
position = self.run(['git', 'footers', '--position', sha])
position = position.strip()
assert position
chromium_commit = ChromiumCommit(self.host, position=position)
return sha, chromium_commit
def fetch_current_revision_commit(self, host):
"""Fetches the git commit for the latest revision of CL.
This method fetches the commit corresponding to the latest revision of
CL to local Chromium repository, but does not checkout the commit to the
working tree. All changes in the CL are squashed into this one commit,
regardless of how many revisions have been uploaded.
Args:
host: A Host object for git invocation.
Returns:
A ChromiumCommit object (the fetched commit).
"""
git = host.git(absolute_chromium_dir(host))
url = self.current_revision['fetch']['http']['url']
ref = self.current_revision['fetch']['http']['ref']
git.run(['fetch', url, ref])
sha = git.run(['rev-parse', 'FETCH_HEAD']).strip()
return ChromiumCommit(host, sha=sha)
def exportable_commits_since(self, commit):
"""Returns SHAs of exportable commits since `commit` in chronological order.
Args:
commit: The SHA of the Chromium commit from which this method will look.
"""
repo_root = self.host.executive.run_command(
['git', 'rev-parse', '--show-toplevel']).strip()
commits = self.host.executive.run_command([
'git', 'rev-list', '{}..HEAD'.format(commit), '--reverse', '--',
repo_root + '/' + CHROMIUM_WPT_DIR
]).splitlines()
chromium_commits = [ChromiumCommit(self.host, sha=c) for c in commits]
def is_exportable(chromium_commit):
patch = chromium_commit.format_patch()
return (patch and self.local_wpt.test_patch(patch)
and 'NOEXPORT=true' not in chromium_commit.message()
and not chromium_commit.message().startswith('Import '))
return [c for c in chromium_commits if is_exportable(c)]
def test_validates_sha(self):
with self.assertRaises(AssertionError):
ChromiumCommit(MockHost(), sha='rutabaga')
def test_creates_and_merges_pull_requests(self):
# This tests 4 exportable commits:
# 1. #458475 has an in-flight PR associated with it but cannot be merged.
# 2. #458476 has no PR associated with it and should have one created.
# 3. #458477 has a closed PR associated with it and should be skipped.
# 4. #458478 has an in-flight PR associated with it and should be merged successfully.
host = MockHost()
host.executive = mock_git_commands({
'show':
'git show text\nCr-Commit-Position: refs/heads/master@{#458476}',
'crrev-parse':
'c2087acb00eee7960339a0be34ea27d6b20e1131',
})
test_exporter = TestExporter(host,
'gh-username',
'gh-token',
gerrit_user=None,
gerrit_token=None)
test_exporter.wpt_github = MockWPTGitHub(pull_requests=[
PullRequest(
title='Open PR',
number=1234,
body=
'rutabaga\nCr-Commit-Position: refs/heads/master@{#458475}',
state='open'),
PullRequest(
title='Merged PR',
number=2345,
body=
'rutabaga\nCr-Commit-Position: refs/heads/master@{#458477}',
state='closed'),
PullRequest(
title='Open PR',
number=3456,
body=
'rutabaga\nCr-Commit-Position: refs/heads/master@{#458478}',
state='open'),
],
unsuccessful_merge_index=0)
test_exporter.gerrit = MockGerritAPI(host, 'gerrit-username',
'gerrit-token')
test_exporter.get_exportable_commits = lambda limit: [
ChromiumCommit(host, position='refs/heads/master@{#458475}'),
ChromiumCommit(host, position='refs/heads/master@{#458476}'),
ChromiumCommit(host, position='refs/heads/master@{#458477}'),
ChromiumCommit(host, position='refs/heads/master@{#458478}'),
]
test_exporter.run()
self.assertEqual(test_exporter.wpt_github.calls, [
'pr_with_position',
'get_pr_branch',
'merge_pull_request',
'pr_with_position',
'create_pr',
'add_label',
'pr_with_position',
'pr_with_position',
'get_pr_branch',
'merge_pull_request',
'delete_remote_branch',
])
self.assertEqual(test_exporter.wpt_github.pull_requests_created, [
('chromium-export-c2087acb00',
'git show text\nCr-Commit-Position: refs/heads/master@{#458476}',
'git show text\nCr-Commit-Position: refs/heads/master@{#458476}'),
])
def test_short_sha(self):
chromium_commit = ChromiumCommit(
MockHost(), sha='c881563d734a86f7d9cd57ac509653a61c45c240')
self.assertEqual(chromium_commit.short_sha, 'c881563d73')
def run(self):
# First, poll for an in-flight pull request and merge if exists
pull_requests = self.wpt_github.in_flight_pull_requests()
if len(pull_requests) == 1:
pull_request = pull_requests.pop()
_log.info('In-flight PR found: #%d', pull_request['number'])
_log.info(pull_request['title'])
# TODO(jeffcarp): Check the PR status here
if self.dry_run:
_log.info('[dry_run] Would have attempted to merge PR')
else:
_log.info('Merging...')
self.wpt_github.merge_pull_request(pull_request['number'])
_log.info('PR merged!')
elif len(pull_requests) > 1:
_log.error(pull_requests)
# TODO(jeffcarp): Print links to PRs
raise Exception('More than two in-flight PRs!')
# Second, look for exportable commits in Chromium
# At this point, no in-flight PRs should exist
# If there was an issue merging, it should have errored out
local_wpt = LocalWPT(self.host, use_github=False)
chromium_wpt = ChromiumWPT(self.host)
# TODO(jeffcarp): have the script running this fetch Chromium origin/master
# TODO(jeffcarp): move WPT fetch out of its constructor to match planned ChromiumWPT pattern
wpt_commit, chromium_commit = local_wpt.most_recent_chromium_commit()
assert chromium_commit, 'No Chromium commit found, this is impossible'
wpt_behind_master = local_wpt.commits_behind_master(wpt_commit)
_log.info('\nLast Chromium export commit in web-platform-tests:')
_log.info('web-platform-tests@%s', wpt_commit)
_log.info('(%d behind web-platform-tests@origin/master)', wpt_behind_master)
_log.info('\nThe above WPT commit points to the following Chromium commit:')
_log.info('chromium@%s', chromium_commit.sha)
_log.info('(%d behind chromium@origin/master)', chromium_commit.num_behind_master())
# TODO(jeffcarp): Have this function return ChromiumCommits
exportable_commits = chromium_wpt.exportable_commits_since(chromium_commit.sha)
if not exportable_commits:
_log.info('No exportable commits found in Chromium, stopping.')
return
_log.info('Found %d exportable commits in Chromium:', len(exportable_commits))
for commit in exportable_commits:
_log.info('- %s %s', commit, chromium_wpt.subject(commit))
outbound_commit = ChromiumCommit(self.host, sha=exportable_commits[0])
_log.info('Picking the earliest commit and creating a PR')
_log.info('- %s %s', outbound_commit.sha, outbound_commit.subject())
patch = outbound_commit.format_patch()
message = outbound_commit.message()
# TODO: now do a test comparison of patch against local WPT
if self.dry_run:
_log.info('[dry_run] Stopping before creating PR')
_log.info('\n\n[dry_run] message:')
_log.info(message)
_log.info('\n\n[dry_run] patch:')
_log.info(patch)
return
local_branch_name = local_wpt.create_branch_with_patch(message, patch)
response_data = self.wpt_github.create_pr(
local_branch_name=local_branch_name,
desc_title=outbound_commit.subject(),
body=outbound_commit.body())
_log.info('Create PR response: %s', response_data)
def test_accepts_sha(self):
chromium_commit = ChromiumCommit(MockHost(), sha='deadbeefcafe')
self.assertEqual(chromium_commit.sha, 'deadbeefcafe')
self.assertIsNone(chromium_commit.position)
def test_url(self):
chromium_commit = ChromiumCommit(
MockHost(), sha='c881563d734a86f7d9cd57ac509653a61c45c240')
self.assertEqual(
chromium_commit.url(),
'https://chromium.googlesource.com/chromium/src/+/c881563d73')
