def test_create_branch_with_patch(self): host = MockHost() host.filesystem = MockFileSystem() local_wpt = LocalWPT(host, 'token') local_wpt.fetch() local_wpt.create_branch_with_patch('chromium-export-decafbad', 'message', 'patch', 'author <*****@*****.**>') self.assertEqual( host.executive.calls, [[ 'git', 'clone', 'https://[email protected]/w3c/web-platform-tests.git', '/tmp/wpt' ], ['git', 'reset', '--hard', 'HEAD'], ['git', 'clean', '-fdx'], ['git', 'checkout', 'origin/master'], ['git', 'branch', '-D', 'chromium-export-decafbad'], ['git', 'checkout', '-b', 'chromium-export-decafbad'], ['git', 'apply', '-'], ['git', 'add', '.'], [ 'git', 'commit', '--author', 'author <*****@*****.**>', '-am', 'message' ], ['git', 'push', 'origin', 'chromium-export-decafbad']])
def test_create_branch_with_patch(self): host = MockHost() host.filesystem = MockFileSystem() local_wpt = LocalWPT(host) local_wpt.fetch() local_branch_name = local_wpt.create_branch_with_patch( 'message', 'patch', 'author') self.assertEqual(local_branch_name, 'chromium-export-try') self.assertEqual( host.executive.calls, [[ 'git', 'clone', 'https://chromium.googlesource.com/external/w3c/web-platform-tests.git', '/tmp/wpt' ], ['git', 'remote'], [ 'git', 'remote', 'add', 'github', '[email protected]:w3c/web-platform-tests.git' ], ['git', 'reset', '--hard', 'HEAD'], ['git', 'clean', '-fdx'], ['git', 'checkout', 'origin/master'], ['git', 'branch', '-a'], ['git', 'branch', '-a'], ['git', 'checkout', '-b', 'chromium-export-try'], ['git', 'apply', '-'], ['git', 'commit', '--author', 'author', '-am', 'message'], ['git', 'push', 'github', 'chromium-export-try']])
def test_fetch_when_wpt_dir_does_not_exist(self): host = MockHost() local_wpt = LocalWPT(host, 'token') local_wpt.fetch() self.assertEqual(host.executive.calls, [ ['git', 'clone', 'https://[email protected]/w3c/web-platform-tests.git', '/tmp/wpt'], ])
def test_fetch_when_wpt_dir_does_not_exist(self): host = MockHost() host.filesystem = MockFileSystem() local_wpt = LocalWPT(host) local_wpt.fetch() self.assertEqual(len(host.executive.calls), 3) self.assertEqual(host.executive.calls[0][1], 'clone')
def test_fetch_when_wpt_dir_exists(self): host = MockHost() host.filesystem = MockFileSystem(files={'/tmp/wpt': ''}) local_wpt = LocalWPT(host, 'token') local_wpt.fetch() self.assertEqual(host.executive.calls, [ ['git', 'fetch', 'origin'], ['git', 'checkout', 'origin/master'], ])
def test_fetch_when_wpt_dir_exists(self): host = MockHost() host.filesystem = MockFileSystem(files={'/tmp/wpt': ''}) local_wpt = LocalWPT(host) local_wpt.fetch() self.assertEqual( host.executive.calls, [['git', 'fetch', '--all'], ['git', 'checkout', 'origin/master'], ['git', 'remote'], [ 'git', 'remote', 'add', 'github', '[email protected]:w3c/web-platform-tests.git' ]])
def test_fetch_requires_gh_token(self): host = MockHost() local_wpt = LocalWPT(host) with self.assertRaises(AssertionError): local_wpt.fetch()
class TestExporter(object): def __init__(self, host, gh_user, gh_token, dry_run=False): self.host = host self.wpt_github = WPTGitHub(host, gh_user, gh_token) self.dry_run = dry_run self.local_wpt = LocalWPT(self.host, gh_token) self.local_wpt.fetch() def run(self): """Query in-flight pull requests, then merges PR or creates one. This script assumes it will be run on a regular interval. On each invocation, it will either attempt to merge or attempt to create a PR, never both. """ pull_requests = self.wpt_github.in_flight_pull_requests() self.merge_all_pull_requests(pull_requests) # TODO(jeffcarp): The below line will enforce draining all open PRs before # adding any more to the queue, which mirrors current behavior. After this # change lands, modify the following to: # - for each exportable commit # - check if there's a corresponding PR # - if not, create one if not pull_requests: _log.info('No in-flight PRs found, looking for exportable commits.') self.export_first_exportable_commit() def merge_all_pull_requests(self, pull_requests): for pr in pull_requests: self.merge_pull_request(pr) def merge_pull_request(self, pull_request): _log.info('In-flight PR found: %s', pull_request.title) _log.info('https://github.com/w3c/web-platform-tests/pull/%d', pull_request.number) _log.info('Attempting to merge...') if self.dry_run: _log.info('[dry_run] Would have attempted to merge PR') return branch = self.wpt_github.get_pr_branch(pull_request.number) try: self.wpt_github.merge_pull_request(pull_request.number) self.wpt_github.delete_remote_branch(branch) except MergeError: _log.info('Could not merge PR.') def export_first_exportable_commit(self): """Looks for exportable commits in Chromium, creates PR if found.""" wpt_commit, chromium_commit = self.local_wpt.most_recent_chromium_commit() assert chromium_commit, 'No Chromium commit found, this is impossible' wpt_behind_master = self.local_wpt.commits_behind_master(wpt_commit) _log.info('\nLast Chromium export commit in web-platform-tests:') _log.info('web-platform-tests@%s', wpt_commit) _log.info('(%d behind web-platform-tests@origin/master)', wpt_behind_master) _log.info('\nThe above WPT commit points to the following Chromium commit:') _log.info('chromium@%s', chromium_commit.sha) _log.info('(%d behind chromium@origin/master)', chromium_commit.num_behind_master()) exportable_commits = exportable_commits_since(chromium_commit.sha, self.host, self.local_wpt) if not exportable_commits: _log.info('No exportable commits found in Chromium, stopping.') return _log.info('Found %d exportable commits in Chromium:', len(exportable_commits)) for commit in exportable_commits: _log.info('- %s %s', commit, commit.subject()) outbound_commit = exportable_commits[0] _log.info('Picking the earliest commit and creating a PR') _log.info('- %s %s', outbound_commit.sha, outbound_commit.subject()) patch = outbound_commit.format_patch() message = outbound_commit.message() author = outbound_commit.author() if self.dry_run: _log.info('[dry_run] Stopping before creating PR') _log.info('\n\n[dry_run] message:') _log.info(message) _log.info('\n\n[dry_run] patch:') _log.info(patch) return remote_branch_name = self.local_wpt.create_branch_with_patch(message, patch, author) response_data = self.wpt_github.create_pr( remote_branch_name=remote_branch_name, desc_title=outbound_commit.subject(), body=outbound_commit.body()) _log.info('Create PR response: %s', response_data) if response_data: data, status_code = self.wpt_github.add_label(response_data['number']) _log.info('Add label response (status %s): %s', status_code, data)
class TestExporter(object): def __init__(self, host, wpt_github, dry_run=False): self.host = host self.wpt_github = wpt_github self.dry_run = dry_run self.local_wpt = LocalWPT(self.host) self.local_wpt.fetch() def run(self): """Query in-flight pull requests, then merge PR or create one. This script assumes it will be run on a regular interval. On each invocation, it will either attempt to merge or attempt to create a PR, never both. """ pull_requests = self.wpt_github.in_flight_pull_requests() if len(pull_requests) == 1: self.merge_in_flight_pull_request(pull_requests.pop()) elif len(pull_requests) > 1: _log.error(pull_requests) # TODO(jeffcarp): Print links to PRs raise Exception('More than two in-flight PRs!') else: self.export_first_exportable_commit() def merge_in_flight_pull_request(self, pull_request): """Attempt to merge an in-flight PR. Args: pull_request: a PR object returned from the GitHub API. """ _log.info('In-flight PR found: #%d', pull_request['number']) _log.info(pull_request['title']) # TODO(jeffcarp): Check the PR status here (for Travis CI, etc.) if self.dry_run: _log.info('[dry_run] Would have attempted to merge PR') return _log.info('Merging...') self.wpt_github.merge_pull_request(pull_request['number']) _log.info('PR merged! Deleting branch.') self.wpt_github.delete_remote_branch('chromium-export-try') _log.info('Branch deleted!') def export_first_exportable_commit(self): """Looks for exportable commits in Chromium, creates PR if found.""" wpt_commit, chromium_commit = self.local_wpt.most_recent_chromium_commit( ) assert chromium_commit, 'No Chromium commit found, this is impossible' wpt_behind_master = self.local_wpt.commits_behind_master(wpt_commit) _log.info('\nLast Chromium export commit in web-platform-tests:') _log.info('web-platform-tests@%s', wpt_commit) _log.info('(%d behind web-platform-tests@origin/master)', wpt_behind_master) _log.info( '\nThe above WPT commit points to the following Chromium commit:') _log.info('chromium@%s', chromium_commit.sha) _log.info('(%d behind chromium@origin/master)', chromium_commit.num_behind_master()) exportable_commits = exportable_commits_since(chromium_commit.sha, self.host, self.local_wpt) if not exportable_commits: _log.info('No exportable commits found in Chromium, stopping.') return _log.info('Found %d exportable commits in Chromium:', len(exportable_commits)) for commit in exportable_commits: _log.info('- %s %s', commit, commit.subject()) outbound_commit = exportable_commits[0] _log.info('Picking the earliest commit and creating a PR') _log.info('- %s %s', outbound_commit.sha, outbound_commit.subject()) patch = outbound_commit.format_patch() message = outbound_commit.message() author = outbound_commit.author() if self.dry_run: _log.info('[dry_run] Stopping before creating PR') _log.info('\n\n[dry_run] message:') _log.info(message) _log.info('\n\n[dry_run] patch:') _log.info(patch) return remote_branch_name = self.local_wpt.create_branch_with_patch( message, patch, author) response_data = self.wpt_github.create_pr( remote_branch_name=remote_branch_name, desc_title=outbound_commit.subject(), body=outbound_commit.body()) _log.info('Create PR response: %s', response_data) if response_data: data, status_code = self.wpt_github.add_label( response_data['number']) _log.info('Add label response (status %s): %s', status_code, data)
def main(self, argv=None): options = self.parse_args(argv) self.verbose = options.verbose log_level = logging.DEBUG if self.verbose else logging.INFO logging.basicConfig(level=log_level, format='%(message)s') if not self.checkout_is_okay(): return 1 credentials = read_credentials(self.host, options.credentials_json) gh_user = credentials.get('GH_USER') gh_token = credentials.get('GH_TOKEN') self.wpt_github = self.wpt_github or WPTGitHub(self.host, gh_user, gh_token) local_wpt = LocalWPT(self.host, gh_token=gh_token) self.git_cl = GitCL( self.host, auth_refresh_token_json=options.auth_refresh_token_json) _log.debug('Noting the current Chromium commit.') # TODO(qyearsley): Use Git (self.host.git) to run git commands. _, show_ref_output = self.run(['git', 'show-ref', 'HEAD']) chromium_commit = show_ref_output.split()[0] local_wpt.fetch() if options.revision is not None: _log.info('Checking out %s', options.revision) self.run(['git', 'checkout', options.revision], cwd=local_wpt.path) _log.info('Noting the revision we are importing.') _, show_ref_output = self.run(['git', 'show-ref', 'origin/master'], cwd=local_wpt.path) import_commit = 'wpt@%s' % show_ref_output.split()[0] commit_message = self._commit_message(chromium_commit, import_commit) if not options.ignore_exportable_commits: commits = self.apply_exportable_commits_locally(local_wpt) if commits is None: _log.error('Could not apply some exportable commits cleanly.') _log.error('Aborting import to prevent clobbering commits.') return 1 commit_message = self._commit_message( chromium_commit, import_commit, locally_applied_commits=commits) dest_path = self.finder.path_from_layout_tests('external', 'wpt') self._clear_out_dest_path(dest_path) _log.info('Copying the tests from the temp repo to the destination.') test_copier = TestCopier(self.host, local_wpt.path) test_copier.do_import() self.run(['git', 'add', '--all', 'external/wpt']) self._delete_orphaned_baselines(dest_path) # TODO(qyearsley): Consider updating manifest after adding baselines. self._generate_manifest(dest_path) # TODO(qyearsley): Consider running the imported tests with # `run-webkit-tests --reset-results external/wpt` to get some baselines # before the try jobs are started. _log.info( 'Updating TestExpectations for any removed or renamed tests.') self.update_all_test_expectations_files(self._list_deleted_tests(), self._list_renamed_tests()) has_changes = self._has_changes() if not has_changes: _log.info('Done: no changes to import.') return 0 self.run(['git', 'commit', '--all', '-F', '-'], stdin=commit_message) self._commit_changes(commit_message) _log.info('Changes imported and committed.') if not options.auto_update: return 0 self._upload_cl() _log.info('Issue: %s', self.git_cl.run(['issue']).strip()) if not self.update_expectations_for_cl(): return 1 if not self.run_commit_queue_for_cl(): return 1 return 0
def main(self, argv=None): # TODO(robertma): Test this method! Split it to make it easier to test # if necessary. options = self.parse_args(argv) self.verbose = options.verbose log_level = logging.DEBUG if self.verbose else logging.INFO configure_logging(logging_level=log_level, include_time=True) if options.verbose: # Print out the full output when executive.run_command fails. self.host.executive.error_output_limit = None if not self.checkout_is_okay(): return 1 credentials = read_credentials(self.host, options.credentials_json) gh_user = credentials.get('GH_USER') gh_token = credentials.get('GH_TOKEN') self.wpt_github = self.wpt_github or WPTGitHub(self.host, gh_user, gh_token) self.git_cl = GitCL( self.host, auth_refresh_token_json=options.auth_refresh_token_json) _log.debug('Noting the current Chromium revision.') chromium_revision = self.chromium_git.latest_git_commit() # Instantiate Git after local_wpt.fetch() to make sure the path exists. local_wpt = LocalWPT(self.host, gh_token=gh_token) local_wpt.fetch() self.wpt_git = self.host.git(local_wpt.path) if options.revision is not None: _log.info('Checking out %s', options.revision) self.wpt_git.run(['checkout', options.revision]) _log.debug('Noting the revision we are importing.') self.wpt_revision = self.wpt_git.latest_git_commit() self.last_wpt_revision = self._get_last_imported_wpt_revision() import_commit = 'wpt@%s' % self.wpt_revision _log.info('Importing %s to Chromium %s', import_commit, chromium_revision) if options.ignore_exportable_commits: commit_message = self._commit_message(chromium_revision, import_commit) else: commits = self.apply_exportable_commits_locally(local_wpt) if commits is None: _log.error('Could not apply some exportable commits cleanly.') _log.error('Aborting import to prevent clobbering commits.') return 1 commit_message = self._commit_message( chromium_revision, import_commit, locally_applied_commits=commits) self._clear_out_dest_path() _log.info('Copying the tests from the temp repo to the destination.') test_copier = TestCopier(self.host, local_wpt.path) test_copier.do_import() # TODO(robertma): Implement `add --all` in Git (it is different from `commit --all`). self.chromium_git.run(['add', '--all', self.dest_path]) self._generate_manifest() self._delete_orphaned_baselines() # TODO(qyearsley): Consider running the imported tests with # `run-webkit-tests --reset-results external/wpt` to get some baselines # before the try jobs are started. _log.info( 'Updating TestExpectations for any removed or renamed tests.') self.update_all_test_expectations_files(self._list_deleted_tests(), self._list_renamed_tests()) if not self.chromium_git.has_working_directory_changes(): _log.info('Done: no changes to import.') return 0 if self._only_wpt_manifest_changed(): _log.info( 'Only WPT_BASE_MANIFEST.json was updated; skipping the import.' ) return 0 self._commit_changes(commit_message) _log.info('Changes imported and committed.') if not options.auto_update: return 0 self._upload_cl() _log.info('Issue: %s', self.git_cl.run(['issue']).strip()) if not self.update_expectations_for_cl(): return 1 if not self.run_commit_queue_for_cl(): return 1 if not self.send_notifications(local_wpt, options.auto_file_bugs, options.monorail_auth_json): return 1 return 0
class TestExporter(object): def __init__(self, host, gh_user, gh_token, gerrit_user, gerrit_token, dry_run=False): self.host = host self.wpt_github = WPTGitHub(host, gh_user, gh_token) self.gerrit = GerritAPI(self.host, gerrit_user, gerrit_token) self.dry_run = dry_run self.local_wpt = LocalWPT(self.host, gh_token) self.local_wpt.fetch() def run(self): """For last n commits on Chromium master, create or try to merge a PR. The exporter will look in chronological order at every commit in Chromium. """ open_gerrit_cls = self.gerrit.query_exportable_open_cls() self.process_gerrit_cls(open_gerrit_cls) exportable_commits = self.get_exportable_commits() for exportable_commit in exportable_commits: pull_request = self.wpt_github.pr_for_chromium_commit( exportable_commit) if pull_request: if pull_request.state == 'open': self.merge_pull_request(pull_request) else: _log.info('Pull request is not open: #%d %s', pull_request.number, pull_request.title) else: self.create_pull_request(exportable_commit) def process_gerrit_cls(self, gerrit_cls): """Creates or updates PRs for Gerrit CLs.""" for cl in gerrit_cls: _log.info('Found Gerrit in-flight CL: "%s" %s', cl.subject, cl.url) if not cl.has_review_started: _log.info('CL review has not started, skipping.') continue # Check if CL already has a corresponding PR pull_request = self.wpt_github.pr_with_change_id(cl.change_id) if pull_request: pr_url = '{}pull/{}'.format(WPT_GH_URL, pull_request.number) _log.info('In-flight PR found: %s', pr_url) pr_cl_revision = self.wpt_github.extract_metadata( WPT_REVISION_FOOTER + ' ', pull_request.body) if cl.current_revision_sha == pr_cl_revision: _log.info( 'PR revision matches CL revision. Nothing to do here.') continue _log.info('New revision found, updating PR...') self.create_or_update_pull_request_from_cl(cl, pull_request) else: _log.info('No in-flight PR found for CL. Creating...') self.create_or_update_pull_request_from_cl(cl) def get_exportable_commits(self): return exportable_commits_over_last_n_commits(self.host, self.local_wpt, self.wpt_github) def merge_pull_request(self, pull_request): _log.info('In-flight PR found: %s', pull_request.title) _log.info('%spull/%d', WPT_GH_URL, pull_request.number) if self.dry_run: _log.info('[dry_run] Would have attempted to merge PR') return if PROVISIONAL_PR_LABEL in pull_request.labels: _log.info('Removing provisional label "%s"...', PROVISIONAL_PR_LABEL) self.wpt_github.remove_label(pull_request.number, PROVISIONAL_PR_LABEL) _log.info('Attempting to merge...') # This is outside of the try block because if there's a problem communicating # with the GitHub API, we should hard fail. branch = self.wpt_github.get_pr_branch(pull_request.number) try: self.wpt_github.merge_pull_request(pull_request.number) # This is in the try block because if a PR can't be merged, we shouldn't # delete its branch. self.wpt_github.delete_remote_branch(branch) change_id = self.wpt_github.extract_metadata( 'Change-Id: ', pull_request.body) if change_id: cl = GerritCL(data={'change_id': change_id}, api=self.gerrit) pr_url = '{}pull/{}'.format(WPT_GH_URL, pull_request.number) cl.post_comment(( 'The WPT PR for this CL has been merged upstream! {pr_url}' ).format(pr_url=pr_url)) except MergeError: _log.info('Could not merge PR.') def export_first_exportable_commit(self): """Looks for exportable commits in Chromium, creates PR if found.""" wpt_commit, chromium_commit = self.local_wpt.most_recent_chromium_commit( ) assert chromium_commit, 'No Chromium commit found, this is impossible' wpt_behind_master = self.local_wpt.commits_behind_master(wpt_commit) _log.info('\nLast Chromium export commit in web-platform-tests:') _log.info('web-platform-tests@%s', wpt_commit) _log.info('(%d behind web-platform-tests@origin/master)', wpt_behind_master) _log.info( '\nThe above WPT commit points to the following Chromium commit:') _log.info('chromium@%s', chromium_commit.sha) _log.info('(%d behind chromium@origin/master)', chromium_commit.num_behind_master()) exportable_commits = exportable_commits_over_last_n_commits( chromium_commit.sha, self.host, self.local_wpt, self.wpt_github) if not exportable_commits: _log.info('No exportable commits found in Chromium, stopping.') return _log.info('Found %d exportable commits in Chromium:', len(exportable_commits)) for commit in exportable_commits: _log.info('- %s %s', commit, commit.subject()) outbound_commit = exportable_commits[0] _log.info('Picking the earliest commit and creating a PR') _log.info('- %s %s', outbound_commit.sha, outbound_commit.subject()) self.create_pull_request(outbound_commit) def create_pull_request(self, outbound_commit): patch = outbound_commit.format_patch() message = outbound_commit.message() author = outbound_commit.author() if self.dry_run: _log.info('[dry_run] Stopping before creating PR') _log.info('\n\n[dry_run] message:') _log.info(message) _log.info('\n\n[dry_run] patch:') _log.info(patch) return branch_name = 'chromium-export-{sha}'.format( sha=outbound_commit.short_sha) self.local_wpt.create_branch_with_patch(branch_name, message, patch, author) response_data = self.wpt_github.create_pr( remote_branch_name=branch_name, desc_title=outbound_commit.subject(), body=outbound_commit.body()) _log.info('Create PR response: %s', response_data) if response_data: data, status_code = self.wpt_github.add_label( response_data['number'], EXPORT_PR_LABEL) _log.info('Add label response (status %s): %s', status_code, data) return response_data def create_or_update_pull_request_from_cl(self, cl, pull_request=None): patch = cl.get_patch() updating = bool(pull_request) action_str = 'updating' if updating else 'creating' if self.local_wpt.test_patch(patch) == '': _log.error('Gerrit CL patch did not apply cleanly.') _log.error('First 500 characters of patch: %s', patch[0:500]) return if self.dry_run: _log.info('[dry_run] Stopping before %s PR from CL', action_str) _log.info('\n\n[dry_run] subject:') _log.info(cl.subject) _log.debug('\n\n[dry_run] patch[0:500]:') _log.debug(patch[0:500]) return message = cl.latest_commit_message_with_footers() # Annotate revision footer for Exporter's later use. message = '\n'.join([ line for line in message.split('\n') if WPT_REVISION_FOOTER not in line ]) message += '\n{} {}'.format(WPT_REVISION_FOOTER, cl.current_revision_sha) branch_name = 'chromium-export-cl-{id}'.format(id=cl.change_id) self.local_wpt.create_branch_with_patch(branch_name, message, patch, cl.owner_email, force_push=True) if updating: response_data = self.wpt_github.update_pr(pull_request.number, cl.subject, message) _log.debug('Update PR response: %s', response_data) # TODO(jeffcarp): Turn PullRequest into a class with a .url method cl.post_comment( ('Successfully updated WPT GitHub pull request with ' 'new revision "{subject}": {pr_url}').format( subject=cl.current_revision_description, pr_url='%spull/%d' % (WPT_GH_URL, pull_request.number), )) else: response_data = self.wpt_github.create_pr(branch_name, cl.subject, message) _log.debug('Create PR response: %s', response_data) self.wpt_github.add_label(response_data['number'], EXPORT_PR_LABEL) self.wpt_github.add_label(response_data['number'], PROVISIONAL_PR_LABEL) cl.post_comment(( 'Exportable changes to web-platform-tests were detected in this CL ' 'and a pull request in the upstream repo has been made: {pr_url}.\n\n' 'If this CL lands and Travis CI upstream is green, we will auto-merge the PR.\n\n' 'Note: Please check the Travis CI status (at the bottom of the PR) ' 'before landing this CL and only land this CL if the status is green. ' 'Otherwise a human needs to step in and resolve it manually, during which time ' 'the WPT Importer is blocked from operating.\n\n' '(There is ongoing work to 1. prevent CLs with red upstream PRs from landing ' '(https://crbug.com/711447) and 2. prevent the importer from being blocked on ' 'stuck exportable changes (https://crbug.com/734121))\n\n' 'WPT Export docs:\n' 'https://chromium.googlesource.com/chromium/src/+/master' '/docs/testing/web_platform_tests.md#Automatic-export-process' ).format(pr_url='%spull/%d' % (WPT_GH_URL, response_data['number']))) return response_data
class TestExporter(object): def __init__(self, host, gh_user, gh_token, gerrit_user, gerrit_token, dry_run=False): self.host = host self.wpt_github = WPTGitHub(host, gh_user, gh_token, pr_history_window=PR_HISTORY_WINDOW) self.gerrit = GerritAPI(self.host, gerrit_user, gerrit_token) self.dry_run = dry_run self.local_wpt = LocalWPT(self.host, gh_token) self.local_wpt.fetch() def run(self): """For last n commits on Chromium master, create or try to merge a PR. The exporter will look in chronological order at every commit in Chromium. """ open_gerrit_cls = self.gerrit.query_exportable_open_cls() self.process_gerrit_cls(open_gerrit_cls) exportable_commits = self.get_exportable_commits(limit=COMMIT_HISTORY_WINDOW) for exportable_commit in exportable_commits: pull_request = self.corresponding_pull_request_for_commit(exportable_commit) if pull_request: if pull_request.state == 'open': self.merge_pull_request(pull_request) else: _log.info('Pull request is not open: #%d %s', pull_request.number, pull_request.title) else: self.create_pull_request(exportable_commit) def process_gerrit_cls(self, gerrit_cls): """Creates or updates PRs for Gerrit CLs.""" for cl in gerrit_cls: _log.info('Found Gerrit in-flight CL: "%s" %s', cl.subject, cl.url) # Check if CL already has a corresponding PR pull_request = self.wpt_github.pr_with_change_id(cl.change_id) if pull_request: pr_url = '{}pull/{}'.format(WPT_GH_URL, pull_request.number) _log.info('In-flight PR found: %s', pr_url) pr_cl_revision = self.wpt_github.extract_metadata(WPT_REVISION_FOOTER + ' ', pull_request.body) if cl.current_revision_sha == pr_cl_revision: _log.info('PR revision matches CL revision. Nothing to do here.') continue _log.info('New revision found, updating PR...') self.create_or_update_pull_request_from_cl(cl, pull_request) else: _log.info('No in-flight PR found for CL. Creating...') self.create_or_update_pull_request_from_cl(cl) def get_exportable_commits(self, limit): return exportable_commits_over_last_n_commits(limit, self.host, self.local_wpt) def merge_pull_request(self, pull_request): _log.info('In-flight PR found: %s', pull_request.title) _log.info('%spull/%d', WPT_GH_URL, pull_request.number) if self.dry_run: _log.info('[dry_run] Would have attempted to merge PR') return _log.info('Attempting to merge...') # This is outside of the try block because if there's a problem communicating # with the GitHub API, we should hard fail. branch = self.wpt_github.get_pr_branch(pull_request.number) try: self.wpt_github.merge_pull_request(pull_request.number) # This is in the try block because if a PR can't be merged, we shouldn't # delete its branch. self.wpt_github.delete_remote_branch(branch) change_id = self.wpt_github.extract_metadata('Change-Id: ', pull_request.body) if change_id: cl = GerritCL(data={'change_id': change_id}, api=self.gerrit) pr_url = '{}pull/{}'.format(WPT_GH_URL, pull_request.number) cl.post_comment(( 'The WPT PR for this CL has been merged upstream! {pr_url}' ).format( pr_url=pr_url )) except MergeError: _log.info('Could not merge PR.') def export_first_exportable_commit(self): """Looks for exportable commits in Chromium, creates PR if found.""" wpt_commit, chromium_commit = self.local_wpt.most_recent_chromium_commit() assert chromium_commit, 'No Chromium commit found, this is impossible' wpt_behind_master = self.local_wpt.commits_behind_master(wpt_commit) _log.info('\nLast Chromium export commit in web-platform-tests:') _log.info('web-platform-tests@%s', wpt_commit) _log.info('(%d behind web-platform-tests@origin/master)', wpt_behind_master) _log.info('\nThe above WPT commit points to the following Chromium commit:') _log.info('chromium@%s', chromium_commit.sha) _log.info('(%d behind chromium@origin/master)', chromium_commit.num_behind_master()) exportable_commits = exportable_commits_since(chromium_commit.sha, self.host, self.local_wpt) if not exportable_commits: _log.info('No exportable commits found in Chromium, stopping.') return _log.info('Found %d exportable commits in Chromium:', len(exportable_commits)) for commit in exportable_commits: _log.info('- %s %s', commit, commit.subject()) outbound_commit = exportable_commits[0] _log.info('Picking the earliest commit and creating a PR') _log.info('- %s %s', outbound_commit.sha, outbound_commit.subject()) self.create_pull_request(outbound_commit) def create_pull_request(self, outbound_commit): patch = outbound_commit.format_patch() message = outbound_commit.message() author = outbound_commit.author() if self.dry_run: _log.info('[dry_run] Stopping before creating PR') _log.info('\n\n[dry_run] message:') _log.info(message) _log.info('\n\n[dry_run] patch:') _log.info(patch) return branch_name = 'chromium-export-{sha}'.format(sha=outbound_commit.short_sha) self.local_wpt.create_branch_with_patch(branch_name, message, patch, author) response_data = self.wpt_github.create_pr( remote_branch_name=branch_name, desc_title=outbound_commit.subject(), body=outbound_commit.body()) _log.info('Create PR response: %s', response_data) if response_data: data, status_code = self.wpt_github.add_label(response_data['number']) _log.info('Add label response (status %s): %s', status_code, data) return response_data def create_or_update_pull_request_from_cl(self, cl, pull_request=None): patch = cl.get_patch() updating = bool(pull_request) action_str = 'updating' if updating else 'creating' if self.local_wpt.test_patch(patch) == '': _log.error('Gerrit CL patch did not apply cleanly.') _log.error('First 500 characters of patch: %s', patch[0:500]) return if self.dry_run: _log.info('[dry_run] Stopping before %s PR from CL', action_str) _log.info('\n\n[dry_run] subject:') _log.info(cl.subject) _log.debug('\n\n[dry_run] patch[0:500]:') _log.debug(patch[0:500]) return message = cl.latest_commit_message_with_footers() # Annotate revision footer for Exporter's later use. message = '\n'.join([line for line in message.split('\n') if WPT_REVISION_FOOTER not in line]) message += '\n{} {}'.format(WPT_REVISION_FOOTER, cl.current_revision_sha) branch_name = 'chromium-export-cl-{id}'.format(id=cl.change_id) self.local_wpt.create_branch_with_patch(branch_name, message, patch, cl.owner_email, force_push=True) if updating: response_data = self.wpt_github.update_pr(pull_request.number, cl.subject, message) _log.debug('Update PR response: %s', response_data) # TODO(jeffcarp): Turn PullRequest into a class with a .url method cl.post_comment(( 'Successfully updated WPT GitHub pull request with ' 'new revision "{subject}": {pr_url}' ).format( subject=cl.current_revision_description, pr_url='%spull/%d' % (WPT_GH_URL, pull_request.number), )) else: response_data = self.wpt_github.create_pr(branch_name, cl.subject, message) _log.debug('Create PR response: %s', response_data) data, status_code = self.wpt_github.add_label(response_data['number']) _log.info('Add label response (status %s): %s', status_code, data) cl.post_comment(( 'Exportable changes to web-platform-tests were detected in this CL ' 'and a pull request in the upstream repo has been made: {pr_url}.\n\n' 'Travis CI has been kicked off and if it fails, we will let you know here. ' 'If this CL lands and Travis CI is green, we will auto-merge the PR.' ).format( pr_url='%spull/%d' % (WPT_GH_URL, response_data['number']) )) return response_data def corresponding_pull_request_for_commit(self, exportable_commit): """Search pull requests for one that corresponds to exportable_commit. Returns the pull_request if found, else returns None. """ # Check for PRs created by commits on master. pull_request = self.wpt_github.pr_with_position(exportable_commit.position) if pull_request: return pull_request # Check for PRs created by open Gerrit CLs. change_id = exportable_commit.change_id() if change_id: return self.wpt_github.pr_with_change_id(change_id) return None