def _run_trigger_test(self, regenerate): host = MockHost() git = host.git() git_cl = MockGitCL(host) finder = PathFinder(host.filesystem) flag_file = finder.path_from_layout_tests( 'additional-driver-flag.setting') flag_expectations_file = finder.path_from_layout_tests( 'FlagExpectations', 'foo') cmd = ['trigger', '--flag=--foo'] if regenerate: cmd.append('--regenerate') TryFlag(cmd, host, git_cl).run() expected_added_paths = {flag_file} expected_commits = [[ 'Flag try job: force --foo for run_web_tests.py.' ]] if regenerate: expected_added_paths.add(flag_expectations_file) expected_commits.append( ['Flag try job: clear expectations for --foo.']) self.assertEqual(git.added_paths, expected_added_paths) self.assertEqual(git.local_commits(), expected_commits) self.assertEqual( git_cl.calls, [[ 'git', 'cl', 'upload', '--bypass-hooks', '-f', '-m', 'Flag try job for --foo.' ], [ 'git', 'cl', 'try', '-B', 'luci.chromium.try', '-b', 'linux_chromium_rel_ng' ], [ 'git', 'cl', 'try', '-B', 'master.tryserver.chromium.mac', '-b', 'mac_chromium_rel_ng' ], [ 'git', 'cl', 'try', '-B', 'master.tryserver.chromium.win', '-b', 'win7_chromium_rel_ng' ]])
class ImportNotifierTest(unittest.TestCase): def setUp(self): self.host = MockHost() # Mock a virtual test suite at virtual/gpu/external/wpt/foo. self.host.filesystem = MockFileSystem({ MOCK_WEB_TESTS + 'VirtualTestSuites': '[{"prefix": "gpu", "bases": ["external/wpt/foo"], "args": ["--foo"]}]' }) self.git = self.host.git() self.local_wpt = MockLocalWPT() self.notifier = ImportNotifier(self.host, self.git, self.local_wpt) def test_find_changed_baselines_of_tests(self): changed_files = [ RELATIVE_WEB_TESTS + 'external/wpt/foo/bar.html', RELATIVE_WEB_TESTS + 'external/wpt/foo/bar-expected.txt', RELATIVE_WEB_TESTS + 'platform/linux/external/wpt/foo/bar-expected.txt', RELATIVE_WEB_TESTS + 'external/wpt/random_stuff.html', ] self.git.changed_files = lambda: changed_files self.assertEqual( self.notifier.find_changed_baselines_of_tests( ['external/wpt/foo/bar.html']), { 'external/wpt/foo/bar.html': [ RELATIVE_WEB_TESTS + 'external/wpt/foo/bar-expected.txt', RELATIVE_WEB_TESTS + 'platform/linux/external/wpt/foo/bar-expected.txt', ] }) self.assertEqual(self.notifier.find_changed_baselines_of_tests(set()), {}) def test_more_failures_in_baseline_more_fails(self): # Replacing self.host.executive won't work here, because ImportNotifier # has been instantiated with a MockGit backed by an empty MockExecutive. executive = mock_git_commands({ 'diff': ('diff --git a/foo-expected.txt b/foo-expected.txt\n' '--- a/foo-expected.txt\n' '+++ b/foo-expected.txt\n' '-FAIL an old failure\n' '+FAIL new failure 1\n' '+FAIL new failure 2\n') }) self.notifier.git = MockGit(executive=executive) self.assertTrue( self.notifier.more_failures_in_baseline('foo-expected.txt')) self.assertEqual( executive.calls, [['git', 'diff', '-U0', 'origin/master', '--', 'foo-expected.txt'] ]) def test_more_failures_in_baseline_fewer_fails(self): executive = mock_git_commands({ 'diff': ('diff --git a/foo-expected.txt b/foo-expected.txt\n' '--- a/foo-expected.txt\n' '+++ b/foo-expected.txt\n' '-FAIL an old failure\n' '-FAIL new failure 1\n' '+FAIL new failure 2\n') }) self.notifier.git = MockGit(executive=executive) self.assertFalse( self.notifier.more_failures_in_baseline('foo-expected.txt')) def test_more_failures_in_baseline_same_fails(self): executive = mock_git_commands({ 'diff': ('diff --git a/foo-expected.txt b/foo-expected.txt\n' '--- a/foo-expected.txt\n' '+++ b/foo-expected.txt\n' '-FAIL an old failure\n' '+FAIL a new failure\n') }) self.notifier.git = MockGit(executive=executive) self.assertFalse( self.notifier.more_failures_in_baseline('foo-expected.txt')) def test_examine_baseline_changes(self): self.host.filesystem.write_text_file( MOCK_WEB_TESTS + 'external/wpt/foo/OWNERS', '*****@*****.**') changed_test_baselines = { 'external/wpt/foo/bar.html': [ RELATIVE_WEB_TESTS + 'external/wpt/foo/bar-expected.txt', RELATIVE_WEB_TESTS + 'platform/linux/external/wpt/foo/bar-expected.txt', ] } gerrit_url_with_ps = 'https://crrev.com/c/12345/3/' self.notifier.more_failures_in_baseline = lambda _: True self.notifier.examine_baseline_changes(changed_test_baselines, gerrit_url_with_ps) self.assertEqual( self.notifier.new_failures_by_directory, { 'external/wpt/foo': [ TestFailure(TestFailure.BASELINE_CHANGE, 'external/wpt/foo/bar.html', baseline_path=RELATIVE_WEB_TESTS + 'external/wpt/foo/bar-expected.txt', gerrit_url_with_ps=gerrit_url_with_ps), TestFailure( TestFailure.BASELINE_CHANGE, 'external/wpt/foo/bar.html', baseline_path=RELATIVE_WEB_TESTS + 'platform/linux/external/wpt/foo/bar-expected.txt', gerrit_url_with_ps=gerrit_url_with_ps), ] }) def test_examine_new_test_expectations(self): self.host.filesystem.write_text_file( MOCK_WEB_TESTS + 'external/wpt/foo/OWNERS', '*****@*****.**') test_expectations = { 'external/wpt/foo/bar.html': [ 'crbug.com/12345 [ Linux ] external/wpt/foo/bar.html [ Fail ]', 'crbug.com/12345 [ Win ] external/wpt/foo/bar.html [ Timeout ]', ] } self.notifier.examine_new_test_expectations(test_expectations) self.assertEqual( self.notifier.new_failures_by_directory, { 'external/wpt/foo': [ TestFailure( TestFailure.NEW_EXPECTATION, 'external/wpt/foo/bar.html', expectation_line= 'crbug.com/12345 [ Linux ] external/wpt/foo/bar.html [ Fail ]' ), TestFailure( TestFailure.NEW_EXPECTATION, 'external/wpt/foo/bar.html', expectation_line= 'crbug.com/12345 [ Win ] external/wpt/foo/bar.html [ Timeout ]' ), ] }) self.notifier.new_failures_by_directory = {} self.notifier.examine_new_test_expectations({}) self.assertEqual(self.notifier.new_failures_by_directory, {}) def test_format_commit_list(self): imported_commits = [ ('SHA1', 'Subject 1'), # Use non-ASCII chars to really test Unicode handling. ('SHA2', u'ABC~‾¥≈¤・・•∙·☼★星🌟星★☼·∙•・・¤≈¥‾~XYZ' ) ] def _is_commit_affecting_directory(commit, directory): self.assertIn(commit, ('SHA1', 'SHA2')) self.assertEqual(directory, 'foo') return commit == 'SHA1' self.local_wpt.is_commit_affecting_directory = _is_commit_affecting_directory self.assertEqual( self.notifier.format_commit_list( imported_commits, MOCK_WEB_TESTS + 'external/wpt/foo'), u'Subject 1: https://github.com/web-platform-tests/wpt/commit/SHA1 [affecting this directory]\n' u'ABC~‾¥≈¤・・•∙·☼★星🌟星★☼·∙•・・¤≈¥‾~XYZ: https://github.com/web-platform-tests/wpt/commit/SHA2\n' ) def test_find_owned_directory_non_virtual(self): self.host.filesystem.write_text_file( MOCK_WEB_TESTS + 'external/wpt/foo/OWNERS', '*****@*****.**') self.assertEqual( self.notifier.find_owned_directory('external/wpt/foo/bar.html'), 'external/wpt/foo') self.assertEqual( self.notifier.find_owned_directory( 'external/wpt/foo/bar/baz.html'), 'external/wpt/foo') def test_find_owned_directory_virtual(self): self.host.filesystem.write_text_file( MOCK_WEB_TESTS + 'external/wpt/foo/OWNERS', '*****@*****.**') self.assertEqual( self.notifier.find_owned_directory( 'virtual/gpu/external/wpt/foo/bar.html'), 'external/wpt/foo') def test_create_bugs_from_new_failures(self): self.host.filesystem.write_text_file( MOCK_WEB_TESTS + 'external/wpt/foo/OWNERS', '# COMPONENT: Blink>Infra>Ecosystem\n' '# WPT-NOTIFY: true\n' '[email protected]\n') self.host.filesystem.write_text_file( MOCK_WEB_TESTS + 'external/wpt/bar/OWNERS', '*****@*****.**') self.notifier.new_failures_by_directory = { 'external/wpt/foo': [ TestFailure( TestFailure.NEW_EXPECTATION, 'external/wpt/foo/baz.html', expectation_line= 'crbug.com/12345 external/wpt/foo/baz.html [ Fail ]') ], 'external/wpt/bar': [ TestFailure( TestFailure.NEW_EXPECTATION, 'external/wpt/bar/baz.html', expectation_line= 'crbug.com/12345 external/wpt/bar/baz.html [ Fail ]') ] } bugs = self.notifier.create_bugs_from_new_failures( 'SHA_START', 'SHA_END', 'https://crrev.com/c/12345') # Only one directory has WPT-NOTIFY enabled. self.assertEqual(len(bugs), 1) # The formatting of imported commits and new failures are already tested. self.assertEqual(bugs[0].body['cc'], ['*****@*****.**']) self.assertEqual(bugs[0].body['components'], ['Blink>Infra>Ecosystem']) self.assertEqual( bugs[0].body['summary'], '[WPT] New failures introduced in external/wpt/foo by import https://crrev.com/c/12345' ) def test_no_bugs_filed_in_dry_run(self): def unreachable(_): self.fail('MonorailAPI should not be instantiated in dry_run.') self.notifier._get_monorail_api = unreachable # pylint: disable=protected-access self.notifier.file_bugs([], True) def test_file_bugs_calls_luci_auth(self): test = self class FakeAPI(object): def __init__(self, service_account_key_json=None, access_token=None): test.assertIsNone(service_account_key_json) test.assertEqual(access_token, 'MOCK output of child process') self.notifier._monorail_api = FakeAPI # pylint: disable=protected-access self.notifier.file_bugs([], False) self.assertEqual(self.host.executive.calls, [['luci-auth', 'token']])
class ExportNotifierTest(LoggingTestCase): def setUp(self): super(ExportNotifierTest, self).setUp() self.host = MockHost() self.git = self.host.git() self.gerrit = MockGerritAPI() self.notifier = ExportNotifier(self.host, self.git, self.gerrit) def test_get_gerrit_sha_from_comment_success(self): gerrit_comment = self.generate_notifier_comment( 123, 'checks', 'SHA', None) actual = PRStatusInfo.get_gerrit_sha_from_comment(gerrit_comment) self.assertEqual(actual, 'SHA') def test_get_gerrit_sha_from_comment_fail(self): gerrit_comment = 'ABC' actual = PRStatusInfo.get_gerrit_sha_from_comment(gerrit_comment) self.assertIsNone(actual) def test_to_gerrit_comment(self): checks_results = {'key1': 'val1', 'key2': 'val2'} result_comment = '\nkey2 (val2)\nkey1 (val1)' pr_status_info = PRStatusInfo(checks_results, 123, 'SHA') expected = self.generate_notifier_comment(123, result_comment, 'SHA', None) actual = pr_status_info.to_gerrit_comment() self.assertEqual(expected, actual) def test_to_gerrit_comment_latest(self): checks_results = {'key1': 'val1', 'key2': 'val2'} result_comment = '\nkey2 (val2)\nkey1 (val1)' pr_status_info = PRStatusInfo(checks_results, 123, None) expected = self.generate_notifier_comment(123, result_comment, 'Latest', None) actual = pr_status_info.to_gerrit_comment() self.assertEqual(expected, actual) def test_to_gerrit_comment_with_patchset(self): checks_results = {'key1': 'val1', 'key2': 'val2'} result_comment = '\nkey2 (val2)\nkey1 (val1)' pr_status_info = PRStatusInfo(checks_results, 123, 'SHA') expected = self.generate_notifier_comment(123, result_comment, 'SHA', 3) actual = pr_status_info.to_gerrit_comment(3) self.assertEqual(expected, actual) def test_get_relevant_failed_taskcluster_checks_success(self): check_runs = [ { "id": "1", "conclusion": "failure", "name": "wpt-chrome-dev-stability" }, { "id": "2", "conclusion": "failure", "name": "wpt-firefox-nightly-stability" }, { "id": "3", "conclusion": "failure", "name": "lint" }, { "id": "4", "conclusion": "failure", "name": "infrastructure/ tests" }, ] expected = { 'wpt-chrome-dev-stability': 'https://github.com/web-platform-tests/wpt/pull/123/checks?check_run_id=1', 'wpt-firefox-nightly-stability': 'https://github.com/web-platform-tests/wpt/pull/123/checks?check_run_id=2', 'lint': 'https://github.com/web-platform-tests/wpt/pull/123/checks?check_run_id=3', 'infrastructure/ tests': 'https://github.com/web-platform-tests/wpt/pull/123/checks?check_run_id=4', } self.assertEqual( self.notifier.get_relevant_failed_taskcluster_checks( check_runs, 123), expected) def test_get_relevant_failed_taskcluster_checks_empty(self): check_runs = [ { "id": "1", "conclusion": "success", "name": "wpt-chrome-dev-stability" }, { "id": "2", "conclusion": "failure", "name": "infra" }, ] self.assertEqual( self.notifier.get_relevant_failed_taskcluster_checks( check_runs, 123), {}) def test_has_latest_taskcluster_status_commented_false(self): pr_status_info = PRStatusInfo('bar', 123, 'SHA') messages = [{ "date": "2019-08-20 17:42:05.000000000", "message": "Uploaded patch set 1.\nInitial upload", "_revision_number": 1 }] actual = self.notifier.has_latest_taskcluster_status_commented( messages, pr_status_info) self.assertFalse(actual) def test_has_latest_taskcluster_status_commented_true(self): pr_status_info = PRStatusInfo({'a': 'b'}, 123, 'SHA') messages = [ { "date": "2019-08-20 17:42:05.000000000", "message": "Uploaded patch set 1.\nInitial upload", "_revision_number": 1 }, { "date": "2019-08-21 17:41:05.000000000", "message": self.generate_notifier_comment(123, 'result', 'SHA', 3), "_revision_number": 2 }, ] actual = self.notifier.has_latest_taskcluster_status_commented( messages, pr_status_info) self.assertTrue(actual) def test_get_check_runs_success(self): self.notifier.wpt_github = MockWPTGitHub(pull_requests=[]) self.notifier.wpt_github.check_runs = [ { "id": "123", "conclusion": "failure", "name": "wpt-chrome-dev-stability" }, { "id": "456", "conclusion": "success", "name": "firefox" }, ] actual = self.notifier.get_check_runs(123) self.assertEqual(len(actual), 2) self.assertEqual(self.notifier.wpt_github.calls, [ 'get_pr_branch', 'get_branch_check_runs', ]) def test_process_failing_prs_success(self): checks_results = {'key1': 'val1', 'key2': 'val2'} result_comment = '\nkey2 (val2)\nkey1 (val1)' self.notifier.dry_run = False self.notifier.gerrit = MockGerritAPI() self.notifier.gerrit.cl = MockGerritCL(data={ 'change_id': 'abc', 'messages': [ { "date": "2019-08-20 17:42:05.000000000", "message": "Uploaded patch set 1.\nInitial upload", "_revision_number": 1 }, { "date": "2019-08-21 17:41:05.000000000", "message": self.generate_notifier_comment(123, 'notbar', 'notnum', 3), "_revision_number": 2 }, ], 'revisions': { 'SHA': { '_number': 1 } } }, api=self.notifier.gerrit) gerrit_dict = {'abc': PRStatusInfo(checks_results, 123, 'SHA')} expected = self.generate_notifier_comment(123, result_comment, 'SHA', 1) self.notifier.process_failing_prs(gerrit_dict) self.assertEqual(self.notifier.gerrit.cls_queried, ['abc']) self.assertEqual(self.notifier.gerrit.request_posted, [('/a/changes/abc/revisions/current/review', { 'message': expected })]) def test_process_failing_prs_has_commented(self): self.notifier.dry_run = False self.notifier.gerrit = MockGerritAPI() self.notifier.gerrit.cl = MockGerritCL(data={ 'change_id': 'abc', 'messages': [ { "date": "2019-08-20 17:42:05.000000000", "message": "Uploaded patch set 1.\nInitial upload", "_revision_number": 1 }, { "date": "2019-08-21 17:41:05.000000000", "message": self.generate_notifier_comment(123, 'bar', 'SHA', 3), "_revision_number": 2 }, ], 'revisions': { 'SHA': { '_number': 1 } } }, api=self.notifier.gerrit) gerrit_dict = {'abc': PRStatusInfo('bar', 123, 'SHA')} self.notifier.process_failing_prs(gerrit_dict) self.assertEqual(self.notifier.gerrit.cls_queried, ['abc']) self.assertEqual(self.notifier.gerrit.request_posted, []) def test_process_failing_prs_with_latest_sha(self): self.notifier.dry_run = False self.notifier.gerrit = MockGerritAPI() self.notifier.gerrit.cl = MockGerritCL(data={ 'change_id': 'abc', 'messages': [ { "date": "2019-08-20 17:42:05.000000000", "message": "Uploaded patch set 1.\nInitial upload", "_revision_number": 1 }, { "date": "2019-08-21 17:41:05.000000000", "message": self.generate_notifier_comment(123, 'notbar', 'notnum', 3), "_revision_number": 2 }, ], 'revisions': { 'SHA': { '_number': 1 } } }, api=self.notifier.gerrit) checks_results = {'key1': 'val1', 'key2': 'val2'} result_comment = '\nkey2 (val2)\nkey1 (val1)' expected = self.generate_notifier_comment(123, result_comment, 'Latest') gerrit_dict = {'abc': PRStatusInfo(checks_results, 123, None)} self.notifier.process_failing_prs(gerrit_dict) self.assertEqual(self.notifier.gerrit.cls_queried, ['abc']) self.assertEqual(self.notifier.gerrit.request_posted, [('/a/changes/abc/revisions/current/review', { 'message': expected })]) def test_process_failing_prs_raise_gerrit_error(self): self.notifier.dry_run = False self.notifier.gerrit = MockGerritAPI(raise_error=True) gerrit_dict = {'abc': PRStatusInfo('bar', 'SHA')} self.notifier.process_failing_prs(gerrit_dict) self.assertEqual(self.notifier.gerrit.cls_queried, ['abc']) self.assertEqual(self.notifier.gerrit.request_posted, []) self.assertLog([ 'INFO: Processing 1 CLs with failed Taskcluster checks.\n', 'INFO: Change-Id: abc\n', 'ERROR: Could not process Gerrit CL abc: Error from query_cl\n', ]) def test_export_notifier_success(self): self.notifier.wpt_github = MockWPTGitHub(pull_requests=[]) self.notifier.wpt_github.recent_failing_pull_requests = [ PullRequest( title='title1', number=1234, body= 'description\nWPT-Export-Revision: hash\nChange-Id: decafbad', state='open', labels=['']) ] self.notifier.wpt_github.check_runs = [ { "id": "123", "conclusion": "failure", "name": "wpt-chrome-dev-stability" }, { "id": "456", "conclusion": "success", "name": "firefox" }, ] checks_results_comment = ('\nwpt-chrome-dev-stability (' 'https://github.com/web-platform-tests/wpt' '/pull/1234/checks?check_run_id=123)') self.notifier.dry_run = False self.notifier.gerrit = MockGerritAPI() self.notifier.gerrit.cl = MockGerritCL(data={ 'change_id': 'decafbad', 'messages': [ { "date": "2019-08-20 17:42:05.000000000", "message": "Uploaded patch set 1.\nInitial upload", "_revision_number": 1 }, { "date": "2019-08-21 17:41:05.000000000", "message": self.generate_notifier_comment(1234, 'notbar', 'notnum', 3), "_revision_number": 2 }, ], 'revisions': { 'hash': { '_number': 2 } } }, api=self.notifier.gerrit) expected = self.generate_notifier_comment(1234, checks_results_comment, 'hash', 2) exit_code = self.notifier.main() self.assertFalse(exit_code) self.assertEqual(self.notifier.wpt_github.calls, [ 'recent_failing_chromium_exports', 'get_pr_branch', 'get_branch_check_runs', ]) self.assertEqual(self.notifier.gerrit.cls_queried, ['decafbad']) self.assertEqual(self.notifier.gerrit.request_posted, [('/a/changes/decafbad/revisions/current/review', { 'message': expected })]) def generate_notifier_comment(self, pr_number, checks_results_comment, sha, patchset=None): if patchset: comment = ( 'The exported PR, https://github.com/web-platform-tests/wpt/pull/{}, ' 'has failed the following check(s) on GitHub:\n{}' '\n\nThese failures will block the export. ' 'They may represent new or existing problems; please take ' 'a look at the output and see if it can be fixed. ' 'Unresolved failures will be looked at by the Ecosystem-Infra ' 'sheriff after this CL has been landed in Chromium; if you ' 'need earlier help please contact [email protected].\n\n' 'Any suggestions to improve this service are welcome; ' 'crbug.com/1027618.\n\n' 'Gerrit CL SHA: {}\n' 'Patchset Number: {}').format(pr_number, checks_results_comment, sha, patchset) else: comment = ( 'The exported PR, https://github.com/web-platform-tests/wpt/pull/{}, ' 'has failed the following check(s) on GitHub:\n{}' '\n\nThese failures will block the export. ' 'They may represent new or existing problems; please take ' 'a look at the output and see if it can be fixed. ' 'Unresolved failures will be looked at by the Ecosystem-Infra ' 'sheriff after this CL has been landed in Chromium; if you ' 'need earlier help please contact [email protected].\n\n' 'Any suggestions to improve this service are welcome; ' 'crbug.com/1027618.\n\n' 'Gerrit CL SHA: {}').format(pr_number, checks_results_comment, sha) return comment
class ExportNotifierTest(LoggingTestCase): def setUp(self): super(ExportNotifierTest, self).setUp() self.host = MockHost() self.git = self.host.git() self.gerrit = MockGerritAPI() self.notifier = ExportNotifier(self.host, self.git, self.gerrit) def test_get_gerrit_sha_from_comment_success(self): gerrit_comment = self.generate_notifier_comment( 123, 'bar', 'num', None) actual = PRStatusInfo.get_gerrit_sha_from_comment(gerrit_comment) self.assertEqual(actual, 'num') def test_get_gerrit_sha_from_comment_fail(self): gerrit_comment = 'ABC' actual = PRStatusInfo.get_gerrit_sha_from_comment(gerrit_comment) self.assertIsNone(actual) def test_to_gerrit_comment(self): pr_status_info = PRStatusInfo('bar', 123, 'num') expected = self.generate_notifier_comment(123, 'bar', 'num', None) actual = pr_status_info.to_gerrit_comment() self.assertEqual(expected, actual) def test_to_gerrit_comment_latest(self): pr_status_info = PRStatusInfo('bar', 123, None) expected = self.generate_notifier_comment(123, 'bar', 'Latest', None) actual = pr_status_info.to_gerrit_comment() self.assertEqual(expected, actual) def test_to_gerrit_comment_with_patchset(self): pr_status_info = PRStatusInfo('bar', 123, 'num') expected = self.generate_notifier_comment(123, 'bar', 'num', 3) actual = pr_status_info.to_gerrit_comment(3) self.assertEqual(expected, actual) def test_get_failure_taskcluster_status_success(self): taskcluster_status = [{ "state": "failure", "context": "Community-TC (pull_request)", }, { "state": "success", "context": "random", }] self.assertEqual( self.notifier.get_failure_taskcluster_status( taskcluster_status, 123), { "state": "failure", "context": "Community-TC (pull_request)", }) def test_get_failure_taskcluster_status_fail(self): taskcluster_status = [ { "state": "success", "context": "Community-TC (pull_request)", }, ] self.assertEqual( self.notifier.get_failure_taskcluster_status( taskcluster_status, 123), None) def test_get_failure_taskcluster_pending_status(self): taskcluster_status = [ { "state": "pending", "context": "Community-TC (pull_request)", }, ] self.assertEqual( self.notifier.get_failure_taskcluster_status( taskcluster_status, 123), None) def test_has_latest_taskcluster_status_commented_false(self): pr_status_info = PRStatusInfo('bar', 123, 'num') messages = [{ "date": "2019-08-20 17:42:05.000000000", "message": "Uploaded patch set 1.\nInitial upload", "_revision_number": 1 }] actual = self.notifier.has_latest_taskcluster_status_commented( messages, pr_status_info) self.assertFalse(actual) def test_has_latest_taskcluster_status_commented_true(self): pr_status_info = PRStatusInfo('bar', 123, 'num') messages = [ { "date": "2019-08-20 17:42:05.000000000", "message": "Uploaded patch set 1.\nInitial upload", "_revision_number": 1 }, { "date": "2019-08-21 17:41:05.000000000", "message": self.generate_notifier_comment(123, 'bar', 'num', 3), "_revision_number": 2 }, ] actual = self.notifier.has_latest_taskcluster_status_commented( messages, pr_status_info) self.assertTrue(actual) self.assertTrue(actual) def test_get_taskcluster_statuses_success(self): self.notifier.wpt_github = MockWPTGitHub(pull_requests=[ PullRequest(title='title1', number=1234, body='description\nWPT-Export-Revision: 1', state='open', labels=[]), ]) status = [{"description": "foo"}] self.notifier.wpt_github.status = status actual = self.notifier.get_taskcluster_statuses(123) self.assertEqual(actual, status) self.assertEqual(self.notifier.wpt_github.calls, [ 'get_pr_branch', 'get_branch_statuses', ]) def test_process_failing_prs_success(self): self.notifier.dry_run = False self.notifier.gerrit = MockGerritAPI() self.notifier.gerrit.cl = MockGerritCL(data={ 'change_id': 'abc', 'messages': [ { "date": "2019-08-20 17:42:05.000000000", "message": "Uploaded patch set 1.\nInitial upload", "_revision_number": 1 }, { "date": "2019-08-21 17:41:05.000000000", "message": self.generate_notifier_comment(123, 'notbar', 'notnum', 3), "_revision_number": 2 }, ], 'revisions': { 'num': { '_number': 1 } } }, api=self.notifier.gerrit) gerrit_dict = {'abc': PRStatusInfo('bar', 123, 'num')} expected = self.generate_notifier_comment(123, 'bar', 'num', 1) self.notifier.process_failing_prs(gerrit_dict) self.assertEqual(self.notifier.gerrit.cls_queried, ['abc']) self.assertEqual(self.notifier.gerrit.request_posted, [('/a/changes/abc/revisions/current/review', { 'message': expected })]) def test_process_failing_prs_has_commented(self): self.notifier.dry_run = False self.notifier.gerrit = MockGerritAPI() self.notifier.gerrit.cl = MockGerritCL(data={ 'change_id': 'abc', 'messages': [ { "date": "2019-08-20 17:42:05.000000000", "message": "Uploaded patch set 1.\nInitial upload", "_revision_number": 1 }, { "date": "2019-08-21 17:41:05.000000000", "message": self.generate_notifier_comment(123, 'bar', 'num', 3), "_revision_number": 2 }, ], 'revisions': { 'num': { '_number': 1 } } }, api=self.notifier.gerrit) gerrit_dict = {'abc': PRStatusInfo('bar', 123, 'num')} self.notifier.process_failing_prs(gerrit_dict) self.assertEqual(self.notifier.gerrit.cls_queried, ['abc']) self.assertEqual(self.notifier.gerrit.request_posted, []) def test_process_failing_prs_with_latest_sha(self): self.notifier.dry_run = False self.notifier.gerrit = MockGerritAPI() self.notifier.gerrit.cl = MockGerritCL(data={ 'change_id': 'abc', 'messages': [ { "date": "2019-08-20 17:42:05.000000000", "message": "Uploaded patch set 1.\nInitial upload", "_revision_number": 1 }, { "date": "2019-08-21 17:41:05.000000000", "message": self.generate_notifier_comment(123, 'notbar', 'notnum', 3), "_revision_number": 2 }, ], 'revisions': { 'num': { '_number': 1 } } }, api=self.notifier.gerrit) expected = self.generate_notifier_comment(123, 'bar', 'Latest') gerrit_dict = {'abc': PRStatusInfo('bar', 123, None)} self.notifier.process_failing_prs(gerrit_dict) self.assertEqual(self.notifier.gerrit.cls_queried, ['abc']) self.assertEqual(self.notifier.gerrit.request_posted, [('/a/changes/abc/revisions/current/review', { 'message': expected })]) def test_process_failing_prs_raise_gerrit_error(self): self.notifier.dry_run = False self.notifier.gerrit = MockGerritAPI(raise_error=True) gerrit_dict = {'abc': PRStatusInfo('bar', 'num')} self.notifier.process_failing_prs(gerrit_dict) self.assertEqual(self.notifier.gerrit.cls_queried, ['abc']) self.assertEqual(self.notifier.gerrit.request_posted, []) self.assertLog([ 'INFO: Processing 1 CLs with failed Taskcluster status.\n', 'ERROR: Could not process Gerrit CL abc: Error from query_cl\n' ]) def test_export_notifier_success(self): self.notifier.wpt_github = MockWPTGitHub(pull_requests=[]) self.notifier.wpt_github.recent_failing_pull_requests = [ PullRequest( title='title1', number=1234, body= 'description\nWPT-Export-Revision: hash\nChange-Id: decafbad', state='open', labels=['']) ] status = [{ "state": "failure", "context": "Community-TC (pull_request)", "node_id": "foo", "target_url": "bar" }] self.notifier.wpt_github.status = status self.notifier.dry_run = False self.notifier.gerrit = MockGerritAPI() self.notifier.gerrit.cl = MockGerritCL(data={ 'change_id': 'decafbad', 'messages': [ { "date": "2019-08-20 17:42:05.000000000", "message": "Uploaded patch set 1.\nInitial upload", "_revision_number": 1 }, { "date": "2019-08-21 17:41:05.000000000", "message": self.generate_notifier_comment(1234, 'notbar', 'notnum', 3), "_revision_number": 2 }, ], 'revisions': { 'hash': { '_number': 2 } } }, api=self.notifier.gerrit) expected = self.generate_notifier_comment(1234, 'bar', 'hash', 2) exit_code = self.notifier.main() self.assertFalse(exit_code) self.assertEqual(self.notifier.wpt_github.calls, [ 'recent_failing_chromium_exports', 'get_pr_branch', 'get_branch_statuses', ]) self.assertEqual(self.notifier.gerrit.cls_queried, ['decafbad']) self.assertEqual(self.notifier.gerrit.request_posted, [('/a/changes/decafbad/revisions/current/review', { 'message': expected })]) def generate_notifier_comment(self, pr_number, link, sha, patchset=None): if patchset: comment = ( 'The exported PR, https://github.com/web-platform-tests/wpt/pull/{}, ' 'has failed Taskcluster check(s) on GitHub, which could indicate ' 'cross-browser failures on the exported changes. Please contact ' '[email protected] for more information.\n\n' 'Taskcluster Link: {}\n' 'Gerrit CL SHA: {}\n' 'Patchset Number: {}' '\n\nAny suggestions to improve this service are welcome; ' 'crbug.com/1027618.').format(pr_number, link, sha, patchset) else: comment = ( 'The exported PR, https://github.com/web-platform-tests/wpt/pull/{}, ' 'has failed Taskcluster check(s) on GitHub, which could indicate ' 'cross-browser failures on the exported changes. Please contact ' '[email protected] for more information.\n\n' 'Taskcluster Link: {}\n' 'Gerrit CL SHA: {}' '\n\nAny suggestions to improve this service are welcome; ' 'crbug.com/1027618.').format(pr_number, link, sha) return comment
class ExportNotifierTest(LoggingTestCase): def setUp(self): super(ExportNotifierTest, self).setUp() self.host = MockHost() self.git = self.host.git() self.gerrit = MockGerritAPI() self.notifier = ExportNotifier(self.host, self.git, self.gerrit) def test_from_gerrit_comment_success(self): gerrit_comment = ( 'The exported PR for the current patch failed Taskcluster check(s) ' 'on GitHub, which could indict cross-broswer failures on the ' 'exportable changes. Please contact ecosystem-infra@ team for ' 'more information.\n\n' 'Taskcluster Link: bar\n' 'Gerrit CL SHA: num') actual = PRStatusInfo.from_gerrit_comment(gerrit_comment) self.assertEqual(actual.link, 'bar') self.assertEqual(actual.gerrit_sha, 'num') def test_from_gerrit_comment_missing_info(self): gerrit_comment = ( 'The exported PR for the current patch failed Taskcluster check(s) ' 'on GitHub, which could indict cross-broswer failures on the ' 'exportable changes. Please contact ecosystem-infra@ team for ' 'more information.\n\n' 'Taskcluster Link: \n' 'Gerrit CL SHA: num') actual = PRStatusInfo.from_gerrit_comment(gerrit_comment) self.assertIsNone(actual) def test_from_gerrit_comment_fail(self): gerrit_comment = 'ABC' actual = PRStatusInfo.from_gerrit_comment(gerrit_comment) self.assertIsNone(actual) def test_to_gerrit_comment(self): pr_status_info = PRStatusInfo('bar', 'num') expected = ( 'The exported PR for the current patch failed Taskcluster check(s) ' 'on GitHub, which could indict cross-broswer failures on the ' 'exportable changes. Please contact ecosystem-infra@ team for ' 'more information.\n\n' 'Taskcluster Link: bar\n' 'Gerrit CL SHA: num') actual = pr_status_info.to_gerrit_comment() self.assertEqual(expected, actual) def test_to_gerrit_comment_latest(self): pr_status_info = PRStatusInfo('bar', None) expected = ( 'The exported PR for the current patch failed Taskcluster check(s) ' 'on GitHub, which could indict cross-broswer failures on the ' 'exportable changes. Please contact ecosystem-infra@ team for ' 'more information.\n\n' 'Taskcluster Link: bar\n' 'Gerrit CL SHA: Latest') actual = pr_status_info.to_gerrit_comment() self.assertEqual(expected, actual) def test_to_gerrit_comment_with_patchset(self): pr_status_info = PRStatusInfo('bar', 'num') expected = ( 'The exported PR for the current patch failed Taskcluster check(s) ' 'on GitHub, which could indict cross-broswer failures on the ' 'exportable changes. Please contact ecosystem-infra@ team for ' 'more information.\n\n' 'Taskcluster Link: bar\n' 'Gerrit CL SHA: num\n' 'Patchset Number: 3') actual = pr_status_info.to_gerrit_comment(3) self.assertEqual(expected, actual) def test_get_failure_taskcluster_status_success(self): taskcluster_status = [{ "state": "failure", "context": "Community-TC (pull_request)", }, { "state": "success", "context": "random", }] self.assertEqual( self.notifier.get_failure_taskcluster_status( taskcluster_status, 123), { "state": "failure", "context": "Community-TC (pull_request)", }) def test_get_failure_taskcluster_status_fail(self): taskcluster_status = [ { "state": "success", "context": "Community-TC (pull_request)", }, ] self.assertEqual( self.notifier.get_failure_taskcluster_status( taskcluster_status, 123), None) def test_has_latest_taskcluster_status_commented_false(self): pr_status_info = PRStatusInfo('bar', 'num') messages = [{ "date": "2019-08-20 17:42:05.000000000", "message": "Uploaded patch set 1.\nInitial upload", "_revision_number": 1 }] actual = self.notifier.has_latest_taskcluster_status_commented( messages, pr_status_info) self.assertFalse(actual) def test_has_latest_taskcluster_status_commented_true(self): pr_status_info = PRStatusInfo('bar', 'num') messages = [ { "date": "2019-08-20 17:42:05.000000000", "message": "Uploaded patch set 1.\nInitial upload", "_revision_number": 1 }, { "date": "2019-08-21 17:41:05.000000000", "message": ('The exported PR for the current patch failed Taskcluster check(s) ' 'on GitHub, which could indict cross-broswer failures on the ' 'exportable changes. Please contact ecosystem-infra@ team for ' 'more information.\n\n' 'Taskcluster Link: bar\n' 'Gerrit CL SHA: num\n' 'Patchset Number: 3'), "_revision_number": 2 }, ] actual = self.notifier.has_latest_taskcluster_status_commented( messages, pr_status_info) self.assertTrue(actual) self.assertTrue(actual) def test_get_taskcluster_statuses_success(self): self.notifier.wpt_github = MockWPTGitHub(pull_requests=[ PullRequest(title='title1', number=1234, body='description\nWPT-Export-Revision: 1', state='open', labels=[]), ]) status = [{"description": "foo"}] self.notifier.wpt_github.status = status actual = self.notifier.get_taskcluster_statuses(123) self.assertEqual(actual, status) self.assertEqual(self.notifier.wpt_github.calls, [ 'get_pr_branch', 'get_branch_statuses', ]) def test_process_failing_prs_success(self): self.notifier.dry_run = False self.notifier.gerrit = MockGerritAPI() self.notifier.gerrit.cl = MockGerritCL(data={ 'change_id': 'abc', 'messages': [ { "date": "2019-08-20 17:42:05.000000000", "message": "Uploaded patch set 1.\nInitial upload", "_revision_number": 1 }, { "date": "2019-08-21 17:41:05.000000000", "message": ('The exported PR for the current patch failed Taskcluster check(s) ' 'on GitHub, which could indict cross-broswer failures on the ' 'exportable changes. Please contact ecosystem-infra@ team for ' 'more information.\n\n' 'Taskcluster Link: notbar\n' 'Gerrit CL SHA: notnum\n' 'Patchset Number: 3'), "_revision_number": 2 }, ], 'revisions': { 'num': { '_number': 1 } } }, api=self.notifier.gerrit) gerrit_dict = {'abc': PRStatusInfo('bar', 'num')} expected = ( 'The exported PR for the current patch failed Taskcluster check(s) ' 'on GitHub, which could indict cross-broswer failures on the ' 'exportable changes. Please contact ecosystem-infra@ team for ' 'more information.\n\n' 'Taskcluster Link: bar\n' 'Gerrit CL SHA: num\n' 'Patchset Number: 1') self.notifier.process_failing_prs(gerrit_dict) self.assertEqual(self.notifier.gerrit.cls_queried, ['abc']) self.assertEqual(self.notifier.gerrit.request_posted, [('/a/changes/abc/revisions/current/review', { 'message': expected })]) def test_process_failing_prs_has_commented(self): self.notifier.dry_run = False self.notifier.gerrit = MockGerritAPI() self.notifier.gerrit.cl = MockGerritCL(data={ 'change_id': 'abc', 'messages': [ { "date": "2019-08-20 17:42:05.000000000", "message": "Uploaded patch set 1.\nInitial upload", "_revision_number": 1 }, { "date": "2019-08-21 17:41:05.000000000", "message": ('The exported PR for the current patch failed Taskcluster check(s) ' 'on GitHub, which could indict cross-broswer failures on the ' 'exportable changes. Please contact ecosystem-infra@ team for ' 'more information.\n\n' 'Taskcluster Link: bar\n' 'Gerrit CL SHA: num\n' 'Patchset Number: 3'), "_revision_number": 2 }, ], 'revisions': { 'num': { '_number': 1 } } }, api=self.notifier.gerrit) gerrit_dict = {'abc': PRStatusInfo('bar', 'num')} self.notifier.process_failing_prs(gerrit_dict) self.assertEqual(self.notifier.gerrit.cls_queried, ['abc']) self.assertEqual(self.notifier.gerrit.request_posted, []) def test_process_failing_prs_with_latest_sha(self): self.notifier.dry_run = False self.notifier.gerrit = MockGerritAPI() self.notifier.gerrit.cl = MockGerritCL(data={ 'change_id': 'abc', 'messages': [ { "date": "2019-08-20 17:42:05.000000000", "message": "Uploaded patch set 1.\nInitial upload", "_revision_number": 1 }, { "date": "2019-08-21 17:41:05.000000000", "message": ('The exported PR for the current patch failed Taskcluster check(s) ' 'on GitHub, which could indict cross-broswer failures on the ' 'exportable changes. Please contact ecosystem-infra@ team for ' 'more information.\n\n' 'Taskcluster Link: notbar\n' 'Gerrit CL SHA: notnum\n' 'Patchset Number: 3'), "_revision_number": 2 }, ], 'revisions': { 'num': { '_number': 1 } } }, api=self.notifier.gerrit) expected = ( 'The exported PR for the current patch failed Taskcluster check(s) ' 'on GitHub, which could indict cross-broswer failures on the ' 'exportable changes. Please contact ecosystem-infra@ team for ' 'more information.\n\n' 'Taskcluster Link: bar\n' 'Gerrit CL SHA: Latest') gerrit_dict = {'abc': PRStatusInfo('bar', None)} self.notifier.process_failing_prs(gerrit_dict) self.assertEqual(self.notifier.gerrit.cls_queried, ['abc']) self.assertEqual(self.notifier.gerrit.request_posted, [('/a/changes/abc/revisions/current/review', { 'message': expected })]) def test_process_failing_prs_raise_gerrit_error(self): self.notifier.dry_run = False self.notifier.gerrit = MockGerritAPI(raise_error=True) gerrit_dict = {'abc': PRStatusInfo('bar', 'num')} self.notifier.process_failing_prs(gerrit_dict) self.assertEqual(self.notifier.gerrit.cls_queried, ['abc']) self.assertEqual(self.notifier.gerrit.request_posted, []) self.assertLog([ 'INFO: Processing 1 CLs with failed Taskcluster status.\n', 'ERROR: Could not process Gerrit CL abc: Error from query_cl\n' ]) def test_export_notifier_success(self): self.notifier.wpt_github = MockWPTGitHub(pull_requests=[]) self.notifier.wpt_github.recent_failing_pull_requests = [ PullRequest( title='title1', number=1234, body= 'description\nWPT-Export-Revision: hash\nChange-Id: decafbad', state='open', labels=['']) ] status = [{ "state": "failure", "context": "Community-TC (pull_request)", "node_id": "foo", "target_url": "bar" }] self.notifier.wpt_github.status = status self.notifier.dry_run = False self.notifier.gerrit = MockGerritAPI() self.notifier.gerrit.cl = MockGerritCL(data={ 'change_id': 'decafbad', 'messages': [ { "date": "2019-08-20 17:42:05.000000000", "message": "Uploaded patch set 1.\nInitial upload", "_revision_number": 1 }, { "date": "2019-08-21 17:41:05.000000000", "message": ('The exported PR for the current patch failed Taskcluster check(s) ' 'on GitHub, which could indict cross-broswer failures on the ' 'exportable changes. Please contact ecosystem-infra@ team for ' 'more information.\n\n' 'Taskcluster Link: notbar\n' 'Gerrit CL SHA: notnum\n' 'Patchset Number: 3'), "_revision_number": 2 }, ], 'revisions': { 'hash': { '_number': 2 } } }, api=self.notifier.gerrit) expected = ( 'The exported PR for the current patch failed Taskcluster check(s) ' 'on GitHub, which could indict cross-broswer failures on the ' 'exportable changes. Please contact ecosystem-infra@ team for ' 'more information.\n\n' 'Taskcluster Link: bar\n' 'Gerrit CL SHA: hash\n' 'Patchset Number: 2') exit_code = self.notifier.main() self.assertFalse(exit_code) self.assertEqual(self.notifier.wpt_github.calls, [ 'recent_failing_chromium_exports', 'get_pr_branch', 'get_branch_statuses', ]) self.assertEqual(self.notifier.gerrit.cls_queried, ['decafbad']) self.assertEqual(self.notifier.gerrit.request_posted, [('/a/changes/decafbad/revisions/current/review', { 'message': expected })])