def test_phabricator_doc_upload(mock_config, mock_phabricator, phab, mock_try_task, mock_task): """ Test Phabricator reporter publication on a mock clang-format issue """ with mock_phabricator as api: revision = Revision.from_try(mock_try_task, api) revision.mercurial_revision = "deadbeef1234" revision.repository = "https://hg.mozilla.org/try" revision.repository_try_name = "try" revision.lines = { # Add dummy lines diff "test.rst": [41, 42, 43], } reporter = PhabricatorReporter({"analyzers": ["doc-upload"]}, api=api) reporter.publish( [], revision, [], [ "http://gecko-docs.mozilla.org-l1.s3-website.us-west-2.amazonaws.com/59dc75b0-e207-11ea-8fa5-0242ac110004/index.html" ], ) # Check the comment has been posted assert phab.comments[51] == [VALID_DOC_UPLOAD_MESSAGE]
def test_phabricator_clang_tidy_build_error( mock_phabricator, phab, mock_try_task, mock_task ): """ Test Phabricator Lint for a ClangTidyIssue with build error """ from code_review_bot import Level with mock_phabricator as api: revision = Revision.from_try(mock_try_task, api) revision.lines = { # Add dummy lines diff "test.cpp": [41, 42, 43] } revision.build_target_phid = "PHID-HMBD-deadbeef12456" revision.repository = "https://hg.mozilla.org/try" revision.repository_try_name = "try" revision.mercurial_revision = "deadbeef1234" reporter = PhabricatorReporter({}, api=api) issue = ClangTidyIssue( analyzer=mock_task(ClangTidyTask, "mock-clang-tidy"), revision=revision, path="dom/animation/Animation.cpp", line=57, column=46, check="clang-diagnostic-error", level=Level.Error, message="Some Error Message", publish=True, ) assert issue.is_publishable() issues, patches = reporter.publish([issue], revision, []) assert len(issues) == 1 assert len(patches) == 0 # Check the callback has been used assert phab.build_messages["PHID-HMBD-deadbeef12456"] == [ { "buildTargetPHID": "PHID-HMBD-deadbeef12456", "lint": [ { "code": "clang-diagnostic-error", "description": "(IMPORTANT) ERROR: Some Error Message", "line": 57, "char": 46, "name": "Build Error", "path": "dom/animation/Animation.cpp", "severity": "error", } ], "type": "work", "unit": [], } ] assert phab.comments[51] == [VALID_BUILD_ERROR_MESSAGE]
def test_phabricator_clang_tidy(mock_phabricator, phab, mock_try_task, mock_task): """ Test Phabricator reporter publication on a mock clang-tidy issue """ with mock_phabricator as api: revision = Revision.from_try(mock_try_task, api) revision.mercurial_revision = "deadbeef1234" revision.repository = "https://hg.mozilla.org/try" revision.repository_try_name = "try" revision.lines = { # Add dummy lines diff "another_test.cpp": [41, 42, 43] } revision.files = ["another_test.cpp"] reporter = PhabricatorReporter({"analyzers": ["clang-tidy"]}, api=api) issue = ClangTidyIssue( mock_task(ClangTidyTask, "source-test-clang-tidy"), revision, "another_test.cpp", "42", "51", "modernize-use-nullptr", "dummy message", ) assert issue.is_publishable() issues, patches = reporter.publish([issue], revision, []) assert len(issues) == 1 assert len(patches) == 0 # Check the comment has been posted assert phab.comments[51] == [VALID_CLANG_TIDY_MESSAGE]
def test_phabricator_clang_format(mock_config, mock_phabricator, mock_try_task): """ Test Phabricator reporter publication on a mock clang-format issue """ from code_review_bot.report.phabricator import PhabricatorReporter from code_review_bot.revisions import Revision, ImprovementPatch from code_review_bot.tasks.clang_format import ClangFormatIssue def _check_comment(request): # Check the Phabricator main comment is well formed payload = urllib.parse.parse_qs(request.body) assert payload["output"] == ["json"] assert len(payload["params"]) == 1 details = json.loads(payload["params"][0]) assert details["message"] == VALID_CLANG_FORMAT_MESSAGE.format( results=mock_config.taskcluster.results_dir ) # Outputs dummy empty response resp = {"error_code": None, "result": None} return ( 201, {"Content-Type": "application/json", "unittest": "clang-format"}, json.dumps(resp), ) responses.add_callback( responses.POST, "http://phabricator.test/api/differential.createcomment", callback=_check_comment, ) with mock_phabricator as api: revision = Revision.from_try(mock_try_task, api) revision.lines = { # Add dummy lines diff "test.cpp": [41, 42, 43], "dom/test.cpp": [42], } reporter = PhabricatorReporter({"analyzers": ["clang-format"]}, api=api) issue = ClangFormatIssue( "source-test-clang-format", "dom/test.cpp", 42, 1, revision ) assert issue.is_publishable() revision.improvement_patches = [ ImprovementPatch("clang-format", repr(revision), "Some lint fixes") ] list(map(lambda p: p.write(), revision.improvement_patches)) # trigger local write issues, patches = reporter.publish([issue], revision, []) assert len(issues) == 1 assert len(patches) == 1 # Check the callback has been used assert len(responses.calls) > 0 call = responses.calls[-1] assert call.request.url == "http://phabricator.test/api/differential.createcomment" assert call.response.headers.get("unittest") == "clang-format"
def test_task_failures(mock_phabricator, phab, mock_try_task): """ Test Phabricator reporter publication with some task failures """ with mock_phabricator as api: revision = Revision.from_try(mock_try_task, api) revision.mercurial_revision = "aabbccddee" revision.repository = "https://hg.mozilla.org/try" revision.repository_try_name = "try" reporter = PhabricatorReporter({"analyzers": ["clang-tidy"]}, api=api) status = { "task": { "metadata": { "name": "mock-infer" } }, "status": { "runs": [{ "runId": 0 }] }, } task = ClangTidyTask("ab3NrysvSZyEwsOHL2MZfw", status) issues, patches = reporter.publish([], revision, [task], []) assert len(issues) == 0 assert len(patches) == 0 # Check the callback has been used to post comments assert phab.comments[51] == [VALID_TASK_FAILURES_MESSAGE]
def _test_reporter(api, analyzers_skipped): # Always use the same setup, only varies the analyzers revision = Revision.from_try(mock_try_task, api) revision.lines = {"test.cpp": [0, 41, 42, 43], "dom/test.cpp": [42]} reporter = PhabricatorReporter( {"analyzers_skipped": analyzers_skipped}, api=api ) issues = [ ClangFormatIssue("mock-clang-format", "dom/test.cpp", 42, 1, revision), ClangTidyIssue( "mock-clang-tidy", revision, "test.cpp", "42", "51", "modernize-use-nullptr", "dummy message", ), InferIssue( "mock-infer", { "file": "test.cpp", "line": 42, "column": 1, "bug_type": "dummy", "kind": "WARNING", "qualifier": "dummy message.", }, revision, ), MozLintIssue( "mock-lint-flake8", "test.cpp", 1, "error", 42, "flake8", "Python error", "EXXX", revision, ), CoverageIssue("test.cpp", 0, "This file is uncovered", revision), ] assert all(i.is_publishable() for i in issues) revision.improvement_patches = [ ImprovementPatch("dummy", repr(revision), "Whatever"), ImprovementPatch("mock-clang-tidy", repr(revision), "Some C fixes"), ImprovementPatch("mock-clang-format", repr(revision), "Some lint fixes"), ImprovementPatch("mock-infer", repr(revision), "Some java fixes"), ImprovementPatch("mock-lint-flake8", repr(revision), "Some js fixes"), ] list( map(lambda p: p.write(), revision.improvement_patches) ) # trigger local write return reporter.publish(issues, revision, [])
def mock_revision(mock_phabricator, mock_try_task, mock_config): """ Mock a mercurial revision """ from code_review_bot.revisions import Revision with mock_phabricator as api: return Revision.from_try(mock_try_task, api)
def test_task_failures(mock_phabricator, mock_try_task, mock_treeherder): """ Test Phabricator reporter publication with some task failures """ from code_review_bot.report.phabricator import PhabricatorReporter from code_review_bot.revisions import Revision from code_review_bot.tasks.clang_tidy import ClangTidyTask def _check_comment(request): # Check the Phabricator main comment is well formed payload = urllib.parse.parse_qs(request.body) assert payload["output"] == ["json"] assert len(payload["params"]) == 1 details = json.loads(payload["params"][0]) assert details == { "revision_id": 51, "message": VALID_TASK_FAILURES_MESSAGE, "attach_inlines": 1, "__conduit__": {"token": "deadbeef"}, } # Outputs dummy empty response resp = {"error_code": None, "result": None} return ( 201, {"Content-Type": "application/json", "unittest": "task-failure"}, json.dumps(resp), ) responses.add_callback( responses.POST, "http://phabricator.test/api/differential.createcomment", callback=_check_comment, ) with mock_phabricator as api: revision = Revision.from_try(mock_try_task, api) revision.mercurial_revision = "aabbccddee" reporter = PhabricatorReporter({"analyzers": ["clang-tidy"]}, api=api) status = { "task": {"metadata": {"name": "mock-infer"}}, "status": {"runs": [{"runId": 0}]}, } task = ClangTidyTask("ab3NrysvSZyEwsOHL2MZfw", status) issues, patches = reporter.publish([], revision, [task]) assert len(issues) == 0 assert len(patches) == 0 # Check the callback has been used assert len(responses.calls) > 0 call = responses.calls[-1] assert call.request.url == "http://phabricator.test/api/differential.createcomment" assert call.response.headers.get("unittest") == "task-failure"
def test_phabricator_coverage( mock_config, mock_phabricator, phab, mock_try_task, mock_task ): """ Test Phabricator reporter publication on a mock coverage issue """ with mock_phabricator as api: revision = Revision.from_try(mock_try_task, api) revision.mercurial_revision = "deadbeef1234" revision.repository = "https://hg.mozilla.org/try" revision.repository_try_name = "try" revision.lines = { # Add dummy lines diff "test.txt": [0], "path/to/test.cpp": [0], "dom/test.cpp": [42], } reporter = PhabricatorReporter({"analyzers": ["coverage"]}, api=api) issue = CoverageIssue( mock_task(ZeroCoverageTask, "coverage"), "path/to/test.cpp", 0, "This file is uncovered", revision, ) assert issue.is_publishable() issues, patches = reporter.publish([issue], revision, []) assert len(issues) == 1 assert len(patches) == 0 # Check the lint results assert phab.build_messages["PHID-HMBT-test"] == [ { "buildTargetPHID": "PHID-HMBT-test", "lint": [ { "code": "no-coverage", "description": "WARNING: This file is uncovered", "line": 1, "name": "code coverage analysis", "path": "path/to/test.cpp", "severity": "warning", } ], "type": "work", "unit": [], } ] # Check the callback has been used assert phab.comments[51] == [VALID_COVERAGE_MESSAGE]
def test_phabricator_coverage(mock_config, mock_phabricator, mock_try_task): """ Test Phabricator reporter publication on a mock coverage issue """ from code_review_bot.report.phabricator import PhabricatorReporter from code_review_bot.revisions import Revision from code_review_bot.tasks.coverage import CoverageIssue def _check_comment(request): # Check the Phabricator main comment is well formed payload = urllib.parse.parse_qs(request.body) assert payload["output"] == ["json"] assert len(payload["params"]) == 1 details = json.loads(payload["params"][0]) assert details["message"] == VALID_COVERAGE_MESSAGE # Outputs dummy empty response resp = {"error_code": None, "result": None} return ( 201, {"Content-Type": "application/json", "unittest": "coverage"}, json.dumps(resp), ) responses.add_callback( responses.POST, "http://phabricator.test/api/differential.createcomment", callback=_check_comment, ) with mock_phabricator as api: revision = Revision.from_try(mock_try_task, api) revision.lines = { # Add dummy lines diff "test.txt": [0], "path/to/test.cpp": [0], "dom/test.cpp": [42], } reporter = PhabricatorReporter({"analyzers": ["coverage"]}, api=api) issue = CoverageIssue("path/to/test.cpp", 0, "This file is uncovered", revision) assert issue.is_publishable() issues, patches = reporter.publish([issue], revision, []) assert len(issues) == 1 assert len(patches) == 0 # Check the callback has been used assert len(responses.calls) > 0 call = responses.calls[-1] assert call.request.url == "http://phabricator.test/api/differential.createcomment" assert call.response.headers.get("unittest") == "coverage"
def test_phabricator_external_tidy(mock_phabricator, phab, mock_try_task, mock_task): """ Test Phabricator reporter publication on a mock external-tidy issue """ with mock_phabricator as api: revision = Revision.from_try(mock_try_task, api) revision.mercurial_revision = "deadbeef1234" revision.repository = "https://hg.mozilla.org/try" revision.repository_try_name = "try" revision.lines = { # Add dummy lines diff "another_test.cpp": [41, 42, 43] } revision.files = ["another_test.cpp"] reporter = PhabricatorReporter({"analyzers": ["clang-tidy-external"]}, api=api) issue_clang_diagnostic = ExternalTidyIssue( mock_task(ExternalTidyTask, "source-test-clang-external"), revision, "another_test.cpp", "42", "51", "clang-diagnostic-unused-variable", "dummy message", publish=False, ) issue_civet_warning = ExternalTidyIssue( mock_task(ExternalTidyTask, "source-test-clang-external"), revision, "another_test.cpp", "43", "9", "mozilla-civet-private-checker-1", "dummy message", publish=True, ) assert issue_civet_warning.is_publishable() assert not issue_clang_diagnostic.is_publishable() issues, patches = reporter.publish( [issue_civet_warning, issue_clang_diagnostic], revision, [], []) assert len(issues) == 1 assert len(patches) == 0 # Check the comment has been posted assert phab.comments[51] == [VALID_EXTERNAL_TIDY_MESSAGE]
def mock_revision(mock_phabricator, mock_try_task, mock_config): """ Mock a mercurial revision """ from code_review_bot.revisions import Revision with mock_phabricator as api: revision = Revision.from_try(mock_try_task, api) # Setup mercurial information manually instead of calling setup_try revision.mercurial_revision = "deadbeef123456" revision.repository = "https://hg.mozilla.org/try" revision.target_repository = "https://hg.mozilla.org/mozilla-central" return revision
def test_phabricator_clang_format(mock_config, mock_phabricator, phab, mock_try_task, mock_task): """ Test Phabricator reporter publication on a mock clang-format issue """ with mock_phabricator as api: revision = Revision.from_try(mock_try_task, api) revision.mercurial_revision = "deadbeef1234" revision.repository = "https://hg.mozilla.org/try" revision.repository_try_name = "try" revision.lines = { # Add dummy lines diff "test.cpp": [41, 42, 43], "dom/test.cpp": [42], } reporter = PhabricatorReporter({"analyzers": ["clang-format"]}, api=api) task = mock_task(ClangFormatTask, "source-test-clang-format") lines = [ (41, 41, b"no change"), (42, None, b"deletion"), (None, 42, b"change here"), ] issue = ClangFormatIssue(task, "dom/test.cpp", lines, revision) assert issue.is_publishable() revision.improvement_patches = [ ImprovementPatch(task, repr(revision), "Some lint fixes") ] list(map(lambda p: p.write(), revision.improvement_patches)) # trigger local write issues, patches = reporter.publish([issue], revision, [], []) assert len(issues) == 1 assert len(patches) == 1 # Check the comment has been posted assert phab.comments[51] == [ VALID_CLANG_FORMAT_MESSAGE.format( results=mock_config.taskcluster.results_dir) ]
def test_phabricator_mozlint( reporter_config, errors_reported, mock_config, mock_phabricator, mock_try_task ): """ Test Phabricator reporter publication on a mock mozlint issue """ from code_review_bot.report.phabricator import PhabricatorReporter from code_review_bot.revisions import Revision from code_review_bot.tasks.lint import MozLintIssue def _check_inline(request): payload = urllib.parse.parse_qs(request.body) assert payload["output"] == ["json"] assert len(payload["params"]) == 1 details = json.loads(payload["params"][0]) assert details == { "__conduit__": {"token": "deadbeef"}, "content": "Error: A bad bad error [flake8: EXXX]", "diffID": 42, "filePath": "python/test.py", "isNewFile": 1, "lineLength": 0, "lineNumber": 42, } # Outputs dummy empty response resp = {"error_code": None, "result": {"id": "PHID-XXXX-YYYYY"}} return ( 201, {"Content-Type": "application/json", "unittest": "flake8-inline"}, json.dumps(resp), ) def _check_comment(request): # Check the Phabricator main comment is well formed payload = urllib.parse.parse_qs(request.body) assert payload["output"] == ["json"] assert len(payload["params"]) == 1 details = json.loads(payload["params"][0]) assert details["message"] == VALID_FLAKE8_MESSAGE.format( results=mock_config.taskcluster.results_dir ) # Outputs dummy empty response resp = {"error_code": None, "result": None} return ( 201, {"Content-Type": "application/json", "unittest": "flake8-comment"}, json.dumps(resp), ) def _check_message(request): payload = urllib.parse.parse_qs(request.body) assert payload["output"] == ["json"] assert len(payload["params"]) == 1 details = json.loads(payload["params"][0]) assert details == { "buildTargetPHID": "PHID-HMBT-test", "lint": [ { "char": 1, "code": "EXXX", "description": "A bad bad error", "line": 42, "name": "source-test-mozlint-py-flake8", "path": "python/test.py", "severity": "error", } ], "unit": [], "type": "work", "__conduit__": {"token": "deadbeef"}, } # Outputs dummy empty response resp = {"error_code": None, "result": None} return ( 201, {"Content-Type": "application/json", "unittest": "flake8-error"}, json.dumps(resp), ) responses.add_callback( responses.POST, "http://phabricator.test/api/differential.createinline", callback=_check_inline, ) responses.add_callback( responses.POST, "http://phabricator.test/api/differential.createcomment", callback=_check_comment, ) responses.add_callback( responses.POST, "http://phabricator.test/api/harbormaster.sendmessage", callback=_check_message, ) with mock_phabricator as api: revision = Revision.from_try(mock_try_task, api) revision.lines = { # Add dummy lines diff "python/test.py": [41, 42, 43], "dom/test.cpp": [42], } revision.files = revision.lines.keys() reporter = PhabricatorReporter(reporter_config, api=api) issue = MozLintIssue( analyzer="source-test-mozlint-py-flake8", path="python/test.py", lineno=42, column=1, message="A bad bad error", level="error", revision=revision, linter="flake8", check="EXXX", ) assert issue.is_publishable() issues, patches = reporter.publish([issue], revision, []) assert len(issues) == 1 assert len(patches) == 0 # Check the callbacks have been used to publish either: # - an inline comment + summary comment when publish_errors is False # - a lint result + summary comment when publish_errors is True assert len(responses.calls) > 0 if errors_reported: urls = [ "http://phabricator.test/api/differential.createcomment", "http://phabricator.test/api/harbormaster.sendmessage", ] markers = ["flake8-comment", "flake8-error"] else: urls = [ "http://phabricator.test/api/differential.createinline", "http://phabricator.test/api/differential.createcomment", ] markers = ["flake8-inline", "flake8-comment"] assert [r.request.url for r in responses.calls[-2:]] == urls assert [r.response.headers.get("unittest") for r in responses.calls[-2:]] == markers
def test_extra_errors(mock_phabricator, mock_try_task, phab, mock_task): """ Test Phabricator reporter publication with some errors outside of patch """ with mock_phabricator as api: revision = Revision.from_try(mock_try_task, api) revision.mercurial_revision = "deadbeef1234" revision.repository = "https://hg.mozilla.org/try" revision.repository_try_name = "try" revision.lines = {"path/to/file.py": [1, 2, 3]} revision.files = ["path/to/file.py"] reporter = PhabricatorReporter({}, api=api) task = mock_task(MozLintTask, "source-test-mozlint-dummy") all_issues = [ # Warning in patch MozLintIssue( analyzer=task, path="path/to/file.py", column=25, level=Level.Warning, lineno=2, linter="flake8", message="Some not so bad python mistake", check="EYYY", revision=revision, ), # Error outside of patch MozLintIssue( analyzer=task, path="path/to/file.py", column=12, level=Level.Error, lineno=10, linter="flake8", message="Some bad python typo", check="EXXX", revision=revision, ), # Warning outside of patch MozLintIssue( analyzer=task, path="path/to/file.py", column=1, level=Level.Warning, lineno=25, linter="flake8", message="Random mistake that will be ignored", check="EZZZ", revision=revision, ), ] published_issues, patches = reporter.publish(all_issues, revision, []) assert len(published_issues) == 2 assert len(patches) == 0 # Check the callbacks have been used to publish: # - a top comment to summarize issues # - a lint result for the error outside of patch # - a lint result for the warning inside patch assert phab.build_messages["PHID-HMBT-test"] == [ { "buildTargetPHID": "PHID-HMBT-test", "lint": [ { "char": 25, "code": "EYYY", "description": "WARNING: Some not so bad python mistake", "line": 2, "name": "dummy (Mozlint)", "path": "path/to/file.py", "severity": "warning", }, { "char": 12, "code": "EXXX", "description": "(IMPORTANT) ERROR: Some bad python typo", "line": 10, "name": "dummy (Mozlint)", "path": "path/to/file.py", "severity": "error", }, ], "unit": [], "type": "work", } ] assert phab.comments[51] == [VALID_MOZLINT_MESSAGE]
def test_full_file(mock_config, mock_phabricator, phab, mock_try_task, mock_task): """ Test Phabricator reporter supports an issue on a full file """ with mock_phabricator as api: revision = Revision.from_try(mock_try_task, api) revision.mercurial_revision = "deadbeef1234" revision.repository = "https://hg.mozilla.org/try" revision.repository_try_name = "try" revision.lines = { # Add dummy lines diff "xx.cpp": [123, 124, 125] } revision.files = list(revision.lines.keys()) reporter = PhabricatorReporter(api=api) issue = DefaultIssue( analyzer=mock_task(DefaultTask, "full-file-analyzer"), revision=revision, path="xx.cpp", line=-1, nb_lines=0, check="a-huge-issue", message="Something bad happened on the whole file !", ) assert issue.line is None # check auto conversion assert ( str(issue) == "full-file-analyzer issue a-huge-issue@warning xx.cpp full file" ) assert issue.is_publishable() assert revision.has_file(issue.path) assert revision.contains(issue) issues, patches = reporter.publish([issue], revision, []) assert len(issues) == 1 assert len(patches) == 0 # Check the comment callback has been used assert phab.comments[51] == [ VALID_DEFAULT_MESSAGE.format(results=mock_config.taskcluster.results_dir) ] # Check the inline callback has been used assert phab.build_messages["PHID-HMBT-test"] == [ { "buildTargetPHID": "PHID-HMBT-test", "lint": [ { "code": "a-huge-issue", "description": "WARNING: Something bad happened on the " "whole file !", "line": 1, "name": "full-file-analyzer", "path": "xx.cpp", "severity": "warning", } ], "type": "work", "unit": [], } ]
def test_phabricator_coverity_build_error( mock_phabricator, phab, mock_try_task, mock_task ): """ Test Phabricator Lint for a CoverityIssue """ with mock_phabricator as api: revision = Revision.from_try(mock_try_task, api) revision.lines = { # Add dummy lines diff "test.cpp": [41, 42, 43] } revision.build_target_phid = "PHID-HMBD-deadbeef12456" revision.repository = "https://hg.mozilla.org/try" revision.repository_try_name = "try" revision.mercurial_revision = "deadbeef1234" reporter = PhabricatorReporter({}, api=api) issue_dict = { "line": 41, "reliability": "medium", "message": 'Dereferencing a pointer that might be "nullptr" "env" when calling "lookupImport".', "flag": "NULL_RETURNS", "build_error": True, "extra": { "category": "Null pointer dereferences", "stateOnServer": { "ownerLdapServerName": "local", "stream": "Firefox", "cid": 95687, "cached": False, "retrievalDateTime": "2019-05-13T10:20:22+00:00", "firstDetectedDateTime": "2019-04-08T12:57:07+00:00", "presentInReferenceSnapshot": False, "components": ["js"], "customTriage": {}, "triage": { "fixTarget": "Untargeted", "severity": "Unspecified", "classification": "Unclassified", "owner": "try", "legacy": "False", "action": "Undecided", "externalReference": "", }, }, }, } issue = CoverityIssue( mock_task(CoverityTask, "mock-coverity"), revision, issue_dict, "test.cpp" ) assert issue.is_publishable() issues, patches = reporter.publish([issue], revision, []) assert len(issues) == 1 assert len(patches) == 0 # Check the callback has been used assert phab.build_messages["PHID-HMBD-deadbeef12456"] == [ { "buildTargetPHID": "PHID-HMBD-deadbeef12456", "lint": [ { "code": "NULL_RETURNS", "description": '(IMPORTANT) ERROR: Dereferencing a pointer that might be "nullptr" ' '"env" when calling "lookupImport".', "line": 41, "name": "Build Error", "path": "test.cpp", "severity": "error", } ], "type": "work", "unit": [], } ] assert phab.comments[51] == [VALID_COVERITY_BUILD_ERROR_MESSAGE]
def test_phabricator_analyzers( analyzers_skipped, valid_issues, valid_patches, mock_config, mock_phabricator, mock_try_task, mock_task, ): """ Test analyzers filtering on phabricator reporter """ with mock_phabricator as api: # Skip commenting on phabricator # we only care about filtering issues api.comment = unittest.mock.Mock(return_value=True) # Always use the same setup, only varies the analyzers revision = Revision.from_try(mock_try_task, api) revision.mercurial_revision = "deadbeef1234" revision.repository = "https://hg.mozilla.org/try" revision.repository_try_name = "try" revision.lines = {"test.cpp": [0, 41, 42, 43], "dom/test.cpp": [42]} reporter = PhabricatorReporter( {"analyzers_skipped": analyzers_skipped}, api=api ) issues = [ ClangFormatIssue( mock_task(ClangFormatTask, "mock-clang-format"), "dom/test.cpp", [ (41, 41, b"no change"), (42, None, b"deletion"), (None, 42, b"change here"), ], revision, ), ClangTidyIssue( mock_task(ClangTidyTask, "mock-clang-tidy"), revision, "test.cpp", "42", "51", "modernize-use-nullptr", "dummy message", ), InferIssue( mock_task(InferTask, "mock-infer"), { "file": "test.cpp", "line": 42, "column": 1, "bug_type": "dummy", "kind": "WARNING", "qualifier": "dummy message.", }, revision, ), MozLintIssue( mock_task(MozLintTask, "mock-lint-flake8"), "test.cpp", 1, "error", 42, "flake8", "Python error", "EXXX", revision, ), CoverageIssue( mock_task(ZeroCoverageTask, "coverage"), "test.cpp", 0, "This file is uncovered", revision, ), ] assert all(i.is_publishable() for i in issues) revision.improvement_patches = [ ImprovementPatch(mock_task(DefaultTask, "dummy"), repr(revision), "Whatever"), ImprovementPatch( mock_task(ClangTidyTask, "mock-clang-tidy"), repr(revision), "Some C fixes" ), ImprovementPatch( mock_task(ClangFormatTask, "mock-clang-format"), repr(revision), "Some lint fixes", ), ImprovementPatch( mock_task(InferTask, "mock-infer"), repr(revision), "Some java fixes" ), ImprovementPatch( mock_task(MozLintTask, "mock-lint-flake8"), repr(revision), "Some js fixes" ), ] list(map(lambda p: p.write(), revision.improvement_patches)) # trigger local write issues, patches = reporter.publish(issues, revision, []) # Check issues & patches analyzers assert len(issues) == len(valid_issues) assert len(patches) == len(valid_patches) assert [i.analyzer.name for i in issues] == valid_issues assert [p.analyzer.name for p in patches] == valid_patches
def test_phabricator_clang_tidy_and_coverage( mock_config, mock_phabricator, phab, mock_try_task, mock_task ): """ Test Phabricator reporter publication on a mock coverage issue """ with mock_phabricator as api: revision = Revision.from_try(mock_try_task, api) revision.mercurial_revision = "deadbeef1234" revision.repository = "https://hg.mozilla.org/try" revision.repository_try_name = "try" revision.lines = { # Add dummy lines diff "test.txt": [0], "path/to/test.cpp": [0], "another_test.cpp": [41, 42, 43], } revision.files = ["test.txt", "test.cpp", "another_test.cpp"] reporter = PhabricatorReporter( {"analyzers": ["coverage", "clang-tidy"]}, api=api ) issue_clang_tidy = ClangTidyIssue( mock_task(ClangTidyTask, "source-test-clang-tidy"), revision, "another_test.cpp", "42", "51", "modernize-use-nullptr", "dummy message", ) assert issue_clang_tidy.is_publishable() issue_coverage = CoverageIssue( mock_task(ZeroCoverageTask, "coverage"), "path/to/test.cpp", 0, "This file is uncovered", revision, ) assert issue_coverage.is_publishable() issues, patches = reporter.publish([issue_clang_tidy, issue_coverage], revision, []) assert len(issues) == 2 assert len(patches) == 0 # Check the lint results assert phab.build_messages["PHID-HMBT-test"] == [ { "buildTargetPHID": "PHID-HMBT-test", "lint": [ { "char": 51, "code": "modernize-use-nullptr", "description": "WARNING: dummy message", "line": 42, "name": "clang-tidy", "path": "another_test.cpp", "severity": "warning", }, { "code": "no-coverage", "description": "WARNING: This file is uncovered", "line": 1, "name": "code coverage analysis", "path": "path/to/test.cpp", "severity": "warning", }, ], "type": "work", "unit": [], } ] # Check the callback has been used to post unique comment assert phab.comments[51] == [VALID_CLANG_TIDY_COVERAGE_MESSAGE]
def test_phabricator_unitresult(mock_phabricator, mock_try_task): """ Test Phabricator UnitResult for a CoverityIssue """ from code_review_bot.tasks.coverity import CoverityIssue from code_review_bot.report.phabricator import PhabricatorReporter from code_review_bot.revisions import Revision def _check_message(request): # Check the Phabricator main comment is well formed payload = urllib.parse.parse_qs(request.body) assert payload["output"] == ["json"] assert len(payload["params"]) == 1 details = json.loads(payload["params"][0]) assert details == { "buildTargetPHID": "PHID-HMBD-deadbeef12456", "lint": [], "unit": [ { "details": 'Code review bot found a **build error**: \nDereferencing a pointer that might be "nullptr" "env" when calling "lookupImport".', "format": "remarkup", "name": "general", "namespace": "code-review", "result": "fail", } ], "type": "work", "__conduit__": {"token": "deadbeef"}, } # Outputs dummy empty response resp = {"error_code": None, "result": None} return ( 201, {"Content-Type": "application/json", "unittest": "coverity"}, json.dumps(resp), ) # Catch the publication of a UnitResult failure responses.add_callback( responses.POST, "http://phabricator.test/api/harbormaster.sendmessage", callback=_check_message, ) with mock_phabricator as api: revision = Revision.from_try(mock_try_task, api) revision.lines = { # Add dummy lines diff "test.cpp": [41, 42, 43] } revision.build_target_phid = "PHID-HMBD-deadbeef12456" reporter = PhabricatorReporter( {"analyzers": ["coverity"], "publish_build_errors": True}, api=api ) issue_dict = { "line": 41, "reliability": "medium", "message": 'Dereferencing a pointer that might be "nullptr" "env" when calling "lookupImport".', "flag": "NULL_RETURNS", "build_error": True, "extra": { "category": "Null pointer dereferences", "stateOnServer": { "ownerLdapServerName": "local", "stream": "Firefox", "cid": 95687, "cached": False, "retrievalDateTime": "2019-05-13T10:20:22+00:00", "firstDetectedDateTime": "2019-04-08T12:57:07+00:00", "presentInReferenceSnapshot": False, "components": ["js"], "customTriage": {}, "triage": { "fixTarget": "Untargeted", "severity": "Unspecified", "classification": "Unclassified", "owner": "try", "legacy": "False", "action": "Undecided", "externalReference": "", }, }, }, } issue = CoverityIssue("mock-coverity", revision, issue_dict, "test.cpp") assert issue.is_publishable() issues, patches = reporter.publish([issue], revision, []) assert len(issues) == 1 assert len(patches) == 0 # Check the callback has been used assert len(responses.calls) > 0 # Only check the last call print(responses.calls[-1].request.url) assert responses.calls[-1].response.headers.get("unittest") == "coverity"
def test_phabricator_mozlint( mock_config, mock_phabricator, phab, mock_try_task, mock_task ): """ Test Phabricator reporter publication on two mock mozlint issues using two different analyzers """ with mock_phabricator as api: revision = Revision.from_try(mock_try_task, api) revision.mercurial_revision = "deadbeef1234" revision.repository = "https://hg.mozilla.org/try" revision.repository_try_name = "try" revision.lines = { # Add dummy lines diff "python/test.py": [41, 42, 43], "js/test.js": [10, 11, 12], "dom/test.cpp": [42], } revision.files = revision.lines.keys() reporter = PhabricatorReporter({}, api=api) issue_flake = MozLintIssue( analyzer=mock_task(MozLintTask, "source-test-mozlint-py-flake8"), path="python/test.py", lineno=42, column=1, message="A bad bad error", level="error", revision=revision, linter="flake8", check="EXXX", ) assert issue_flake.is_publishable() issue_eslint = MozLintIssue( analyzer=mock_task(MozLintTask, "source-test-mozlint-eslint"), path="js/test.js", lineno=10, column=4, message="A bad error", level="warning", revision=revision, linter="eslint", check="YYY", ) assert issue_eslint.is_publishable() issues, patches = reporter.publish([issue_flake, issue_eslint], revision, []) assert len(issues) == 2 assert len(patches) == 0 # Check the callbacks have been used to publish either a lint result + summary comment assert phab.comments[51] == [ VALID_FLAKE8_MESSAGE.format(results=mock_config.taskcluster.results_dir) ] assert phab.build_messages["PHID-HMBT-test"] == [ { "buildTargetPHID": "PHID-HMBT-test", "lint": [ { "char": 1, "code": "EXXX", "description": "(IMPORTANT) ERROR: A bad bad error", "line": 42, "name": "py-flake8 (Mozlint)", "path": "python/test.py", "severity": "error", }, { "char": 4, "code": "YYY", "description": "WARNING: A bad error", "line": 10, "name": "eslint (Mozlint)", "path": "js/test.js", "severity": "warning", }, ], "unit": [], "type": "work", } ]
def test_full_file(mock_config, mock_phabricator, mock_try_task): """ Test Phabricator reporter supports an issue on a full file """ from code_review_bot.report.phabricator import PhabricatorReporter from code_review_bot.revisions import Revision from code_review_bot.tasks.default import DefaultIssue def _check_comment(request): # Check the Phabricator main comment is well formed payload = urllib.parse.parse_qs(request.body) assert payload["output"] == ["json"] assert len(payload["params"]) == 1 details = json.loads(payload["params"][0]) assert details["message"] == VALID_DEFAULT_MESSAGE.format( results=mock_config.taskcluster.results_dir ) # Outputs dummy empty response resp = {"error_code": None, "result": None} return ( 201, {"Content-Type": "application/json", "unittest": "full-file-comment"}, json.dumps(resp), ) def _check_inline(request): # Check the inline does not have a null value for line payload = urllib.parse.parse_qs(request.body) assert payload["output"] == ["json"] assert len(payload["params"]) == 1 details = json.loads(payload["params"][0]) assert details == { "__conduit__": {"token": "deadbeef"}, "content": "Warning: Something bad happened on the whole file ! [a-huge-issue]", "diffID": 42, "filePath": "xx.cpp", "isNewFile": 1, "lineLength": -1, # Cannot be null for a full file as it's not supported by phabricator "lineNumber": 1, } # Outputs dummy empty response resp = {"error_code": None, "result": {"id": "PHID-XXXX-YYYYY"}} return ( 201, {"Content-Type": "application/json", "unittest": "full-file-inline"}, json.dumps(resp), ) responses.add_callback( responses.POST, "http://phabricator.test/api/differential.createinline", callback=_check_inline, ) responses.add_callback( responses.POST, "http://phabricator.test/api/differential.createcomment", callback=_check_comment, ) with mock_phabricator as api: revision = Revision.from_try(mock_try_task, api) revision.lines = { # Add dummy lines diff "xx.cpp": [123, 124, 125] } revision.files = list(revision.lines.keys()) reporter = PhabricatorReporter(api=api) issue = DefaultIssue( analyzer="full-file-analyzer", revision=revision, path="xx.cpp", line=-1, nb_lines=0, check="a-huge-issue", message="Something bad happened on the whole file !", ) assert issue.line is None # check auto conversion assert ( str(issue) == "full-file-analyzer issue a-huge-issue@warning xx.cpp full file" ) assert issue.is_publishable() assert revision.has_file(issue.path) assert revision.contains(issue) issues, patches = reporter.publish([issue], revision, []) assert len(issues) == 1 assert len(patches) == 0 # Check the comment callback has been used assert len(responses.calls) > 0 call = responses.calls[-1] assert call.request.url == "http://phabricator.test/api/differential.createcomment" assert call.response.headers.get("unittest") == "full-file-comment" # Check the inline callback has been used call = responses.calls[-2] assert call.request.url == "http://phabricator.test/api/differential.createinline" assert call.response.headers.get("unittest") == "full-file-inline"
def test_phabricator_clang_tidy(mock_phabricator, mock_try_task): """ Test Phabricator reporter publication on a mock clang-tidy issue """ from code_review_bot.report.phabricator import PhabricatorReporter from code_review_bot.revisions import Revision from code_review_bot.tasks.clang_tidy import ClangTidyIssue def _check_comment(request): # Check the Phabricator main comment is well formed payload = urllib.parse.parse_qs(request.body) assert payload["output"] == ["json"] assert len(payload["params"]) == 1 details = json.loads(payload["params"][0]) assert details == { "revision_id": 51, "message": VALID_CLANG_TIDY_MESSAGE, "attach_inlines": 1, "__conduit__": {"token": "deadbeef"}, } # Outputs dummy empty response resp = {"error_code": None, "result": None} return ( 201, {"Content-Type": "application/json", "unittest": "clang-tidy"}, json.dumps(resp), ) responses.add_callback( responses.POST, "http://phabricator.test/api/differential.createcomment", callback=_check_comment, ) with mock_phabricator as api: revision = Revision.from_try(mock_try_task, api) revision.lines = { # Add dummy lines diff "another_test.cpp": [41, 42, 43] } revision.files = ["another_test.cpp"] reporter = PhabricatorReporter({"analyzers": ["clang-tidy"]}, api=api) issue = ClangTidyIssue( "source-test-clang-tidy", revision, "another_test.cpp", "42", "51", "modernize-use-nullptr", "dummy message", ) assert issue.is_publishable() issues, patches = reporter.publish([issue], revision, []) assert len(issues) == 1 assert len(patches) == 0 # Check the callback has been used assert len(responses.calls) > 0 call = responses.calls[-1] assert call.request.url == "http://phabricator.test/api/differential.createcomment" assert call.response.headers.get("unittest") == "clang-tidy"
def main(): args = parse_cli() taskcluster.auth(args.taskcluster_client_id, args.taskcluster_access_token) taskcluster.load_secrets( args.taskcluster_secret, prefixes=["common", "code-review-bot", "bot"], required=( "APP_CHANNEL", "REPORTERS", "PHABRICATOR", "ALLOWED_PATHS", "repositories", ), existing={ "APP_CHANNEL": "development", "REPORTERS": [], "ZERO_COVERAGE_ENABLED": True, "ALLOWED_PATHS": ["*"], "task_failures_ignored": [], }, local_secrets=yaml.safe_load(args.configuration) if args.configuration else None, ) init_logger( "bot", channel=taskcluster.secrets.get("APP_CHANNEL", "dev"), PAPERTRAIL_HOST=taskcluster.secrets.get("PAPERTRAIL_HOST"), PAPERTRAIL_PORT=taskcluster.secrets.get("PAPERTRAIL_PORT"), SENTRY_DSN=taskcluster.secrets.get("SENTRY_DSN"), ) # Setup settings before stats settings.setup( taskcluster.secrets["APP_CHANNEL"], taskcluster.secrets["ALLOWED_PATHS"], taskcluster.secrets["repositories"], ) # Setup statistics influx_conf = taskcluster.secrets.get("influxdb") if influx_conf: stats.auth(influx_conf) # Load reporters reporters = get_reporters(taskcluster.secrets["REPORTERS"]) # Load index service index_service = taskcluster.get_service("index") # Load queue service queue_service = taskcluster.get_service("queue") # Load Phabricator API phabricator = taskcluster.secrets["PHABRICATOR"] phabricator_reporting_enabled = "phabricator" in reporters phabricator_api = PhabricatorAPI(phabricator["api_key"], phabricator["url"]) if phabricator_reporting_enabled: reporters["phabricator"].setup_api(phabricator_api) # Load unique revision try: if settings.autoland_group_id: revision = Revision.from_autoland( queue_service.task(settings.autoland_group_id), phabricator_api ) else: revision = Revision.from_try( queue_service.task(settings.try_task_id), phabricator_api ) except Exception as e: # Report revision loading failure on production only # On testing or dev instances, we can use different Phabricator # configuration that do not match all the pulse messages sent if settings.on_production: raise else: logger.info( "Failed to load revision", task=settings.try_task_id, error=str(e), phabricator=phabricator["url"], ) return 1 # Run workflow according to source w = Workflow( reporters, index_service, queue_service, phabricator_api, taskcluster.secrets["ZERO_COVERAGE_ENABLED"], # Update build status only when phabricator reporting is enabled update_build=phabricator_reporting_enabled, task_failures_ignored=taskcluster.secrets["task_failures_ignored"], ) try: if revision.repository == REPO_AUTOLAND: w.ingest_autoland(revision) else: w.run(revision) except Exception as e: # Log errors to papertrail logger.error("Static analysis failure", revision=revision, error=e) # Index analysis state extras = {} if isinstance(e, AnalysisException): extras["error_code"] = e.code extras["error_message"] = str(e) w.index(revision, state="error", **extras) # Update Harbormaster status w.update_status(revision, state=BuildState.Fail) # Then raise to mark task as erroneous raise return 0
def test_extra_errors( reporter_config, errors_reported, mock_phabricator, mock_try_task ): """ Test Phabricator reporter publication with some errors outside of patch """ from code_review_bot.report.phabricator import PhabricatorReporter from code_review_bot.revisions import Revision from code_review_bot.tasks.lint import MozLintIssue def _check_inline(request): payload = urllib.parse.parse_qs(request.body) assert payload["output"] == ["json"] assert len(payload["params"]) == 1 details = json.loads(payload["params"][0]) assert details == { "__conduit__": {"token": "deadbeef"}, "content": "Warning: Some not so bad python mistake [flake8: EYYY]", "diffID": 42, "filePath": "path/to/file.py", "isNewFile": 1, "lineLength": 0, "lineNumber": 2, } # Outputs dummy empty response resp = {"error_code": None, "result": {"id": "PHID-XXXX-YYYYY"}} return ( 201, {"Content-Type": "application/json", "unittest": "mozlint-warning"}, json.dumps(resp), ) def _check_comment(request): payload = urllib.parse.parse_qs(request.body) assert payload["output"] == ["json"] assert len(payload["params"]) == 1 details = json.loads(payload["params"][0]) assert details == { "revision_id": 51, "message": VALID_MOZLINT_MESSAGE, "attach_inlines": 1, "__conduit__": {"token": "deadbeef"}, } # Outputs dummy empty response resp = {"error_code": None, "result": None} return ( 201, {"Content-Type": "application/json", "unittest": "mozlint-comment"}, json.dumps(resp), ) def _check_message(request): payload = urllib.parse.parse_qs(request.body) assert payload["output"] == ["json"] assert len(payload["params"]) == 1 details = json.loads(payload["params"][0]) assert details == { "buildTargetPHID": "PHID-HMBT-test", "lint": [ { "char": 12, "code": "EXXX", "description": "Some bad python typo", "line": 10, "name": "source-test-mozlint-dummy", "path": "path/to/file.py", "severity": "error", } ], "unit": [], "type": "work", "__conduit__": {"token": "deadbeef"}, } # Outputs dummy empty response resp = {"error_code": None, "result": None} return ( 201, {"Content-Type": "application/json", "unittest": "mozlint-error"}, json.dumps(resp), ) responses.add_callback( responses.POST, "http://phabricator.test/api/differential.createinline", callback=_check_inline, ) responses.add_callback( responses.POST, "http://phabricator.test/api/harbormaster.sendmessage", callback=_check_message, ) responses.add_callback( responses.POST, "http://phabricator.test/api/differential.createcomment", callback=_check_comment, ) with mock_phabricator as api: revision = Revision.from_try(mock_try_task, api) revision.lines = {"path/to/file.py": [1, 2, 3]} revision.files = ["path/to/file.py"] reporter = PhabricatorReporter(reporter_config, api=api) all_issues = [ # Warning in patch MozLintIssue( analyzer="source-test-mozlint-dummy", path="path/to/file.py", column=25, level=Level.Warning, lineno=2, linter="flake8", message="Some not so bad python mistake", check="EYYY", revision=revision, ), # Error outside of patch MozLintIssue( analyzer="source-test-mozlint-dummy", path="path/to/file.py", column=12, level=Level.Error, lineno=10, linter="flake8", message="Some bad python typo", check="EXXX", revision=revision, ), # Warning outside of patch MozLintIssue( analyzer="source-test-mozlint-dummy", path="path/to/file.py", column=1, level=Level.Warning, lineno=25, linter="flake8", message="Random mistake that will be ignored", check="EZZZ", revision=revision, ), ] published_issues, patches = reporter.publish(all_issues, revision, []) assert len(published_issues) == 2 if errors_reported else 1 assert len(patches) == 0 # Check the callbacks have been used to publish: # - an inline comment for the warning in patch # - a top comment to summarize issues # - a lint result for the error outside of patch (when errors are set to be reported) assert len(responses.calls) > 0 urls = [ "http://phabricator.test/api/differential.createinline", "http://phabricator.test/api/differential.createcomment", ] markers = ["mozlint-warning", "mozlint-comment"] if errors_reported: urls.append("http://phabricator.test/api/harbormaster.sendmessage") markers.append("mozlint-error") assert [r.request.url for r in responses.calls[-len(urls) :]] == urls assert [ r.response.headers.get("unittest") for r in responses.calls[-len(urls) :] ] == markers