def test_as_text(mock_config, mock_revision): """ Test text export for ClangTidyIssue """ from code_review_bot.tasks.lint import MozLintIssue issue = MozLintIssue( "test.py", 1, "error", 1, "flake8", "dummy test withUppercaseChars", "dummy rule", mock_revision, ) assert (issue.as_text() == "Error: Dummy test withUppercaseChars [flake8: dummy rule]") assert issue.as_phabricator_lint() == { "char": 1, "code": "flake8.dummy rule", "line": 1, "name": "MozLint Flake8 - dummy rule", "description": "dummy test withUppercaseChars", "path": "test.py", "severity": "error", }
def test_build_hash(mock_revision, mock_hgmo): """ Test build hash algorithm """ # Hardcode revision & repo mock_revision.repository = "test-try" mock_revision.mercurial_revision = "deadbeef1234" issue = MozLintIssue( "mock-analyzer-eslint", "path/to/file.cpp", 42, "error", 123, "eslint", "A random & fake linting issue", "EXXX", mock_revision, ) assert (str(issue) == "mock-analyzer-eslint issue EXXX@error path/to/file.cpp line 123") # Check the mock file retrieval for that file raw_file = mock_revision.load_file(issue.path) assert raw_file == "\n".join( f"test-try:deadbeef1234:path/to/file.cpp:{i+1}" for i in range(1000)) # Build hash in the unit test by re-creating the payload payload = "mock-analyzer-eslint:path/to/file.cpp:error:EXXX:{}:test-try:deadbeef1234:path/to/file.cpp:123" hash_check = hashlib.md5(payload.encode("utf-8")).hexdigest() assert hash_check == "045f57ef8ee111d0c8c475bd7a617564" == issue.build_hash( )
def _test_reporter(api, analyzers_skipped): # Always use the same setup, only varies the analyzers revision = Revision.from_try(mock_try_task, api) revision.lines = {"test.cpp": [0, 41, 42, 43], "dom/test.cpp": [42]} reporter = PhabricatorReporter( {"analyzers_skipped": analyzers_skipped}, api=api ) issues = [ ClangFormatIssue("mock-clang-format", "dom/test.cpp", 42, 1, revision), ClangTidyIssue( "mock-clang-tidy", revision, "test.cpp", "42", "51", "modernize-use-nullptr", "dummy message", ), InferIssue( "mock-infer", { "file": "test.cpp", "line": 42, "column": 1, "bug_type": "dummy", "kind": "WARNING", "qualifier": "dummy message.", }, revision, ), MozLintIssue( "mock-lint-flake8", "test.cpp", 1, "error", 42, "flake8", "Python error", "EXXX", revision, ), CoverageIssue("test.cpp", 0, "This file is uncovered", revision), ] assert all(i.is_publishable() for i in issues) revision.improvement_patches = [ ImprovementPatch("dummy", repr(revision), "Whatever"), ImprovementPatch("mock-clang-tidy", repr(revision), "Some C fixes"), ImprovementPatch("mock-clang-format", repr(revision), "Some lint fixes"), ImprovementPatch("mock-infer", repr(revision), "Some java fixes"), ImprovementPatch("mock-lint-flake8", repr(revision), "Some js fixes"), ] list( map(lambda p: p.write(), revision.improvement_patches) ) # trigger local write return reporter.publish(issues, revision, [])
def test_as_text(mock_config, mock_revision, mock_hgmo, mock_task): """ Test text export for ClangTidyIssue """ issue = MozLintIssue( mock_task(MozLintTask, "mock-lint-flake8"), "test.py", 1, "error", 1, "flake8", "dummy test withUppercaseChars", "dummy rule", mock_revision, ) assert (issue.as_text() == "Error: Dummy test withUppercaseChars [flake8: dummy rule]") assert issue.as_phabricator_lint() == { "char": 1, "code": "dummy rule", "line": 1, "name": "mock-lint-flake8", "description": "(IMPORTANT) ERROR: dummy test withUppercaseChars", "path": "test.py", "severity": "error", } assert issue.as_dict() == { "analyzer": "mock-lint-flake8", "check": "dummy rule", "column": 1, "in_patch": False, "level": "error", "line": 1, "message": "dummy test withUppercaseChars", "nb_lines": 1, "path": "test.py", "publishable": True, "validates": True, "hash": "f8d818d42677f3ffdc0be647453278b8", "fix": None, }
def test_flake8_checks(mock_config, mock_revision, mock_hgmo, mock_task): """ Check flake8 check detection """ # Valid issue issue = MozLintIssue( mock_task(MozLintTask, "mock-lint-flake8"), "test.py", 1, "error", 1, "flake8", "Dummy test", "dummy rule", mock_revision, ) assert not issue.is_disabled_check() assert issue.validates() # Flake8 bad quotes issue = MozLintIssue( mock_task(MozLintTask, "mock-lint-flake8"), "test.py", 1, "error", 1, "flake8", "Remove bad quotes or whatever.", "Q000", mock_revision, ) assert issue.is_disabled_check() assert not issue.validates() assert issue.as_dict() == { "analyzer": "mock-lint-flake8", "check": "Q000", "column": 1, "in_patch": False, "level": "error", "line": 1, "message": "Remove bad quotes or whatever.", "nb_lines": 1, "path": "test.py", "publishable": False, "validates": False, "hash": "28e40e8a562fa8ebea98f984abd503fd", "fix": None, }
def test_as_text(mock_config, mock_revision, mock_hgmo): """ Test text export for ClangTidyIssue """ issue = MozLintIssue( "mock-lint-flake8", "test.py", 1, "error", 1, "flake8", "dummy test withUppercaseChars", "dummy rule", mock_revision, ) assert (issue.as_text() == "Error: Dummy test withUppercaseChars [flake8: dummy rule]") assert issue.as_phabricator_lint() == { "char": 1, "code": "dummy rule", "line": 1, "name": "mock-lint-flake8", "description": "dummy test withUppercaseChars", "path": "test.py", "severity": "error", } assert issue.as_dict() == { "analyzer": "mock-lint-flake8", "check": "dummy rule", "column": 1, "in_patch": False, "level": "error", "line": 1, "message": "dummy test withUppercaseChars", "nb_lines": 1, "path": "test.py", "publishable": True, "validates": True, "hash": "34c27d119c21ea5a2cd3f6ac230d8c4e", }
def test_flake8_checks(mock_config, mock_revision, mock_hgmo): """ Check flake8 check detection """ # Valid issue issue = MozLintIssue( "mock-lint-flake8", "test.py", 1, "error", 1, "flake8", "Dummy test", "dummy rule", mock_revision, ) assert not issue.is_disabled_check() assert issue.validates() # Flake8 bad quotes issue = MozLintIssue( "mock-lint-flake8", "test.py", 1, "error", 1, "flake8", "Remove bad quotes or whatever.", "Q000", mock_revision, ) assert issue.is_disabled_check() assert not issue.validates() assert issue.as_dict() == { "analyzer": "mock-lint-flake8", "check": "Q000", "column": 1, "in_patch": False, "level": "error", "line": 1, "message": "Remove bad quotes or whatever.", "nb_lines": 1, "path": "test.py", "publishable": False, "validates": False, "hash": "57a49008d6a3ec23f987c0bf62c43d75", }
def test_full_file(mock_revision, mock_hgmo, mock_task): """ Test build hash algorithm when using a full file (line is -1) """ # Hardcode revision & repo mock_revision.repository = "test-try" mock_revision.mercurial_revision = "deadbeef1234" issue = MozLintIssue( mock_task(MozLintTask, "mock-analyzer-fullfile"), "path/to/afile.py", 0, "error", -1, "fullfile", "Some issue found on a file", "EXXX", mock_revision, ) assert ( str(issue) == "mock-analyzer-fullfile issue EXXX@error path/to/afile.py full file") assert issue.line is None # Build hash should use the full file assert issue.build_hash() == "76a76ba6e023d933acba9e07ae2897f6" # Check positive integers or None are used in report assert issue.as_dict() == { "analyzer": "mock-analyzer-fullfile", "check": "EXXX", "column": 0, "hash": "76a76ba6e023d933acba9e07ae2897f6", "in_patch": False, "level": "error", "line": None, "message": "Some issue found on a file", "nb_lines": 1, "path": "path/to/afile.py", "publishable": True, "validates": True, "fix": None, }
def test_indentation_effect(mock_revision, mock_hgmo): """ Test indentation does not affect the hash 2 lines with same content in a file, triggering the same error should produce the same hash """ # Hardcode revision & repo mock_revision.repository = "test-try" mock_revision.mercurial_revision = "deadbeef1234" issue_indent = MozLintIssue( "mock-analyzer-flake8", "hello1", 2, "error", 1, "flake8", "A random & fake linting issue", "EXXX", mock_revision, ) issue_no_indent = MozLintIssue( "mock-analyzer-flake8", "hello1", 5, "error", 1, "flake8", "A random & fake linting issue", "EXXX", mock_revision, ) # Check raw file content lines = mock_revision.load_file("hello1").splitlines() assert lines[1] == ' print("Hello !")' assert lines[4] == 'print("Hello !")' # Check the hashes are equal assert ( issue_indent.build_hash() == issue_no_indent.build_hash() == "9a9b08552e9e2a5f1da5c103c3a4657d" )
def test_extra_errors(mock_phabricator, mock_try_task, phab, mock_task): """ Test Phabricator reporter publication with some errors outside of patch """ with mock_phabricator as api: revision = Revision.from_try(mock_try_task, api) revision.mercurial_revision = "deadbeef1234" revision.repository = "https://hg.mozilla.org/try" revision.repository_try_name = "try" revision.lines = {"path/to/file.py": [1, 2, 3]} revision.files = ["path/to/file.py"] reporter = PhabricatorReporter({}, api=api) task = mock_task(MozLintTask, "source-test-mozlint-dummy") all_issues = [ # Warning in patch MozLintIssue( analyzer=task, path="path/to/file.py", column=25, level=Level.Warning, lineno=2, linter="flake8", message="Some not so bad python mistake", check="EYYY", revision=revision, ), # Error outside of patch MozLintIssue( analyzer=task, path="path/to/file.py", column=12, level=Level.Error, lineno=10, linter="flake8", message="Some bad python typo", check="EXXX", revision=revision, ), # Warning outside of patch MozLintIssue( analyzer=task, path="path/to/file.py", column=1, level=Level.Warning, lineno=25, linter="flake8", message="Random mistake that will be ignored", check="EZZZ", revision=revision, ), ] published_issues, patches = reporter.publish(all_issues, revision, []) assert len(published_issues) == 2 assert len(patches) == 0 # Check the callbacks have been used to publish: # - a top comment to summarize issues # - a lint result for the error outside of patch # - a lint result for the warning inside patch assert phab.build_messages["PHID-HMBT-test"] == [ { "buildTargetPHID": "PHID-HMBT-test", "lint": [ { "char": 25, "code": "EYYY", "description": "WARNING: Some not so bad python mistake", "line": 2, "name": "dummy (Mozlint)", "path": "path/to/file.py", "severity": "warning", }, { "char": 12, "code": "EXXX", "description": "(IMPORTANT) ERROR: Some bad python typo", "line": 10, "name": "dummy (Mozlint)", "path": "path/to/file.py", "severity": "error", }, ], "unit": [], "type": "work", } ] assert phab.comments[51] == [VALID_MOZLINT_MESSAGE]
def test_phabricator_analyzers( analyzers_skipped, valid_issues, valid_patches, mock_config, mock_phabricator, mock_try_task, mock_task, ): """ Test analyzers filtering on phabricator reporter """ with mock_phabricator as api: # Skip commenting on phabricator # we only care about filtering issues api.comment = unittest.mock.Mock(return_value=True) # Always use the same setup, only varies the analyzers revision = Revision.from_try(mock_try_task, api) revision.mercurial_revision = "deadbeef1234" revision.repository = "https://hg.mozilla.org/try" revision.repository_try_name = "try" revision.lines = {"test.cpp": [0, 41, 42, 43], "dom/test.cpp": [42]} reporter = PhabricatorReporter( {"analyzers_skipped": analyzers_skipped}, api=api ) issues = [ ClangFormatIssue( mock_task(ClangFormatTask, "mock-clang-format"), "dom/test.cpp", [ (41, 41, b"no change"), (42, None, b"deletion"), (None, 42, b"change here"), ], revision, ), ClangTidyIssue( mock_task(ClangTidyTask, "mock-clang-tidy"), revision, "test.cpp", "42", "51", "modernize-use-nullptr", "dummy message", ), InferIssue( mock_task(InferTask, "mock-infer"), { "file": "test.cpp", "line": 42, "column": 1, "bug_type": "dummy", "kind": "WARNING", "qualifier": "dummy message.", }, revision, ), MozLintIssue( mock_task(MozLintTask, "mock-lint-flake8"), "test.cpp", 1, "error", 42, "flake8", "Python error", "EXXX", revision, ), CoverageIssue( mock_task(ZeroCoverageTask, "coverage"), "test.cpp", 0, "This file is uncovered", revision, ), ] assert all(i.is_publishable() for i in issues) revision.improvement_patches = [ ImprovementPatch(mock_task(DefaultTask, "dummy"), repr(revision), "Whatever"), ImprovementPatch( mock_task(ClangTidyTask, "mock-clang-tidy"), repr(revision), "Some C fixes" ), ImprovementPatch( mock_task(ClangFormatTask, "mock-clang-format"), repr(revision), "Some lint fixes", ), ImprovementPatch( mock_task(InferTask, "mock-infer"), repr(revision), "Some java fixes" ), ImprovementPatch( mock_task(MozLintTask, "mock-lint-flake8"), repr(revision), "Some js fixes" ), ] list(map(lambda p: p.write(), revision.improvement_patches)) # trigger local write issues, patches = reporter.publish(issues, revision, []) # Check issues & patches analyzers assert len(issues) == len(valid_issues) assert len(patches) == len(valid_patches) assert [i.analyzer.name for i in issues] == valid_issues assert [p.analyzer.name for p in patches] == valid_patches
def test_phabricator_mozlint( mock_config, mock_phabricator, phab, mock_try_task, mock_task ): """ Test Phabricator reporter publication on two mock mozlint issues using two different analyzers """ with mock_phabricator as api: revision = Revision.from_try(mock_try_task, api) revision.mercurial_revision = "deadbeef1234" revision.repository = "https://hg.mozilla.org/try" revision.repository_try_name = "try" revision.lines = { # Add dummy lines diff "python/test.py": [41, 42, 43], "js/test.js": [10, 11, 12], "dom/test.cpp": [42], } revision.files = revision.lines.keys() reporter = PhabricatorReporter({}, api=api) issue_flake = MozLintIssue( analyzer=mock_task(MozLintTask, "source-test-mozlint-py-flake8"), path="python/test.py", lineno=42, column=1, message="A bad bad error", level="error", revision=revision, linter="flake8", check="EXXX", ) assert issue_flake.is_publishable() issue_eslint = MozLintIssue( analyzer=mock_task(MozLintTask, "source-test-mozlint-eslint"), path="js/test.js", lineno=10, column=4, message="A bad error", level="warning", revision=revision, linter="eslint", check="YYY", ) assert issue_eslint.is_publishable() issues, patches = reporter.publish([issue_flake, issue_eslint], revision, []) assert len(issues) == 2 assert len(patches) == 0 # Check the callbacks have been used to publish either a lint result + summary comment assert phab.comments[51] == [ VALID_FLAKE8_MESSAGE.format(results=mock_config.taskcluster.results_dir) ] assert phab.build_messages["PHID-HMBT-test"] == [ { "buildTargetPHID": "PHID-HMBT-test", "lint": [ { "char": 1, "code": "EXXX", "description": "(IMPORTANT) ERROR: A bad bad error", "line": 42, "name": "py-flake8 (Mozlint)", "path": "python/test.py", "severity": "error", }, { "char": 4, "code": "YYY", "description": "WARNING: A bad error", "line": 10, "name": "eslint (Mozlint)", "path": "js/test.js", "severity": "warning", }, ], "unit": [], "type": "work", } ]
def test_extra_errors( reporter_config, errors_reported, mock_phabricator, mock_try_task ): """ Test Phabricator reporter publication with some errors outside of patch """ from code_review_bot.report.phabricator import PhabricatorReporter from code_review_bot.revisions import Revision from code_review_bot.tasks.lint import MozLintIssue def _check_inline(request): payload = urllib.parse.parse_qs(request.body) assert payload["output"] == ["json"] assert len(payload["params"]) == 1 details = json.loads(payload["params"][0]) assert details == { "__conduit__": {"token": "deadbeef"}, "content": "Warning: Some not so bad python mistake [flake8: EYYY]", "diffID": 42, "filePath": "path/to/file.py", "isNewFile": 1, "lineLength": 0, "lineNumber": 2, } # Outputs dummy empty response resp = {"error_code": None, "result": {"id": "PHID-XXXX-YYYYY"}} return ( 201, {"Content-Type": "application/json", "unittest": "mozlint-warning"}, json.dumps(resp), ) def _check_comment(request): payload = urllib.parse.parse_qs(request.body) assert payload["output"] == ["json"] assert len(payload["params"]) == 1 details = json.loads(payload["params"][0]) assert details == { "revision_id": 51, "message": VALID_MOZLINT_MESSAGE, "attach_inlines": 1, "__conduit__": {"token": "deadbeef"}, } # Outputs dummy empty response resp = {"error_code": None, "result": None} return ( 201, {"Content-Type": "application/json", "unittest": "mozlint-comment"}, json.dumps(resp), ) def _check_message(request): payload = urllib.parse.parse_qs(request.body) assert payload["output"] == ["json"] assert len(payload["params"]) == 1 details = json.loads(payload["params"][0]) assert details == { "buildTargetPHID": "PHID-HMBT-test", "lint": [ { "char": 12, "code": "EXXX", "description": "Some bad python typo", "line": 10, "name": "source-test-mozlint-dummy", "path": "path/to/file.py", "severity": "error", } ], "unit": [], "type": "work", "__conduit__": {"token": "deadbeef"}, } # Outputs dummy empty response resp = {"error_code": None, "result": None} return ( 201, {"Content-Type": "application/json", "unittest": "mozlint-error"}, json.dumps(resp), ) responses.add_callback( responses.POST, "http://phabricator.test/api/differential.createinline", callback=_check_inline, ) responses.add_callback( responses.POST, "http://phabricator.test/api/harbormaster.sendmessage", callback=_check_message, ) responses.add_callback( responses.POST, "http://phabricator.test/api/differential.createcomment", callback=_check_comment, ) with mock_phabricator as api: revision = Revision.from_try(mock_try_task, api) revision.lines = {"path/to/file.py": [1, 2, 3]} revision.files = ["path/to/file.py"] reporter = PhabricatorReporter(reporter_config, api=api) all_issues = [ # Warning in patch MozLintIssue( analyzer="source-test-mozlint-dummy", path="path/to/file.py", column=25, level=Level.Warning, lineno=2, linter="flake8", message="Some not so bad python mistake", check="EYYY", revision=revision, ), # Error outside of patch MozLintIssue( analyzer="source-test-mozlint-dummy", path="path/to/file.py", column=12, level=Level.Error, lineno=10, linter="flake8", message="Some bad python typo", check="EXXX", revision=revision, ), # Warning outside of patch MozLintIssue( analyzer="source-test-mozlint-dummy", path="path/to/file.py", column=1, level=Level.Warning, lineno=25, linter="flake8", message="Random mistake that will be ignored", check="EZZZ", revision=revision, ), ] published_issues, patches = reporter.publish(all_issues, revision, []) assert len(published_issues) == 2 if errors_reported else 1 assert len(patches) == 0 # Check the callbacks have been used to publish: # - an inline comment for the warning in patch # - a top comment to summarize issues # - a lint result for the error outside of patch (when errors are set to be reported) assert len(responses.calls) > 0 urls = [ "http://phabricator.test/api/differential.createinline", "http://phabricator.test/api/differential.createcomment", ] markers = ["mozlint-warning", "mozlint-comment"] if errors_reported: urls.append("http://phabricator.test/api/harbormaster.sendmessage") markers.append("mozlint-error") assert [r.request.url for r in responses.calls[-len(urls) :]] == urls assert [ r.response.headers.get("unittest") for r in responses.calls[-len(urls) :] ] == markers
def test_phabricator_mozlint( reporter_config, errors_reported, mock_config, mock_phabricator, mock_try_task ): """ Test Phabricator reporter publication on a mock mozlint issue """ from code_review_bot.report.phabricator import PhabricatorReporter from code_review_bot.revisions import Revision from code_review_bot.tasks.lint import MozLintIssue def _check_inline(request): payload = urllib.parse.parse_qs(request.body) assert payload["output"] == ["json"] assert len(payload["params"]) == 1 details = json.loads(payload["params"][0]) assert details == { "__conduit__": {"token": "deadbeef"}, "content": "Error: A bad bad error [flake8: EXXX]", "diffID": 42, "filePath": "python/test.py", "isNewFile": 1, "lineLength": 0, "lineNumber": 42, } # Outputs dummy empty response resp = {"error_code": None, "result": {"id": "PHID-XXXX-YYYYY"}} return ( 201, {"Content-Type": "application/json", "unittest": "flake8-inline"}, json.dumps(resp), ) def _check_comment(request): # Check the Phabricator main comment is well formed payload = urllib.parse.parse_qs(request.body) assert payload["output"] == ["json"] assert len(payload["params"]) == 1 details = json.loads(payload["params"][0]) assert details["message"] == VALID_FLAKE8_MESSAGE.format( results=mock_config.taskcluster.results_dir ) # Outputs dummy empty response resp = {"error_code": None, "result": None} return ( 201, {"Content-Type": "application/json", "unittest": "flake8-comment"}, json.dumps(resp), ) def _check_message(request): payload = urllib.parse.parse_qs(request.body) assert payload["output"] == ["json"] assert len(payload["params"]) == 1 details = json.loads(payload["params"][0]) assert details == { "buildTargetPHID": "PHID-HMBT-test", "lint": [ { "char": 1, "code": "EXXX", "description": "A bad bad error", "line": 42, "name": "source-test-mozlint-py-flake8", "path": "python/test.py", "severity": "error", } ], "unit": [], "type": "work", "__conduit__": {"token": "deadbeef"}, } # Outputs dummy empty response resp = {"error_code": None, "result": None} return ( 201, {"Content-Type": "application/json", "unittest": "flake8-error"}, json.dumps(resp), ) responses.add_callback( responses.POST, "http://phabricator.test/api/differential.createinline", callback=_check_inline, ) responses.add_callback( responses.POST, "http://phabricator.test/api/differential.createcomment", callback=_check_comment, ) responses.add_callback( responses.POST, "http://phabricator.test/api/harbormaster.sendmessage", callback=_check_message, ) with mock_phabricator as api: revision = Revision.from_try(mock_try_task, api) revision.lines = { # Add dummy lines diff "python/test.py": [41, 42, 43], "dom/test.cpp": [42], } revision.files = revision.lines.keys() reporter = PhabricatorReporter(reporter_config, api=api) issue = MozLintIssue( analyzer="source-test-mozlint-py-flake8", path="python/test.py", lineno=42, column=1, message="A bad bad error", level="error", revision=revision, linter="flake8", check="EXXX", ) assert issue.is_publishable() issues, patches = reporter.publish([issue], revision, []) assert len(issues) == 1 assert len(patches) == 0 # Check the callbacks have been used to publish either: # - an inline comment + summary comment when publish_errors is False # - a lint result + summary comment when publish_errors is True assert len(responses.calls) > 0 if errors_reported: urls = [ "http://phabricator.test/api/differential.createcomment", "http://phabricator.test/api/harbormaster.sendmessage", ] markers = ["flake8-comment", "flake8-error"] else: urls = [ "http://phabricator.test/api/differential.createinline", "http://phabricator.test/api/differential.createcomment", ] markers = ["flake8-inline", "flake8-comment"] assert [r.request.url for r in responses.calls[-2:]] == urls assert [r.response.headers.get("unittest") for r in responses.calls[-2:]] == markers
def test_flake8_rules(mock_config, mock_revision): """ Check flake8 rule detection """ from code_review_bot.tasks.lint import MozLintIssue # Valid issue issue = MozLintIssue("test.py", 1, "error", 1, "flake8", "Dummy test", "dummy rule", mock_revision) assert not issue.is_disabled_rule() assert issue.validates() # 3rd party issue = MozLintIssue( "test/dummy/XXX.py", 1, "error", 1, "flake8", "Dummy test", "dummy rule", mock_revision, ) assert not issue.is_disabled_rule() assert issue.is_third_party() assert not issue.validates() # Flake8 bad quotes issue = MozLintIssue( "test.py", 1, "error", 1, "flake8", "Remove bad quotes or whatever.", "Q000", mock_revision, ) assert issue.is_disabled_rule() assert not issue.validates()