def test_phabricator_coverage(mock_config, mock_phabricator, mock_try_task):
'''
Test Phabricator reporter publication on a mock coverage issue
'''
from static_analysis_bot.report.phabricator import PhabricatorReporter
from static_analysis_bot.revisions import Revision
from static_analysis_bot.tasks.coverage import CoverageIssue
def _check_comment(request):
# Check the Phabricator main comment is well formed
payload = urllib.parse.parse_qs(request.body)
assert payload['output'] == ['json']
assert len(payload['params']) == 1
details = json.loads(payload['params'][0])
assert details['message'] == VALID_COVERAGE_MESSAGE.format(
results=mock_config.taskcluster.results_dir)
# Outputs dummy empty response
resp = {
'error_code': None,
'result': None,
}
return 201, {
'Content-Type': 'application/json',
'unittest': 'coverage'
}, json.dumps(resp)
responses.add_callback(
responses.POST,
'http://phabricator.test/api/differential.createcomment',
callback=_check_comment,
)
with mock_phabricator as api:
revision = Revision(api, mock_try_task)
revision.lines = {
# Add dummy lines diff
'test.txt': [0],
'test.cpp': [0],
'dom/test.cpp': [
42,
],
}
reporter = PhabricatorReporter({'analyzers': ['coverage']}, api=api)
issue = CoverageIssue('test.cpp', 0, 'This file is uncovered', revision)
assert issue.is_publishable()
issues, patches = reporter.publish([
issue,
], revision)
assert len(issues) == 1
assert len(patches) == 0
# Check the callback has been used
assert len(responses.calls) > 0
call = responses.calls[-1]
assert call.request.url == 'http://phabricator.test/api/differential.createcomment'
assert call.response.headers.get('unittest') == 'coverage'
def mock_revision():
'''
Mock a mercurial revision
'''
from static_analysis_bot.revisions import Revision
rev = Revision()
rev.mercurial = 'a6ce14f59749c3388ffae2459327a323b6179ef0'
return rev
def test_taskcluster_index(mock_config, mock_workflow):
'''
Test the Taskcluster indexing API
by mocking an online taskcluster state
'''
from static_analysis_bot.config import TaskCluster
from static_analysis_bot.revisions import Revision
mock_config.taskcluster = TaskCluster('/tmp/dummy', '12345deadbeef', 0,
False)
mock_workflow.index_service = mock.Mock()
rev = Revision()
rev.namespaces = ['mock.1234']
rev.as_dict = lambda: {
'id': '1234',
'someData': 'mock',
'state': 'done',
}
mock_workflow.index(rev, test='dummy')
assert mock_workflow.index_service.insertTask.call_count == 3
calls = mock_workflow.index_service.insertTask.call_args_list
# First call with namespace
namespace, args = calls[0][0]
assert namespace == 'project.releng.services.project.test.static_analysis_bot.mock.1234'
assert args['taskId'] == '12345deadbeef'
assert args['data']['test'] == 'dummy'
assert args['data']['id'] == '1234'
assert args['data']['source'] == 'try'
assert args['data']['try_task_id'] == 'remoteTryTask'
assert args['data']['try_group_id'] == 'remoteTryGroup'
assert args['data']['someData'] == 'mock'
assert 'indexed' in args['data']
# Second call with sub namespace
namespace, args = calls[1][0]
assert namespace == 'project.releng.services.project.test.static_analysis_bot.mock.1234.12345deadbeef'
assert args['taskId'] == '12345deadbeef'
assert args['data']['test'] == 'dummy'
assert args['data']['id'] == '1234'
assert args['data']['source'] == 'try'
assert args['data']['try_task_id'] == 'remoteTryTask'
assert args['data']['try_group_id'] == 'remoteTryGroup'
assert args['data']['someData'] == 'mock'
assert 'indexed' in args['data']
# Third call for monitoring
namespace, args = calls[2][0]
assert namespace == 'project.releng.services.tasks.12345deadbeef'
assert args['taskId'] == '12345deadbeef'
assert args['data']['test'] == 'dummy'
assert args['data']['id'] == '1234'
assert args['data']['source'] == 'try'
assert args['data']['try_task_id'] == 'remoteTryTask'
assert args['data']['try_group_id'] == 'remoteTryGroup'
assert args['data']['monitoring_restart'] is False
def test_taskcluster_index(mock_try_config, mock_try_workflow):
'''
Test the Taskcluster indexing API
by mocking an online taskcluster state
'''
from static_analysis_bot.config import TaskCluster
from static_analysis_bot.revisions import Revision
mock_try_config.taskcluster = TaskCluster('/tmp/dummy', '12345deadbeef', 0, False)
mock_try_workflow.index_service = mock.Mock()
rev = Revision()
rev.namespaces = ['mock.1234']
rev.as_dict = lambda: {'id': '1234', 'someData': 'mock', 'state': 'done', }
mock_try_workflow.index(rev, test='dummy')
assert mock_try_workflow.index_service.insertTask.call_count == 3
calls = mock_try_workflow.index_service.insertTask.call_args_list
# First call with namespace
namespace, args = calls[0][0]
assert namespace == 'project.releng.services.project.test.static_analysis_bot.mock.1234'
assert args['taskId'] == '12345deadbeef'
assert args['data']['test'] == 'dummy'
assert args['data']['id'] == '1234'
assert args['data']['source'] == 'try'
assert args['data']['try_task_id'] == 'remoteTryTask'
assert args['data']['try_group_id'] == 'remoteTryGroup'
assert args['data']['someData'] == 'mock'
assert 'indexed' in args['data']
# Second call with sub namespace
namespace, args = calls[1][0]
assert namespace == 'project.releng.services.project.test.static_analysis_bot.mock.1234.12345deadbeef'
assert args['taskId'] == '12345deadbeef'
assert args['data']['test'] == 'dummy'
assert args['data']['id'] == '1234'
assert args['data']['source'] == 'try'
assert args['data']['try_task_id'] == 'remoteTryTask'
assert args['data']['try_group_id'] == 'remoteTryGroup'
assert args['data']['someData'] == 'mock'
assert 'indexed' in args['data']
# Third call for monitoring
namespace, args = calls[2][0]
assert namespace == 'project.releng.services.tasks.12345deadbeef'
assert args['taskId'] == '12345deadbeef'
assert args['data']['test'] == 'dummy'
assert args['data']['id'] == '1234'
assert args['data']['source'] == 'try'
assert args['data']['try_task_id'] == 'remoteTryTask'
assert args['data']['try_group_id'] == 'remoteTryGroup'
assert args['data']['monitoring_restart'] is False
def mock_revision(mock_phabricator, mock_try_task, mock_config):
'''
Mock a mercurial revision
'''
from static_analysis_bot.revisions import Revision
with mock_phabricator as api:
return Revision(api, mock_try_task, update_build=False)
def _test_reporter(api, analyzers):
# Always use the same setup, only varies the analyzers
revision = Revision(api, mock_try_task)
revision.lines = {
'test.cpp': [0, 41, 42, 43],
'dom/test.cpp': [
42,
],
}
reporter = PhabricatorReporter({'analyzers': analyzers}, api=api)
issues = [
ClangFormatIssue('dom/test.cpp', 42, 1, revision),
ClangTidyIssue(revision, 'test.cpp', '42', '51',
'modernize-use-nullptr', 'dummy message', 'error'),
InferIssue(
{
'file': 'test.cpp',
'line': 42,
'column': 1,
'bug_type': 'dummy',
'kind': 'whatever',
'qualifier': 'dummy message.',
}, revision),
MozLintIssue('test.cpp', 1, 'danger', 42, 'flake8', 'Python error',
'EXXX', revision),
CoverageIssue('test.cpp', 0, 'This file is uncovered', revision),
]
assert all(i.is_publishable() for i in issues)
revision.improvement_patches = [
ImprovementPatch('dummy', repr(revision), 'Whatever'),
ImprovementPatch('clang-tidy', repr(revision), 'Some C fixes'),
ImprovementPatch('clang-format', repr(revision),
'Some lint fixes'),
ImprovementPatch('infer', repr(revision), 'Some java fixes'),
ImprovementPatch('mozlint', repr(revision), 'Some js fixes'),
]
list(map(lambda p: p.write(),
revision.improvement_patches)) # trigger local write
return reporter.publish(issues, revision)
def test_monitoring_restart(mock_config, mock_workflow):
'''
Test the Taskcluster indexing API and restart capabilities
'''
from static_analysis_bot.config import TaskCluster
from static_analysis_bot.revisions import Revision
mock_config.taskcluster = TaskCluster('/tmp/dummy', 'someTaskId', 0, False)
mock_workflow.index_service = mock.Mock()
rev = Revision()
rev.as_dict = dict
rev.namespaces = []
# Unsupported error code
mock_workflow.index(rev, test='dummy', error_code='nope', state='error')
assert mock_workflow.index_service.insertTask.call_count == 1
calls = mock_workflow.index_service.insertTask.call_args_list
namespace, args = calls[0][0]
assert namespace == 'project.releng.services.tasks.someTaskId'
assert args['taskId'] == 'someTaskId'
assert args['data']['monitoring_restart'] is False
# watchdog should be restated
mock_workflow.index(rev,
test='dummy',
error_code='watchdog',
state='error')
assert mock_workflow.index_service.insertTask.call_count == 2
calls = mock_workflow.index_service.insertTask.call_args_list
namespace, args = calls[1][0]
assert namespace == 'project.releng.services.tasks.someTaskId'
assert args['taskId'] == 'someTaskId'
assert args['data']['monitoring_restart'] is True
# Invalid state
mock_workflow.index(rev, test='dummy', state='running')
assert mock_workflow.index_service.insertTask.call_count == 3
calls = mock_workflow.index_service.insertTask.call_args_list
namespace, args = calls[2][0]
assert namespace == 'project.releng.services.tasks.someTaskId'
assert args['taskId'] == 'someTaskId'
assert args['data']['monitoring_restart'] is False
def test_monitoring_restart(mock_try_config, mock_try_workflow):
'''
Test the Taskcluster indexing API and restart capabilities
'''
from static_analysis_bot.config import TaskCluster
from static_analysis_bot.revisions import Revision
mock_try_config.taskcluster = TaskCluster('/tmp/dummy', 'someTaskId', 0, False)
mock_try_workflow.index_service = mock.Mock()
rev = Revision()
rev.as_dict = dict
rev.namespaces = []
# Unsupported error code
mock_try_workflow.index(rev, test='dummy', error_code='nope', state='error')
assert mock_try_workflow.index_service.insertTask.call_count == 1
calls = mock_try_workflow.index_service.insertTask.call_args_list
namespace, args = calls[0][0]
assert namespace == 'project.releng.services.tasks.someTaskId'
assert args['taskId'] == 'someTaskId'
assert args['data']['monitoring_restart'] is False
# watchdog should be restated
mock_try_workflow.index(rev, test='dummy', error_code='watchdog', state='error')
assert mock_try_workflow.index_service.insertTask.call_count == 2
calls = mock_try_workflow.index_service.insertTask.call_args_list
namespace, args = calls[1][0]
assert namespace == 'project.releng.services.tasks.someTaskId'
assert args['taskId'] == 'someTaskId'
assert args['data']['monitoring_restart'] is True
# Invalid state
mock_try_workflow.index(rev, test='dummy', state='running')
assert mock_try_workflow.index_service.insertTask.call_count == 3
calls = mock_try_workflow.index_service.insertTask.call_args_list
namespace, args = calls[2][0]
assert namespace == 'project.releng.services.tasks.someTaskId'
assert args['taskId'] == 'someTaskId'
assert args['data']['monitoring_restart'] is False
def test_phabricator_clang_tidy(mock_phabricator, mock_try_task):
'''
Test Phabricator reporter publication on a mock clang-tidy issue
'''
from static_analysis_bot.report.phabricator import PhabricatorReporter
from static_analysis_bot.revisions import Revision
from static_analysis_bot.tasks.clang_tidy import ClangTidyIssue
def _check_comment(request):
# Check the Phabricator main comment is well formed
payload = urllib.parse.parse_qs(request.body)
assert payload['output'] == ['json']
assert len(payload['params']) == 1
details = json.loads(payload['params'][0])
assert details == {
'revision_id': 51,
'message': VALID_CLANG_TIDY_MESSAGE,
'attach_inlines': 1,
'__conduit__': {
'token': 'deadbeef'
},
}
# Outputs dummy empty response
resp = {
'error_code': None,
'result': None,
}
return 201, {
'Content-Type': 'application/json',
'unittest': 'clang-tidy'
}, json.dumps(resp)
responses.add_callback(
responses.POST,
'http://phabricator.test/api/differential.createcomment',
callback=_check_comment,
)
with mock_phabricator as api:
revision = Revision(api, mock_try_task)
revision.lines = {
# Add dummy lines diff
'another_test.cpp': [41, 42, 43],
}
revision.files = ['another_test.cpp']
reporter = PhabricatorReporter(
{
'analyzers': ['clang-tidy'],
'modes': ('comment')
}, api=api)
issue = ClangTidyIssue(revision, 'another_test.cpp', '42', '51',
'modernize-use-nullptr', 'dummy message', 'error')
assert issue.is_publishable()
issues, patches = reporter.publish([
issue,
], revision)
assert len(issues) == 1
assert len(patches) == 0
# Check the callback has been used
assert len(responses.calls) > 0
call = responses.calls[-1]
assert call.request.url == 'http://phabricator.test/api/differential.createcomment'
assert call.response.headers.get('unittest') == 'clang-tidy'
def test_phabricator_harbormaster(mock_phabricator, mock_try_task):
'''
Test Phabricator reporter publication on a mock clang-tidy issue
using harbormaster
'''
from static_analysis_bot.report.phabricator import PhabricatorReporter
from static_analysis_bot.revisions import Revision
from static_analysis_bot.tasks.clang_tidy import ClangTidyIssue
def _check_message(request):
# Check the Phabricator main comment is well formed
payload = urllib.parse.parse_qs(request.body)
assert payload['output'] == ['json']
assert len(payload['params']) == 1
details = json.loads(payload['params'][0])
assert details == {
'buildTargetPHID':
'PHID-HMBD-deadbeef12456',
'lint': [{
'char': 51,
'code': 'clang-tidy.modernize-use-nullptr',
'name': 'Clang-Tidy - modernize-use-nullptr',
'line': 42,
'path': 'test.cpp',
'severity': 'warning',
'description': 'dummy message'
}],
'unit': [],
'type':
'work',
'__conduit__': {
'token': 'deadbeef'
},
}
# Outputs dummy empty response
resp = {
'error_code': None,
'result': None,
}
return 201, {
'Content-Type': 'application/json',
'unittest': 'clang-tidy'
}, json.dumps(resp)
responses.add_callback(
responses.POST,
'http://phabricator.test/api/harbormaster.sendmessage',
callback=_check_message,
)
with mock_phabricator as api:
revision = Revision(api, mock_try_task)
revision.lines = {
# Add dummy lines diff
'test.cpp': [41, 42, 43],
}
revision.build_target_phid = 'PHID-HMBD-deadbeef12456'
reporter = PhabricatorReporter(
{
'analyzers': ['clang-tidy'],
'mode': 'harbormaster'
}, api=api)
issue = ClangTidyIssue(revision, 'test.cpp', '42', '51',
'modernize-use-nullptr', 'dummy message', 'error')
assert issue.is_publishable()
issues, patches = reporter.publish([
issue,
], revision)
assert len(issues) == 1
assert len(patches) == 0
# Check the callback has been used
assert len(responses.calls) > 0
call = responses.calls[-1]
assert call.request.url == 'http://phabricator.test/api/harbormaster.sendmessage'
assert call.response.headers.get('unittest') == 'clang-tidy'
def test_phabricator_clang_format(mock_config, mock_phabricator,
mock_try_task):
'''
Test Phabricator reporter publication on a mock clang-format issue
'''
from static_analysis_bot.report.phabricator import PhabricatorReporter
from static_analysis_bot.revisions import Revision, ImprovementPatch
from static_analysis_bot.tasks.clang_format import ClangFormatIssue
def _check_comment(request):
# Check the Phabricator main comment is well formed
payload = urllib.parse.parse_qs(request.body)
assert payload['output'] == ['json']
assert len(payload['params']) == 1
details = json.loads(payload['params'][0])
assert details['message'] == VALID_CLANG_FORMAT_MESSAGE.format(
results=mock_config.taskcluster.results_dir)
# Outputs dummy empty response
resp = {
'error_code': None,
'result': None,
}
return 201, {
'Content-Type': 'application/json',
'unittest': 'clang-format'
}, json.dumps(resp)
responses.add_callback(
responses.POST,
'http://phabricator.test/api/differential.createcomment',
callback=_check_comment,
)
with mock_phabricator as api:
revision = Revision(api, mock_try_task)
revision.lines = {
# Add dummy lines diff
'test.cpp': [41, 42, 43],
'dom/test.cpp': [
42,
],
}
reporter = PhabricatorReporter({'analyzers': ['clang-format']},
api=api)
issue = ClangFormatIssue('dom/test.cpp', 42, 1, revision)
assert issue.is_publishable()
revision.improvement_patches = [
ImprovementPatch('clang-format', repr(revision), 'Some lint fixes'),
]
list(map(lambda p: p.write(),
revision.improvement_patches)) # trigger local write
issues, patches = reporter.publish([
issue,
], revision)
assert len(issues) == 1
assert len(patches) == 1
# Check the callback has been used
assert len(responses.calls) > 0
call = responses.calls[-1]
assert call.request.url == 'http://phabricator.test/api/differential.createcomment'
assert call.response.headers.get('unittest') == 'clang-format'
def test_analyze_patch():
from static_analysis_bot.revisions import Revision
from static_analysis_bot import Issue
class MyIssue(Issue):
def __init__(self, path, line):
self.path = path
self.line = line
self.nb_lines = 1
def as_dict():
return {}
def as_markdown():
return ''
def as_text():
return ''
def validates():
return True
def as_phabricator_lint():
return {}
issue_in_new_file = MyIssue('new.txt', 1)
issue_in_existing_file_touched_line = MyIssue('modified.txt', 3)
issue_in_existing_file_not_changed_line = MyIssue('modified.txt', 1)
issue_in_existing_file_added_line = MyIssue('added.txt', 4)
issue_in_not_changed_file = MyIssue('notexisting.txt', 1)
rev = Revision()
rev.patch = '''
diff --git a/new.txt b/new.txt
new file mode 100644
index 00000000..83db48f8
--- /dev/null
+++ b/new.txt
@@ -0,0 +1,3 @@
+line1
+line2
+line3
diff --git a/modified.txt b/modified.txt
index 84275f99..cbc9b72a 100644
--- a/modified.txt
+++ b/modified.txt
@@ -1,4 +1,4 @@
line1
line2
-line3
+line7
line4
diff --git a/added.txt b/added.txt
index 83db48f8..84275f99 100644
--- a/added.txt
+++ b/added.txt
@@ -1,3 +1,4 @@
line1
line2
line3
+line4
'''
rev.analyze_patch()
assert 'new.txt' in rev.lines
assert rev.lines['new.txt'] == [1, 2, 3]
assert 'modified.txt' in rev.lines
assert rev.lines['modified.txt'] == [3]
assert 'added.txt' in rev.lines
assert rev.lines['added.txt'] == [4]
assert 'new.txt' in rev.files
assert 'modified.txt' in rev.files
assert 'added.txt' in rev.files
assert rev.has_file('new.txt')
assert rev.has_file('modified.txt')
assert rev.has_file('added.txt')
assert not rev.has_file('notexisting.txt')
assert rev.contains(issue_in_new_file)
assert rev.contains(issue_in_existing_file_touched_line)
assert not rev.contains(issue_in_existing_file_not_changed_line)
assert rev.contains(issue_in_existing_file_added_line)
assert not rev.contains(issue_in_not_changed_file)
def main(taskcluster_secret,
taskcluster_client_id,
taskcluster_access_token,
):
secrets = get_secrets(taskcluster_secret,
config.PROJECT_NAME,
required=(
'APP_CHANNEL',
'REPORTERS',
'PHABRICATOR',
'ALLOWED_PATHS',
),
existing={
'APP_CHANNEL': 'development',
'REPORTERS': [],
'PUBLICATION': 'IN_PATCH',
'ALLOWED_PATHS': ['*', ],
},
taskcluster_client_id=taskcluster_client_id,
taskcluster_access_token=taskcluster_access_token,
)
init_logger(config.PROJECT_NAME,
PAPERTRAIL_HOST=secrets.get('PAPERTRAIL_HOST'),
PAPERTRAIL_PORT=secrets.get('PAPERTRAIL_PORT'),
SENTRY_DSN=secrets.get('SENTRY_DSN'),
MOZDEF=secrets.get('MOZDEF'),
timestamp=True,
)
# Setup settings before stats
phabricator = secrets['PHABRICATOR']
settings.setup(
secrets['APP_CHANNEL'],
secrets['PUBLICATION'],
secrets['ALLOWED_PATHS'],
)
# Setup statistics
datadog_api_key = secrets.get('DATADOG_API_KEY')
if datadog_api_key:
stats.auth(datadog_api_key)
# Load reporters
reporters = get_reporters(
secrets['REPORTERS'],
taskcluster_client_id,
taskcluster_access_token,
)
# Load index service
index_service = get_service(
'index',
taskcluster_client_id,
taskcluster_access_token,
)
# Load queue service
queue_service = get_service(
'queue',
taskcluster_client_id,
taskcluster_access_token,
)
# Load Phabricator API
phabricator_reporting_enabled = 'phabricator' in reporters
phabricator_api = PhabricatorAPI(phabricator['api_key'], phabricator['url'])
if phabricator_reporting_enabled:
reporters['phabricator'].setup_api(phabricator_api)
# Load unique revision
revision = Revision(
phabricator_api,
try_task=queue_service.task(settings.try_task_id),
# Update build status only when phabricator reporting is enabled
update_build=phabricator_reporting_enabled,
)
# Run workflow according to source
w = Workflow(reporters, index_service, queue_service, phabricator_api)
try:
w.run(revision)
except Exception as e:
# Log errors to papertrail
logger.error(
'Static analysis failure',
revision=revision,
error=e,
)
# Index analysis state
extras = {}
if isinstance(e, AnalysisException):
extras['error_code'] = e.code
extras['error_message'] = str(e)
w.index(revision, state='error', **extras)
# Update Harbormaster status
revision.update_status(state=BuildState.Fail)
# Then raise to mark task as erroneous
raise