def load(self, repo):
'''
Load full stack of patches from Phabricator:
* setup repo to base revision from Mozilla Central
* Apply previous needed patches from Phabricator
'''
try:
_, patches = self.api.load_patches_stack(repo, self.diff)
except Exception as e:
raise AnalysisException('mercurial', str(e))
# Expose current patch to workflow
self.patch = dict(patches)[self.diff_phid]
# Apply all patches from base to top
# except our current (top) patch
for diff_phid, patch in patches[:-1]:
logger.info('Applying parent diff', phid=diff_phid)
try:
repo.import_(
patches=io.BytesIO(patch.encode('utf-8')),
message='SA Imported patch {}'.format(diff_phid),
user='******',
)
except hglib.error.CommandError:
raise AnalysisException('mercurial', 'Failed to import parent patch {}'.format(diff_phid))
def load(self, repo):
'''
Load full raw patch from Phabricator API then load and apply
the dependant stack of patches from Phabricator
when the patch is not already in the repository
'''
try:
_, patches = self.api.load_patches_stack(repo, self.diff)
except Exception as e:
raise AnalysisException('mercurial', str(e))
# Expose current patch to workflow
self.patch = dict(patches)[self.diff_phid]
# Skip patch application when repo already has the patch
if self.mercurial_revision is not None:
return
# Apply all patches from base to top
# except our current (top) patch
for diff_phid, patch in patches[:-1]:
logger.info('Applying parent diff', phid=diff_phid)
try:
repo.import_(
patches=io.BytesIO(patch.encode('utf-8')),
message='SA Imported patch {}'.format(diff_phid),
user='******',
)
except hglib.error.CommandError:
raise AnalysisException(
'mercurial',
'Failed to import parent patch {}'.format(diff_phid))
def setup(index):
'''
Setup Taskcluster Coverity build for static-analysis
'''
assert settings.cov_url, 'Missing secret COVERITY_CONFIG:server_url'
assert settings.cov_analysis_url, 'Missing secret COVERITY_CONFIG:package_url'
assert settings.cov_auth, 'Missing secret COVERITY_CONFIG:auth_key'
target = os.path.join(
os.environ['MOZBUILD_STATE_PATH'], 'coverity')
# Generate the coverity.conf and auth files
cov_auth_path = os.path.join(target, 'auth')
cov_setup_path = os.path.join(target, 'coverity.conf')
cov_conf = COVERITY_CONFIG % (settings.cov_url, cov_auth_path)
logger.info('Downloading from {}.'.format(settings.cov_analysis_url))
cli_common.utils.retry(lambda: download(settings.cov_analysis_url, target))
if not os.path.exists(target):
raise AnalysisException('artifact', 'Setup failed for {}'.format(target))
with open(cov_auth_path, 'w') as f:
f.write(settings.cov_auth)
# Modify it's permission to 600
os.chmod(cov_auth_path, 0o600)
with open(cov_setup_path, 'a') as f:
f.write(cov_conf)
def return_issues(self, coverity_results_path, revision):
'''
Parse Coverity json into structured issues
'''
if not os.path.isfile(coverity_results_path):
raise AnalysisException(
'coverity',
'Coverity Analysis did not generate an analysis report.')
with open(coverity_results_path) as f:
result = json.load(f)
if 'issues' not in result:
logger.info('Coverity did not find any issues')
return []
def _is_local_issue(issue):
'''
The given coverity issue should be only locally stored and not in the
remote snapshot
'''
stateOnServer = issue['stateOnServer']
# According to Coverity manual:
# presentInReferenceSnapshot - True if the issue is present in the reference
# snapshot specified in the cov-run-desktop command, false if not.
return stateOnServer is not None and 'presentInReferenceSnapshot' in stateOnServer \
and stateOnServer['presentInReferenceSnapshot'] is False
return [
CoverityIssue(issue, revision) for issue in result['issues']
if _is_local_issue(issue)
]
def do_build_setup(self):
# Mach pre-setup with mozconfig
try:
logger.info('Mach delete any existing obj dir')
obj_dir = os.path.join(settings.repo_dir,
'obj-x86_64-pc-linux-gnu')
if (os.path.exists(obj_dir)):
shutil.rmtree(obj_dir)
logger.info('Mach configure...')
with stats.api.timer('runtime.mach.configure'):
run_check(['gecko-env', './mach', 'configure'],
cwd=settings.repo_dir)
logger.info('Mach compile db...')
with stats.api.timer('runtime.mach.build-backend'):
run_check([
'gecko-env', './mach', 'build-backend',
'--backend=CompileDB'
],
cwd=settings.repo_dir)
logger.info('Mach pre-export...')
with stats.api.timer('runtime.mach.pre-export'):
run_check(['gecko-env', './mach', 'build', 'pre-export'],
cwd=settings.repo_dir)
logger.info('Mach export...')
with stats.api.timer('runtime.mach.export'):
run_check(['gecko-env', './mach', 'build', 'export'],
cwd=settings.repo_dir)
except Exception as e:
raise AnalysisException('mach', str(e))
def __init__(self, issue, revision):
assert not settings.repo_dir.endswith('/')
self.revision = revision
# We look only for main event
event_path = next(
(event for event in issue['events'] if event['main'] is True),
None)
if event_path is None:
raise AnalysisException(
'coverity',
'Coverity Analysis did not find main event for mergeKey {}'.
format(issue['mergeKey']))
checker_properties = issue['checkerProperties']
# Strip the leading slash
self.path = issue['strippedMainEventFilePathname'].strip('/')
self.line = issue['mainEventLineNumber']
self.bug_type = checker_properties['category']
self.kind = issue['checkerName']
self.message = event_path['eventDescription']
self.body = None
self.nb_lines = 1
if settings.cov_full_stack:
self.message += ISSUE_RELATION
# Embed all events into message
for event in issue['events']:
self.message += ISSUE_ELEMENT_IN_STACK.format(
file_path=event['strippedFilePathname'],
line_number=event['lineNumber'],
path_type=event['eventTag'],
description=event['eventDescription'])
def setup(index,
job_name='linux64-infer',
revision='latest',
artifact='public/build/infer.tar.xz'):
'''
Setup Taskcluster infer build for static-analysis
Defaults values are from https://dxr.mozilla.org/mozilla-central/source/taskcluster/ci/toolchain/linux.yml
- Download the artifact from latest Taskcluster build
- Extracts it into the MOZBUILD_STATE_PATH as expected by mach
'''
if job_name == 'linux64-infer':
jobs = [{
'job-name':
'linux64-infer',
'artifact':
'public/build/infer.tar.xz',
'namespace':
'gecko.v2.autoland.latest.static-analysis.linux64-infer'
}, {
'job-name':
'linux64-android-sdk-linux-repack',
'artifact':
'project/gecko/android-sdk/android-sdk-linux.tar.xz',
'namespace':
'gecko.cache.level-1.toolchains.v2.linux64-android-sdk-linux-repack.latest'
}, {
'job-name':
'linux64-android-ndk-linux-repack',
'artifact':
'project/gecko/android-ndk/android-ndk.tar.xz',
'namespace':
'gecko.cache.level-1.toolchains.v2.linux64-android-ndk-linux-repack.latest'
}]
for element in jobs:
namespace = element['namespace']
artifact = element['artifact']
# on staging buildSignedUrl will fail, because the artifacts are downloaded from
# a proxy, therefore we need to use buildUrl in case the signed version fails
try:
artifact_url = index.buildSignedUrl('findArtifactFromTask',
indexPath=namespace,
name=artifact,
expiration=7200)
except taskcluster.exceptions.TaskclusterAuthFailure:
artifact_url = index.buildUrl('findArtifactFromTask',
indexPath=namespace,
name=artifact)
target = os.path.join(
os.environ['MOZBUILD_STATE_PATH'],
os.path.basename(artifact).split('.')[0],
)
logger.info('Downloading {}.'.format(artifact))
cli_common.utils.retry(lambda: download(artifact_url, target))
if not os.path.exists(target):
raise AnalysisException('artifact',
'Setup failed for {}'.format(target))
def _download():
# Download Taskcluster archive
resp = requests.get(artifact_url, stream=True)
if not resp.ok:
raise AnalysisException('artifact',
'Download failed {}'.format(artifact_url))
# Extract archive into destination
with tarfile.open(fileobj=io.BytesIO(resp.content)) as tar:
tar.extractall(target)
def run(self, revision):
'''
Run modified files with specified checks through clang-tidy
using threaded workers (communicate through queues)
Output a list of ClangTidyIssue
'''
assert isinstance(revision, Revision)
self.revision = revision
# Run all files in a single command
# through mach static-analysis
cmd = [
'gecko-env',
'./mach',
'--log-no-times',
'static-analysis',
'check',
# Limit warnings to current files
'--header-filter={}'.format('|'.join(
os.path.basename(filename) for filename in revision.files)),
'--checks={}'.format(','.join(c['name']
for c in settings.clang_checkers)),
] + list(revision.files)
logger.info('Running static-analysis', cmd=' '.join(cmd))
# Run command, without checking its exit code as clang-tidy 7+
# exits with an error code when finding errors (which we want to report !)
try:
clang = subprocess.run(cmd,
cwd=settings.repo_dir,
check=False,
stdout=subprocess.PIPE)
except subprocess.CalledProcessError as e:
raise AnalysisException(
'clang-tidy',
'Mach static analysis failed: {}'.format(e.output))
clang_output = clang.stdout.decode('utf-8')
# Dump raw clang-tidy output as a Taskcluster artifact (for debugging)
clang_output_path = os.path.join(
settings.taskcluster.results_dir,
'{}-clang-tidy.txt'.format(repr(revision)),
)
with open(clang_output_path, 'w') as f:
f.write(clang_output)
issues = self.parse_issues(clang_output, revision)
# Report stats for these issues
stats.report_issues('clang-tidy', issues)
return issues
def find_issues(self, path, revision):
'''
Run mozlint through mach, using gecko-env
'''
# Check file exists (before mode)
full_path = os.path.join(settings.repo_dir, path)
if not os.path.exists(full_path):
logger.info('Modified file not found {}'.format(full_path))
return
# Run mozlint on a file
command = [
'gecko-env',
'./mach', 'lint',
'-f', 'json',
'--quiet',
path
]
returncode, output, error = run(' '.join(command), cwd=settings.repo_dir)
output = output.decode('utf-8')
# Dump raw mozlint output as a Taskcluster artifact (for debugging)
output_path = os.path.join(
settings.taskcluster.results_dir,
'{}-mozlint.txt'.format(repr(revision)),
)
with open(output_path, 'a') as f:
f.write(output)
if returncode == 0:
logger.debug('No Mozlint errors', path=path)
return
assert 'error: problem with lint setup' not in output, \
'Mach lint setup failed'
# Load output as json
# Only consider last line, as ./mach lint may output
# linter setup output on stdout :/
try:
lines = list(filter(None, output.split('\n')))
payload = json.loads(lines[-1])
except json.decoder.JSONDecodeError:
raise AnalysisException('mozlint', 'Invalid json output', path=path, lines=lines)
if full_path not in payload and path not in payload:
logger.warn('Missing path in linter output', path=path)
return
# Mozlint uses both full & relative path to index issues
return [
MozLintIssue(revision=revision, **issue)
for p in (path, full_path)
for issue in payload.get(p, [])
]
def apply(self, repo):
'''
Apply patch from Phabricator to Mercurial local repository
'''
assert isinstance(repo, hglib.client.hgclient)
# Apply the patch on top of repository
try:
repo.import_(
patches=io.BytesIO(self.patch.encode('utf-8')),
nocommit=True,
)
logger.info('Applied target patch', phid=self.diff_phid)
except hglib.error.CommandError as e:
raise AnalysisException('mercurial', 'Failed to import target patch')
def run(self, revision):
'''
Run modified files with specified checks through infer
using threaded workers (communicate through queues)
Output a list of InferIssue
'''
assert isinstance(revision, Revision)
self.revision = revision
with AndroidConfig():
# Mach pre-setup with mozconfig
logger.info('Mach configure for infer...')
run_check(['gecko-env', './mach', 'configure'],
cwd=settings.repo_dir)
# Run all files in a single command
# through mach static-analysis
cmd = [
'gecko-env', './mach', '--log-no-times', 'static-analysis',
'check-java'
] + list(revision.files)
logger.info('Running static-analysis', cmd=' '.join(cmd))
# Run command
try:
infer_output = subprocess.check_output(cmd,
cwd=settings.repo_dir)
except subprocess.CalledProcessError as e:
raise AnalysisException(
'infer',
'Mach static analysis failed: {}'.format(e.output))
report_file = os.path.join(settings.repo_dir, 'infer-out',
'report.json')
infer_output = json.load(open(report_file))
# Dump raw infer output as a Taskcluster artifact (for debugging)
infer_output_path = os.path.join(
settings.taskcluster.results_dir,
'{}-infer.txt'.format(repr(revision)),
)
with open(infer_output_path, 'w') as f:
f.write(json.dumps(infer_output, indent=2))
issues = self.parse_issues(infer_output, revision)
# Report stats for these issues
stats.report_issues('infer', issues)
return issues
def return_issues(self, coverity_results_path, revision):
'''
Parse Coverity json into structured issues
'''
if not os.path.isfile(coverity_results_path):
raise AnalysisException(
'coverity',
'Coverity Analysis did not generate an analysis report.')
with open(coverity_results_path) as f:
result = json.load(f)
if 'issues' not in result:
logger.info('Coverity did not find any issues')
return []
return [
CoverityIssue(issue, revision) for issue in result['issues']
]
def __init__(self, issue, revision):
assert not settings.repo_dir.endswith('/')
self.revision = revision
# We look only for main event
event_path = next(
(event for event in issue['events'] if event['main'] is True),
None)
if event_path is None:
raise AnalysisException(
'coverity',
'Coverity Analysis did not find main event for mergeKey {}'.
format(issue['mergeKey']))
checker_properties = issue['checkerProperties']
# Strip the leading slash
self.path = issue['strippedMainEventFilePathname'].strip('/')
self.line = issue['mainEventLineNumber']
self.bug_type = checker_properties['category']
self.kind = issue['checkerName']
self.message = event_path['eventDescription']
self.body = None
self.nb_lines = 1
def apply(self, repo):
'''
Apply patch from Phabricator to Mercurial local repository
'''
assert isinstance(repo, hglib.client.hgclient)
if self.mercurial_revision:
# Apply the existing commit when available
repo.update(
rev=self.mercurial_revision,
clean=True,
)
else:
# Apply the patch on top of repository
try:
repo.import_(
patches=io.BytesIO(self.patch.encode('utf-8')),
message='SA Analyzed patch',
user='******',
)
logger.info('Applied target patch', phid=self.diff_phid)
except hglib.error.CommandError:
raise AnalysisException('mercurial',
'Failed to import target patch')
def run(self, revision):
'''
Run the static analysis workflow:
* Pull revision from review
* Checkout revision
* Run static analysis
* Publish results
'''
analyzers = []
# Index ASAP Taskcluster task for this revision
self.index(revision, state='started')
# Add log to find Taskcluster task in papertrail
logger.info(
'New static analysis',
taskcluster_task=settings.taskcluster.task_id,
taskcluster_run=settings.taskcluster.run_id,
channel=settings.app_channel,
publication=settings.publication.name,
revision=str(revision),
)
stats.api.event(
title='Static analysis on {} for {}'.format(settings.app_channel, revision),
text='Task {} #{}'.format(settings.taskcluster.task_id, settings.taskcluster.run_id),
)
stats.api.increment('analysis')
with stats.api.timer('runtime.mercurial'):
try:
# Start by cloning the mercurial repository
self.hg = self.clone()
self.index(revision, state='cloned')
# Force cleanup to reset top of MU
# otherwise previous pull are there
self.hg.update(rev=self.top_revision, clean=True)
logger.info('Set repo back to Mozilla unified top', rev=self.hg.identify())
except hglib.error.CommandError as e:
raise AnalysisException('mercurial', str(e))
# Load and analyze revision patch
revision.load(self.hg)
revision.analyze_patch()
with stats.api.timer('runtime.mach'):
# Only run mach if revision has any C/C++ or Java files
if revision.has_clang_files:
# Mach pre-setup with mozconfig
try:
logger.info('Mach configure...')
with stats.api.timer('runtime.mach.configure'):
run_check(['gecko-env', './mach', 'configure'], cwd=settings.repo_dir)
logger.info('Mach compile db...')
with stats.api.timer('runtime.mach.build-backend'):
run_check(['gecko-env', './mach', 'build-backend', '--backend=CompileDB'], cwd=settings.repo_dir)
logger.info('Mach pre-export...')
with stats.api.timer('runtime.mach.pre-export'):
run_check(['gecko-env', './mach', 'build', 'pre-export'], cwd=settings.repo_dir)
logger.info('Mach export...')
with stats.api.timer('runtime.mach.export'):
run_check(['gecko-env', './mach', 'build', 'export'], cwd=settings.repo_dir)
except Exception as e:
raise AnalysisException('mach', str(e))
# Download clang build from Taskcluster
# Use new clang-tidy paths, https://bugzilla.mozilla.org/show_bug.cgi?id=1495641
logger.info('Setup Taskcluster clang build...')
setup_clang(repository='mozilla-inbound', revision='revision.874a07fdb045b725edc2aaa656a8620ff439ec10')
# Use clang-tidy & clang-format
if CLANG_TIDY in self.analyzers:
analyzers.append(ClangTidy)
else:
logger.info('Skip clang-tidy')
if CLANG_FORMAT in self.analyzers:
analyzers.append(ClangFormat)
else:
logger.info('Skip clang-format')
if revision.has_infer_files:
if INFER in self.analyzers:
analyzers.append(Infer)
logger.info('Setup Taskcluster infer build...')
setup_infer(self.index_service)
else:
logger.info('Skip infer')
if not (revision.has_clang_files or revision.has_infer_files):
logger.info('No clang or java files detected, skipping mach, infer and clang-*')
# Setup python environment
logger.info('Mach lint setup...')
cmd = ['gecko-env', './mach', 'lint', '--list']
with stats.api.timer('runtime.mach.lint'):
out = run_check(cmd, cwd=settings.repo_dir)
if 'error: problem with lint setup' in out.decode('utf-8'):
raise AnalysisException('mach', 'Mach lint setup failed')
# Always use mozlint
if MOZLINT in self.analyzers:
analyzers.append(MozLint)
else:
logger.info('Skip mozlint')
if not analyzers:
logger.error('No analyzers to use on revision')
return
self.index(revision, state='analyzing')
with stats.api.timer('runtime.issues'):
# Detect initial issues
if settings.publication == Publication.BEFORE_AFTER:
before_patch, _ = self.detect_issues(analyzers, revision)
logger.info('Detected {} issue(s) before patch'.format(len(before_patch)))
stats.api.increment('analysis.issues.before', len(before_patch))
# Apply patch
revision.apply(self.hg)
# Detect new issues
issues = self.detect_issues(analyzers, revision)
logger.info('Detected {} issue(s) after patch'.format(len(issues)))
stats.api.increment('analysis.issues.after', len(issues))
# Mark newly found issues
if settings.publication == Publication.BEFORE_AFTER:
for issue in issues:
issue.is_new = issue not in before_patch
# Avoid duplicates
issues = set(issues)
if not issues:
logger.info('No issues, stopping there.')
self.index(revision, state='done', issues=0)
return
# Report issues publication stats
nb_issues = len(issues)
nb_publishable = len([i for i in issues if i.is_publishable()])
self.index(revision, state='analyzed', issues=nb_issues, issues_publishable=nb_publishable)
stats.api.increment('analysis.issues.publishable', nb_publishable)
# Publish reports about these issues
with stats.api.timer('runtime.reports'):
for reporter in self.reporters.values():
reporter.publish(issues, revision)
self.index(revision, state='done', issues=nb_issues, issues_publishable=nb_publishable)
def run(self, revision):
'''
Run coverity
'''
assert isinstance(revision, Revision)
self.revision = revision
# Based on our previous configs we should already have generated compile_commands.json
self.compile_commands_path = os.path.join(settings.repo_dir,
'obj-x86_64-pc-linux-gnu',
'compile_commands.json')
assert os.path.exists(self.compile_commands_path), \
'Missing Coverity in {}'.format(self.compile_commands_path)
logger.info('Building command files from compile_commands.json')
# Retrieve the revision files with build commands associated
commands_list = self.get_files_with_commands()
assert commands_list is not [], 'Commands List is empty'
logger.info('Built commands for {} files'.format(len(commands_list)))
cmd = ['gecko-env', self.cov_run_desktop, '--setup']
logger.info('Running Coverity Setup', cmd=cmd)
try:
run_check(cmd, cwd=self.cov_path)
except subprocess.CalledProcessError as e:
raise AnalysisException(
'coverity', 'Coverity Setup failed: {}'.format(e.output))
cmd = ['gecko-env', self.cov_configure, '--clang']
logger.info('Running Coverity Configure', cmd=cmd)
try:
run_check(cmd, cwd=self.cov_path)
except subprocess.CalledProcessError as e:
raise AnalysisException(
'coverity', 'Coverity Configure failed: {}'.format(e.output))
# For each element in commands_list run `cov-translate`
for element in commands_list:
cmd = [
'gecko-env', self.cov_translate, '--dir', self.cov_idir_path,
element['command']
]
logger.info('Running Coverity Tranlate', cmd=cmd)
try:
run_check(cmd, cwd=element['directory'])
except subprocess.CalledProcessError as e:
raise AnalysisException(
'coverity',
'Coverity Translate failed: {}'.format(e.output))
# Once the capture is performed we need to do the actual Coverity Desktop analysis
cmd = [
'gecko-env', self.cov_run_desktop, '--json-output-v6',
'cov-results.json', '--strip-path', settings.repo_dir
]
cmd += [element['file'] for element in commands_list]
logger.info('Running Coverity Analysis', cmd=cmd)
try:
run_check(cmd, cwd=self.cov_state_path)
except subprocess.CalledProcessError as e:
raise AnalysisException(
'coverity', 'Coverity Analysis failed: {}'.format(e.output))
# Write the results.json to the artifact directory to have it later on for debug
coverity_results_path = os.path.join(self.cov_state_path,
'cov-results.json')
coverity_results_path_on_tc = os.path.join(
settings.taskcluster.results_dir, 'cov-results.json')
shutil.copyfile(coverity_results_path, coverity_results_path_on_tc)
# Parsing the issues from coverity_results_path
logger.info('Parsing Coverity issues')
return self.return_issues(coverity_results_path, revision)
def run(self, revision):
'''
Run the local static analysis workflow:
* Pull revision from review
* Checkout revision
* Run static analyzers
'''
analyzers = []
# Add log to find Taskcluster task in papertrail
logger.info(
'New static analysis',
taskcluster_task=settings.taskcluster.task_id,
taskcluster_run=settings.taskcluster.run_id,
channel=settings.app_channel,
publication=settings.publication.name,
revision=str(revision),
)
stats.api.event(
title='Static analysis on {} for {}'.format(
settings.app_channel, revision),
text='Task {} #{}'.format(settings.taskcluster.task_id,
settings.taskcluster.run_id),
)
stats.api.increment('analysis')
with stats.api.timer('runtime.mercurial'):
try:
# Clone in a controllable process
# and kill this new process if it exceeds the maximum allowed runtime
clone = multiprocessing.Process(target=self.clone,
args=(revision, ))
clone.start()
clone.join(settings.max_clone_runtime)
if clone.is_alive():
logger.error(
'Clone watchdog expired, stopping immediately')
# Kill the clone process
clone.terminate()
# Stop workflow
raise AnalysisException('watchdog',
'Clone watchdog expired')
# Open a mercurial client in main process
self.hg = self.open_repository()
# Start by cloning the mercurial repository
self.parent.index(revision, state='cloned')
# Force cleanup to reset top of MU
# otherwise previous pull are there
self.hg.update(rev=self.top_revision, clean=True)
logger.info('Set repo back to Mozilla unified top',
rev=self.hg.identify())
except hglib.error.CommandError as e:
raise AnalysisException('mercurial', str(e))
# Load and analyze revision patch
revision.load(self.hg)
revision.analyze_patch()
with stats.api.timer('runtime.mach'):
# Only run mach if revision has any C/C++ or Java files
if revision.has_clang_files:
self.do_build_setup()
# Download clang build from Taskcluster
# Use new clang-tidy paths, https://bugzilla.mozilla.org/show_bug.cgi?id=1495641
logger.info('Setup Taskcluster clang build...')
setup_clang(
repository='mozilla-inbound',
revision='revision.874a07fdb045b725edc2aaa656a8620ff439ec10'
)
# Use clang-tidy & clang-format
if CLANG_TIDY in self.analyzers:
analyzers.append(ClangTidy)
else:
logger.info('Skip clang-tidy')
if CLANG_FORMAT in self.analyzers:
analyzers.append(ClangFormat)
else:
logger.info('Skip clang-format')
# Run Coverity Scan
if COVERITY in self.analyzers:
logger.info('Setup Taskcluster coverity build...')
setup_coverity(self.index_service)
analyzers.append(Coverity)
else:
logger.info('Skip Coverity')
if COVERAGE in self.analyzers:
analyzers.append(Coverage)
else:
logger.info('Skip coverage analysis')
if revision.has_infer_files:
if INFER in self.analyzers:
analyzers.append(Infer)
logger.info('Setup Taskcluster infer build...')
setup_infer(self.index_service)
else:
logger.info('Skip infer')
if not (revision.has_clang_files or revision.has_infer_files):
logger.info(
'No clang or java files detected, skipping mach, infer and clang-*'
)
# Setup python environment
logger.info('Mach lint setup...')
cmd = ['gecko-env', './mach', 'lint', '--list']
with stats.api.timer('runtime.mach.lint'):
out = run_check(cmd, cwd=settings.repo_dir)
if 'error: problem with lint setup' in out.decode('utf-8'):
raise AnalysisException('mach', 'Mach lint setup failed')
# Always use mozlint
if MOZLINT in self.analyzers:
analyzers.append(MozLint)
else:
logger.info('Skip mozlint')
if not analyzers:
logger.error('No analyzers to use on revision')
return
self.parent.index(revision, state='analyzing')
with stats.api.timer('runtime.issues'):
# Detect initial issues
if settings.publication == Publication.BEFORE_AFTER:
before_patch = self.detect_issues(analyzers, revision, True)
logger.info('Detected {} issue(s) before patch'.format(
len(before_patch)))
stats.api.increment('analysis.issues.before',
len(before_patch))
revision.reset()
# Apply patch
revision.apply(self.hg)
if settings.publication == Publication.BEFORE_AFTER and revision.has_clang_files \
and (revision.has_clang_header_files or revision.has_idl_files):
self.do_build_setup()
# Detect new issues
issues = self.detect_issues(analyzers, revision)
logger.info('Detected {} issue(s) after patch'.format(len(issues)))
stats.api.increment('analysis.issues.after', len(issues))
# Mark newly found issues
if settings.publication == Publication.BEFORE_AFTER:
for issue in issues:
issue.is_new = issue not in before_patch
# Avoid duplicates
# but still output a list to be compatible with LocalWorkflow
return list(set(issues))
def parse_issues(self, clang_output, revision):
'''
Parse clang-tidy output into structured issues
'''
# Detect end of file warnings count
# When an invalid file is used, this line does not appear
has_warnings = REGEX_HAS_WARNINGS.search(clang_output)
if has_warnings is None:
logger.info('Empty clang output, skipping analysis.')
return []
nb_warnings = int(has_warnings.group(1))
if nb_warnings == 0:
logger.info('Mach static analysis did not find any issue')
return []
logger.info('Mach static analysis found some issues', nb=nb_warnings)
# Sort headers by positions
headers = sorted(REGEX_HEADER.finditer(clang_output),
key=lambda h: h.start())
if not headers:
raise AnalysisException(
'clang-tidy',
'No clang-tidy header was found even though a clang output was provided'
)
def _remove_footer(start_pos, end_pos):
'''
Build an issue body from clang-tidy output
and stops when an extra paylaod is detected (footer)
'''
assert isinstance(start_pos, int)
assert isinstance(end_pos, int)
body = clang_output[start_pos:end_pos]
footer = REGEX_FOOTER.search(body)
if footer is None:
return body
return body[:footer.start() - 1]
issues = []
for i, header in enumerate(headers):
issue = ClangTidyIssue(header.groups(), revision)
# Get next header
if i + 1 < len(headers):
# Parse body until next header
next_header = headers[i + 1]
issue.body = _remove_footer(header.end(),
next_header.start() - 1)
else:
# Limit last element to 3 lines to avoid parsing extra metadatas
issue.body = _remove_footer(header.end(), header.end() + 3)
if issue.is_problem():
# Save problem to append notes
# Skip diagnostic errors, but warn through Sentry
if issue.check == 'clang-diagnostic-error':
logger.error(
'Encountered a clang-diagnostic-error: {}'.format(
issue))
else:
issues.append(issue)
mode = issue.is_third_party() and '3rd party' or 'in-tree'
logger.info('Found {} code issue {}'.format(mode, issue))
elif issues:
# Link notes to last problem
issues[-1].notes.append(issue)
return issues
def load(self, repo):
'''
Load full stack of patches from Phabricator:
* setup repo to base revision from Mozilla Central
* Apply previous needed patches from Phabricator
'''
assert isinstance(repo, hglib.client.hgclient)
# Diff PHIDs from our patch to its base
patches = OrderedDict()
patches[self.diff_phid] = self.diff_id
parents = self.api.load_parents(self.phid)
if parents:
# Load all parent diffs
for parent in parents:
logger.info('Loading parent diff', phid=parent)
# Sort parent diffs by their id to load the most recent patch
parent_diffs = sorted(
self.api.search_diffs(revision_phid=parent),
key=lambda x: x['id'],
)
last_diff = parent_diffs[-1]
patches[last_diff['phid']] = last_diff['id']
# Use base revision of last parent
hg_base = last_diff['baseRevision']
else:
# Use base revision from top diff
hg_base = self.diff['baseRevision']
# When base revision is missing, update to top of Central
if hg_base is None or not revision_available(repo, hg_base):
logger.warning('Missing base revision from Phabricator')
hg_base = 'central'
# Load all patches from their numerical ID
for diff_phid, diff_id in patches.items():
patches[diff_phid] = self.api.load_raw_diff(diff_id)
# Expose current patch to workflow
self.patch = patches[self.diff_phid]
# Update the repo to base revision
try:
logger.info('Updating repo to revision', rev=hg_base)
repo.update(
rev=hg_base,
clean=True,
)
except hglib.error.CommandError:
raise AnalysisException('mercurial', 'Failed to update to revision {}'.format(hg_base))
# Apply all patches from base to top
# except our current (top) patch
for diff_phid, patch in reversed(list(patches.items())[1:]):
logger.info('Applying parent diff', phid=diff_phid)
try:
repo.import_(
patches=io.BytesIO(patch.encode('utf-8')),
message='SA Imported patch {}'.format(diff_phid),
user='******',
)
except hglib.error.CommandError:
raise AnalysisException('mercurial', 'Failed to import parent patch {}'.format(diff_phid))
