def load(self, repo): ''' Load full stack of patches from Phabricator: * setup repo to base revision from Mozilla Central * Apply previous needed patches from Phabricator ''' try: _, patches = self.api.load_patches_stack(repo, self.diff) except Exception as e: raise AnalysisException('mercurial', str(e)) # Expose current patch to workflow self.patch = dict(patches)[self.diff_phid] # Apply all patches from base to top # except our current (top) patch for diff_phid, patch in patches[:-1]: logger.info('Applying parent diff', phid=diff_phid) try: repo.import_( patches=io.BytesIO(patch.encode('utf-8')), message='SA Imported patch {}'.format(diff_phid), user='******', ) except hglib.error.CommandError: raise AnalysisException('mercurial', 'Failed to import parent patch {}'.format(diff_phid))
def load(self, repo): ''' Load full raw patch from Phabricator API then load and apply the dependant stack of patches from Phabricator when the patch is not already in the repository ''' try: _, patches = self.api.load_patches_stack(repo, self.diff) except Exception as e: raise AnalysisException('mercurial', str(e)) # Expose current patch to workflow self.patch = dict(patches)[self.diff_phid] # Skip patch application when repo already has the patch if self.mercurial_revision is not None: return # Apply all patches from base to top # except our current (top) patch for diff_phid, patch in patches[:-1]: logger.info('Applying parent diff', phid=diff_phid) try: repo.import_( patches=io.BytesIO(patch.encode('utf-8')), message='SA Imported patch {}'.format(diff_phid), user='******', ) except hglib.error.CommandError: raise AnalysisException( 'mercurial', 'Failed to import parent patch {}'.format(diff_phid))
def setup(index): ''' Setup Taskcluster Coverity build for static-analysis ''' assert settings.cov_url, 'Missing secret COVERITY_CONFIG:server_url' assert settings.cov_analysis_url, 'Missing secret COVERITY_CONFIG:package_url' assert settings.cov_auth, 'Missing secret COVERITY_CONFIG:auth_key' target = os.path.join( os.environ['MOZBUILD_STATE_PATH'], 'coverity') # Generate the coverity.conf and auth files cov_auth_path = os.path.join(target, 'auth') cov_setup_path = os.path.join(target, 'coverity.conf') cov_conf = COVERITY_CONFIG % (settings.cov_url, cov_auth_path) logger.info('Downloading from {}.'.format(settings.cov_analysis_url)) cli_common.utils.retry(lambda: download(settings.cov_analysis_url, target)) if not os.path.exists(target): raise AnalysisException('artifact', 'Setup failed for {}'.format(target)) with open(cov_auth_path, 'w') as f: f.write(settings.cov_auth) # Modify it's permission to 600 os.chmod(cov_auth_path, 0o600) with open(cov_setup_path, 'a') as f: f.write(cov_conf)
def return_issues(self, coverity_results_path, revision): ''' Parse Coverity json into structured issues ''' if not os.path.isfile(coverity_results_path): raise AnalysisException( 'coverity', 'Coverity Analysis did not generate an analysis report.') with open(coverity_results_path) as f: result = json.load(f) if 'issues' not in result: logger.info('Coverity did not find any issues') return [] def _is_local_issue(issue): ''' The given coverity issue should be only locally stored and not in the remote snapshot ''' stateOnServer = issue['stateOnServer'] # According to Coverity manual: # presentInReferenceSnapshot - True if the issue is present in the reference # snapshot specified in the cov-run-desktop command, false if not. return stateOnServer is not None and 'presentInReferenceSnapshot' in stateOnServer \ and stateOnServer['presentInReferenceSnapshot'] is False return [ CoverityIssue(issue, revision) for issue in result['issues'] if _is_local_issue(issue) ]
def do_build_setup(self): # Mach pre-setup with mozconfig try: logger.info('Mach delete any existing obj dir') obj_dir = os.path.join(settings.repo_dir, 'obj-x86_64-pc-linux-gnu') if (os.path.exists(obj_dir)): shutil.rmtree(obj_dir) logger.info('Mach configure...') with stats.api.timer('runtime.mach.configure'): run_check(['gecko-env', './mach', 'configure'], cwd=settings.repo_dir) logger.info('Mach compile db...') with stats.api.timer('runtime.mach.build-backend'): run_check([ 'gecko-env', './mach', 'build-backend', '--backend=CompileDB' ], cwd=settings.repo_dir) logger.info('Mach pre-export...') with stats.api.timer('runtime.mach.pre-export'): run_check(['gecko-env', './mach', 'build', 'pre-export'], cwd=settings.repo_dir) logger.info('Mach export...') with stats.api.timer('runtime.mach.export'): run_check(['gecko-env', './mach', 'build', 'export'], cwd=settings.repo_dir) except Exception as e: raise AnalysisException('mach', str(e))
def __init__(self, issue, revision): assert not settings.repo_dir.endswith('/') self.revision = revision # We look only for main event event_path = next( (event for event in issue['events'] if event['main'] is True), None) if event_path is None: raise AnalysisException( 'coverity', 'Coverity Analysis did not find main event for mergeKey {}'. format(issue['mergeKey'])) checker_properties = issue['checkerProperties'] # Strip the leading slash self.path = issue['strippedMainEventFilePathname'].strip('/') self.line = issue['mainEventLineNumber'] self.bug_type = checker_properties['category'] self.kind = issue['checkerName'] self.message = event_path['eventDescription'] self.body = None self.nb_lines = 1 if settings.cov_full_stack: self.message += ISSUE_RELATION # Embed all events into message for event in issue['events']: self.message += ISSUE_ELEMENT_IN_STACK.format( file_path=event['strippedFilePathname'], line_number=event['lineNumber'], path_type=event['eventTag'], description=event['eventDescription'])
def setup(index, job_name='linux64-infer', revision='latest', artifact='public/build/infer.tar.xz'): ''' Setup Taskcluster infer build for static-analysis Defaults values are from https://dxr.mozilla.org/mozilla-central/source/taskcluster/ci/toolchain/linux.yml - Download the artifact from latest Taskcluster build - Extracts it into the MOZBUILD_STATE_PATH as expected by mach ''' if job_name == 'linux64-infer': jobs = [{ 'job-name': 'linux64-infer', 'artifact': 'public/build/infer.tar.xz', 'namespace': 'gecko.v2.autoland.latest.static-analysis.linux64-infer' }, { 'job-name': 'linux64-android-sdk-linux-repack', 'artifact': 'project/gecko/android-sdk/android-sdk-linux.tar.xz', 'namespace': 'gecko.cache.level-1.toolchains.v2.linux64-android-sdk-linux-repack.latest' }, { 'job-name': 'linux64-android-ndk-linux-repack', 'artifact': 'project/gecko/android-ndk/android-ndk.tar.xz', 'namespace': 'gecko.cache.level-1.toolchains.v2.linux64-android-ndk-linux-repack.latest' }] for element in jobs: namespace = element['namespace'] artifact = element['artifact'] # on staging buildSignedUrl will fail, because the artifacts are downloaded from # a proxy, therefore we need to use buildUrl in case the signed version fails try: artifact_url = index.buildSignedUrl('findArtifactFromTask', indexPath=namespace, name=artifact, expiration=7200) except taskcluster.exceptions.TaskclusterAuthFailure: artifact_url = index.buildUrl('findArtifactFromTask', indexPath=namespace, name=artifact) target = os.path.join( os.environ['MOZBUILD_STATE_PATH'], os.path.basename(artifact).split('.')[0], ) logger.info('Downloading {}.'.format(artifact)) cli_common.utils.retry(lambda: download(artifact_url, target)) if not os.path.exists(target): raise AnalysisException('artifact', 'Setup failed for {}'.format(target))
def _download(): # Download Taskcluster archive resp = requests.get(artifact_url, stream=True) if not resp.ok: raise AnalysisException('artifact', 'Download failed {}'.format(artifact_url)) # Extract archive into destination with tarfile.open(fileobj=io.BytesIO(resp.content)) as tar: tar.extractall(target)
def run(self, revision): ''' Run modified files with specified checks through clang-tidy using threaded workers (communicate through queues) Output a list of ClangTidyIssue ''' assert isinstance(revision, Revision) self.revision = revision # Run all files in a single command # through mach static-analysis cmd = [ 'gecko-env', './mach', '--log-no-times', 'static-analysis', 'check', # Limit warnings to current files '--header-filter={}'.format('|'.join( os.path.basename(filename) for filename in revision.files)), '--checks={}'.format(','.join(c['name'] for c in settings.clang_checkers)), ] + list(revision.files) logger.info('Running static-analysis', cmd=' '.join(cmd)) # Run command, without checking its exit code as clang-tidy 7+ # exits with an error code when finding errors (which we want to report !) try: clang = subprocess.run(cmd, cwd=settings.repo_dir, check=False, stdout=subprocess.PIPE) except subprocess.CalledProcessError as e: raise AnalysisException( 'clang-tidy', 'Mach static analysis failed: {}'.format(e.output)) clang_output = clang.stdout.decode('utf-8') # Dump raw clang-tidy output as a Taskcluster artifact (for debugging) clang_output_path = os.path.join( settings.taskcluster.results_dir, '{}-clang-tidy.txt'.format(repr(revision)), ) with open(clang_output_path, 'w') as f: f.write(clang_output) issues = self.parse_issues(clang_output, revision) # Report stats for these issues stats.report_issues('clang-tidy', issues) return issues
def find_issues(self, path, revision): ''' Run mozlint through mach, using gecko-env ''' # Check file exists (before mode) full_path = os.path.join(settings.repo_dir, path) if not os.path.exists(full_path): logger.info('Modified file not found {}'.format(full_path)) return # Run mozlint on a file command = [ 'gecko-env', './mach', 'lint', '-f', 'json', '--quiet', path ] returncode, output, error = run(' '.join(command), cwd=settings.repo_dir) output = output.decode('utf-8') # Dump raw mozlint output as a Taskcluster artifact (for debugging) output_path = os.path.join( settings.taskcluster.results_dir, '{}-mozlint.txt'.format(repr(revision)), ) with open(output_path, 'a') as f: f.write(output) if returncode == 0: logger.debug('No Mozlint errors', path=path) return assert 'error: problem with lint setup' not in output, \ 'Mach lint setup failed' # Load output as json # Only consider last line, as ./mach lint may output # linter setup output on stdout :/ try: lines = list(filter(None, output.split('\n'))) payload = json.loads(lines[-1]) except json.decoder.JSONDecodeError: raise AnalysisException('mozlint', 'Invalid json output', path=path, lines=lines) if full_path not in payload and path not in payload: logger.warn('Missing path in linter output', path=path) return # Mozlint uses both full & relative path to index issues return [ MozLintIssue(revision=revision, **issue) for p in (path, full_path) for issue in payload.get(p, []) ]
def apply(self, repo): ''' Apply patch from Phabricator to Mercurial local repository ''' assert isinstance(repo, hglib.client.hgclient) # Apply the patch on top of repository try: repo.import_( patches=io.BytesIO(self.patch.encode('utf-8')), nocommit=True, ) logger.info('Applied target patch', phid=self.diff_phid) except hglib.error.CommandError as e: raise AnalysisException('mercurial', 'Failed to import target patch')
def run(self, revision): ''' Run modified files with specified checks through infer using threaded workers (communicate through queues) Output a list of InferIssue ''' assert isinstance(revision, Revision) self.revision = revision with AndroidConfig(): # Mach pre-setup with mozconfig logger.info('Mach configure for infer...') run_check(['gecko-env', './mach', 'configure'], cwd=settings.repo_dir) # Run all files in a single command # through mach static-analysis cmd = [ 'gecko-env', './mach', '--log-no-times', 'static-analysis', 'check-java' ] + list(revision.files) logger.info('Running static-analysis', cmd=' '.join(cmd)) # Run command try: infer_output = subprocess.check_output(cmd, cwd=settings.repo_dir) except subprocess.CalledProcessError as e: raise AnalysisException( 'infer', 'Mach static analysis failed: {}'.format(e.output)) report_file = os.path.join(settings.repo_dir, 'infer-out', 'report.json') infer_output = json.load(open(report_file)) # Dump raw infer output as a Taskcluster artifact (for debugging) infer_output_path = os.path.join( settings.taskcluster.results_dir, '{}-infer.txt'.format(repr(revision)), ) with open(infer_output_path, 'w') as f: f.write(json.dumps(infer_output, indent=2)) issues = self.parse_issues(infer_output, revision) # Report stats for these issues stats.report_issues('infer', issues) return issues
def return_issues(self, coverity_results_path, revision): ''' Parse Coverity json into structured issues ''' if not os.path.isfile(coverity_results_path): raise AnalysisException( 'coverity', 'Coverity Analysis did not generate an analysis report.') with open(coverity_results_path) as f: result = json.load(f) if 'issues' not in result: logger.info('Coverity did not find any issues') return [] return [ CoverityIssue(issue, revision) for issue in result['issues'] ]
def __init__(self, issue, revision): assert not settings.repo_dir.endswith('/') self.revision = revision # We look only for main event event_path = next( (event for event in issue['events'] if event['main'] is True), None) if event_path is None: raise AnalysisException( 'coverity', 'Coverity Analysis did not find main event for mergeKey {}'. format(issue['mergeKey'])) checker_properties = issue['checkerProperties'] # Strip the leading slash self.path = issue['strippedMainEventFilePathname'].strip('/') self.line = issue['mainEventLineNumber'] self.bug_type = checker_properties['category'] self.kind = issue['checkerName'] self.message = event_path['eventDescription'] self.body = None self.nb_lines = 1
def apply(self, repo): ''' Apply patch from Phabricator to Mercurial local repository ''' assert isinstance(repo, hglib.client.hgclient) if self.mercurial_revision: # Apply the existing commit when available repo.update( rev=self.mercurial_revision, clean=True, ) else: # Apply the patch on top of repository try: repo.import_( patches=io.BytesIO(self.patch.encode('utf-8')), message='SA Analyzed patch', user='******', ) logger.info('Applied target patch', phid=self.diff_phid) except hglib.error.CommandError: raise AnalysisException('mercurial', 'Failed to import target patch')
def run(self, revision): ''' Run the static analysis workflow: * Pull revision from review * Checkout revision * Run static analysis * Publish results ''' analyzers = [] # Index ASAP Taskcluster task for this revision self.index(revision, state='started') # Add log to find Taskcluster task in papertrail logger.info( 'New static analysis', taskcluster_task=settings.taskcluster.task_id, taskcluster_run=settings.taskcluster.run_id, channel=settings.app_channel, publication=settings.publication.name, revision=str(revision), ) stats.api.event( title='Static analysis on {} for {}'.format(settings.app_channel, revision), text='Task {} #{}'.format(settings.taskcluster.task_id, settings.taskcluster.run_id), ) stats.api.increment('analysis') with stats.api.timer('runtime.mercurial'): try: # Start by cloning the mercurial repository self.hg = self.clone() self.index(revision, state='cloned') # Force cleanup to reset top of MU # otherwise previous pull are there self.hg.update(rev=self.top_revision, clean=True) logger.info('Set repo back to Mozilla unified top', rev=self.hg.identify()) except hglib.error.CommandError as e: raise AnalysisException('mercurial', str(e)) # Load and analyze revision patch revision.load(self.hg) revision.analyze_patch() with stats.api.timer('runtime.mach'): # Only run mach if revision has any C/C++ or Java files if revision.has_clang_files: # Mach pre-setup with mozconfig try: logger.info('Mach configure...') with stats.api.timer('runtime.mach.configure'): run_check(['gecko-env', './mach', 'configure'], cwd=settings.repo_dir) logger.info('Mach compile db...') with stats.api.timer('runtime.mach.build-backend'): run_check(['gecko-env', './mach', 'build-backend', '--backend=CompileDB'], cwd=settings.repo_dir) logger.info('Mach pre-export...') with stats.api.timer('runtime.mach.pre-export'): run_check(['gecko-env', './mach', 'build', 'pre-export'], cwd=settings.repo_dir) logger.info('Mach export...') with stats.api.timer('runtime.mach.export'): run_check(['gecko-env', './mach', 'build', 'export'], cwd=settings.repo_dir) except Exception as e: raise AnalysisException('mach', str(e)) # Download clang build from Taskcluster # Use new clang-tidy paths, https://bugzilla.mozilla.org/show_bug.cgi?id=1495641 logger.info('Setup Taskcluster clang build...') setup_clang(repository='mozilla-inbound', revision='revision.874a07fdb045b725edc2aaa656a8620ff439ec10') # Use clang-tidy & clang-format if CLANG_TIDY in self.analyzers: analyzers.append(ClangTidy) else: logger.info('Skip clang-tidy') if CLANG_FORMAT in self.analyzers: analyzers.append(ClangFormat) else: logger.info('Skip clang-format') if revision.has_infer_files: if INFER in self.analyzers: analyzers.append(Infer) logger.info('Setup Taskcluster infer build...') setup_infer(self.index_service) else: logger.info('Skip infer') if not (revision.has_clang_files or revision.has_infer_files): logger.info('No clang or java files detected, skipping mach, infer and clang-*') # Setup python environment logger.info('Mach lint setup...') cmd = ['gecko-env', './mach', 'lint', '--list'] with stats.api.timer('runtime.mach.lint'): out = run_check(cmd, cwd=settings.repo_dir) if 'error: problem with lint setup' in out.decode('utf-8'): raise AnalysisException('mach', 'Mach lint setup failed') # Always use mozlint if MOZLINT in self.analyzers: analyzers.append(MozLint) else: logger.info('Skip mozlint') if not analyzers: logger.error('No analyzers to use on revision') return self.index(revision, state='analyzing') with stats.api.timer('runtime.issues'): # Detect initial issues if settings.publication == Publication.BEFORE_AFTER: before_patch, _ = self.detect_issues(analyzers, revision) logger.info('Detected {} issue(s) before patch'.format(len(before_patch))) stats.api.increment('analysis.issues.before', len(before_patch)) # Apply patch revision.apply(self.hg) # Detect new issues issues = self.detect_issues(analyzers, revision) logger.info('Detected {} issue(s) after patch'.format(len(issues))) stats.api.increment('analysis.issues.after', len(issues)) # Mark newly found issues if settings.publication == Publication.BEFORE_AFTER: for issue in issues: issue.is_new = issue not in before_patch # Avoid duplicates issues = set(issues) if not issues: logger.info('No issues, stopping there.') self.index(revision, state='done', issues=0) return # Report issues publication stats nb_issues = len(issues) nb_publishable = len([i for i in issues if i.is_publishable()]) self.index(revision, state='analyzed', issues=nb_issues, issues_publishable=nb_publishable) stats.api.increment('analysis.issues.publishable', nb_publishable) # Publish reports about these issues with stats.api.timer('runtime.reports'): for reporter in self.reporters.values(): reporter.publish(issues, revision) self.index(revision, state='done', issues=nb_issues, issues_publishable=nb_publishable)
def run(self, revision): ''' Run coverity ''' assert isinstance(revision, Revision) self.revision = revision # Based on our previous configs we should already have generated compile_commands.json self.compile_commands_path = os.path.join(settings.repo_dir, 'obj-x86_64-pc-linux-gnu', 'compile_commands.json') assert os.path.exists(self.compile_commands_path), \ 'Missing Coverity in {}'.format(self.compile_commands_path) logger.info('Building command files from compile_commands.json') # Retrieve the revision files with build commands associated commands_list = self.get_files_with_commands() assert commands_list is not [], 'Commands List is empty' logger.info('Built commands for {} files'.format(len(commands_list))) cmd = ['gecko-env', self.cov_run_desktop, '--setup'] logger.info('Running Coverity Setup', cmd=cmd) try: run_check(cmd, cwd=self.cov_path) except subprocess.CalledProcessError as e: raise AnalysisException( 'coverity', 'Coverity Setup failed: {}'.format(e.output)) cmd = ['gecko-env', self.cov_configure, '--clang'] logger.info('Running Coverity Configure', cmd=cmd) try: run_check(cmd, cwd=self.cov_path) except subprocess.CalledProcessError as e: raise AnalysisException( 'coverity', 'Coverity Configure failed: {}'.format(e.output)) # For each element in commands_list run `cov-translate` for element in commands_list: cmd = [ 'gecko-env', self.cov_translate, '--dir', self.cov_idir_path, element['command'] ] logger.info('Running Coverity Tranlate', cmd=cmd) try: run_check(cmd, cwd=element['directory']) except subprocess.CalledProcessError as e: raise AnalysisException( 'coverity', 'Coverity Translate failed: {}'.format(e.output)) # Once the capture is performed we need to do the actual Coverity Desktop analysis cmd = [ 'gecko-env', self.cov_run_desktop, '--json-output-v6', 'cov-results.json', '--strip-path', settings.repo_dir ] cmd += [element['file'] for element in commands_list] logger.info('Running Coverity Analysis', cmd=cmd) try: run_check(cmd, cwd=self.cov_state_path) except subprocess.CalledProcessError as e: raise AnalysisException( 'coverity', 'Coverity Analysis failed: {}'.format(e.output)) # Write the results.json to the artifact directory to have it later on for debug coverity_results_path = os.path.join(self.cov_state_path, 'cov-results.json') coverity_results_path_on_tc = os.path.join( settings.taskcluster.results_dir, 'cov-results.json') shutil.copyfile(coverity_results_path, coverity_results_path_on_tc) # Parsing the issues from coverity_results_path logger.info('Parsing Coverity issues') return self.return_issues(coverity_results_path, revision)
def run(self, revision): ''' Run the local static analysis workflow: * Pull revision from review * Checkout revision * Run static analyzers ''' analyzers = [] # Add log to find Taskcluster task in papertrail logger.info( 'New static analysis', taskcluster_task=settings.taskcluster.task_id, taskcluster_run=settings.taskcluster.run_id, channel=settings.app_channel, publication=settings.publication.name, revision=str(revision), ) stats.api.event( title='Static analysis on {} for {}'.format( settings.app_channel, revision), text='Task {} #{}'.format(settings.taskcluster.task_id, settings.taskcluster.run_id), ) stats.api.increment('analysis') with stats.api.timer('runtime.mercurial'): try: # Clone in a controllable process # and kill this new process if it exceeds the maximum allowed runtime clone = multiprocessing.Process(target=self.clone, args=(revision, )) clone.start() clone.join(settings.max_clone_runtime) if clone.is_alive(): logger.error( 'Clone watchdog expired, stopping immediately') # Kill the clone process clone.terminate() # Stop workflow raise AnalysisException('watchdog', 'Clone watchdog expired') # Open a mercurial client in main process self.hg = self.open_repository() # Start by cloning the mercurial repository self.parent.index(revision, state='cloned') # Force cleanup to reset top of MU # otherwise previous pull are there self.hg.update(rev=self.top_revision, clean=True) logger.info('Set repo back to Mozilla unified top', rev=self.hg.identify()) except hglib.error.CommandError as e: raise AnalysisException('mercurial', str(e)) # Load and analyze revision patch revision.load(self.hg) revision.analyze_patch() with stats.api.timer('runtime.mach'): # Only run mach if revision has any C/C++ or Java files if revision.has_clang_files: self.do_build_setup() # Download clang build from Taskcluster # Use new clang-tidy paths, https://bugzilla.mozilla.org/show_bug.cgi?id=1495641 logger.info('Setup Taskcluster clang build...') setup_clang( repository='mozilla-inbound', revision='revision.874a07fdb045b725edc2aaa656a8620ff439ec10' ) # Use clang-tidy & clang-format if CLANG_TIDY in self.analyzers: analyzers.append(ClangTidy) else: logger.info('Skip clang-tidy') if CLANG_FORMAT in self.analyzers: analyzers.append(ClangFormat) else: logger.info('Skip clang-format') # Run Coverity Scan if COVERITY in self.analyzers: logger.info('Setup Taskcluster coverity build...') setup_coverity(self.index_service) analyzers.append(Coverity) else: logger.info('Skip Coverity') if COVERAGE in self.analyzers: analyzers.append(Coverage) else: logger.info('Skip coverage analysis') if revision.has_infer_files: if INFER in self.analyzers: analyzers.append(Infer) logger.info('Setup Taskcluster infer build...') setup_infer(self.index_service) else: logger.info('Skip infer') if not (revision.has_clang_files or revision.has_infer_files): logger.info( 'No clang or java files detected, skipping mach, infer and clang-*' ) # Setup python environment logger.info('Mach lint setup...') cmd = ['gecko-env', './mach', 'lint', '--list'] with stats.api.timer('runtime.mach.lint'): out = run_check(cmd, cwd=settings.repo_dir) if 'error: problem with lint setup' in out.decode('utf-8'): raise AnalysisException('mach', 'Mach lint setup failed') # Always use mozlint if MOZLINT in self.analyzers: analyzers.append(MozLint) else: logger.info('Skip mozlint') if not analyzers: logger.error('No analyzers to use on revision') return self.parent.index(revision, state='analyzing') with stats.api.timer('runtime.issues'): # Detect initial issues if settings.publication == Publication.BEFORE_AFTER: before_patch = self.detect_issues(analyzers, revision, True) logger.info('Detected {} issue(s) before patch'.format( len(before_patch))) stats.api.increment('analysis.issues.before', len(before_patch)) revision.reset() # Apply patch revision.apply(self.hg) if settings.publication == Publication.BEFORE_AFTER and revision.has_clang_files \ and (revision.has_clang_header_files or revision.has_idl_files): self.do_build_setup() # Detect new issues issues = self.detect_issues(analyzers, revision) logger.info('Detected {} issue(s) after patch'.format(len(issues))) stats.api.increment('analysis.issues.after', len(issues)) # Mark newly found issues if settings.publication == Publication.BEFORE_AFTER: for issue in issues: issue.is_new = issue not in before_patch # Avoid duplicates # but still output a list to be compatible with LocalWorkflow return list(set(issues))
def parse_issues(self, clang_output, revision): ''' Parse clang-tidy output into structured issues ''' # Detect end of file warnings count # When an invalid file is used, this line does not appear has_warnings = REGEX_HAS_WARNINGS.search(clang_output) if has_warnings is None: logger.info('Empty clang output, skipping analysis.') return [] nb_warnings = int(has_warnings.group(1)) if nb_warnings == 0: logger.info('Mach static analysis did not find any issue') return [] logger.info('Mach static analysis found some issues', nb=nb_warnings) # Sort headers by positions headers = sorted(REGEX_HEADER.finditer(clang_output), key=lambda h: h.start()) if not headers: raise AnalysisException( 'clang-tidy', 'No clang-tidy header was found even though a clang output was provided' ) def _remove_footer(start_pos, end_pos): ''' Build an issue body from clang-tidy output and stops when an extra paylaod is detected (footer) ''' assert isinstance(start_pos, int) assert isinstance(end_pos, int) body = clang_output[start_pos:end_pos] footer = REGEX_FOOTER.search(body) if footer is None: return body return body[:footer.start() - 1] issues = [] for i, header in enumerate(headers): issue = ClangTidyIssue(header.groups(), revision) # Get next header if i + 1 < len(headers): # Parse body until next header next_header = headers[i + 1] issue.body = _remove_footer(header.end(), next_header.start() - 1) else: # Limit last element to 3 lines to avoid parsing extra metadatas issue.body = _remove_footer(header.end(), header.end() + 3) if issue.is_problem(): # Save problem to append notes # Skip diagnostic errors, but warn through Sentry if issue.check == 'clang-diagnostic-error': logger.error( 'Encountered a clang-diagnostic-error: {}'.format( issue)) else: issues.append(issue) mode = issue.is_third_party() and '3rd party' or 'in-tree' logger.info('Found {} code issue {}'.format(mode, issue)) elif issues: # Link notes to last problem issues[-1].notes.append(issue) return issues
def load(self, repo): ''' Load full stack of patches from Phabricator: * setup repo to base revision from Mozilla Central * Apply previous needed patches from Phabricator ''' assert isinstance(repo, hglib.client.hgclient) # Diff PHIDs from our patch to its base patches = OrderedDict() patches[self.diff_phid] = self.diff_id parents = self.api.load_parents(self.phid) if parents: # Load all parent diffs for parent in parents: logger.info('Loading parent diff', phid=parent) # Sort parent diffs by their id to load the most recent patch parent_diffs = sorted( self.api.search_diffs(revision_phid=parent), key=lambda x: x['id'], ) last_diff = parent_diffs[-1] patches[last_diff['phid']] = last_diff['id'] # Use base revision of last parent hg_base = last_diff['baseRevision'] else: # Use base revision from top diff hg_base = self.diff['baseRevision'] # When base revision is missing, update to top of Central if hg_base is None or not revision_available(repo, hg_base): logger.warning('Missing base revision from Phabricator') hg_base = 'central' # Load all patches from their numerical ID for diff_phid, diff_id in patches.items(): patches[diff_phid] = self.api.load_raw_diff(diff_id) # Expose current patch to workflow self.patch = patches[self.diff_phid] # Update the repo to base revision try: logger.info('Updating repo to revision', rev=hg_base) repo.update( rev=hg_base, clean=True, ) except hglib.error.CommandError: raise AnalysisException('mercurial', 'Failed to update to revision {}'.format(hg_base)) # Apply all patches from base to top # except our current (top) patch for diff_phid, patch in reversed(list(patches.items())[1:]): logger.info('Applying parent diff', phid=diff_phid) try: repo.import_( patches=io.BytesIO(patch.encode('utf-8')), message='SA Imported patch {}'.format(diff_phid), user='******', ) except hglib.error.CommandError: raise AnalysisException('mercurial', 'Failed to import parent patch {}'.format(diff_phid))