def is_ctu_capable(extdef_map_cmd): """ Detects if the current (or given) clang and external definition mapping executables are CTU compatible. """ try: run_command([extdef_map_cmd, '-version']) except (OSError, subprocess.CalledProcessError): return False return True
def report_failure(opts): # type: (Dict[str, Any]) -> None """ Create report when analyzer failed. The major report is the preprocessor output. The output filename generated randomly. The compiler output also captured into '.stderr.txt' file. And some more execution context also saved into '.info.txt' file. """ def extension(): # type: () -> str """ Generate preprocessor file extension. """ mapping = {'objective-c++': '.mii', 'objective-c': '.mi', 'c++': '.ii'} return mapping.get(opts['language'], '.i') def destination(): # type: () -> str """ Creates failures directory if not exits yet. """ failures_dir = os.path.join(opts['output_dir'], 'failures') if not os.path.isdir(failures_dir): os.makedirs(failures_dir) return failures_dir # Classify error type: when Clang terminated by a signal it's a 'Crash'. # (python subprocess Popen.returncode is negative when child terminated # by signal.) Everything else is 'Other Error'. error = 'crash' if opts['exit_code'] < 0 else 'other_error' # Create preprocessor output file name. (This is blindly following the # Perl implementation.) (fd, name) = tempfile.mkstemp(suffix=extension(), prefix='clang_' + error + '_', dir=destination()) os.close(fd) # Execute Clang again, but run the syntax check only. try: cwd = opts['directory'] cmd = get_arguments([opts['clang'], '-fsyntax-only', '-E'] + opts['flags'] + [opts['source'], '-o', name], cwd) run_command(cmd, cwd=cwd) # write general information about the crash with open(name + '.info.txt', 'w') as handle: handle.write(opts['source'] + os.linesep) handle.write(error.title().replace('_', ' ') + os.linesep) handle.write(' '.join(cmd) + os.linesep) handle.write(' '.join(platform.uname()) + os.linesep) handle.write(get_version(opts['clang'])) handle.close() # write the captured output too with open(name + '.stderr.txt', 'w') as handle: for line in opts['error_output']: handle.write(line) handle.close() except (OSError, subprocess.CalledProcessError): logging.warning('failed to report failure', exc_info=True)
def map_extdefs(triple_arch): """ Generate external definition map file for the current source. """ args = opts['direct_args'] + opts['flags'] extdefmap_command = [opts['ctu'].extdef_map_cmd] extdefmap_command.append(opts['file']) extdefmap_command.append('--') extdefmap_command.extend(args) logging.debug("Generating external definition map using '%s'", extdefmap_command) extdef_src_list = run_command(extdefmap_command, cwd=opts['directory']) extdef_ast_list = extdef_map_list_src_to_ast(extdef_src_list) extern_defs_map_folder = os.path.join(opts['ctu'].dir, triple_arch, CTU_TEMP_DEFMAP_FOLDER) if not os.path.isdir(extern_defs_map_folder): try: os.makedirs(extern_defs_map_folder) except OSError: # In case an other process already created it. pass if extdef_ast_list: with tempfile.NamedTemporaryFile(mode='w', dir=extern_defs_map_folder, delete=False) as out_file: out_file.write("\n".join(extdef_ast_list) + "\n")
def run_analyzer(opts, continuation=report_failure): """ It assembles the analysis command line and executes it. Capture the output of the analysis and returns with it. If failure reports are requested, it calls the continuation to generate it. """ def target(): """ Creates output file name for reports. """ if opts['output_format'] in { 'plist', 'plist-html', 'plist-multi-file'}: (handle, name) = tempfile.mkstemp(prefix='report-', suffix='.plist', dir=opts['output_dir']) os.close(handle) return name return opts['output_dir'] try: cwd = opts['directory'] cmd = get_arguments([opts['clang'], '--analyze'] + opts['direct_args'] + opts['flags'] + [opts['file'], '-o', target()], cwd) output = run_command(cmd, cwd=cwd) return {'error_output': output, 'exit_code': 0} except subprocess.CalledProcessError as ex: result = {'error_output': ex.output, 'exit_code': ex.returncode} if opts.get('output_failures', False): opts.update(result) continuation(opts) return result
def run_analyzer(opts, continuation=report_failure): # type: (...) -> Dict[str, Any] """ It assembles the analysis command line and executes it. Capture the output of the analysis and returns with it. If failure reports are requested, it calls the continuation to generate it. """ def target(): # type: () -> str """ Creates output file name for reports. """ if opts['output_format'].startswith('plist'): (handle, name) = tempfile.mkstemp(prefix='report-', suffix='.plist', dir=opts['output_dir']) os.close(handle) return name return opts['output_dir'] try: cwd = opts['directory'] cmd = get_arguments([opts['clang'], '--analyze'] + opts['direct_args'] + opts['flags'] + [opts['source'], '-o', target()], cwd) output = run_command(cmd, cwd=cwd) return {'error_output': output, 'exit_code': 0} except OSError: message = 'failed to execute "{0}"'.format(opts['clang']) return {'error_output': [message], 'exit_code': 127} except subprocess.CalledProcessError as ex: logging.warning('analysis failed', exc_info=True) result = {'error_output': ex.output, 'exit_code': ex.returncode} if opts.get('output_failures', False): opts.update(result) continuation(opts) return result
def get_checkers(clang, plugins): # type: (str, List[str]) -> Dict[str, Tuple[str, bool]] """ Get all the available checkers from default and from the plugins. :param clang: the compiler we are using :param plugins: list of plugins which was requested by the user :return: a dictionary of all available checkers and its status {<checker name>: (<checker description>, <is active by default>)} """ load = [elem for plugin in plugins for elem in ['-load', plugin]] cmd = [clang, '-cc1'] + load + ['-analyzer-checker-help'] lines = run_command(cmd) is_active_checker = is_active(get_active_checkers(clang, plugins)) checkers = { name: (description, is_active_checker(name)) for name, description in parse_checkers(lines) } if not checkers: raise Exception('Could not query Clang for available checkers.') return checkers
def get_version(clang): """ Returns the compiler version as string. :param clang: the compiler we are using :return: the version string printed to stderr """ output = run_command([clang, '-v']) # the relevant version info is in the first line return output[0]
def find_compiler_config(compiler, tmp_file_name, tmp_dir): filePath = os.path.join(tmp_dir, tmp_file_name) with open(filePath, 'w'): pass config_output = run_command([compiler, '-E', '-dM', '-v', tmp_file_name], cwd=tmp_dir) logging.debug('config_output is %s' % config_output) compiler_conf = recognize_config(config_output) logging.debug('Recognized config: %s' % compiler_conf) return compiler_conf
def get_version(clang): # type: (str) -> str """ Returns the compiler version as string. :param clang: the compiler we are using :return: the version string printed to stderr """ output = run_command([clang, '-v']) # the relevant version info is in the first line return output[0]
def generate_ast(triple_arch): """ Generates ASTs for the current compilation command. """ args = opts['direct_args'] + opts['flags'] ast_joined_path = os.path.join(opts['ctu'].dir, triple_arch, 'ast', os.path.realpath(opts['file'])[1:] + '.ast') ast_path = os.path.abspath(ast_joined_path) ast_dir = os.path.dirname(ast_path) if not os.path.isdir(ast_dir): try: os.makedirs(ast_dir) except OSError: pass ast_command = [opts['clang'], '-emit-ast'] ast_command.extend(args) ast_command.append('-w') ast_command.append(opts['file']) ast_command.append('-o') ast_command.append(ast_path) logging.debug("Generating AST using '%s'", ast_command) run_command(ast_command, cwd=opts['directory'])
def generate_ast(triple_arch): """ Generates ASTs for the current compilation command. """ args = opts['direct_args'] + opts['flags'] ast_joined_path = os.path.join(opts['ctu'].dir, triple_arch, 'ast', os.path.realpath(opts['file'])[1:] + '.ast') ast_path = os.path.abspath(ast_joined_path) ast_dir = os.path.dirname(ast_path) if not os.path.isdir(ast_dir): try: os.makedirs(ast_dir) except OSError: # In case an other process already created it. pass ast_command = [opts['clang'], '-emit-ast'] ast_command.extend(args) ast_command.append('-w') ast_command.append(opts['file']) ast_command.append('-o') ast_command.append(ast_path) logging.debug("Generating AST using '%s'", ast_command) run_command(ast_command, cwd=opts['directory'])
def get_mpi_call(wrapper): # type: (str) -> List[str] """ Provide information on how the underlying compiler would have been invoked without the MPI compiler wrapper. """ for query_flags in [['-show'], ['--showme']]: try: output = run_command([wrapper] + query_flags) if output: return shell_split(output[0]) except (OSError, subprocess.CalledProcessError): pass # Fail loud raise RuntimeError("Could not determinate MPI flags.")
def get_arguments(command, cwd): """ Capture Clang invocation. :param command: the compilation command :param cwd: the current working directory :return: the detailed front-end invocation command """ cmd = command[:] cmd.insert(1, '-###') output = run_command(cmd, cwd=cwd) # The relevant information is in the last line of the output. # Don't check if finding last line fails, would throw exception anyway. last_line = output[-1] if re.search(r'clang(.*): error:', last_line): raise Exception(last_line) return decode(last_line)
def run_analyzer(opts, continuation=report_failure): """ It assembles the analysis command line and executes it. Capture the output of the analysis and returns with it. If failure reports are requested, it calls the continuation to generate it. """ def target(): """ Creates output file name for reports. """ if opts['output_format'] in { 'plist', 'plist-html', 'plist-multi-file'}: (handle, name) = tempfile.mkstemp(prefix='report-', suffix='.plist', dir=opts['output_dir']) os.close(handle) return name elif opts['output_format'] in { 'sarif', 'sarif-html'}: (handle, name) = tempfile.mkstemp(prefix='result-', suffix='.sarif', dir=opts['output_dir']) os.close(handle) return name return opts['output_dir'] try: cwd = opts['directory'] cmd = get_arguments([opts['clang'], '--analyze'] + opts['direct_args'] + opts['flags'] + [opts['file'], '-o', target()], cwd) output = run_command(cmd, cwd=cwd) return {'error_output': output, 'exit_code': 0} except subprocess.CalledProcessError as ex: result = {'error_output': ex.output, 'exit_code': ex.returncode} if opts.get('output_failures', False): opts.update(result) continuation(opts) return result except ClangErrorException as ex: result = {'error_output': ex.error, 'exit_code': 0} if opts.get('output_failures', False): opts.update(result) continuation(opts) return result
def is_preload_disabled(platform): """ Library-based interposition will fail silently if SIP is enabled, so this should be detected. You can detect whether SIP is enabled on Darwin by checking whether (1) there is a binary called 'csrutil' in the path and, if so, (2) whether the output of executing 'csrutil status' contains 'System Integrity Protection status: enabled'. :param platform: name of the platform (returned by sys.platform), :return: True if library preload will fail by the dynamic linker. """ if platform in WRAPPER_ONLY_PLATFORMS: return True elif platform == 'darwin': command = ['csrutil', 'status'] pattern = re.compile(r'System Integrity Protection status:\s+enabled') try: return any(pattern.match(line) for line in run_command(command)) except: return False else: return False
def map_functions(triple_arch): """ Generate function map file for the current source. """ args = opts['direct_args'] + opts['flags'] funcmap_command = [opts['ctu'].func_map_cmd] funcmap_command.append(opts['file']) funcmap_command.append('--') funcmap_command.extend(args) logging.debug("Generating function map using '%s'", funcmap_command) func_src_list = run_command(funcmap_command, cwd=opts['directory']) func_ast_list = func_map_list_src_to_ast(func_src_list, triple_arch) extern_fns_map_folder = os.path.join(opts['ctu'].dir, triple_arch, CTU_TEMP_FNMAP_FOLDER) if not os.path.isdir(extern_fns_map_folder): try: os.makedirs(extern_fns_map_folder) except OSError: pass if func_ast_list: with tempfile.NamedTemporaryFile(mode='w', dir=extern_fns_map_folder, delete=False) as out_file: out_file.write("\n".join(func_ast_list) + "\n")
def get_checkers(clang, plugins): """ Get all the available checkers from default and from the plugins. :param clang: the compiler we are using :param plugins: list of plugins which was requested by the user :return: a dictionary of all available checkers and its status {<checker name>: (<checker description>, <is active by default>)} """ load = [elem for plugin in plugins for elem in ['-load', plugin]] cmd = [clang, '-cc1'] + load + ['-analyzer-checker-help'] lines = run_command(cmd) is_active_checker = is_active(get_active_checkers(clang, plugins)) checkers = { name: (description, is_active_checker(name)) for name, description in parse_checkers(lines) } if not checkers: raise Exception('Could not query Clang for available checkers.') return checkers