def is_ctu_capable(extdef_map_cmd):
""" Detects if the current (or given) clang and external definition mapping
executables are CTU compatible. """
try:
run_command([extdef_map_cmd, '-version'])
except (OSError, subprocess.CalledProcessError):
return False
return True
def report_failure(opts):
# type: (Dict[str, Any]) -> None
""" Create report when analyzer failed.
The major report is the preprocessor output. The output filename generated
randomly. The compiler output also captured into '.stderr.txt' file.
And some more execution context also saved into '.info.txt' file. """
def extension():
# type: () -> str
""" Generate preprocessor file extension. """
mapping = {'objective-c++': '.mii', 'objective-c': '.mi', 'c++': '.ii'}
return mapping.get(opts['language'], '.i')
def destination():
# type: () -> str
""" Creates failures directory if not exits yet. """
failures_dir = os.path.join(opts['output_dir'], 'failures')
if not os.path.isdir(failures_dir):
os.makedirs(failures_dir)
return failures_dir
# Classify error type: when Clang terminated by a signal it's a 'Crash'.
# (python subprocess Popen.returncode is negative when child terminated
# by signal.) Everything else is 'Other Error'.
error = 'crash' if opts['exit_code'] < 0 else 'other_error'
# Create preprocessor output file name. (This is blindly following the
# Perl implementation.)
(fd, name) = tempfile.mkstemp(suffix=extension(),
prefix='clang_' + error + '_',
dir=destination())
os.close(fd)
# Execute Clang again, but run the syntax check only.
try:
cwd = opts['directory']
cmd = get_arguments([opts['clang'], '-fsyntax-only', '-E'] +
opts['flags'] + [opts['source'], '-o', name], cwd)
run_command(cmd, cwd=cwd)
# write general information about the crash
with open(name + '.info.txt', 'w') as handle:
handle.write(opts['source'] + os.linesep)
handle.write(error.title().replace('_', ' ') + os.linesep)
handle.write(' '.join(cmd) + os.linesep)
handle.write(' '.join(platform.uname()) + os.linesep)
handle.write(get_version(opts['clang']))
handle.close()
# write the captured output too
with open(name + '.stderr.txt', 'w') as handle:
for line in opts['error_output']:
handle.write(line)
handle.close()
except (OSError, subprocess.CalledProcessError):
logging.warning('failed to report failure', exc_info=True)
def map_extdefs(triple_arch):
""" Generate external definition map file for the current source. """
args = opts['direct_args'] + opts['flags']
extdefmap_command = [opts['ctu'].extdef_map_cmd]
extdefmap_command.append(opts['file'])
extdefmap_command.append('--')
extdefmap_command.extend(args)
logging.debug("Generating external definition map using '%s'",
extdefmap_command)
extdef_src_list = run_command(extdefmap_command, cwd=opts['directory'])
extdef_ast_list = extdef_map_list_src_to_ast(extdef_src_list)
extern_defs_map_folder = os.path.join(opts['ctu'].dir, triple_arch,
CTU_TEMP_DEFMAP_FOLDER)
if not os.path.isdir(extern_defs_map_folder):
try:
os.makedirs(extern_defs_map_folder)
except OSError:
# In case an other process already created it.
pass
if extdef_ast_list:
with tempfile.NamedTemporaryFile(mode='w',
dir=extern_defs_map_folder,
delete=False) as out_file:
out_file.write("\n".join(extdef_ast_list) + "\n")
def run_analyzer(opts, continuation=report_failure):
""" It assembles the analysis command line and executes it. Capture the
output of the analysis and returns with it. If failure reports are
requested, it calls the continuation to generate it. """
def target():
""" Creates output file name for reports. """
if opts['output_format'] in {
'plist',
'plist-html',
'plist-multi-file'}:
(handle, name) = tempfile.mkstemp(prefix='report-',
suffix='.plist',
dir=opts['output_dir'])
os.close(handle)
return name
return opts['output_dir']
try:
cwd = opts['directory']
cmd = get_arguments([opts['clang'], '--analyze'] +
opts['direct_args'] + opts['flags'] +
[opts['file'], '-o', target()],
cwd)
output = run_command(cmd, cwd=cwd)
return {'error_output': output, 'exit_code': 0}
except subprocess.CalledProcessError as ex:
result = {'error_output': ex.output, 'exit_code': ex.returncode}
if opts.get('output_failures', False):
opts.update(result)
continuation(opts)
return result
def run_analyzer(opts, continuation=report_failure):
# type: (...) -> Dict[str, Any]
""" It assembles the analysis command line and executes it. Capture the
output of the analysis and returns with it. If failure reports are
requested, it calls the continuation to generate it. """
def target():
# type: () -> str
""" Creates output file name for reports. """
if opts['output_format'].startswith('plist'):
(handle, name) = tempfile.mkstemp(prefix='report-',
suffix='.plist',
dir=opts['output_dir'])
os.close(handle)
return name
return opts['output_dir']
try:
cwd = opts['directory']
cmd = get_arguments([opts['clang'], '--analyze'] +
opts['direct_args'] + opts['flags'] +
[opts['source'], '-o', target()],
cwd)
output = run_command(cmd, cwd=cwd)
return {'error_output': output, 'exit_code': 0}
except OSError:
message = 'failed to execute "{0}"'.format(opts['clang'])
return {'error_output': [message], 'exit_code': 127}
except subprocess.CalledProcessError as ex:
logging.warning('analysis failed', exc_info=True)
result = {'error_output': ex.output, 'exit_code': ex.returncode}
if opts.get('output_failures', False):
opts.update(result)
continuation(opts)
return result
def get_checkers(clang, plugins):
# type: (str, List[str]) -> Dict[str, Tuple[str, bool]]
""" Get all the available checkers from default and from the plugins.
:param clang: the compiler we are using
:param plugins: list of plugins which was requested by the user
:return: a dictionary of all available checkers and its status
{<checker name>: (<checker description>, <is active by default>)} """
load = [elem for plugin in plugins for elem in ['-load', plugin]]
cmd = [clang, '-cc1'] + load + ['-analyzer-checker-help']
lines = run_command(cmd)
is_active_checker = is_active(get_active_checkers(clang, plugins))
checkers = {
name: (description, is_active_checker(name))
for name, description in parse_checkers(lines)
}
if not checkers:
raise Exception('Could not query Clang for available checkers.')
return checkers
def get_version(clang):
""" Returns the compiler version as string.
:param clang: the compiler we are using
:return: the version string printed to stderr """
output = run_command([clang, '-v'])
# the relevant version info is in the first line
return output[0]
def find_compiler_config(compiler, tmp_file_name, tmp_dir):
filePath = os.path.join(tmp_dir, tmp_file_name)
with open(filePath, 'w'):
pass
config_output = run_command([compiler, '-E', '-dM', '-v', tmp_file_name],
cwd=tmp_dir)
logging.debug('config_output is %s' % config_output)
compiler_conf = recognize_config(config_output)
logging.debug('Recognized config: %s' % compiler_conf)
return compiler_conf
def get_version(clang):
# type: (str) -> str
""" Returns the compiler version as string.
:param clang: the compiler we are using
:return: the version string printed to stderr """
output = run_command([clang, '-v'])
# the relevant version info is in the first line
return output[0]
def generate_ast(triple_arch):
""" Generates ASTs for the current compilation command. """
args = opts['direct_args'] + opts['flags']
ast_joined_path = os.path.join(opts['ctu'].dir, triple_arch, 'ast',
os.path.realpath(opts['file'])[1:] +
'.ast')
ast_path = os.path.abspath(ast_joined_path)
ast_dir = os.path.dirname(ast_path)
if not os.path.isdir(ast_dir):
try:
os.makedirs(ast_dir)
except OSError:
pass
ast_command = [opts['clang'], '-emit-ast']
ast_command.extend(args)
ast_command.append('-w')
ast_command.append(opts['file'])
ast_command.append('-o')
ast_command.append(ast_path)
logging.debug("Generating AST using '%s'", ast_command)
run_command(ast_command, cwd=opts['directory'])
def generate_ast(triple_arch):
""" Generates ASTs for the current compilation command. """
args = opts['direct_args'] + opts['flags']
ast_joined_path = os.path.join(opts['ctu'].dir, triple_arch, 'ast',
os.path.realpath(opts['file'])[1:] +
'.ast')
ast_path = os.path.abspath(ast_joined_path)
ast_dir = os.path.dirname(ast_path)
if not os.path.isdir(ast_dir):
try:
os.makedirs(ast_dir)
except OSError:
# In case an other process already created it.
pass
ast_command = [opts['clang'], '-emit-ast']
ast_command.extend(args)
ast_command.append('-w')
ast_command.append(opts['file'])
ast_command.append('-o')
ast_command.append(ast_path)
logging.debug("Generating AST using '%s'", ast_command)
run_command(ast_command, cwd=opts['directory'])
def get_mpi_call(wrapper):
# type: (str) -> List[str]
""" Provide information on how the underlying compiler would have been
invoked without the MPI compiler wrapper. """
for query_flags in [['-show'], ['--showme']]:
try:
output = run_command([wrapper] + query_flags)
if output:
return shell_split(output[0])
except (OSError, subprocess.CalledProcessError):
pass
# Fail loud
raise RuntimeError("Could not determinate MPI flags.")
def get_arguments(command, cwd):
""" Capture Clang invocation.
:param command: the compilation command
:param cwd: the current working directory
:return: the detailed front-end invocation command """
cmd = command[:]
cmd.insert(1, '-###')
output = run_command(cmd, cwd=cwd)
# The relevant information is in the last line of the output.
# Don't check if finding last line fails, would throw exception anyway.
last_line = output[-1]
if re.search(r'clang(.*): error:', last_line):
raise Exception(last_line)
return decode(last_line)
def get_arguments(command, cwd):
""" Capture Clang invocation.
:param command: the compilation command
:param cwd: the current working directory
:return: the detailed front-end invocation command """
cmd = command[:]
cmd.insert(1, '-###')
output = run_command(cmd, cwd=cwd)
# The relevant information is in the last line of the output.
# Don't check if finding last line fails, would throw exception anyway.
last_line = output[-1]
if re.search(r'clang(.*): error:', last_line):
raise Exception(last_line)
return decode(last_line)
def run_analyzer(opts, continuation=report_failure):
""" It assembles the analysis command line and executes it. Capture the
output of the analysis and returns with it. If failure reports are
requested, it calls the continuation to generate it. """
def target():
""" Creates output file name for reports. """
if opts['output_format'] in {
'plist',
'plist-html',
'plist-multi-file'}:
(handle, name) = tempfile.mkstemp(prefix='report-',
suffix='.plist',
dir=opts['output_dir'])
os.close(handle)
return name
elif opts['output_format'] in {
'sarif',
'sarif-html'}:
(handle, name) = tempfile.mkstemp(prefix='result-',
suffix='.sarif',
dir=opts['output_dir'])
os.close(handle)
return name
return opts['output_dir']
try:
cwd = opts['directory']
cmd = get_arguments([opts['clang'], '--analyze'] +
opts['direct_args'] + opts['flags'] +
[opts['file'], '-o', target()],
cwd)
output = run_command(cmd, cwd=cwd)
return {'error_output': output, 'exit_code': 0}
except subprocess.CalledProcessError as ex:
result = {'error_output': ex.output, 'exit_code': ex.returncode}
if opts.get('output_failures', False):
opts.update(result)
continuation(opts)
return result
except ClangErrorException as ex:
result = {'error_output': ex.error, 'exit_code': 0}
if opts.get('output_failures', False):
opts.update(result)
continuation(opts)
return result
def is_preload_disabled(platform):
""" Library-based interposition will fail silently if SIP is enabled,
so this should be detected. You can detect whether SIP is enabled on
Darwin by checking whether (1) there is a binary called 'csrutil' in
the path and, if so, (2) whether the output of executing 'csrutil status'
contains 'System Integrity Protection status: enabled'.
:param platform: name of the platform (returned by sys.platform),
:return: True if library preload will fail by the dynamic linker. """
if platform in WRAPPER_ONLY_PLATFORMS:
return True
elif platform == 'darwin':
command = ['csrutil', 'status']
pattern = re.compile(r'System Integrity Protection status:\s+enabled')
try:
return any(pattern.match(line) for line in run_command(command))
except:
return False
else:
return False
def map_functions(triple_arch):
""" Generate function map file for the current source. """
args = opts['direct_args'] + opts['flags']
funcmap_command = [opts['ctu'].func_map_cmd]
funcmap_command.append(opts['file'])
funcmap_command.append('--')
funcmap_command.extend(args)
logging.debug("Generating function map using '%s'", funcmap_command)
func_src_list = run_command(funcmap_command, cwd=opts['directory'])
func_ast_list = func_map_list_src_to_ast(func_src_list, triple_arch)
extern_fns_map_folder = os.path.join(opts['ctu'].dir, triple_arch,
CTU_TEMP_FNMAP_FOLDER)
if not os.path.isdir(extern_fns_map_folder):
try:
os.makedirs(extern_fns_map_folder)
except OSError:
pass
if func_ast_list:
with tempfile.NamedTemporaryFile(mode='w',
dir=extern_fns_map_folder,
delete=False) as out_file:
out_file.write("\n".join(func_ast_list) + "\n")
def get_checkers(clang, plugins):
""" Get all the available checkers from default and from the plugins.
:param clang: the compiler we are using
:param plugins: list of plugins which was requested by the user
:return: a dictionary of all available checkers and its status
{<checker name>: (<checker description>, <is active by default>)} """
load = [elem for plugin in plugins for elem in ['-load', plugin]]
cmd = [clang, '-cc1'] + load + ['-analyzer-checker-help']
lines = run_command(cmd)
is_active_checker = is_active(get_active_checkers(clang, plugins))
checkers = {
name: (description, is_active_checker(name))
for name, description in parse_checkers(lines)
}
if not checkers:
raise Exception('Could not query Clang for available checkers.')
return checkers