def test_tc_detect(self):
"""
Parse gcc-toolchain argument from clang compile command.
"""
compilation_action = "clang -x c -O3 " \
"--gcc-toolchain=/home/user/my_gcc/toolchain -c main.cpp "
toolchain = \
gcc_toolchain.toolchain_in_args(shlex.split(compilation_action))
print(toolchain)
self.assertEqual(toolchain, "/home/user/my_gcc/toolchain")
def test_get_tc_gcc_compiler(self):
"""
Get gcc compiler from the toolchain path.
"""
compilation_action = \
"clang --gcc-toolchain=/home/user/my_gcc/toolchain"
toolchain = \
gcc_toolchain.toolchain_in_args(shlex.split(compilation_action))
print(toolchain)
self.assertEqual(toolchain, "/home/user/my_gcc/toolchain")
tc_compiler_c = gcc_toolchain.get_toolchain_compiler(toolchain, "c")
print(tc_compiler_c)
self.assertEqual(tc_compiler_c, "/home/user/my_gcc/toolchain/bin/gcc")
def get_dependent_headers(buildaction, archive):
LOG.debug("Generating dependent headers via compiler...")
command = shlex.split(buildaction.original_command)
try:
dependencies = set(create_dependencies(command, buildaction.directory))
except Exception as ex:
LOG.debug("Couldn't create dependencies:")
LOG.debug(str(ex))
# TODO append with buildaction
archive.writestr("no-sources", str(ex))
dependencies = set()
toolchain = gcc_toolchain.toolchain_in_args(
shlex.split(buildaction.original_command))
if toolchain:
LOG.debug("Generating gcc-toolchain headers via toolchain compiler...")
try:
# Change the original compiler to the compiler from the
# toolchain.
toolchain_compiler = gcc_toolchain.get_toolchain_compiler(
toolchain, buildaction.lang)
if not toolchain_compiler:
raise Exception("Could not get the compiler "
"from the gcc-toolchain.")
command[0] = toolchain_compiler
dependencies = dependencies.union(
set(create_dependencies(command, buildaction.directory)))
except Exception as ex:
LOG.debug("Couldn't create dependencies:")
LOG.debug(str(ex))
# TODO append with buildaction
archive.writestr("no-sources", str(ex))
dependencies = set()
return dependencies
def collect_debug_data(zip_file, other_files, buildaction, out, err,
original_command, analyzer_cmd, analyzer_returncode,
action_directory, action_target, actions_map):
"""
Collect analysis and build system information which can be used
to reproduce the failed analysis.
"""
LOG.debug("Collecting debug data")
with zipfile.ZipFile(zip_file, 'w') as archive:
LOG.debug("[ZIP] Writing analyzer STDOUT to /stdout")
archive.writestr("stdout", out)
LOG.debug("[ZIP] Writing analyzer STDERR to /stderr")
archive.writestr("stderr", err)
dependencies = get_dependent_headers(buildaction,
archive)
mentioned_files_dependent_files = set()
for of in other_files:
mentioned_file = os.path.abspath(
os.path.join(action_directory, of))
# Use the target of the original build action.
key = mentioned_file, action_target
mentioned_file_action = actions_map.get(key)
if mentioned_file_action is not None:
mentioned_file_deps = get_dependent_headers(
mentioned_file_action, archive)
mentioned_files_dependent_files.\
update(mentioned_file_deps)
else:
LOG.debug("Could not find {0} in build actions."
.format(key))
dependencies.update(other_files)
dependencies.update(mentioned_files_dependent_files)
# `dependencies` might contain absolute and relative paths
# for the same file, so make everything absolute by
# joining with the the absolute action.directory.
# If `dependent_source` is absolute it remains absolute
# after the join, as documented on docs.python.org .
dependencies_copy = set()
for dependent_source in dependencies:
dependent_source = os.path.join(action_directory,
dependent_source)
dependencies_copy.add(dependent_source)
dependencies = dependencies_copy
LOG.debug("Writing dependent files to archive.")
for dependent_source in dependencies:
LOG.debug("[ZIP] Writing '" + dependent_source + "' "
"to the archive.")
archive_subpath = dependent_source.lstrip('/')
archive_path = os.path.join('sources-root',
archive_subpath)
try:
_ = archive.getinfo(archive_path)
LOG.debug("[ZIP] '{0}' is already in the ZIP "
"file, won't add it again!"
.format(archive_path))
continue
except KeyError:
# If the file is already contained in the ZIP,
# a valid object is returned, otherwise a KeyError
# is raised.
pass
try:
archive.write(dependent_source,
archive_path,
zipfile.ZIP_DEFLATED)
except Exception as ex:
# In certain cases, the output could contain
# invalid tokens (such as error messages that were
# printed even though the dependency generation
# returned 0).
LOG.debug("[ZIP] Couldn't write, because " +
str(ex))
archive.writestr(
os.path.join('failed-sources-root',
archive_subpath),
"Couldn't write this file, because:\n" +
str(ex))
LOG.debug("[ZIP] Writing extra information...")
archive.writestr("build-action", original_command)
archive.writestr("analyzer-command", ' '.join(analyzer_cmd))
archive.writestr("return-code", str(analyzer_returncode))
toolchain = gcc_toolchain.toolchain_in_args(
shlex.split(buildaction.original_command))
if toolchain:
archive.writestr("gcc-toolchain-path", toolchain)
def parse_compile_commands_json(logfile, parseLogOptions):
"""
logfile: is a compile command json
"""
output_path = parseLogOptions.output_path
if output_path is not None:
remove_file_if_exists(
os.path.join(output_path, compiler_includes_dump_file))
remove_file_if_exists(
os.path.join(output_path, compiler_target_dump_file))
actions = []
filtered_build_actions = {}
data = json.load(logfile)
compiler_includes = {}
compiler_target = {}
counter = 0
for entry in data:
sourcefile = entry['file']
if not os.path.isabs(sourcefile):
# Newest versions of intercept-build can create the 'file' in the
# JSON Compilation Database as a relative path.
sourcefile = os.path.join(os.path.abspath(entry['directory']),
sourcefile)
lang = option_parser.get_language(sourcefile[sourcefile.rfind('.'):])
if not lang:
continue
action = build_action.BuildAction(counter)
if 'command' in entry:
command = entry['command']
# Old versions of intercept-build (confirmed to those shipping
# with upstream clang-5.0) do escapes in another way:
# -DVARIABLE="a b" becomes -DVARIABLE=\"a b\" in the output.
# This would be messed up later on by options_parser, so need a
# fix here. (Should be removed once we are sure noone uses this
# intercept-build anymore!)
if r'\"' in command:
command = command.replace(r'\"', '"')
elif 'arguments' in entry:
# Newest versions of intercept-build create an argument vector
# instead of a command string.
command = ' '.join(entry['arguments'])
else:
raise KeyError("No valid 'command' or 'arguments' entry found!")
results = option_parser.parse_options(command)
action.original_command = command
# If the original include directory could not be found
# in the filesystem, it is possible that it was provided
# relative to the working directory in the compile json.
compile_opts = results.compile_opts
for i, opt in enumerate(compile_opts):
if opt.startswith('-I'):
inc_dir = opt[2:].strip()
if not os.path.isdir(inc_dir):
compile_opts[i] = '-I' + \
os.path.join(entry['directory'], inc_dir)
action.analyzer_options = compile_opts
action.lang = results.lang
action.target = results.arch
action.output = results.output
add_compiler_defaults = True
# With gcc-toolchain a non default compiler toolchain can be set.
# Clang will search for include paths and libraries based on the
# gcc-toolchain parameter.
# Detecting extra include paths from the host compiler could
# conflict with this.
# For example if the compiler in the compile command is clang
# and gcc-toolchain is set we will get the include paths
# for clang and not for the compiler set in gcc-toolchain.
# This can cause missing headers during the analysis.
toolchain = gcc_toolchain.toolchain_in_args(action.analyzer_options)
if toolchain:
add_compiler_defaults = False
# Store the compiler built in include paths and defines.
if add_compiler_defaults and results.compiler:
if not (results.compiler in compiler_includes):
# Fetch defaults from the compiler,
# make sure we use the correct architecture.
extra_opts = []
for regex in COMPILE_OPTS_FWD_TO_DEFAULTS_GETTER:
pattern = re.compile(regex)
for comp_opt in action.analyzer_options:
if re.match(pattern, comp_opt):
extra_opts.append(comp_opt)
compiler_includes[results.compiler] = \
get_compiler_includes(parseLogOptions, results.compiler,
results.lang, results.compile_opts,
extra_opts)
if not (results.compiler in compiler_target):
compiler_target[results.compiler] = \
get_compiler_target(parseLogOptions, results.compiler)
action.compiler_includes = compiler_includes[results.compiler]
action.target = compiler_target[results.compiler]
if results.action != option_parser.ActionType.COMPILE:
action.skip = True
# TODO: Check arch.
action.directory = entry['directory']
action.sources = sourcefile
# Filter out duplicate compilation commands.
unique_key = action.cmp_key
if filtered_build_actions.get(unique_key) is None:
filtered_build_actions[unique_key] = action
del action
counter += 1
for _, ba in filtered_build_actions.items():
actions.append(ba)
return actions