def test_tc_detect(self): """ Parse gcc-toolchain argument from clang compile command. """ compilation_action = "clang -x c -O3 " \ "--gcc-toolchain=/home/user/my_gcc/toolchain -c main.cpp " toolchain = \ gcc_toolchain.toolchain_in_args(shlex.split(compilation_action)) print(toolchain) self.assertEqual(toolchain, "/home/user/my_gcc/toolchain")
def test_get_tc_gcc_compiler(self): """ Get gcc compiler from the toolchain path. """ compilation_action = \ "clang --gcc-toolchain=/home/user/my_gcc/toolchain" toolchain = \ gcc_toolchain.toolchain_in_args(shlex.split(compilation_action)) print(toolchain) self.assertEqual(toolchain, "/home/user/my_gcc/toolchain") tc_compiler_c = gcc_toolchain.get_toolchain_compiler(toolchain, "c") print(tc_compiler_c) self.assertEqual(tc_compiler_c, "/home/user/my_gcc/toolchain/bin/gcc")
def get_dependent_headers(buildaction, archive): LOG.debug("Generating dependent headers via compiler...") command = shlex.split(buildaction.original_command) try: dependencies = set(create_dependencies(command, buildaction.directory)) except Exception as ex: LOG.debug("Couldn't create dependencies:") LOG.debug(str(ex)) # TODO append with buildaction archive.writestr("no-sources", str(ex)) dependencies = set() toolchain = gcc_toolchain.toolchain_in_args( shlex.split(buildaction.original_command)) if toolchain: LOG.debug("Generating gcc-toolchain headers via toolchain compiler...") try: # Change the original compiler to the compiler from the # toolchain. toolchain_compiler = gcc_toolchain.get_toolchain_compiler( toolchain, buildaction.lang) if not toolchain_compiler: raise Exception("Could not get the compiler " "from the gcc-toolchain.") command[0] = toolchain_compiler dependencies = dependencies.union( set(create_dependencies(command, buildaction.directory))) except Exception as ex: LOG.debug("Couldn't create dependencies:") LOG.debug(str(ex)) # TODO append with buildaction archive.writestr("no-sources", str(ex)) dependencies = set() return dependencies
def collect_debug_data(zip_file, other_files, buildaction, out, err, original_command, analyzer_cmd, analyzer_returncode, action_directory, action_target, actions_map): """ Collect analysis and build system information which can be used to reproduce the failed analysis. """ LOG.debug("Collecting debug data") with zipfile.ZipFile(zip_file, 'w') as archive: LOG.debug("[ZIP] Writing analyzer STDOUT to /stdout") archive.writestr("stdout", out) LOG.debug("[ZIP] Writing analyzer STDERR to /stderr") archive.writestr("stderr", err) dependencies = get_dependent_headers(buildaction, archive) mentioned_files_dependent_files = set() for of in other_files: mentioned_file = os.path.abspath( os.path.join(action_directory, of)) # Use the target of the original build action. key = mentioned_file, action_target mentioned_file_action = actions_map.get(key) if mentioned_file_action is not None: mentioned_file_deps = get_dependent_headers( mentioned_file_action, archive) mentioned_files_dependent_files.\ update(mentioned_file_deps) else: LOG.debug("Could not find {0} in build actions." .format(key)) dependencies.update(other_files) dependencies.update(mentioned_files_dependent_files) # `dependencies` might contain absolute and relative paths # for the same file, so make everything absolute by # joining with the the absolute action.directory. # If `dependent_source` is absolute it remains absolute # after the join, as documented on docs.python.org . dependencies_copy = set() for dependent_source in dependencies: dependent_source = os.path.join(action_directory, dependent_source) dependencies_copy.add(dependent_source) dependencies = dependencies_copy LOG.debug("Writing dependent files to archive.") for dependent_source in dependencies: LOG.debug("[ZIP] Writing '" + dependent_source + "' " "to the archive.") archive_subpath = dependent_source.lstrip('/') archive_path = os.path.join('sources-root', archive_subpath) try: _ = archive.getinfo(archive_path) LOG.debug("[ZIP] '{0}' is already in the ZIP " "file, won't add it again!" .format(archive_path)) continue except KeyError: # If the file is already contained in the ZIP, # a valid object is returned, otherwise a KeyError # is raised. pass try: archive.write(dependent_source, archive_path, zipfile.ZIP_DEFLATED) except Exception as ex: # In certain cases, the output could contain # invalid tokens (such as error messages that were # printed even though the dependency generation # returned 0). LOG.debug("[ZIP] Couldn't write, because " + str(ex)) archive.writestr( os.path.join('failed-sources-root', archive_subpath), "Couldn't write this file, because:\n" + str(ex)) LOG.debug("[ZIP] Writing extra information...") archive.writestr("build-action", original_command) archive.writestr("analyzer-command", ' '.join(analyzer_cmd)) archive.writestr("return-code", str(analyzer_returncode)) toolchain = gcc_toolchain.toolchain_in_args( shlex.split(buildaction.original_command)) if toolchain: archive.writestr("gcc-toolchain-path", toolchain)
def parse_compile_commands_json(logfile, parseLogOptions): """ logfile: is a compile command json """ output_path = parseLogOptions.output_path if output_path is not None: remove_file_if_exists( os.path.join(output_path, compiler_includes_dump_file)) remove_file_if_exists( os.path.join(output_path, compiler_target_dump_file)) actions = [] filtered_build_actions = {} data = json.load(logfile) compiler_includes = {} compiler_target = {} counter = 0 for entry in data: sourcefile = entry['file'] if not os.path.isabs(sourcefile): # Newest versions of intercept-build can create the 'file' in the # JSON Compilation Database as a relative path. sourcefile = os.path.join(os.path.abspath(entry['directory']), sourcefile) lang = option_parser.get_language(sourcefile[sourcefile.rfind('.'):]) if not lang: continue action = build_action.BuildAction(counter) if 'command' in entry: command = entry['command'] # Old versions of intercept-build (confirmed to those shipping # with upstream clang-5.0) do escapes in another way: # -DVARIABLE="a b" becomes -DVARIABLE=\"a b\" in the output. # This would be messed up later on by options_parser, so need a # fix here. (Should be removed once we are sure noone uses this # intercept-build anymore!) if r'\"' in command: command = command.replace(r'\"', '"') elif 'arguments' in entry: # Newest versions of intercept-build create an argument vector # instead of a command string. command = ' '.join(entry['arguments']) else: raise KeyError("No valid 'command' or 'arguments' entry found!") results = option_parser.parse_options(command) action.original_command = command # If the original include directory could not be found # in the filesystem, it is possible that it was provided # relative to the working directory in the compile json. compile_opts = results.compile_opts for i, opt in enumerate(compile_opts): if opt.startswith('-I'): inc_dir = opt[2:].strip() if not os.path.isdir(inc_dir): compile_opts[i] = '-I' + \ os.path.join(entry['directory'], inc_dir) action.analyzer_options = compile_opts action.lang = results.lang action.target = results.arch action.output = results.output add_compiler_defaults = True # With gcc-toolchain a non default compiler toolchain can be set. # Clang will search for include paths and libraries based on the # gcc-toolchain parameter. # Detecting extra include paths from the host compiler could # conflict with this. # For example if the compiler in the compile command is clang # and gcc-toolchain is set we will get the include paths # for clang and not for the compiler set in gcc-toolchain. # This can cause missing headers during the analysis. toolchain = gcc_toolchain.toolchain_in_args(action.analyzer_options) if toolchain: add_compiler_defaults = False # Store the compiler built in include paths and defines. if add_compiler_defaults and results.compiler: if not (results.compiler in compiler_includes): # Fetch defaults from the compiler, # make sure we use the correct architecture. extra_opts = [] for regex in COMPILE_OPTS_FWD_TO_DEFAULTS_GETTER: pattern = re.compile(regex) for comp_opt in action.analyzer_options: if re.match(pattern, comp_opt): extra_opts.append(comp_opt) compiler_includes[results.compiler] = \ get_compiler_includes(parseLogOptions, results.compiler, results.lang, results.compile_opts, extra_opts) if not (results.compiler in compiler_target): compiler_target[results.compiler] = \ get_compiler_target(parseLogOptions, results.compiler) action.compiler_includes = compiler_includes[results.compiler] action.target = compiler_target[results.compiler] if results.action != option_parser.ActionType.COMPILE: action.skip = True # TODO: Check arch. action.directory = entry['directory'] action.sources = sourcefile # Filter out duplicate compilation commands. unique_key = action.cmp_key if filtered_build_actions.get(unique_key) is None: filtered_build_actions[unique_key] = action del action counter += 1 for _, ba in filtered_build_actions.items(): actions.append(ba) return actions