def handle_success(rh, result_file, result_base, skip_handler,
capture_analysis_output, success_dir):
"""
Result postprocessing is required if the analysis was
successful (mainly clang tidy output conversion is done).
Skipping reports for header files is done here too.
"""
if capture_analysis_output:
save_output(os.path.join(success_dir, result_base), rh.analyzer_stdout,
rh.analyzer_stderr)
rh.postprocess_result()
# Generated reports will be handled separately at store.
save_metadata(result_file, rh.analyzer_result_file,
rh.analyzed_source_file)
if skip_handler:
# We need to check the plist content because skipping
# reports in headers can be done only this way.
plist_parser.skip_report_from_plist(result_file, skip_handler)
def check(check_data):
"""
Invoke clang with an action which called by processes.
Different analyzer object belongs to for each build action.
skiplist handler is None if no skip file was configured.
"""
actions_map, action, context, analyzer_config_map, \
output_dir, skip_handler, quiet_output_on_stdout, \
capture_analysis_output, analysis_timeout, \
analyzer_environment, ctu_reanalyze_on_failure, \
output_dirs, statistics_data = check_data
skipped = False
reanalyzed = False
failed_dir = output_dirs["failed"]
success_dir = output_dirs["success"]
try:
# If one analysis fails the check fails.
return_codes = 0
skipped = False
result_file = ''
for source in action.sources:
# If there is no skiplist handler there was no skip list file
# in the command line.
# C++ file skipping is handled here.
source_file_name = os.path.basename(source)
if skip_handler and skip_handler.should_skip(source):
LOG.debug_analyzer(source_file_name + ' is skipped')
skipped = True
continue
source = util.escape_source_path(source)
source_analyzer, analyzer_cmd, rh, reanalyzed = \
prepare_check(source, action, analyzer_config_map,
output_dir, context.severity_map,
skip_handler, statistics_data)
# The analyzer invocation calls __create_timeout as a callback
# when the analyzer starts. This callback creates the timeout
# watcher over the analyzer process, which in turn returns a
# function, that can later be used to check if the analyzer quit
# because we killed it due to a timeout.
#
# We need to capture the "function pointer" returned by
# setup_process_timeout as reference, so that we may call it
# later. To work around scoping issues, we use a list here so the
# "function pointer" is captured by reference.
timeout_cleanup = [lambda: False]
if analysis_timeout and analysis_timeout > 0:
def __create_timeout(analyzer_process):
"""
Once the analyzer process is started, this method is
called. Set up a timeout for the analysis.
"""
timeout_cleanup[0] = util.setup_process_timeout(
analyzer_process, analysis_timeout)
else:
def __create_timeout(analyzer_process):
# If no timeout is given by the client, this callback
# shouldn't do anything.
pass
# Fills up the result handler with the analyzer information.
source_analyzer.analyze(analyzer_cmd, rh, analyzer_environment,
__create_timeout)
# If execution reaches this line, the analyzer process has quit.
if timeout_cleanup[0]():
LOG.warning("Analyzer ran too long, exceeding time limit "
"of {0} seconds.".format(analysis_timeout))
LOG.warning("Considering this analysis as failed...")
rh.analyzer_returncode = -1
rh.analyzer_stderr = (">>> CodeChecker: Analysis timed out "
"after {0} seconds. <<<\n{1}") \
.format(analysis_timeout, rh.analyzer_stderr)
# If source file contains escaped spaces ("\ " tokens), then
# clangSA writes the plist file with removing this escape
# sequence, whereas clang-tidy does not. We rewrite the file
# names to contain no escape sequences for every analyzer.
result_file = rh.analyzer_result_file.replace(r'\ ', ' ')
result_base = os.path.basename(result_file)
ctu_active = is_ctu_active(source_analyzer)
ctu_suffix = 'CTU'
zip_suffix = ctu_suffix if ctu_active else ''
zip_file = result_base + zip_suffix + '.zip'
zip_file = os.path.join(failed_dir, zip_file)
ctu_zip_file = result_base + ctu_suffix + '.zip'
ctu_zip_file = os.path.join(failed_dir, ctu_zip_file)
return_codes = rh.analyzer_returncode
if rh.analyzer_returncode == 0:
# Remove the previously generated error file.
if os.path.exists(zip_file):
os.remove(zip_file)
# Remove the previously generated CTU error file.
if os.path.exists(ctu_zip_file):
os.remove(ctu_zip_file)
handle_success(rh, result_file, result_base,
skip_handler, capture_analysis_output,
success_dir)
LOG.info("[%d/%d] %s analyzed %s successfully." %
(progress_checked_num.value, progress_actions.value,
action.analyzer_type, source_file_name))
if skip_handler:
# We need to check the plist content because skipping
# reports in headers can be done only this way.
plist_parser.skip_report_from_plist(result_file,
skip_handler)
else:
LOG.error("Analyzing '" + source_file_name + "' with " +
action.analyzer_type +
" CTU" if ctu_active else " " + "failed.")
if not quiet_output_on_stdout:
LOG.error('\n' + rh.analyzer_stdout)
LOG.error('\n' + rh.analyzer_stderr)
handle_failure(source_analyzer, rh, action, zip_file,
result_base, actions_map)
if ctu_active and ctu_reanalyze_on_failure:
LOG.error("Try to reanalyze without CTU")
# Try to reanalyze with CTU disabled.
source_analyzer, analyzer_cmd, rh, reanalyzed = \
prepare_check(source, action,
analyzer_config_map,
output_dir,
context.severity_map,
skip_handler,
statistics_data,
True)
# Fills up the result handler with
# the analyzer information.
source_analyzer.analyze(analyzer_cmd,
rh,
analyzer_environment)
return_codes = rh.analyzer_returncode
if rh.analyzer_returncode == 0:
handle_success(rh, result_file, result_base,
skip_handler, capture_analysis_output,
success_dir, zipfile)
msg = "[{0}/{1}] {2} analyzed {3} without" \
" CTU successfully.".format(
progress_checked_num.value,
progress_actions.value,
action.analyzer_type,
source_file_name)
LOG.info(msg)
else:
LOG.error("Analyzing '" + source_file_name +
"' with " + action.analyzer_type +
" without CTU failed.")
zip_file = result_base + '.zip'
zip_file = os.path.join(failed_dir, zip_file)
handle_failure(source_analyzer, rh, action,
zip_file, result_base, actions_map)
if not quiet_output_on_stdout:
if rh.analyzer_returncode:
LOG.error('\n' + rh.analyzer_stdout)
LOG.error('\n' + rh.analyzer_stderr)
else:
LOG.debug_analyzer('\n' + rh.analyzer_stdout)
LOG.debug_analyzer('\n' + rh.analyzer_stderr)
progress_checked_num.value += 1
return return_codes, skipped, reanalyzed, action.analyzer_type, \
result_file, list(action.sources)
except Exception as e:
LOG.debug_analyzer(str(e))
traceback.print_exc(file=sys.stdout)
return 1, skipped, reanalyzed, action.analyzer_type, None, \
list(action.sources)
def check(check_data):
"""
Invoke clang with an action which called by processes.
Different analyzer object belongs to for each build action.
skiplist handler is None if no skip file was configured.
"""
action, context, analyzer_config_map, \
output_dir, skip_handler, quiet_output_on_stdout, \
capture_analysis_output = check_data
skipped = False
reanalyzed = False
try:
# If one analysis fails the check fails.
return_codes = 0
skipped = False
result_file = ''
for source in action.sources:
# If there is no skiplist handler there was no skip list file
# in the command line.
# C++ file skipping is handled here.
source_file_name = os.path.basename(source)
if skip_handler and skip_handler.should_skip(source):
LOG.debug_analyzer(source_file_name + ' is skipped')
skipped = True
continue
# Escape the spaces in the source path, but make sure not to
# over-escape already escaped spaces.
if ' ' in source:
space_locations = [i for i, c in enumerate(source) if c == ' ']
# If a \ is added to the text, the following indexes must be
# shifted by one.
rolling_offset = 0
for orig_idx in space_locations:
idx = orig_idx + rolling_offset
if idx != 0 and source[idx - 1] != '\\':
source = source[:idx] + '\ ' + source[idx + 1:]
rolling_offset += 1
# Construct analyzer env.
analyzer_environment = analyzer_env.get_check_env(
context.path_env_extra, context.ld_lib_path_extra)
# Create a source analyzer.
source_analyzer = \
analyzer_types.construct_analyzer(action,
analyzer_config_map)
# Source is the currently analyzed source file
# there can be more in one buildaction.
source_analyzer.source_file = source
# The result handler for analysis is an empty result handler
# which only returns metadata, but can't process the results.
rh = analyzer_types.construct_analyze_handler(
action, output_dir, context.severity_map, skip_handler)
rh.analyzed_source_file = source
if os.path.exists(rh.analyzer_result_file):
reanalyzed = True
# Fills up the result handler with the analyzer information.
source_analyzer.analyze(rh, analyzer_environment)
# If source file contains escaped spaces ("\ " tokens), then
# clangSA writes the plist file with removing this escape
# sequence, whereas clang-tidy does not. We rewrite the file
# names to contain no escape sequences for every analyzer.
result_file = rh.analyzer_result_file.replace(r'\ ', ' ')
result_base = os.path.basename(result_file)
failed_dir = os.path.join(output_dir, "failed")
if rh.analyzer_returncode == 0:
# Analysis was successful processing results.
if capture_analysis_output:
success_dir = os.path.join(output_dir, "success")
if not os.path.exists(success_dir):
os.makedirs(success_dir)
if len(rh.analyzer_stdout) > 0:
if not quiet_output_on_stdout:
LOG.debug_analyzer('\n' + rh.analyzer_stdout)
if capture_analysis_output:
with open(
os.path.join(success_dir, result_base) +
".stdout.txt", 'w') as outf:
outf.write(rh.analyzer_stdout)
if len(rh.analyzer_stderr) > 0:
if not quiet_output_on_stdout:
LOG.debug_analyzer('\n' + rh.analyzer_stderr)
if capture_analysis_output:
with open(
os.path.join(success_dir, result_base) +
".stderr.txt", 'w') as outf:
outf.write(rh.analyzer_stderr)
rh.postprocess_result()
# Generated reports will be handled separately at store.
# Save some extra information next to the plist, .source
# acting as an extra metadata file.
with open(result_file + ".source", 'w') as orig:
orig.write(
rh.analyzed_source_file.replace(r'\ ', ' ') + "\n")
if os.path.exists(rh.analyzer_result_file) and \
not os.path.exists(result_file):
os.rename(rh.analyzer_result_file, result_file)
LOG.info("[%d/%d] %s analyzed %s successfully." %
(progress_checked_num.value, progress_actions.value,
action.analyzer_type, source_file_name))
# Remove the previously generated error file.
if os.path.exists(failed_dir):
err_file = os.path.join(failed_dir, result_base + '.zip')
if os.path.exists(err_file):
os.remove(err_file)
if skip_handler:
# We need to check the plist content because skipping
# reports in headers can be done only this way.
plist_parser.skip_report_from_plist(
result_file, skip_handler)
else:
# If the analysis has failed, we help debugging.
if not os.path.exists(failed_dir):
os.makedirs(failed_dir)
LOG.debug("Writing error debugging to '" + failed_dir + "'")
zip_file = result_base + '.zip'
with zipfile.ZipFile(os.path.join(failed_dir, zip_file),
'w') as archive:
if len(rh.analyzer_stdout) > 0:
LOG.debug("[ZIP] Writing analyzer STDOUT to /stdout")
archive.writestr("stdout", rh.analyzer_stdout)
if not quiet_output_on_stdout:
LOG.debug_analyzer('\n' + rh.analyzer_stdout)
if len(rh.analyzer_stderr) > 0:
LOG.debug("[ZIP] Writing analyzer STDERR to /stderr")
archive.writestr("stderr", rh.analyzer_stderr)
if not quiet_output_on_stdout:
LOG.debug_analyzer('\n' + rh.analyzer_stderr)
LOG.debug("Generating dependent headers via compiler...")
try:
dependencies = set(create_dependencies(rh.buildaction))
except Exception as ex:
LOG.debug("Couldn't create dependencies:")
LOG.debug(str(ex))
archive.writestr("no-sources", str(ex))
dependencies = set()
LOG.debug("Fetching other dependent files from analyzer "
"output...")
try:
other_files = set()
if len(rh.analyzer_stdout) > 0:
other_files.update(
source_analyzer.get_analyzer_mentioned_files(
rh.analyzer_stdout))
if len(rh.analyzer_stderr) > 0:
other_files.update(
source_analyzer.get_analyzer_mentioned_files(
rh.analyzer_stderr))
except Exception as ex:
LOG.debug("Couldn't generate list of other files "
"from analyzer output:")
LOG.debug(str(ex))
other_files = set()
dependencies.update(other_files)
LOG.debug("Writing dependent files to archive.")
for dependent_source in dependencies:
dependent_source = os.path.join(
action.directory, dependent_source)
if not os.path.isabs(dependent_source):
dependent_source = \
os.path.abspath(dependent_source)
LOG.debug("[ZIP] Writing '" + dependent_source + "' "
"to the archive.")
archive_path = dependent_source.lstrip('/')
try:
archive.write(
dependent_source,
os.path.join("sources-root", archive_path),
zipfile.ZIP_DEFLATED)
except Exception as ex:
# In certain cases, the output could contain
# invalid tokens (such as error messages that were
# printed even though the dependency generation
# returned 0).
LOG.debug("[ZIP] Couldn't write, because " +
str(ex))
archive.writestr(
os.path.join("failed-sources-root",
archive_path),
"Couldn't write this file, because:\n" +
str(ex))
LOG.debug("[ZIP] Writing extra information...")
archive.writestr("build-action",
rh.buildaction.original_command)
archive.writestr("analyzer-command",
' '.join(rh.analyzer_cmd))
archive.writestr("return-code",
str(rh.analyzer_returncode))
LOG.debug("ZIP file written at '" +
os.path.join(failed_dir, zip_file) + "'")
LOG.error("Analyzing '" + source_file_name + "' with " +
action.analyzer_type + " failed.")
if rh.analyzer_stdout != '' and not quiet_output_on_stdout:
LOG.error(rh.analyzer_stdout)
if rh.analyzer_stderr != '' and not quiet_output_on_stdout:
LOG.error(rh.analyzer_stderr)
return_codes = rh.analyzer_returncode
# Remove files that successfully analyzed earlier on.
plist_file = result_base + ".plist"
if os.path.exists(plist_file):
os.remove(plist_file)
progress_checked_num.value += 1
return return_codes, skipped, reanalyzed, action.analyzer_type, \
result_file
except Exception as e:
LOG.debug_analyzer(str(e))
traceback.print_exc(file=sys.stdout)
return 1, skipped, reanalyzed, action.analyzer_type, None
