def prepare_check(source,
action,
analyzer_config_map,
output_dir,
severity_map,
skip_handler,
disable_ctu=False):
"""
Construct the source analyzer build the analysis command
and result handler for the analysis.
"""
reanalyzed = False
# Create a source analyzer.
source_analyzer = \
analyzer_types.construct_analyzer(action,
analyzer_config_map)
if disable_ctu:
# WARNING! can be called only on ClangSA
# Needs to be called before construct_analyzer_cmd
source_analyzer.disable_ctu()
# Source is the currently analyzed source file
# there can be more in one buildaction.
source_analyzer.source_file = source
# The result handler for analysis is an empty result handler
# which only returns metadata, but can't process the results.
rh = analyzer_types.construct_analyze_handler(action, output_dir,
severity_map, skip_handler)
# NOTICE!
# The currently analyzed source file needs to be set before the
# analyzer command is constructed.
# The analyzer output file is based on the currently
# analyzed source.
rh.analyzed_source_file = source
if os.path.exists(rh.analyzer_result_file):
reanalyzed = True
# Construct the analyzer cmd.
analyzer_cmd = source_analyzer.construct_analyzer_cmd(rh)
return source_analyzer, analyzer_cmd, rh, reanalyzed
def prepare_check(source,
action,
analyzer_config_map,
output_dir,
severity_map,
skip_handler,
statistics_data,
disable_ctu=False):
"""
Construct the source analyzer build the analysis command
and result handler for the analysis.
"""
reanalyzed = False
# Create a source analyzer.
source_analyzer = \
analyzer_types.construct_analyzer(action,
analyzer_config_map)
if disable_ctu:
# WARNING! can be called only on ClangSA
# Needs to be called before construct_analyzer_cmd
source_analyzer.disable_ctu()
if action.analyzer_type == analyzer_types.CLANG_SA and statistics_data:
# WARNING! Statistical checkers are only supported by Clang
# Static Analyzer right now.
stats_dir = statistics_data['stats_out_dir']
# WARNING Because both statistical checkers use the same config
# directory it is enough to add it only once. This might change later.
# Configuration arguments should be added before the checkers are
# enabled.
stats_cfg = \
SpecialReturnValueCollector.checker_analyze_cfg(stats_dir)
source_analyzer.add_checker_config(stats_cfg)
source_analyzer.config_handler.enable_checker(
SpecialReturnValueCollector.checker_analyze)
source_analyzer.config_handler.enable_checker(
ReturnValueCollector.checker_analyze)
# Source is the currently analyzed source file
# there can be more in one buildaction.
source_analyzer.source_file = source
# The result handler for analysis is an empty result handler
# which only returns metadata, but can't process the results.
rh = analyzer_types.construct_analyze_handler(action, output_dir,
severity_map, skip_handler)
# NOTICE!
# The currently analyzed source file needs to be set before the
# analyzer command is constructed.
# The analyzer output file is based on the currently
# analyzed source.
rh.analyzed_source_file = source
if os.path.exists(rh.analyzer_result_file):
reanalyzed = True
# Construct the analyzer cmd.
analyzer_cmd = source_analyzer.construct_analyzer_cmd(rh)
return source_analyzer, analyzer_cmd, rh, reanalyzed
def check(check_data):
"""
Invoke clang with an action which called by processes.
Different analyzer object belongs to for each build action.
skiplist handler is None if no skip file was configured.
"""
action, context, analyzer_config_map, \
output_dir, skip_handler, quiet_output_on_stdout, \
capture_analysis_output = check_data
skipped = False
reanalyzed = False
try:
# If one analysis fails the check fails.
return_codes = 0
skipped = False
result_file = ''
for source in action.sources:
# If there is no skiplist handler there was no skip list file
# in the command line.
# C++ file skipping is handled here.
source_file_name = os.path.basename(source)
if skip_handler and skip_handler.should_skip(source):
LOG.debug_analyzer(source_file_name + ' is skipped')
skipped = True
continue
# Escape the spaces in the source path, but make sure not to
# over-escape already escaped spaces.
if ' ' in source:
space_locations = [i for i, c in enumerate(source) if c == ' ']
# If a \ is added to the text, the following indexes must be
# shifted by one.
rolling_offset = 0
for orig_idx in space_locations:
idx = orig_idx + rolling_offset
if idx != 0 and source[idx - 1] != '\\':
source = source[:idx] + '\ ' + source[idx + 1:]
rolling_offset += 1
# Construct analyzer env.
analyzer_environment = analyzer_env.get_check_env(
context.path_env_extra, context.ld_lib_path_extra)
# Create a source analyzer.
source_analyzer = \
analyzer_types.construct_analyzer(action,
analyzer_config_map)
# Source is the currently analyzed source file
# there can be more in one buildaction.
source_analyzer.source_file = source
# The result handler for analysis is an empty result handler
# which only returns metadata, but can't process the results.
rh = analyzer_types.construct_analyze_handler(
action, output_dir, context.severity_map, skip_handler)
rh.analyzed_source_file = source
if os.path.exists(rh.analyzer_result_file):
reanalyzed = True
# Fills up the result handler with the analyzer information.
source_analyzer.analyze(rh, analyzer_environment)
# If source file contains escaped spaces ("\ " tokens), then
# clangSA writes the plist file with removing this escape
# sequence, whereas clang-tidy does not. We rewrite the file
# names to contain no escape sequences for every analyzer.
result_file = rh.analyzer_result_file.replace(r'\ ', ' ')
result_base = os.path.basename(result_file)
failed_dir = os.path.join(output_dir, "failed")
if rh.analyzer_returncode == 0:
# Analysis was successful processing results.
if capture_analysis_output:
success_dir = os.path.join(output_dir, "success")
if not os.path.exists(success_dir):
os.makedirs(success_dir)
if len(rh.analyzer_stdout) > 0:
if not quiet_output_on_stdout:
LOG.debug_analyzer('\n' + rh.analyzer_stdout)
if capture_analysis_output:
with open(
os.path.join(success_dir, result_base) +
".stdout.txt", 'w') as outf:
outf.write(rh.analyzer_stdout)
if len(rh.analyzer_stderr) > 0:
if not quiet_output_on_stdout:
LOG.debug_analyzer('\n' + rh.analyzer_stderr)
if capture_analysis_output:
with open(
os.path.join(success_dir, result_base) +
".stderr.txt", 'w') as outf:
outf.write(rh.analyzer_stderr)
rh.postprocess_result()
# Generated reports will be handled separately at store.
# Save some extra information next to the plist, .source
# acting as an extra metadata file.
with open(result_file + ".source", 'w') as orig:
orig.write(
rh.analyzed_source_file.replace(r'\ ', ' ') + "\n")
if os.path.exists(rh.analyzer_result_file) and \
not os.path.exists(result_file):
os.rename(rh.analyzer_result_file, result_file)
LOG.info("[%d/%d] %s analyzed %s successfully." %
(progress_checked_num.value, progress_actions.value,
action.analyzer_type, source_file_name))
# Remove the previously generated error file.
if os.path.exists(failed_dir):
err_file = os.path.join(failed_dir, result_base + '.zip')
if os.path.exists(err_file):
os.remove(err_file)
if skip_handler:
# We need to check the plist content because skipping
# reports in headers can be done only this way.
plist_parser.skip_report_from_plist(
result_file, skip_handler)
else:
# If the analysis has failed, we help debugging.
if not os.path.exists(failed_dir):
os.makedirs(failed_dir)
LOG.debug("Writing error debugging to '" + failed_dir + "'")
zip_file = result_base + '.zip'
with zipfile.ZipFile(os.path.join(failed_dir, zip_file),
'w') as archive:
if len(rh.analyzer_stdout) > 0:
LOG.debug("[ZIP] Writing analyzer STDOUT to /stdout")
archive.writestr("stdout", rh.analyzer_stdout)
if not quiet_output_on_stdout:
LOG.debug_analyzer('\n' + rh.analyzer_stdout)
if len(rh.analyzer_stderr) > 0:
LOG.debug("[ZIP] Writing analyzer STDERR to /stderr")
archive.writestr("stderr", rh.analyzer_stderr)
if not quiet_output_on_stdout:
LOG.debug_analyzer('\n' + rh.analyzer_stderr)
LOG.debug("Generating dependent headers via compiler...")
try:
dependencies = set(create_dependencies(rh.buildaction))
except Exception as ex:
LOG.debug("Couldn't create dependencies:")
LOG.debug(str(ex))
archive.writestr("no-sources", str(ex))
dependencies = set()
LOG.debug("Fetching other dependent files from analyzer "
"output...")
try:
other_files = set()
if len(rh.analyzer_stdout) > 0:
other_files.update(
source_analyzer.get_analyzer_mentioned_files(
rh.analyzer_stdout))
if len(rh.analyzer_stderr) > 0:
other_files.update(
source_analyzer.get_analyzer_mentioned_files(
rh.analyzer_stderr))
except Exception as ex:
LOG.debug("Couldn't generate list of other files "
"from analyzer output:")
LOG.debug(str(ex))
other_files = set()
dependencies.update(other_files)
LOG.debug("Writing dependent files to archive.")
for dependent_source in dependencies:
dependent_source = os.path.join(
action.directory, dependent_source)
if not os.path.isabs(dependent_source):
dependent_source = \
os.path.abspath(dependent_source)
LOG.debug("[ZIP] Writing '" + dependent_source + "' "
"to the archive.")
archive_path = dependent_source.lstrip('/')
try:
archive.write(
dependent_source,
os.path.join("sources-root", archive_path),
zipfile.ZIP_DEFLATED)
except Exception as ex:
# In certain cases, the output could contain
# invalid tokens (such as error messages that were
# printed even though the dependency generation
# returned 0).
LOG.debug("[ZIP] Couldn't write, because " +
str(ex))
archive.writestr(
os.path.join("failed-sources-root",
archive_path),
"Couldn't write this file, because:\n" +
str(ex))
LOG.debug("[ZIP] Writing extra information...")
archive.writestr("build-action",
rh.buildaction.original_command)
archive.writestr("analyzer-command",
' '.join(rh.analyzer_cmd))
archive.writestr("return-code",
str(rh.analyzer_returncode))
LOG.debug("ZIP file written at '" +
os.path.join(failed_dir, zip_file) + "'")
LOG.error("Analyzing '" + source_file_name + "' with " +
action.analyzer_type + " failed.")
if rh.analyzer_stdout != '' and not quiet_output_on_stdout:
LOG.error(rh.analyzer_stdout)
if rh.analyzer_stderr != '' and not quiet_output_on_stdout:
LOG.error(rh.analyzer_stderr)
return_codes = rh.analyzer_returncode
# Remove files that successfully analyzed earlier on.
plist_file = result_base + ".plist"
if os.path.exists(plist_file):
os.remove(plist_file)
progress_checked_num.value += 1
return return_codes, skipped, reanalyzed, action.analyzer_type, \
result_file
except Exception as e:
LOG.debug_analyzer(str(e))
traceback.print_exc(file=sys.stdout)
return 1, skipped, reanalyzed, action.analyzer_type, None
def check(check_data):
"""
Invoke clang with an action which called by processes.
Different analyzer object belongs to for each build action.
skiplist handler is None if no skip file was configured.
"""
action, context, analyzer_config_map, output_dir, skip_handler = check_data
skipped = False
try:
# If one analysis fails the check fails.
return_codes = 0
skipped = False
for source in action.sources:
# If there is no skiplist handler there was no skip list file
# in the command line.
# C++ file skipping is handled here.
_, source_file_name = os.path.split(source)
if skip_handler and skip_handler.should_skip(source):
LOG.debug_analyzer(source_file_name + ' is skipped')
skipped = True
continue
# Construct analyzer env.
analyzer_environment = analyzer_env.get_check_env(
context.path_env_extra, context.ld_lib_path_extra)
# Create a source analyzer.
source_analyzer = \
analyzer_types.construct_analyzer(action,
analyzer_config_map)
# Source is the currently analyzed source file
# there can be more in one buildaction.
source_analyzer.source_file = source
# The result handler for analysis is an empty result handler
# which only returns metadata, but can't process the results.
rh = analyzer_types.construct_analyze_handler(
action, output_dir, context.severity_map, skip_handler)
# Fills up the result handler with the analyzer information.
source_analyzer.analyze(rh, analyzer_environment)
if rh.analyzer_returncode == 0:
# Analysis was successful processing results.
if rh.analyzer_stdout != '':
LOG.debug_analyzer('\n' + rh.analyzer_stdout)
if rh.analyzer_stderr != '':
LOG.debug_analyzer('\n' + rh.analyzer_stderr)
rh.postprocess_result()
rh.handle_results()
# Save some extra information next to the plist, .source
# acting as an extra metadata file.
with open(rh.analyzer_result_file + ".source", 'w') as orig:
orig.write(rh.analyzed_source_file + "\n")
LOG.info("[%d/%d] %s analyzed %s successfully." %
(progress_checked_num.value, progress_actions.value,
action.analyzer_type, source_file_name))
else:
# Analysis failed.
LOG.error('Analyzing ' + source_file_name + ' with ' +
action.analyzer_type + ' failed.')
if rh.analyzer_stdout != '':
LOG.error(rh.analyzer_stdout)
if rh.analyzer_stderr != '':
LOG.error(rh.analyzer_stderr)
return_codes = rh.analyzer_returncode
progress_checked_num.value += 1
return return_codes, skipped, action.analyzer_type
except Exception as e:
LOG.debug_analyzer(str(e))
traceback.print_exc(file=sys.stdout)
return 1, skipped, action.analyzer_type
