def main(args):
"""
Setup a logger server based on the configuration and
manage the CodeChecker server.
"""
with logger.LOG_CFG_SERVER(args.verbose if 'verbose' in args else None):
try:
cmd_config.check_config_file(args)
except FileNotFoundError as fnerr:
LOG.error(fnerr)
sys.exit(1)
server_init_start(args)
def main(args):
"""
Entry point for parsing some analysis results and printing them to the
stdout in a human-readable format.
"""
logger.setup_logger(args.verbose if 'verbose' in args else None)
try:
cmd_config.check_config_file(args)
except FileNotFoundError as fnerr:
LOG.error(fnerr)
sys.exit(1)
export = args.export if 'export' in args else None
if export == 'html' and 'output_path' not in args:
LOG.error("Argument --export not allowed without argument --output "
"when exporting to HTML.")
sys.exit(1)
if export == 'gerrit' and not gerrit.mandatory_env_var_is_set():
sys.exit(1)
context = analyzer_context.get_context()
# To ensure the help message prints the default folder properly,
# the 'default' for 'args.input' is a string, not a list.
# But we need lists for the foreach here to work.
if isinstance(args.input, str):
args.input = [args.input]
original_cwd = os.getcwd()
src_comment_status_filter = args.review_status
suppr_handler = None
if 'suppress' in args:
__make_handler = False
if not os.path.isfile(args.suppress):
if 'create_suppress' in args:
with open(args.suppress,
'w',
encoding='utf-8',
errors='ignore') as _:
# Just create the file.
__make_handler = True
LOG.info(
"Will write source-code suppressions to "
"suppress file: %s", args.suppress)
else:
LOG.warning(
"Suppress file '%s' given, but it does not exist"
" -- will not suppress anything.", args.suppress)
else:
__make_handler = True
if __make_handler:
suppr_handler = suppress_handler.\
GenericSuppressHandler(args.suppress,
'create_suppress' in args,
src_comment_status_filter)
elif 'create_suppress' in args:
LOG.error("Can't use '--export-source-suppress' unless '--suppress "
"SUPPRESS_FILE' is also given.")
sys.exit(2)
processed_path_hashes = set()
skip_handler = None
if 'skipfile' in args:
with open(args.skipfile, 'r', encoding='utf-8',
errors='ignore') as skip_file:
skip_handler = SkipListHandler(skip_file.read())
trim_path_prefixes = args.trim_path_prefix if \
'trim_path_prefix' in args else None
if export:
if export not in EXPORT_TYPES:
LOG.error(f"Unknown export format: {export}")
return
# The HTML part will be handled separately below.
if export != 'html':
try:
res = parse_convert_reports(args.input, export,
context.severity_map,
trim_path_prefixes)
if 'output_path' in args:
output_path = os.path.abspath(args.output_path)
if not os.path.exists(output_path):
os.mkdir(output_path)
reports_json = os.path.join(output_path, 'reports.json')
with open(reports_json,
mode='w',
encoding='utf-8',
errors="ignore") as output_f:
output_f.write(json.dumps(res))
return print(json.dumps(res))
except Exception as ex:
LOG.error(ex)
sys.exit(1)
def trim_path_prefixes_handler(source_file):
"""
Callback to util.trim_path_prefixes to prevent module dependency
of plist_to_html
"""
return util.trim_path_prefixes(source_file, trim_path_prefixes)
html_builder = None
def skip_html_report_data_handler(report_hash, source_file, report_line,
checker_name, diag, files):
"""
Report handler which skips bugs which were suppressed by source code
comments. This function will return a tuple. The first element
will decide whether the report should be skipped or not and the second
element will be a list of source code comments related to the actual
report.
"""
files_dict = {k: v for k, v in enumerate(files)}
report = Report({'check_name': checker_name},
diag['path'],
files_dict,
metadata=None)
path_hash = get_report_path_hash(report)
if path_hash in processed_path_hashes:
LOG.debug("Skip report because it is a deduplication of an "
"already processed report!")
LOG.debug("Path hash: %s", path_hash)
LOG.debug(diag)
return True, []
skip, source_code_comments = skip_report(report_hash, source_file,
report_line, checker_name,
suppr_handler,
src_comment_status_filter)
if skip_handler:
skip |= skip_handler.should_skip(source_file)
if not skip:
processed_path_hashes.add(path_hash)
return skip, source_code_comments
file_change = set()
severity_stats = defaultdict(int)
file_stats = defaultdict(int)
report_count = 0
for input_path in args.input:
input_path = os.path.abspath(input_path)
os.chdir(original_cwd)
LOG.debug("Parsing input argument: '%s'", input_path)
if export == 'html':
output_path = os.path.abspath(args.output_path)
if not html_builder:
html_builder = \
PlistToHtml.HtmlBuilder(context.path_plist_to_html_dist,
context.severity_map)
LOG.info("Generating html output files:")
PlistToHtml.parse(input_path, output_path,
context.path_plist_to_html_dist,
skip_html_report_data_handler, html_builder,
trim_path_prefixes_handler)
continue
files = []
metadata_dict = {}
if os.path.isfile(input_path):
files.append(input_path)
elif os.path.isdir(input_path):
metadata_file = os.path.join(input_path, "metadata.json")
if os.path.exists(metadata_file):
metadata_dict = util.load_json_or_empty(metadata_file)
LOG.debug(metadata_dict)
if 'working_directory' in metadata_dict:
working_dir = metadata_dict['working_directory']
try:
os.chdir(working_dir)
except OSError as oerr:
LOG.debug(oerr)
LOG.error(
"Working directory %s is missing.\n"
"Can not parse reports safely.", working_dir)
sys.exit(1)
_, _, file_names = next(os.walk(input_path), ([], [], []))
files = [
os.path.join(input_path, file_name) for file_name in file_names
]
file_report_map = defaultdict(list)
plist_pltf = PlistToPlaintextFormatter(suppr_handler, skip_handler,
context.severity_map,
processed_path_hashes,
trim_path_prefixes,
src_comment_status_filter)
plist_pltf.print_steps = 'print_steps' in args
for file_path in files:
f_change = parse_with_plt_formatter(file_path, metadata_dict,
plist_pltf, file_report_map)
file_change = file_change.union(f_change)
report_stats = plist_pltf.write(file_report_map)
sev_stats = report_stats.get('severity')
for severity in sev_stats:
severity_stats[severity] += sev_stats[severity]
f_stats = report_stats.get('files')
for file_path in f_stats:
file_stats[file_path] += f_stats[file_path]
rep_stats = report_stats.get('reports')
report_count += rep_stats.get("report_count", 0)
# Create index.html and statistics.html for the generated html files.
if html_builder:
html_builder.create_index_html(args.output_path)
html_builder.create_statistics_html(args.output_path)
print('\nTo view statistics in a browser run:\n> firefox {0}'.format(
os.path.join(args.output_path, 'statistics.html')))
print('\nTo view the results in a browser run:\n> firefox {0}'.format(
os.path.join(args.output_path, 'index.html')))
else:
print("\n----==== Summary ====----")
if file_stats:
vals = [[os.path.basename(k), v]
for k, v in dict(file_stats).items()]
vals.sort(key=itemgetter(0))
keys = ['Filename', 'Report count']
table = twodim.to_str('table', keys, vals, 1, True)
print(table)
if severity_stats:
vals = [[k, v] for k, v in dict(severity_stats).items()]
vals.sort(key=itemgetter(0))
keys = ['Severity', 'Report count']
table = twodim.to_str('table', keys, vals, 1, True)
print(table)
print("----=================----")
print("Total number of reports: {}".format(report_count))
print("----=================----")
if file_change:
changed_files = '\n'.join([' - ' + f for f in file_change])
LOG.warning(
"The following source file contents changed since the "
"latest analysis:\n%s\nMultiple reports were not "
"shown and skipped from the statistics. Please "
"analyze your project again to update the "
"reports!", changed_files)
os.chdir(original_cwd)
if report_count != 0:
sys.exit(2)
def main(args):
"""
Store the defect results in the specified input list as bug reports in the
database.
"""
logger.setup_logger(args.verbose if 'verbose' in args else None)
try:
cmd_config.check_config_file(args)
except FileNotFoundError as fnerr:
LOG.error(fnerr)
sys.exit(1)
if not host_check.check_zlib():
raise Exception("zlib is not available on the system!")
# To ensure the help message prints the default folder properly,
# the 'default' for 'args.input' is a string, not a list.
# But we need lists for the foreach here to work.
if isinstance(args.input, str):
args.input = [args.input]
if 'name' not in args:
LOG.debug("Generating name for analysis...")
generated = __get_run_name(args.input)
if generated:
setattr(args, 'name', generated)
else:
LOG.error("No suitable name was found in the inputs for the "
"analysis run. Please specify one by passing argument "
"--name run_name in the invocation.")
sys.exit(2) # argparse returns error code 2 for bad invocations.
LOG.info("Storing analysis results for run '" + args.name + "'")
if 'force' in args:
LOG.info("argument --force was specified: the run with name '" +
args.name + "' will be deleted.")
# Setup connection to the remote server.
client = libclient.setup_client(args.product_url)
_, zip_file = tempfile.mkstemp('.zip')
LOG.debug("Will write mass store ZIP to '%s'...", zip_file)
try:
LOG.debug("Assembling zip file.")
try:
assemble_zip(args.input, zip_file, client)
except Exception as ex:
print(ex)
import traceback
traceback.print_stack()
zip_size = os.stat(zip_file).st_size
LOG.debug("Assembling zip done, size is %s",
sizeof_fmt(zip_size))
if zip_size > MAX_UPLOAD_SIZE:
LOG.error("The result list to upload is too big (max: %s).",
sizeof_fmt(MAX_UPLOAD_SIZE))
sys.exit(1)
with open(zip_file, 'rb') as zf:
b64zip = base64.b64encode(zf.read()).decode("utf-8")
context = webserver_context.get_context()
trim_path_prefixes = args.trim_path_prefix if \
'trim_path_prefix' in args else None
description = args.description if 'description' in args else None
LOG.info("Storing results to the server...")
client.massStoreRun(args.name,
args.tag if 'tag' in args else None,
str(context.version),
b64zip,
'force' in args,
trim_path_prefixes,
description)
# Storing analysis statistics if the server allows them.
if client.allowsStoringAnalysisStatistics():
storing_analysis_statistics(client, args.input, args.name)
LOG.info("Storage finished successfully.")
except RequestFailed as reqfail:
if reqfail.errorCode == ErrorCode.SOURCE_FILE:
header = ['File', 'Line', 'Checker name']
table = twodim_to_str('table',
header,
[c.split('|') for c in reqfail.extraInfo])
LOG.warning("Setting the review statuses for some reports failed "
"because of non valid source code comments: "
"%s\n %s", reqfail.message, table)
sys.exit(1)
except Exception as ex:
import traceback
traceback.print_stack()
LOG.info("Storage failed: %s", str(ex))
sys.exit(1)
finally:
os.remove(zip_file)
def main(args):
"""
Perform analysis on the given logfiles and store the results in a machine-
readable format.
"""
logger.setup_logger(args.verbose if 'verbose' in args else None)
# CTU loading mode is only meaningful if CTU itself is enabled.
if 'ctu_ast_mode' in args and 'ctu_phases' not in args:
LOG.error("Analyzer option 'ctu-ast-mode' requires CTU mode enabled")
sys.exit(1)
try:
cmd_config.check_config_file(args)
except FileNotFoundError as fnerr:
LOG.error(fnerr)
sys.exit(1)
if not os.path.exists(args.logfile):
LOG.error("The specified logfile '%s' does not exist!", args.logfile)
sys.exit(1)
args.output_path = os.path.abspath(args.output_path)
if os.path.exists(args.output_path) and \
not os.path.isdir(args.output_path):
LOG.error("The given output path is not a directory: " +
args.output_path)
sys.exit(1)
if 'enable_all' in args:
LOG.info("'--enable-all' was supplied for this analysis.")
config_option_re = re.compile(r'^({}):.+=.+$'.format(
'|'.join(analyzer_types.supported_analyzers)))
# Check the format of analyzer options.
if 'analyzer_config' in args:
for config in args.analyzer_config:
if not re.match(config_option_re, config):
LOG.error("Analyzer option in wrong format: %s", config)
sys.exit(1)
# Check the format of checker options.
if 'checker_config' in args:
for config in args.checker_config:
if not re.match(config_option_re, config):
LOG.error("Checker option in wrong format: %s", config)
sys.exit(1)
# Process the skip list if present.
skip_handler = __get_skip_handler(args)
# Enable alpha uniqueing by default if ctu analysis is used.
if 'none' in args.compile_uniqueing and 'ctu_phases' in args:
args.compile_uniqueing = "alpha"
compiler_info_file = None
if 'compiler_info_file' in args:
LOG.debug("Compiler info is read from: %s", args.compiler_info_file)
if not os.path.exists(args.compiler_info_file):
LOG.error("Compiler info file %s does not exist",
args.compiler_info_file)
sys.exit(1)
compiler_info_file = args.compiler_info_file
ctu_or_stats_enabled = False
# Skip list is applied only in pre-analysis
# if --ctu-collect was called explicitly.
pre_analysis_skip_handler = None
if 'ctu_phases' in args:
ctu_collect = args.ctu_phases[0]
ctu_analyze = args.ctu_phases[1]
if ctu_collect and not ctu_analyze:
pre_analysis_skip_handler = skip_handler
if ctu_collect or ctu_analyze:
ctu_or_stats_enabled = True
# Skip list is applied only in pre-analysis
# if --stats-collect was called explicitly.
if 'stats_output' in args and args.stats_output:
pre_analysis_skip_handler = skip_handler
ctu_or_stats_enabled = True
if 'stats_enabled' in args and args.stats_enabled:
ctu_or_stats_enabled = True
context = analyzer_context.get_context()
analyzer_env = env.extend(context.path_env_extra,
context.ld_lib_path_extra)
compile_commands = load_json_or_empty(args.logfile)
if compile_commands is None:
sys.exit(1)
# Number of all the compilation commands in the parsed log files,
# logged by the logger.
all_cmp_cmd_count = len(compile_commands)
# We clear the output directory in the following cases.
ctu_dir = os.path.join(args.output_path, 'ctu-dir')
if 'ctu_phases' in args and args.ctu_phases[0] and \
os.path.isdir(ctu_dir):
# Clear the CTU-dir if the user turned on the collection phase.
LOG.debug("Previous CTU contents have been deleted.")
shutil.rmtree(ctu_dir)
if 'clean' in args and os.path.isdir(args.output_path):
LOG.info("Previous analysis results in '%s' have been removed, "
"overwriting with current result", args.output_path)
shutil.rmtree(args.output_path)
if not os.path.exists(args.output_path):
os.makedirs(args.output_path)
# TODO: I'm not sure that this directory should be created here.
fixit_dir = os.path.join(args.output_path, 'fixit')
if not os.path.exists(fixit_dir):
os.makedirs(fixit_dir)
LOG.debug("args: %s", str(args))
LOG.debug("Output will be stored to: '%s'", args.output_path)
analyzer_clang_binary = \
context.analyzer_binaries.get(
clangsa.analyzer.ClangSA.ANALYZER_NAME)
analyzer_clang_version = None
if analyzer_clang_binary:
analyzer_clang_version = clangsa.version.get(analyzer_clang_binary,
analyzer_env)
actions, skipped_cmp_cmd_count = log_parser.parse_unique_log(
compile_commands,
args.output_path,
args.compile_uniqueing,
compiler_info_file,
args.keep_gcc_include_fixed,
args.keep_gcc_intrin,
skip_handler,
pre_analysis_skip_handler,
ctu_or_stats_enabled,
analyzer_env,
analyzer_clang_version)
if not actions:
LOG.info("No analysis is required.\nThere were no compilation "
"commands in the provided compilation database or "
"all of them were skipped.")
sys.exit(0)
uniqued_compilation_db_file = os.path.join(
args.output_path, "unique_compile_commands.json")
with open(uniqued_compilation_db_file, 'w',
encoding="utf-8", errors="ignore") as f:
json.dump(actions, f,
cls=log_parser.CompileCommandEncoder)
metadata = {
'version': 2,
'tools': [{
'name': 'codechecker',
'action_num': len(actions),
'command': sys.argv,
'version': "{0} ({1})".format(context.package_git_tag,
context.package_git_hash),
'working_directory': os.getcwd(),
'output_path': args.output_path,
'result_source_files': {},
'analyzers': {}
}]}
metadata_tool = metadata['tools'][0]
if 'name' in args:
metadata_tool['run_name'] = args.name
# Update metadata dictionary with old values.
metadata_file = os.path.join(args.output_path, 'metadata.json')
metadata_prev = None
if os.path.exists(metadata_file):
metadata_prev = load_json_or_empty(metadata_file)
metadata_tool['result_source_files'] = \
__get_result_source_files(metadata_prev)
CompileCmdParseCount = \
collections.namedtuple('CompileCmdParseCount',
'total, analyze, skipped, removed_by_uniqueing')
cmp_cmd_to_be_uniqued = all_cmp_cmd_count - skipped_cmp_cmd_count
# Number of compile commands removed during uniqueing.
removed_during_uniqueing = cmp_cmd_to_be_uniqued - len(actions)
all_to_be_analyzed = cmp_cmd_to_be_uniqued - removed_during_uniqueing
compile_cmd_count = CompileCmdParseCount(
total=all_cmp_cmd_count,
analyze=all_to_be_analyzed,
skipped=skipped_cmp_cmd_count,
removed_by_uniqueing=removed_during_uniqueing)
LOG.debug_analyzer("Total number of compile commands without "
"skipping or uniqueing: %d", compile_cmd_count.total)
LOG.debug_analyzer("Compile commands removed by uniqueing: %d",
compile_cmd_count.removed_by_uniqueing)
LOG.debug_analyzer("Compile commands skipped during log processing: %d",
compile_cmd_count.skipped)
LOG.debug_analyzer("Compile commands forwarded for analysis: %d",
compile_cmd_count.analyze)
analyzer.perform_analysis(args, skip_handler, context, actions,
metadata_tool,
compile_cmd_count)
__update_skip_file(args)
__cleanup_metadata(metadata_prev, metadata)
LOG.debug("Analysis metadata write to '%s'", metadata_file)
with open(metadata_file, 'w',
encoding="utf-8", errors="ignore") as metafile:
json.dump(metadata, metafile)
# WARN: store command will search for this file!!!!
compile_cmd_json = os.path.join(args.output_path, 'compile_cmd.json')
try:
source = os.path.abspath(args.logfile)
target = os.path.abspath(compile_cmd_json)
if source != target:
shutil.copyfile(source, target)
except shutil.Error:
LOG.debug("Compilation database JSON file is the same.")
except Exception:
LOG.debug("Copying compilation database JSON file failed.")
try:
# pylint: disable=no-name-in-module
from codechecker_analyzer import analyzer_statistics
analyzer_statistics.collect(metadata, "analyze")
except Exception:
pass
# Generally exit status is set by sys.exit() call in CodeChecker. However,
# exit code 3 has a special meaning: it returns when the underlying
# analyzer tool fails.
# "CodeChecker analyze" is special in the sense that it can be invoked
# either top-level or through "CodeChecker check". In the latter case
# "CodeChecker check" should have the same exit status. Calling sys.exit()
# at this specific point is not an option, because the remaining statements
# of "CodeChecker check" after the analysis wouldn't execute.
for analyzer_data in metadata_tool['analyzers'].values():
if analyzer_data['analyzer_statistics']['failed'] != 0:
return 3
def main(args):
"""
Entry point for parsing some analysis results and printing them to the
stdout in a human-readable format.
"""
# If the given output format is not 'table', redirect logger's output to
# the stderr.
stream = None
if 'export' in args and args.export not in [None, 'table', 'html']:
stream = 'stderr'
init_logger(args.verbose if 'verbose' in args else None, stream)
try:
cmd_config.check_config_file(args)
except FileNotFoundError as fnerr:
LOG.error(fnerr)
sys.exit(1)
export = args.export if 'export' in args else None
if export == 'html' and 'output_path' not in args:
LOG.error("Argument --export not allowed without argument --output "
"when exporting to HTML.")
sys.exit(1)
if export == 'gerrit' and not gerrit.mandatory_env_var_is_set():
sys.exit(1)
if export and export not in EXPORT_TYPES:
LOG.error("Unknown export format: %s", export)
sys.exit(1)
context = analyzer_context.get_context()
# To ensure the help message prints the default folder properly,
# the 'default' for 'args.input' is a string, not a list.
# But we need lists for the foreach here to work.
if isinstance(args.input, str):
args.input = [args.input]
src_comment_status_filter = args.review_status
suppr_handler = None
if 'suppress' in args:
__make_handler = False
if not os.path.isfile(args.suppress):
if 'create_suppress' in args:
with open(args.suppress,
'w',
encoding='utf-8',
errors='ignore') as _:
# Just create the file.
__make_handler = True
LOG.info(
"Will write source-code suppressions to "
"suppress file: %s", args.suppress)
else:
LOG.warning(
"Suppress file '%s' given, but it does not exist"
" -- will not suppress anything.", args.suppress)
else:
__make_handler = True
if __make_handler:
suppr_handler = suppress_handler.\
GenericSuppressHandler(args.suppress,
'create_suppress' in args,
src_comment_status_filter)
elif 'create_suppress' in args:
LOG.error("Can't use '--export-source-suppress' unless '--suppress "
"SUPPRESS_FILE' is also given.")
sys.exit(1)
output_dir_path = None
output_file_path = None
if 'output_path' in args:
output_path = os.path.abspath(args.output_path)
if export == 'html':
output_dir_path = output_path
else:
if os.path.exists(output_path) and os.path.isdir(output_path):
# For backward compatibility reason we handle the use case
# when directory is provided to this command.
LOG.error(
"Please provide a file path instead of a directory "
"for '%s' export type!", export)
sys.exit(1)
if export == 'baseline' and not baseline.check(output_path):
LOG.error("Baseline files must have '.baseline' extensions.")
sys.exit(1)
output_file_path = output_path
output_dir_path = os.path.dirname(output_file_path)
if not os.path.exists(output_dir_path):
os.makedirs(output_dir_path)
def get_output_file_path(default_file_name: str) -> Optional[str]:
""" Return an output file path. """
if output_file_path:
return output_file_path
if output_dir_path:
return os.path.join(output_dir_path, default_file_name)
skip_handlers = SkipListHandlers()
if 'files' in args:
items = [f"+{file_path}" for file_path in args.files]
items.append("-*")
skip_handlers.append(SkipListHandler("\n".join(items)))
if 'skipfile' in args:
with open(args.skipfile, 'r', encoding='utf-8', errors='ignore') as f:
skip_handlers.append(SkipListHandler(f.read()))
trim_path_prefixes = args.trim_path_prefix if \
'trim_path_prefix' in args else None
all_reports = []
statistics = Statistics()
file_cache = {} # For memory effiency.
changed_files: Set[str] = set()
processed_path_hashes = set()
processed_file_paths = set()
print_steps = 'print_steps' in args
html_builder: Optional[report_to_html.HtmlBuilder] = None
if export == 'html':
html_builder = report_to_html.HtmlBuilder(
context.path_plist_to_html_dist, context.checker_labels)
for dir_path, file_paths in report_file.analyzer_result_files(args.input):
metadata = get_metadata(dir_path)
for file_path in file_paths:
reports = report_file.get_reports(file_path,
context.checker_labels,
file_cache)
reports = reports_helper.skip(reports, processed_path_hashes,
skip_handlers, suppr_handler,
src_comment_status_filter)
statistics.num_of_analyzer_result_files += 1
for report in reports:
if report.changed_files:
changed_files.update(report.changed_files)
statistics.add_report(report)
if trim_path_prefixes:
report.trim_path_prefixes(trim_path_prefixes)
all_reports.extend(reports)
# Print reports continously.
if not export:
file_report_map = plaintext.get_file_report_map(
reports, file_path, metadata)
plaintext.convert(file_report_map, processed_file_paths,
print_steps)
elif export == 'html':
print(f"Parsing input file '{file_path}'.")
report_to_html.convert(file_path, reports, output_dir_path,
html_builder)
if export is None: # Plain text output
statistics.write()
elif export == 'html':
html_builder.finish(output_dir_path, statistics)
elif export == 'json':
data = report_to_json.convert(all_reports)
dump_json_output(data, get_output_file_path("reports.json"))
elif export == 'codeclimate':
data = codeclimate.convert(all_reports)
dump_json_output(data, get_output_file_path("reports.json"))
elif export == 'gerrit':
data = gerrit.convert(all_reports)
dump_json_output(data, get_output_file_path("reports.json"))
elif export == 'baseline':
data = baseline.convert(all_reports)
output_path = get_output_file_path("reports.baseline")
if output_path:
baseline.write(output_path, data)
reports_helper.dump_changed_files(changed_files)
if statistics.num_of_reports:
sys.exit(2)
