def main(args):
"""
Get and print the version information from the version config
file and Thrift API definition.
"""
logger.setup_logger(args.verbose if 'verbose' in args else None)
output_format = args.output_format
has_analyzer_version = False
try:
from libcodechecker.libhandlers import analyzer_version
has_analyzer_version = True
# Print analyzer version information.
print("CodeChecker analyzer version:")
analyzer_version.print_version(output_format)
except Exception:
pass
try:
from libcodechecker.libhandlers import webserver_version
if has_analyzer_version:
print() # Print a new line to separate version information.
# Print web server version information.
print("CodeChecker web server version:")
webserver_version.print_version(output_format)
except Exception:
pass
def main(args):
"""
List the analyzers' basic information supported by CodeChecker.
"""
logger.setup_logger(args.verbose if 'verbose' in args else None)
context = generic_package_context.get_context()
working, errored = \
analyzer_types.check_supported_analyzers(
analyzer_types.supported_analyzers,
context)
if args.dump_config:
binary = context.analyzer_binaries.get(args.dump_config)
if args.dump_config == 'clang-tidy':
subprocess.call([binary, '-dump-config', '-checks=*'])
elif args.dump_config == 'clangsa':
# TODO: Not supported by ClangSA yet!
LOG.warning("'--dump-config clangsa' is not supported yet.")
return
if args.output_format not in ['csv', 'json']:
if 'details' not in args:
header = ['Name']
else:
header = ['Name', 'Path', 'Version']
else:
if 'details' not in args:
header = ['name']
else:
header = ['name', 'path', 'version_string']
rows = []
for analyzer in working:
if 'details' not in args:
rows.append([analyzer])
else:
binary = context.analyzer_binaries.get(analyzer)
try:
version = subprocess.check_output([binary, '--version'])
except (subprocess.CalledProcessError, OSError):
version = 'ERROR'
rows.append([analyzer, binary, version])
if 'all' in args:
for analyzer, err_reason in errored:
if 'details' not in args:
rows.append([analyzer])
else:
rows.append([
analyzer,
context.analyzer_binaries.get(analyzer), err_reason
])
if len(rows) > 0:
print(output_formatters.twodim_to_str(args.output_format, header,
rows))
def main(args):
"""
Get and print the version information from the version config
file and Thrift API definition.
"""
logger.setup_logger(args.verbose if 'verbose' in args else None)
print_version(args.output_format)
def main(args):
"""
Generates a build log by running the original build command.
No analysis is done.
"""
logger.setup_logger(args.verbose if 'verbose' in args else None)
args.logfile = os.path.realpath(args.logfile)
if os.path.exists(args.logfile):
os.remove(args.logfile)
context = generic_package_context.get_context()
build_manager.perform_build_command(args.logfile,
args.command,
context,
silent='quiet' in args)
def main(args):
"""
Get and print the version information from the version config
file and Thrift API definition.
"""
logger.setup_logger(args.verbose if 'verbose' in args else None)
context = generic_package_context.get_context()
server_versions = [
'{0}.{1}'.format(major, minor)
for major, minor in version.SUPPORTED_VERSIONS.items()
]
if args.output_format != 'json':
server_versions = ', '.join(server_versions)
rows = [("Base package version", context.version),
("Package build date", context.package_build_date),
("Git commit ID (hash)", context.package_git_hash),
("Git tag information", context.package_git_tag),
("Configuration schema", str(context.product_db_version_info)),
("Database schema", str(context.run_db_version_info)),
("Server supported API (Thrift)", server_versions),
("Client API (Thrift)", version.CLIENT_API)]
if args.output_format != "json":
print(
output_formatters.twodim_to_str(args.output_format,
["Kind", "Version"], rows))
elif args.output_format == "json":
# Use a special JSON format here, instead of
# [ {"kind": "something", "version": "0.0.0"}, {"kind": "foo", ... } ]
# do
# { "something": "0.0.0", "foo": ... }
print(json.dumps(dict(rows)))
def main(args):
"""
Execute a wrapper over log-analyze-parse, aka 'check'.
"""
logger.setup_logger(args.verbose if 'verbose' in args else None)
def __load_module(name):
"""Loads the given subcommand's definition from the libs."""
try:
module = libhandlers.load_module(name)
except ImportError:
LOG.error("Couldn't import subcommand '" + name + "'.")
raise
return module
def __update_if_key_exists(source, target, key):
"""Append the source Namespace's element with 'key' to target with
the same key, but only if it exists."""
if key in source:
setattr(target, key, getattr(source, key))
# If no output directory is given then the checker results go to a
# temporary directory. This way the subsequent "quick" checks don't pollute
# the result list of a previous check. If the detection status function is
# intended to be used (i.e. by keeping the .plist files) then the output
# directory has to be provided explicitly.
is_temp_output = False
if 'output_dir' in args:
output_dir = args.output_dir
else:
output_dir = tempfile.mkdtemp()
is_temp_output = True
output_dir = os.path.abspath(output_dir)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
logfile = None
is_command = False
try:
# --- Step 1.: Perform logging if build command was specified.
if 'command' in args:
logfile = tempfile.NamedTemporaryFile().name
is_command = True
# Translate the argument list between check and log.
log_args = argparse.Namespace(command=args.command,
logfile=logfile)
__update_if_key_exists(args, log_args, 'quiet')
__update_if_key_exists(args, log_args, 'verbose')
log_module = __load_module('log')
LOG.debug("Calling LOG with args:")
LOG.debug(log_args)
log_module.main(log_args)
elif 'logfile' in args:
logfile = args.logfile
# --- Step 2.: Perform the analysis.
if not os.path.exists(logfile):
raise OSError("The specified logfile '" + logfile + "' does not "
"exist.")
analyze_args = argparse.Namespace(logfile=[logfile],
output_path=output_dir,
output_format='plist',
jobs=args.jobs)
# Some arguments don't have default values.
# We can't set these keys to None because it would result in an error
# after the call.
args_to_update = [
'quiet',
'skipfile',
'analyzers',
'add_compiler_defaults',
'clangsa_args_cfg_file',
'tidy_args_cfg_file',
'tidy_config',
'capture_analysis_output',
'ctu_phases',
'stats_output',
'stats_dir',
'stats_enabled',
'stats_relevance_threshold',
'stats_min_sample_count',
'enable_all',
'ordered_checkers', # --enable and --disable.
'timeout',
'report_hash'
]
for key in args_to_update:
__update_if_key_exists(args, analyze_args, key)
if 'force' in args or 'clean' in args:
setattr(analyze_args, 'clean', True)
__update_if_key_exists(args, analyze_args, 'verbose')
analyze_module = __load_module('analyze')
LOG.debug("Calling ANALYZE with args:")
LOG.debug(analyze_args)
analyze_module.main(analyze_args)
# --- Step 3.: Print to stdout.
parse_args = argparse.Namespace(input=[output_dir],
input_format='plist')
__update_if_key_exists(args, parse_args, 'print_steps')
__update_if_key_exists(args, parse_args, 'verbose')
__update_if_key_exists(args, parse_args, 'skipfile')
parse_module = __load_module('parse')
LOG.debug("Calling PARSE with args:")
LOG.debug(parse_args)
parse_module.main(parse_args)
except ImportError:
LOG.error("Check failed: couldn't import a library.")
except Exception as ex:
LOG.error("Running check failed. " + ex.message)
import traceback
traceback.print_exc()
finally:
if is_temp_output:
shutil.rmtree(output_dir)
if is_command:
os.remove(logfile)
LOG.debug("Check finished.")
def main(args):
"""
Perform analysis on the given logfiles and store the results in a machine-
readable format.
"""
logger.setup_logger(args.verbose if 'verbose' in args else None)
if len(args.logfile) != 1:
LOG.warning("Only one log file can be processed right now!")
sys.exit(1)
args.output_path = os.path.abspath(args.output_path)
if os.path.exists(args.output_path) and \
not os.path.isdir(args.output_path):
LOG.error("The given output path is not a directory: " +
args.output_path)
sys.exit(1)
if 'enable_all' in args:
LOG.info("'--enable-all' was supplied for this analysis.")
# We clear the output directory in the following cases.
ctu_dir = os.path.join(args.output_path, 'ctu-dir')
if 'ctu_phases' in args and args.ctu_phases[0] and \
os.path.isdir(ctu_dir):
# Clear the CTU-dir if the user turned on the collection phase.
LOG.debug("Previous CTU contents have been deleted.")
shutil.rmtree(ctu_dir)
if 'clean' in args and os.path.isdir(args.output_path):
LOG.info("Previous analysis results in '{0}' have been removed, "
"overwriting with current result".format(args.output_path))
shutil.rmtree(args.output_path)
if not os.path.exists(args.output_path):
os.makedirs(args.output_path)
LOG.debug("args: " + str(args))
LOG.debug("Output will be stored to: '" + args.output_path + "'")
# Parse the JSON CCDBs and retrieve the compile commands.
actions = []
for log_file in args.logfile:
if not os.path.exists(log_file):
LOG.error("The specified logfile '" + log_file + "' does not "
"exist!")
continue
parseLogOptions = ParseLogOptions(args)
actions += log_parser.parse_log(log_file, parseLogOptions)
if len(actions) == 0:
LOG.info("None of the specified build log files contained "
"valid compilation commands. No analysis needed...")
sys.exit(1)
context = generic_package_context.get_context()
metadata = {
'action_num': len(actions),
'command': sys.argv,
'versions': {
'codechecker':
"{0} ({1})".format(context.package_git_tag,
context.package_git_hash)
},
'working_directory': os.getcwd(),
'output_path': args.output_path,
'result_source_files': {}
}
if 'name' in args:
metadata['name'] = args.name
# Update metadata dictionary with old values.
metadata_file = os.path.join(args.output_path, 'metadata.json')
if os.path.exists(metadata_file):
with open(metadata_file, 'r') as data:
metadata_prev = json.load(data)
metadata['result_source_files'] =\
metadata_prev['result_source_files']
analyzer.perform_analysis(args, context, actions, metadata)
LOG.debug("Analysis metadata write to '" + metadata_file + "'")
with open(metadata_file, 'w') as metafile:
json.dump(metadata, metafile)
# WARN: store command will search for this file!!!!
compile_cmd_json = os.path.join(args.output_path, 'compile_cmd.json')
try:
source = os.path.abspath(args.logfile[0])
target = os.path.abspath(compile_cmd_json)
if source != target:
shutil.copyfile(source, target)
except shutil.Error:
LOG.debug("Compilation database JSON file is the same.")
except Exception:
LOG.debug("Copying compilation database JSON file failed.")
def main(args):
"""
Entry point for parsing some analysis results and printing them to the
stdout in a human-readable format.
"""
logger.setup_logger(args.verbose if 'verbose' in args else None)
context = package_context.get_context()
# To ensure the help message prints the default folder properly,
# the 'default' for 'args.input' is a string, not a list.
# But we need lists for the foreach here to work.
if isinstance(args.input, str):
args.input = [args.input]
original_cwd = os.getcwd()
suppr_handler = None
if 'suppress' in args:
__make_handler = False
if not os.path.isfile(args.suppress):
if 'create_suppress' in args:
with open(args.suppress, 'w') as _:
# Just create the file.
__make_handler = True
LOG.info("Will write source-code suppressions to "
"suppress file.")
else:
LOG.warning("Suppress file '" + args.suppress + "' given, but "
"it does not exist -- will not suppress anything.")
else:
__make_handler = True
if __make_handler:
suppr_handler = suppress_handler.\
GenericSuppressHandler(args.suppress,
'create_suppress' in args)
elif 'create_suppress' in args:
LOG.error("Can't use '--export-source-suppress' unless '--suppress "
"SUPPRESS_FILE' is also given.")
sys.exit(2)
processed_path_hashes = set()
def skip_html_report_data_handler(report_hash, source_file, report_line,
checker_name, diag, files):
"""
Report handler which skips bugs which were suppressed by source code
comments.
"""
report = Report(None, diag['path'], files)
path_hash = get_report_path_hash(report, files)
if path_hash in processed_path_hashes:
LOG.debug("Skip report because it is a deduplication of an "
"already processed report!")
LOG.debug("Path hash: %s", path_hash)
LOG.debug(diag)
return True
skip = plist_parser.skip_report(report_hash, source_file, report_line,
checker_name, suppr_handler)
if not skip:
processed_path_hashes.add(path_hash)
return skip
skip_handler = None
if 'skipfile' in args:
with open(args.skipfile, 'r') as skip_file:
skip_handler = SkipListHandler(skip_file.read())
html_builder = None
for input_path in args.input:
input_path = os.path.abspath(input_path)
os.chdir(original_cwd)
LOG.debug("Parsing input argument: '" + input_path + "'")
export = args.export if 'export' in args else None
if export is not None and export == 'html':
output_path = os.path.abspath(args.output_path)
if not html_builder:
html_builder = \
PlistToHtml.HtmlBuilder(context.path_plist_to_html_dist,
context.severity_map)
LOG.info("Generating html output files:")
PlistToHtml.parse(input_path, output_path,
context.path_plist_to_html_dist,
skip_html_report_data_handler, html_builder)
continue
files = []
metadata_dict = {}
if os.path.isfile(input_path):
files.append(input_path)
elif os.path.isdir(input_path):
metadata_file = os.path.join(input_path, "metadata.json")
if os.path.exists(metadata_file):
metadata_dict = util.load_json_or_empty(metadata_file)
LOG.debug(metadata_dict)
if 'working_directory' in metadata_dict:
working_dir = metadata_dict['working_directory']
try:
os.chdir(working_dir)
except OSError as oerr:
LOG.debug(oerr)
LOG.error(
"Working directory %s is missing.\n"
"Can not parse reports safely.", working_dir)
sys.exit(1)
_, _, file_names = next(os.walk(input_path), ([], [], []))
files = [
os.path.join(input_path, file_name) for file_name in file_names
]
file_change = set()
file_report_map = defaultdict(list)
rh = plist_parser.PlistToPlaintextFormatter(suppr_handler,
skip_handler,
context.severity_map,
processed_path_hashes)
rh.print_steps = 'print_steps' in args
for file_path in files:
f_change = parse(file_path, metadata_dict, rh, file_report_map)
file_change = file_change.union(f_change)
report_stats = rh.write(file_report_map)
severity_stats = report_stats.get('severity')
file_stats = report_stats.get('files')
reports_stats = report_stats.get('reports')
print("\n----==== Summary ====----")
if file_stats:
vals = [[os.path.basename(k), v]
for k, v in dict(file_stats).items()]
keys = ['Filename', 'Report count']
table = twodim_to_str('table', keys, vals, 1, True)
print(table)
if severity_stats:
vals = [[k, v] for k, v in dict(severity_stats).items()]
keys = ['Severity', 'Report count']
table = twodim_to_str('table', keys, vals, 1, True)
print(table)
report_count = reports_stats.get("report_count", 0)
print("----=================----")
print("Total number of reports: {}".format(report_count))
print("----=================----")
if file_change:
changed_files = '\n'.join([' - ' + f for f in file_change])
LOG.warning("The following source file contents changed since the "
"latest analysis:\n{0}\nMultiple reports were not "
"shown and skipped from the statistics. Please "
"analyze your project again to update the "
"reports!".format(changed_files))
os.chdir(original_cwd)
# Create index.html for the generated html files.
if html_builder:
html_builder.create_index_html(args.output_path)
def main(args):
"""
Store the defect results in the specified input list as bug reports in the
database.
"""
logger.setup_logger(args.verbose if 'verbose' in args else None)
if not host_check.check_zlib():
raise Exception("zlib is not available on the system!")
# To ensure the help message prints the default folder properly,
# the 'default' for 'args.input' is a string, not a list.
# But we need lists for the foreach here to work.
if isinstance(args.input, str):
args.input = [args.input]
if 'name' not in args:
LOG.debug("Generating name for analysis...")
generated = __get_run_name(args.input)
if generated:
setattr(args, 'name', generated)
else:
LOG.error("No suitable name was found in the inputs for the "
"analysis run. Please specify one by passing argument "
"--name run_name in the invocation.")
sys.exit(2) # argparse returns error code 2 for bad invocations.
LOG.info("Storing analysis results for run '" + args.name + "'")
if 'force' in args:
LOG.info("argument --force was specified: the run with name '" +
args.name + "' will be deleted.")
protocol, host, port, product_name = split_product_url(args.product_url)
# Before any transmission happens, check if we have the PRODUCT_STORE
# permission to prevent a possibly long ZIP operation only to get an
# error later on.
product_client = libclient.setup_product_client(protocol, host, port,
product_name)
product_id = product_client.getCurrentProduct().id
auth_client, _ = libclient.setup_auth_client(protocol, host, port)
has_perm = libclient.check_permission(auth_client,
Permission.PRODUCT_STORE,
{'productID': product_id})
if not has_perm:
LOG.error("You are not authorised to store analysis results in "
"product '{0}'".format(product_name))
sys.exit(1)
# Setup connection to the remote server.
client = libclient.setup_client(args.product_url, product_client=False)
LOG.debug("Initializing client connecting to {0}:{1}/{2} done.".format(
host, port, product_name))
_, zip_file = tempfile.mkstemp('.zip')
LOG.debug("Will write mass store ZIP to '{0}'...".format(zip_file))
try:
assemble_zip(args.input, zip_file, client)
if os.stat(zip_file).st_size > MAX_UPLOAD_SIZE:
LOG.error("The result list to upload is too big (max: {}).".format(
sizeof_fmt(MAX_UPLOAD_SIZE)))
sys.exit(1)
with open(zip_file, 'rb') as zf:
b64zip = base64.b64encode(zf.read())
context = generic_package_context.get_context()
trim_path_prefixes = args.trim_path_prefix if \
'trim_path_prefix' in args else None
client.massStoreRun(args.name, args.tag if 'tag' in args else None,
str(context.version), b64zip, 'force' in args,
trim_path_prefixes)
LOG.info("Storage finished successfully.")
except RequestFailed as reqfail:
if reqfail.errorCode == ErrorCode.SOURCE_FILE:
header = ['File', 'Line', 'Checker name']
table = twodim_to_str('table', header,
[c.split('|') for c in reqfail.extraInfo])
LOG.warning("Setting the review statuses for some reports failed "
"because of non valid source code comments: "
"{0}\n {1}".format(reqfail.message, table))
sys.exit(1)
except Exception as ex:
LOG.info("Storage failed: " + str(ex))
sys.exit(1)
finally:
os.remove(zip_file)
def main(args):
"""
List the checkers available in the specified (or all supported) analyzers
alongside with their description or enabled status in various formats.
"""
logger.setup_logger(args.verbose if 'verbose' in args else None)
# If nothing is set, list checkers for all supported analyzers.
analyzers = args.analyzers \
if 'analyzers' in args \
else analyzer_types.supported_analyzers
context = package_context.get_context()
working, errored = analyzer_types.check_supported_analyzers(
analyzers, context)
analyzer_environment = analyzer_env.get_check_env(
context.path_env_extra, context.ld_lib_path_extra)
analyzer_config_map = analyzer_types.build_config_handlers(
args, context, working)
# List available checker profiles.
if 'profile' in args and args.profile == 'list':
if 'details' not in args:
if args.output_format not in ['csv', 'json']:
header = ['Profile name']
else:
header = ['profile_name']
else:
if args.output_format not in ['csv', 'json']:
header = ['Profile name', 'Description']
else:
header = ['profile_name', 'description']
rows = []
for (profile, description) in context.available_profiles.items():
if 'details' not in args:
rows.append([profile])
else:
rows.append([profile, description])
print(output_formatters.twodim_to_str(args.output_format, header,
rows))
return
# Use good looking different headers based on format.
if 'details' not in args:
if args.output_format not in ['csv', 'json']:
header = ['Name']
else:
header = ['name']
else:
if args.output_format not in ['csv', 'json']:
header = ['', 'Name', 'Analyzer', 'Severity', 'Description']
else:
header = ['enabled', 'name', 'analyzer', 'severity', 'description']
rows = []
for analyzer in working:
config_handler = analyzer_config_map.get(analyzer)
analyzer_class = \
analyzer_types.supported_analyzers[analyzer]
checkers = analyzer_class.get_analyzer_checkers(
config_handler, analyzer_environment)
default_checker_cfg = context.checker_config.get(analyzer +
'_checkers')
profile_checkers = None
if 'profile' in args:
if args.profile not in context.available_profiles:
LOG.error("Checker profile '" + args.profile +
"' does not exist!")
LOG.error("To list available profiles, use '--profile list'.")
return
profile_checkers = [(args.profile, True)]
config_handler.initialize_checkers(context.available_profiles,
context.package_root, checkers,
default_checker_cfg,
profile_checkers)
for checker_name, value in config_handler.checks().items():
enabled, description = value
if not enabled and 'profile' in args:
continue
if enabled and 'only_disabled' in args:
continue
elif not enabled and 'only_enabled' in args:
continue
if args.output_format != 'json':
enabled = '+' if enabled else '-'
if 'details' not in args:
rows.append([checker_name])
else:
severity = context.severity_map.get(checker_name)
rows.append(
[enabled, checker_name, analyzer, severity, description])
if len(rows) > 0:
print(output_formatters.twodim_to_str(args.output_format, header,
rows))
for analyzer_binary, reason in errored:
LOG.error("Failed to get checkers for '" + analyzer_binary +
"'! The error reason was: '" + reason + "'")
LOG.error("Please check your installation and the "
"'config/package_layout.json' file!")
def main(args):
"""
Entry point for parsing some analysis results and printing them to the
stdout in a human-readable format.
"""
logger.setup_logger(args.verbose if 'verbose' in args else None)
context = generic_package_context.get_context()
# To ensure the help message prints the default folder properly,
# the 'default' for 'args.input' is a string, not a list.
# But we need lists for the foreach here to work.
if isinstance(args.input, str):
args.input = [args.input]
original_cwd = os.getcwd()
suppress_handler = None
if 'suppress' in args:
__make_handler = False
if not os.path.isfile(args.suppress):
if 'create_suppress' in args:
with open(args.suppress, 'w') as _:
# Just create the file.
__make_handler = True
LOG.info("Will write source-code suppressions to "
"suppress file.")
else:
LOG.warning("Suppress file '" + args.suppress + "' given, but "
"it does not exist -- will not suppress anything.")
else:
__make_handler = True
if __make_handler:
suppress_handler = generic_package_suppress_handler.\
GenericSuppressHandler(args.suppress,
'create_suppress' in args)
elif 'create_suppress' in args:
LOG.error("Can't use '--export-source-suppress' unless '--suppress "
"SUPPRESS_FILE' is also given.")
sys.exit(2)
skip_handler = None
if 'skipfile' in args:
skip_handler = SkipListHandler(args.skipfile)
for input_path in args.input:
input_path = os.path.abspath(input_path)
os.chdir(original_cwd)
LOG.debug("Parsing input argument: '" + input_path + "'")
export = args.export if 'export' in args else None
if export is not None and export == 'html':
output_path = os.path.abspath(args.output_path)
LOG.info("Generating html output files:")
PlistToHtml.parse(input_path, output_path,
context.path_plist_to_html_dist, 'clean' in args)
continue
severity_stats = Counter({})
file_stats = Counter({})
report_count = Counter({})
files = []
metadata_dict = {}
if os.path.isfile(input_path):
files.append(input_path)
elif os.path.isdir(input_path):
metadata_file = os.path.join(input_path, "metadata.json")
if os.path.exists(metadata_file):
with open(metadata_file, 'r') as metadata:
metadata_dict = json.load(metadata)
LOG.debug(metadata_dict)
if 'working_directory' in metadata_dict:
os.chdir(metadata_dict['working_directory'])
_, _, file_names = next(os.walk(input_path), ([], [], []))
files = [
os.path.join(input_path, file_name) for file_name in file_names
]
for file_path in files:
report_stats = parse(file_path, context, metadata_dict,
suppress_handler, skip_handler, 'print_steps'
in args)
severity_stats.update(Counter(report_stats.get('severity', {})))
file_stats.update(Counter(report_stats.get('files', {})))
report_count.update(Counter(report_stats.get('reports', {})))
print("\n----==== Summary ====----")
if file_stats:
vals = [[os.path.basename(k), v]
for k, v in dict(file_stats).items()]
keys = ['Filename', 'Report count']
table = twodim_to_str('table', keys, vals, 1, True)
print(table)
if severity_stats:
vals = [[k, v] for k, v in dict(severity_stats).items()]
keys = ['Severity', 'Report count']
table = twodim_to_str('table', keys, vals, 1, True)
print(table)
report_count = dict(report_count).get("report_count", 0)
print("----=================----")
print("Total number of reports: {}".format(report_count))
print("----=================----")
os.chdir(original_cwd)
def init_logger(level, logger_name='system'):
logger.setup_logger(level)
global LOG
LOG = logger.get_logger(logger_name)
