def test_new_ldlogger(self):
"""
Test log file parsing escape behaviour with after-#631 LD-LOGGER.
"""
logfile = os.path.join(self.__test_files, "ldlogger-new.json")
# LD-LOGGERS after http://github.com/Ericsson/codechecker/pull/631
# now properly log the multiword arguments. When these are parsed by
# the log_parser, the define's value will be passed to the analyzer.
#
# Logfile contains -DVARIABLE="some value"
# and --target=x86_64-linux-gnu.
build_action = log_parser.parse_log(logfile, ParseLogOptions())[0]
self.assertEqual(list(build_action.sources)[0], r'/tmp/a.cpp')
self.assertEqual(len(build_action.analyzer_options), 1)
self.assertTrue(len(build_action.target) > 0)
self.assertEqual(build_action.analyzer_options[0],
r'-DVARIABLE="\"some value"\"')
# Test source file with spaces.
logfile = os.path.join(self.__test_files, "ldlogger-new-space.json")
build_action = log_parser.parse_log(logfile, ParseLogOptions())[0]
self.assertEqual(list(build_action.sources)[0], r'/tmp/a b.cpp')
self.assertEqual(build_action.lang, 'c++')
def test_old_intercept_build(self):
"""
Test log file parsing escape behaviour with clang-5.0 intercept-build.
"""
logfile = os.path.join(self.__test_files, "intercept-old.json")
# Scan-build-py shipping with clang-5.0 makes a logfile that contains:
# -DVARIABLE=\"some value\" and --target=x86_64-linux-gnu
#
# The define is passed to the analyzer properly.
build_action = log_parser.parse_log(logfile, ParseLogOptions())[0]
self.assertEqual(list(build_action.sources)[0], r'/tmp/a.cpp')
self.assertEqual(len(build_action.analyzer_options), 1)
self.assertTrue(len(build_action.target) > 0)
self.assertEqual(build_action.analyzer_options[0],
r'-DVARIABLE="\"some value"\"')
# Test source file with spaces.
logfile = os.path.join(self.__test_files, "intercept-old-space.json")
build_action = log_parser.parse_log(logfile, ParseLogOptions())[0]
self.assertEqual(list(build_action.sources)[0], r'/tmp/a b.cpp')
self.assertEqual(build_action.lang, 'c++')
def test_new_intercept_build(self):
"""
Test log file parsing escapes with upstream (GitHub) intercept-build.
"""
logfile = os.path.join(self.__test_files, "intercept-new.json")
# Upstream scan-build-py creates an argument vector, as opposed to a
# command string. This argument vector contains the define as it's
# element in the following format:
# -DVARIABLE=\"some value\"
# and the target triplet, e.g.:
# --target=x86_64-linux-gnu
#
# The define is passed to the analyzer properly.
build_action = log_parser.parse_log(logfile, ParseLogOptions())[0]
self.assertEqual(list(build_action.sources)[0], r'/tmp/a.cpp')
self.assertEqual(len(build_action.analyzer_options), 1)
self.assertTrue(len(build_action.target) > 0)
self.assertEqual(build_action.analyzer_options[0],
r'-DVARIABLE="\"some value"\"')
# Test source file with spaces.
logfile = os.path.join(self.__test_files, "intercept-new-space.json")
build_action = log_parser.parse_log(logfile, ParseLogOptions())[0]
self.assertEqual(list(build_action.sources)[0], r'/tmp/a b.cpp')
self.assertEqual(build_action.lang, 'c++')
def test_old_ldlogger(self):
"""
Test log file parsing escape behaviour with pre-2017 Q2 LD-LOGGER.
"""
logfile = os.path.join(self.__test_files, "ldlogger-old.json")
# LD-LOGGER before http://github.com/Ericsson/codechecker/pull/631
# used an escape mechanism that, when parsed by the log parser via
# shlex, made CodeChecker parse arguments with multiword string
# literals in them be considered as "file" (instead of compile option),
# eventually ignored by the command builder, thus lessening analysis
# accuracy, as defines were lost.
#
# Logfile contains "-DVARIABLE="some value"".
#
# There is no good way to back-and-forth convert in log_parser or
# option_parser, so here we aim for a non-failing stalemate of the
# define being considered a file and ignored, for now.
build_action = log_parser.parse_log(logfile, ParseLogOptions())[0]
results = option_parser.parse_options(build_action.original_command)
self.assertEqual(' '.join(results.files),
r'"-DVARIABLE="some value"" /tmp/a.cpp')
self.assertEqual(len(build_action.analyzer_options), 0)
def _do_quickcheck(args):
"""
Handles the "quickcheck" command.
For arguments see main function in CodeChecker.py.
It also requires an extra property in args object, namely workspace which
is a directory path as a string.
This function is called from handle_quickcheck.
"""
try:
context = generic_package_context.get_context()
context.codechecker_workspace = args.workspace
args.name = "quickcheck"
log_file, set_in_cmdline = build_manager.check_log_file(args, context)
actions = log_parser.parse_log(log_file, args.add_compiler_defaults)
analyzer.run_quick_check(args, context, actions)
except Exception as ex:
LOG.error("Running quickcheck failed.")
finally:
if not args.keep_tmp:
if log_file and not set_in_cmdline:
LOG.debug('Removing temporary log file: ' + log_file)
os.remove(log_file)
def handle_check(args):
"""
Runs the original build and logs the buildactions.
Based on the log runs the analysis.
"""
try:
if not host_check.check_zlib():
sys.exit(1)
args.workspace = os.path.abspath(args.workspace)
if not os.path.isdir(args.workspace):
os.mkdir(args.workspace)
context = generic_package_context.get_context()
context.codechecker_workspace = args.workspace
context.db_username = args.dbusername
log_file, set_in_cmdline = build_manager.check_log_file(args, context)
if not log_file:
LOG.error("Failed to generate compilation command file: " +
log_file)
sys.exit(1)
actions = log_parser.parse_log(log_file, args.add_compiler_defaults)
check_env = analyzer_env.get_check_env(context.path_env_extra,
context.ld_lib_path_extra)
sql_server = SQLServer.from_cmdline_args(args,
context.codechecker_workspace,
context.migration_root,
check_env)
conn_mgr = client.ConnectionManager(sql_server, 'localhost',
util.get_free_port())
sql_server.start(context.db_version_info,
wait_for_start=True,
init=True)
conn_mgr.start_report_server()
LOG.debug("Checker server started.")
analyzer.run_check(args, actions, context)
LOG.info("Analysis has finished.")
log_startserver_hint(args)
except Exception as ex:
LOG.error(ex)
import traceback
print(traceback.format_exc())
finally:
if not args.keep_tmp:
if log_file and not set_in_cmdline:
LOG.debug('Removing temporary log file: ' + log_file)
os.remove(log_file)
def test_include_rel_to_abs(self):
"""
Test working directory prepending to relative include paths.
"""
logfile = os.path.join(self.__test_files, "include.json")
build_action = log_parser.parse_log(logfile, ParseLogOptions())[0]
self.assertEqual(len(build_action.analyzer_options), 3)
self.assertEqual(build_action.analyzer_options[0], '-I/include')
self.assertEqual(build_action.analyzer_options[1], '-I/include')
self.assertEqual(build_action.analyzer_options[2], '-I/tmp')
def test_old_intercept_build(self):
"""
Test log file parsing escape behaviour with clang-5.0 intercept-build.
"""
logfile = os.path.join(self.__test_files, "intercept-old.json")
# Scan-build-py shipping with clang-5.0 makes a logfile that contains:
# -DVARIABLE=\"some value\"
#
# The define is passed to the analyzer properly.
build_action = log_parser.parse_log(logfile)[0]
self.assertEqual(list(build_action.sources)[0], r'/tmp/a.cpp')
self.assertEqual(len(build_action.analyzer_options), 1)
self.assertEqual(build_action.analyzer_options[0],
r'-DVARIABLE="\"some value"\"')
def test_new_intercept_build(self):
"""
Test log file parsing escapes with upstream (GitHub) intercept-build.
"""
logfile = os.path.join(self.__test_files, "intercept-new.json")
# Upstream scan-build-py creates an argument vector, as opposed to a
# command string. This argument vector contains the define as it's
# element in the following format:
# -DVARIABLE=\"some value\"
#
# The define is passed to the analyzer properly.
build_action = log_parser.parse_log(logfile)[0]
self.assertEqual(list(build_action.sources)[0], r'/tmp/a.cpp')
self.assertEqual(len(build_action.analyzer_options), 1)
self.assertEqual(build_action.analyzer_options[0],
r'-DVARIABLE="\"some value"\"')
def main(args):
"""
Perform analysis on the given logfiles and store the results in a machine-
readable format.
"""
context = generic_package_context.get_context()
# Parse the JSON CCDBs and retrieve the compile commands.
actions = []
for log_file in args.logfile:
if not os.path.exists(log_file):
LOG.error("The specified logfile '" + log_file + "' does not "
"exist!")
continue
actions += log_parser.parse_log(log_file,
args.add_compiler_defaults)
if len(actions) == 0:
LOG.info("None of the specified build log files contained "
"valid compilation commands. No analysis needed...")
return
if 'enable_all' in args:
LOG.info("'--enable-all' was supplied for this analysis.")
# Run the analysis.
args.output_path = os.path.abspath(args.output_path)
if os.path.isdir(args.output_path):
LOG.info("Previous analysis results in '{0}' have been "
"removed, overwriting with current result".
format(args.output_path))
shutil.rmtree(args.output_path)
os.makedirs(args.output_path)
LOG.debug("Output will be stored to: '" + args.output_path + "'")
metadata = {'action_num': len(actions),
'command': sys.argv,
'versions': {
'codechecker': "{0} ({1})".format(context.package_git_tag,
context.package_git_hash)
},
'working_directory': os.getcwd(),
'output_path': args.output_path}
if 'name' in args:
metadata['name'] = args.name
if 'skipfile' in args:
# Skip information needs to be saved because reports in a header
# can only be skipped by the report-server used in 'store' later
# on if this information is persisted.
with open(args.skipfile, 'r') as skipfile:
metadata['skip_data'] = [l.strip() for l in skipfile.readlines()]
analyzer.perform_analysis(args, context, actions, metadata)
metadata_path = os.path.join(args.output_path, 'metadata.json')
LOG.debug("Analysis metadata write to '" + metadata_path + "'")
with open(metadata_path, 'w') as metafile:
json.dump(metadata, metafile)
def main(args):
"""
Perform analysis on the given logfiles and store the results in a machine-
readable format.
"""
logger.setup_logger(args.verbose if 'verbose' in args else None)
if len(args.logfile) != 1:
LOG.warning("Only one log file can be processed right now!")
sys.exit(1)
args.output_path = os.path.abspath(args.output_path)
if os.path.exists(args.output_path) and \
not os.path.isdir(args.output_path):
LOG.error("The given output path is not a directory: " +
args.output_path)
sys.exit(1)
if 'enable_all' in args:
LOG.info("'--enable-all' was supplied for this analysis.")
# We clear the output directory in the following cases.
ctu_dir = os.path.join(args.output_path, 'ctu-dir')
if 'ctu_phases' in args and args.ctu_phases[0] and \
os.path.isdir(ctu_dir):
# Clear the CTU-dir if the user turned on the collection phase.
LOG.debug("Previous CTU contents have been deleted.")
shutil.rmtree(ctu_dir)
if 'clean' in args and os.path.isdir(args.output_path):
LOG.info("Previous analysis results in '{0}' have been removed, "
"overwriting with current result".format(args.output_path))
shutil.rmtree(args.output_path)
if not os.path.exists(args.output_path):
os.makedirs(args.output_path)
LOG.debug("args: " + str(args))
LOG.debug("Output will be stored to: '" + args.output_path + "'")
# Parse the JSON CCDBs and retrieve the compile commands.
actions = []
for log_file in args.logfile:
if not os.path.exists(log_file):
LOG.error("The specified logfile '" + log_file + "' does not "
"exist!")
continue
parseLogOptions = ParseLogOptions(args)
actions += log_parser.parse_log(log_file, parseLogOptions)
if len(actions) == 0:
LOG.info("None of the specified build log files contained "
"valid compilation commands. No analysis needed...")
sys.exit(1)
context = generic_package_context.get_context()
metadata = {
'action_num': len(actions),
'command': sys.argv,
'versions': {
'codechecker':
"{0} ({1})".format(context.package_git_tag,
context.package_git_hash)
},
'working_directory': os.getcwd(),
'output_path': args.output_path,
'result_source_files': {}
}
if 'name' in args:
metadata['name'] = args.name
# Update metadata dictionary with old values.
metadata_file = os.path.join(args.output_path, 'metadata.json')
if os.path.exists(metadata_file):
with open(metadata_file, 'r') as data:
metadata_prev = json.load(data)
metadata['result_source_files'] =\
metadata_prev['result_source_files']
analyzer.perform_analysis(args, context, actions, metadata)
LOG.debug("Analysis metadata write to '" + metadata_file + "'")
with open(metadata_file, 'w') as metafile:
json.dump(metadata, metafile)
# WARN: store command will search for this file!!!!
compile_cmd_json = os.path.join(args.output_path, 'compile_cmd.json')
try:
source = os.path.abspath(args.logfile[0])
target = os.path.abspath(compile_cmd_json)
if source != target:
shutil.copyfile(source, target)
except shutil.Error:
LOG.debug("Compilation database JSON file is the same.")
except Exception:
LOG.debug("Copying compilation database JSON file failed.")
def main(args):
"""
Perform analysis on the given logfiles and store the results in a machine-
readable format.
"""
context = generic_package_context.get_context()
# Parse the JSON CCDBs and retrieve the compile commands.
actions = []
if len(args.logfile) != 1:
LOG.warning("Only one log file can be processed right now!")
sys.exit(1)
for log_file in args.logfile:
if not os.path.exists(log_file):
LOG.error("The specified logfile '" + log_file + "' does not "
"exist!")
continue
actions += log_parser.parse_log(log_file, args.add_compiler_defaults)
if len(actions) == 0:
LOG.info("None of the specified build log files contained "
"valid compilation commands. No analysis needed...")
sys.exit(1)
if 'enable_all' in args:
LOG.info("'--enable-all' was supplied for this analysis.")
# Run the analysis.
args.output_path = os.path.abspath(args.output_path)
if 'ctu_phases' not in args or args.ctu_phases[0] or \
(not args.ctu_phases[0] and not args.ctu_phases[1]):
if os.path.isdir(args.output_path):
LOG.info("Previous analysis results in '{0}' have been "
"removed, overwriting with current result".format(
args.output_path))
shutil.rmtree(args.output_path)
os.makedirs(args.output_path)
LOG.debug("Output will be stored to: '" + args.output_path + "'")
metadata = {
'action_num': len(actions),
'command': sys.argv,
'versions': {
'codechecker':
"{0} ({1})".format(context.package_git_tag,
context.package_git_hash)
},
'working_directory': os.getcwd(),
'output_path': args.output_path
}
if 'name' in args:
metadata['name'] = args.name
if 'skipfile' in args:
# Skip information needs to be saved because reports in a header
# can only be skipped by the report-server used in 'store' later
# on if this information is persisted.
with open(args.skipfile, 'r') as skipfile:
metadata['skip_data'] = [l.strip() for l in skipfile.readlines()]
analyzer.perform_analysis(args, context, actions, metadata)
metadata_path = os.path.join(args.output_path, 'metadata.json')
LOG.debug("Analysis metadata write to '" + metadata_path + "'")
with open(metadata_path, 'w') as metafile:
json.dump(metadata, metafile)
# WARN: store command will search for this file!!!!
compile_cmd_json = os.path.join(args.output_path, 'compile_cmd.json')
try:
source = os.path.abspath(args.logfile[0])
target = os.path.abspath(compile_cmd_json)
shutil.copyfile(source, target)
except shutil.Error as serr:
LOG.debug("Compile command json file is the same")
except Exception as ex:
LOG.debug("Copying compile command json file failed.")
LOG.info("Analysis finished.")
LOG.info("To view results in the terminal use the "
"\"CodeChecker parse\" command.")
LOG.info("To store results use the \"CodeChecker store\" command.")
LOG.info("See --help and the user guide for further options about"
" parsing and storing the reports.")
LOG.info("----=================----")
