def test_new_ldlogger(self): """ Test log file parsing escape behaviour with after-#631 LD-LOGGER. """ logfile = os.path.join(self.__test_files, "ldlogger-new.json") # LD-LOGGERS after http://github.com/Ericsson/codechecker/pull/631 # now properly log the multiword arguments. When these are parsed by # the log_parser, the define's value will be passed to the analyzer. # # Logfile contains -DVARIABLE="some value" # and --target=x86_64-linux-gnu. build_action = log_parser.parse_log(logfile, ParseLogOptions())[0] self.assertEqual(list(build_action.sources)[0], r'/tmp/a.cpp') self.assertEqual(len(build_action.analyzer_options), 1) self.assertTrue(len(build_action.target) > 0) self.assertEqual(build_action.analyzer_options[0], r'-DVARIABLE="\"some value"\"') # Test source file with spaces. logfile = os.path.join(self.__test_files, "ldlogger-new-space.json") build_action = log_parser.parse_log(logfile, ParseLogOptions())[0] self.assertEqual(list(build_action.sources)[0], r'/tmp/a b.cpp') self.assertEqual(build_action.lang, 'c++')
def test_old_intercept_build(self): """ Test log file parsing escape behaviour with clang-5.0 intercept-build. """ logfile = os.path.join(self.__test_files, "intercept-old.json") # Scan-build-py shipping with clang-5.0 makes a logfile that contains: # -DVARIABLE=\"some value\" and --target=x86_64-linux-gnu # # The define is passed to the analyzer properly. build_action = log_parser.parse_log(logfile, ParseLogOptions())[0] self.assertEqual(list(build_action.sources)[0], r'/tmp/a.cpp') self.assertEqual(len(build_action.analyzer_options), 1) self.assertTrue(len(build_action.target) > 0) self.assertEqual(build_action.analyzer_options[0], r'-DVARIABLE="\"some value"\"') # Test source file with spaces. logfile = os.path.join(self.__test_files, "intercept-old-space.json") build_action = log_parser.parse_log(logfile, ParseLogOptions())[0] self.assertEqual(list(build_action.sources)[0], r'/tmp/a b.cpp') self.assertEqual(build_action.lang, 'c++')
def test_new_intercept_build(self): """ Test log file parsing escapes with upstream (GitHub) intercept-build. """ logfile = os.path.join(self.__test_files, "intercept-new.json") # Upstream scan-build-py creates an argument vector, as opposed to a # command string. This argument vector contains the define as it's # element in the following format: # -DVARIABLE=\"some value\" # and the target triplet, e.g.: # --target=x86_64-linux-gnu # # The define is passed to the analyzer properly. build_action = log_parser.parse_log(logfile, ParseLogOptions())[0] self.assertEqual(list(build_action.sources)[0], r'/tmp/a.cpp') self.assertEqual(len(build_action.analyzer_options), 1) self.assertTrue(len(build_action.target) > 0) self.assertEqual(build_action.analyzer_options[0], r'-DVARIABLE="\"some value"\"') # Test source file with spaces. logfile = os.path.join(self.__test_files, "intercept-new-space.json") build_action = log_parser.parse_log(logfile, ParseLogOptions())[0] self.assertEqual(list(build_action.sources)[0], r'/tmp/a b.cpp') self.assertEqual(build_action.lang, 'c++')
def test_old_ldlogger(self): """ Test log file parsing escape behaviour with pre-2017 Q2 LD-LOGGER. """ logfile = os.path.join(self.__test_files, "ldlogger-old.json") # LD-LOGGER before http://github.com/Ericsson/codechecker/pull/631 # used an escape mechanism that, when parsed by the log parser via # shlex, made CodeChecker parse arguments with multiword string # literals in them be considered as "file" (instead of compile option), # eventually ignored by the command builder, thus lessening analysis # accuracy, as defines were lost. # # Logfile contains "-DVARIABLE="some value"". # # There is no good way to back-and-forth convert in log_parser or # option_parser, so here we aim for a non-failing stalemate of the # define being considered a file and ignored, for now. build_action = log_parser.parse_log(logfile, ParseLogOptions())[0] results = option_parser.parse_options(build_action.original_command) self.assertEqual(' '.join(results.files), r'"-DVARIABLE="some value"" /tmp/a.cpp') self.assertEqual(len(build_action.analyzer_options), 0)
def _do_quickcheck(args): """ Handles the "quickcheck" command. For arguments see main function in CodeChecker.py. It also requires an extra property in args object, namely workspace which is a directory path as a string. This function is called from handle_quickcheck. """ try: context = generic_package_context.get_context() context.codechecker_workspace = args.workspace args.name = "quickcheck" log_file, set_in_cmdline = build_manager.check_log_file(args, context) actions = log_parser.parse_log(log_file, args.add_compiler_defaults) analyzer.run_quick_check(args, context, actions) except Exception as ex: LOG.error("Running quickcheck failed.") finally: if not args.keep_tmp: if log_file and not set_in_cmdline: LOG.debug('Removing temporary log file: ' + log_file) os.remove(log_file)
def handle_check(args): """ Runs the original build and logs the buildactions. Based on the log runs the analysis. """ try: if not host_check.check_zlib(): sys.exit(1) args.workspace = os.path.abspath(args.workspace) if not os.path.isdir(args.workspace): os.mkdir(args.workspace) context = generic_package_context.get_context() context.codechecker_workspace = args.workspace context.db_username = args.dbusername log_file, set_in_cmdline = build_manager.check_log_file(args, context) if not log_file: LOG.error("Failed to generate compilation command file: " + log_file) sys.exit(1) actions = log_parser.parse_log(log_file, args.add_compiler_defaults) check_env = analyzer_env.get_check_env(context.path_env_extra, context.ld_lib_path_extra) sql_server = SQLServer.from_cmdline_args(args, context.codechecker_workspace, context.migration_root, check_env) conn_mgr = client.ConnectionManager(sql_server, 'localhost', util.get_free_port()) sql_server.start(context.db_version_info, wait_for_start=True, init=True) conn_mgr.start_report_server() LOG.debug("Checker server started.") analyzer.run_check(args, actions, context) LOG.info("Analysis has finished.") log_startserver_hint(args) except Exception as ex: LOG.error(ex) import traceback print(traceback.format_exc()) finally: if not args.keep_tmp: if log_file and not set_in_cmdline: LOG.debug('Removing temporary log file: ' + log_file) os.remove(log_file)
def test_include_rel_to_abs(self): """ Test working directory prepending to relative include paths. """ logfile = os.path.join(self.__test_files, "include.json") build_action = log_parser.parse_log(logfile, ParseLogOptions())[0] self.assertEqual(len(build_action.analyzer_options), 3) self.assertEqual(build_action.analyzer_options[0], '-I/include') self.assertEqual(build_action.analyzer_options[1], '-I/include') self.assertEqual(build_action.analyzer_options[2], '-I/tmp')
def test_old_intercept_build(self): """ Test log file parsing escape behaviour with clang-5.0 intercept-build. """ logfile = os.path.join(self.__test_files, "intercept-old.json") # Scan-build-py shipping with clang-5.0 makes a logfile that contains: # -DVARIABLE=\"some value\" # # The define is passed to the analyzer properly. build_action = log_parser.parse_log(logfile)[0] self.assertEqual(list(build_action.sources)[0], r'/tmp/a.cpp') self.assertEqual(len(build_action.analyzer_options), 1) self.assertEqual(build_action.analyzer_options[0], r'-DVARIABLE="\"some value"\"')
def test_new_intercept_build(self): """ Test log file parsing escapes with upstream (GitHub) intercept-build. """ logfile = os.path.join(self.__test_files, "intercept-new.json") # Upstream scan-build-py creates an argument vector, as opposed to a # command string. This argument vector contains the define as it's # element in the following format: # -DVARIABLE=\"some value\" # # The define is passed to the analyzer properly. build_action = log_parser.parse_log(logfile)[0] self.assertEqual(list(build_action.sources)[0], r'/tmp/a.cpp') self.assertEqual(len(build_action.analyzer_options), 1) self.assertEqual(build_action.analyzer_options[0], r'-DVARIABLE="\"some value"\"')
def main(args): """ Perform analysis on the given logfiles and store the results in a machine- readable format. """ context = generic_package_context.get_context() # Parse the JSON CCDBs and retrieve the compile commands. actions = [] for log_file in args.logfile: if not os.path.exists(log_file): LOG.error("The specified logfile '" + log_file + "' does not " "exist!") continue actions += log_parser.parse_log(log_file, args.add_compiler_defaults) if len(actions) == 0: LOG.info("None of the specified build log files contained " "valid compilation commands. No analysis needed...") return if 'enable_all' in args: LOG.info("'--enable-all' was supplied for this analysis.") # Run the analysis. args.output_path = os.path.abspath(args.output_path) if os.path.isdir(args.output_path): LOG.info("Previous analysis results in '{0}' have been " "removed, overwriting with current result". format(args.output_path)) shutil.rmtree(args.output_path) os.makedirs(args.output_path) LOG.debug("Output will be stored to: '" + args.output_path + "'") metadata = {'action_num': len(actions), 'command': sys.argv, 'versions': { 'codechecker': "{0} ({1})".format(context.package_git_tag, context.package_git_hash) }, 'working_directory': os.getcwd(), 'output_path': args.output_path} if 'name' in args: metadata['name'] = args.name if 'skipfile' in args: # Skip information needs to be saved because reports in a header # can only be skipped by the report-server used in 'store' later # on if this information is persisted. with open(args.skipfile, 'r') as skipfile: metadata['skip_data'] = [l.strip() for l in skipfile.readlines()] analyzer.perform_analysis(args, context, actions, metadata) metadata_path = os.path.join(args.output_path, 'metadata.json') LOG.debug("Analysis metadata write to '" + metadata_path + "'") with open(metadata_path, 'w') as metafile: json.dump(metadata, metafile)
def main(args): """ Perform analysis on the given logfiles and store the results in a machine- readable format. """ logger.setup_logger(args.verbose if 'verbose' in args else None) if len(args.logfile) != 1: LOG.warning("Only one log file can be processed right now!") sys.exit(1) args.output_path = os.path.abspath(args.output_path) if os.path.exists(args.output_path) and \ not os.path.isdir(args.output_path): LOG.error("The given output path is not a directory: " + args.output_path) sys.exit(1) if 'enable_all' in args: LOG.info("'--enable-all' was supplied for this analysis.") # We clear the output directory in the following cases. ctu_dir = os.path.join(args.output_path, 'ctu-dir') if 'ctu_phases' in args and args.ctu_phases[0] and \ os.path.isdir(ctu_dir): # Clear the CTU-dir if the user turned on the collection phase. LOG.debug("Previous CTU contents have been deleted.") shutil.rmtree(ctu_dir) if 'clean' in args and os.path.isdir(args.output_path): LOG.info("Previous analysis results in '{0}' have been removed, " "overwriting with current result".format(args.output_path)) shutil.rmtree(args.output_path) if not os.path.exists(args.output_path): os.makedirs(args.output_path) LOG.debug("args: " + str(args)) LOG.debug("Output will be stored to: '" + args.output_path + "'") # Parse the JSON CCDBs and retrieve the compile commands. actions = [] for log_file in args.logfile: if not os.path.exists(log_file): LOG.error("The specified logfile '" + log_file + "' does not " "exist!") continue parseLogOptions = ParseLogOptions(args) actions += log_parser.parse_log(log_file, parseLogOptions) if len(actions) == 0: LOG.info("None of the specified build log files contained " "valid compilation commands. No analysis needed...") sys.exit(1) context = generic_package_context.get_context() metadata = { 'action_num': len(actions), 'command': sys.argv, 'versions': { 'codechecker': "{0} ({1})".format(context.package_git_tag, context.package_git_hash) }, 'working_directory': os.getcwd(), 'output_path': args.output_path, 'result_source_files': {} } if 'name' in args: metadata['name'] = args.name # Update metadata dictionary with old values. metadata_file = os.path.join(args.output_path, 'metadata.json') if os.path.exists(metadata_file): with open(metadata_file, 'r') as data: metadata_prev = json.load(data) metadata['result_source_files'] =\ metadata_prev['result_source_files'] analyzer.perform_analysis(args, context, actions, metadata) LOG.debug("Analysis metadata write to '" + metadata_file + "'") with open(metadata_file, 'w') as metafile: json.dump(metadata, metafile) # WARN: store command will search for this file!!!! compile_cmd_json = os.path.join(args.output_path, 'compile_cmd.json') try: source = os.path.abspath(args.logfile[0]) target = os.path.abspath(compile_cmd_json) if source != target: shutil.copyfile(source, target) except shutil.Error: LOG.debug("Compilation database JSON file is the same.") except Exception: LOG.debug("Copying compilation database JSON file failed.")
def main(args): """ Perform analysis on the given logfiles and store the results in a machine- readable format. """ context = generic_package_context.get_context() # Parse the JSON CCDBs and retrieve the compile commands. actions = [] if len(args.logfile) != 1: LOG.warning("Only one log file can be processed right now!") sys.exit(1) for log_file in args.logfile: if not os.path.exists(log_file): LOG.error("The specified logfile '" + log_file + "' does not " "exist!") continue actions += log_parser.parse_log(log_file, args.add_compiler_defaults) if len(actions) == 0: LOG.info("None of the specified build log files contained " "valid compilation commands. No analysis needed...") sys.exit(1) if 'enable_all' in args: LOG.info("'--enable-all' was supplied for this analysis.") # Run the analysis. args.output_path = os.path.abspath(args.output_path) if 'ctu_phases' not in args or args.ctu_phases[0] or \ (not args.ctu_phases[0] and not args.ctu_phases[1]): if os.path.isdir(args.output_path): LOG.info("Previous analysis results in '{0}' have been " "removed, overwriting with current result".format( args.output_path)) shutil.rmtree(args.output_path) os.makedirs(args.output_path) LOG.debug("Output will be stored to: '" + args.output_path + "'") metadata = { 'action_num': len(actions), 'command': sys.argv, 'versions': { 'codechecker': "{0} ({1})".format(context.package_git_tag, context.package_git_hash) }, 'working_directory': os.getcwd(), 'output_path': args.output_path } if 'name' in args: metadata['name'] = args.name if 'skipfile' in args: # Skip information needs to be saved because reports in a header # can only be skipped by the report-server used in 'store' later # on if this information is persisted. with open(args.skipfile, 'r') as skipfile: metadata['skip_data'] = [l.strip() for l in skipfile.readlines()] analyzer.perform_analysis(args, context, actions, metadata) metadata_path = os.path.join(args.output_path, 'metadata.json') LOG.debug("Analysis metadata write to '" + metadata_path + "'") with open(metadata_path, 'w') as metafile: json.dump(metadata, metafile) # WARN: store command will search for this file!!!! compile_cmd_json = os.path.join(args.output_path, 'compile_cmd.json') try: source = os.path.abspath(args.logfile[0]) target = os.path.abspath(compile_cmd_json) shutil.copyfile(source, target) except shutil.Error as serr: LOG.debug("Compile command json file is the same") except Exception as ex: LOG.debug("Copying compile command json file failed.") LOG.info("Analysis finished.") LOG.info("To view results in the terminal use the " "\"CodeChecker parse\" command.") LOG.info("To store results use the \"CodeChecker store\" command.") LOG.info("See --help and the user guide for further options about" " parsing and storing the reports.") LOG.info("----=================----")