def main(args): """ Setup a logger server based on the configuration and manage the CodeChecker server. """ with logger.LOG_CFG_SERVER(args.verbose if 'verbose' in args else None): try: cmd_config.check_config_file(args) except FileNotFoundError as fnerr: LOG.error(fnerr) sys.exit(1) server_init_start(args)
def main(args): """ Entry point for parsing some analysis results and printing them to the stdout in a human-readable format. """ logger.setup_logger(args.verbose if 'verbose' in args else None) try: cmd_config.check_config_file(args) except FileNotFoundError as fnerr: LOG.error(fnerr) sys.exit(1) export = args.export if 'export' in args else None if export == 'html' and 'output_path' not in args: LOG.error("Argument --export not allowed without argument --output " "when exporting to HTML.") sys.exit(1) if export == 'gerrit' and not gerrit.mandatory_env_var_is_set(): sys.exit(1) context = analyzer_context.get_context() # To ensure the help message prints the default folder properly, # the 'default' for 'args.input' is a string, not a list. # But we need lists for the foreach here to work. if isinstance(args.input, str): args.input = [args.input] original_cwd = os.getcwd() src_comment_status_filter = args.review_status suppr_handler = None if 'suppress' in args: __make_handler = False if not os.path.isfile(args.suppress): if 'create_suppress' in args: with open(args.suppress, 'w', encoding='utf-8', errors='ignore') as _: # Just create the file. __make_handler = True LOG.info( "Will write source-code suppressions to " "suppress file: %s", args.suppress) else: LOG.warning( "Suppress file '%s' given, but it does not exist" " -- will not suppress anything.", args.suppress) else: __make_handler = True if __make_handler: suppr_handler = suppress_handler.\ GenericSuppressHandler(args.suppress, 'create_suppress' in args, src_comment_status_filter) elif 'create_suppress' in args: LOG.error("Can't use '--export-source-suppress' unless '--suppress " "SUPPRESS_FILE' is also given.") sys.exit(2) processed_path_hashes = set() skip_handler = None if 'skipfile' in args: with open(args.skipfile, 'r', encoding='utf-8', errors='ignore') as skip_file: skip_handler = SkipListHandler(skip_file.read()) trim_path_prefixes = args.trim_path_prefix if \ 'trim_path_prefix' in args else None if export: if export not in EXPORT_TYPES: LOG.error(f"Unknown export format: {export}") return # The HTML part will be handled separately below. if export != 'html': try: res = parse_convert_reports(args.input, export, context.severity_map, trim_path_prefixes) if 'output_path' in args: output_path = os.path.abspath(args.output_path) if not os.path.exists(output_path): os.mkdir(output_path) reports_json = os.path.join(output_path, 'reports.json') with open(reports_json, mode='w', encoding='utf-8', errors="ignore") as output_f: output_f.write(json.dumps(res)) return print(json.dumps(res)) except Exception as ex: LOG.error(ex) sys.exit(1) def trim_path_prefixes_handler(source_file): """ Callback to util.trim_path_prefixes to prevent module dependency of plist_to_html """ return util.trim_path_prefixes(source_file, trim_path_prefixes) html_builder = None def skip_html_report_data_handler(report_hash, source_file, report_line, checker_name, diag, files): """ Report handler which skips bugs which were suppressed by source code comments. This function will return a tuple. The first element will decide whether the report should be skipped or not and the second element will be a list of source code comments related to the actual report. """ files_dict = {k: v for k, v in enumerate(files)} report = Report({'check_name': checker_name}, diag['path'], files_dict, metadata=None) path_hash = get_report_path_hash(report) if path_hash in processed_path_hashes: LOG.debug("Skip report because it is a deduplication of an " "already processed report!") LOG.debug("Path hash: %s", path_hash) LOG.debug(diag) return True, [] skip, source_code_comments = skip_report(report_hash, source_file, report_line, checker_name, suppr_handler, src_comment_status_filter) if skip_handler: skip |= skip_handler.should_skip(source_file) if not skip: processed_path_hashes.add(path_hash) return skip, source_code_comments file_change = set() severity_stats = defaultdict(int) file_stats = defaultdict(int) report_count = 0 for input_path in args.input: input_path = os.path.abspath(input_path) os.chdir(original_cwd) LOG.debug("Parsing input argument: '%s'", input_path) if export == 'html': output_path = os.path.abspath(args.output_path) if not html_builder: html_builder = \ PlistToHtml.HtmlBuilder(context.path_plist_to_html_dist, context.severity_map) LOG.info("Generating html output files:") PlistToHtml.parse(input_path, output_path, context.path_plist_to_html_dist, skip_html_report_data_handler, html_builder, trim_path_prefixes_handler) continue files = [] metadata_dict = {} if os.path.isfile(input_path): files.append(input_path) elif os.path.isdir(input_path): metadata_file = os.path.join(input_path, "metadata.json") if os.path.exists(metadata_file): metadata_dict = util.load_json_or_empty(metadata_file) LOG.debug(metadata_dict) if 'working_directory' in metadata_dict: working_dir = metadata_dict['working_directory'] try: os.chdir(working_dir) except OSError as oerr: LOG.debug(oerr) LOG.error( "Working directory %s is missing.\n" "Can not parse reports safely.", working_dir) sys.exit(1) _, _, file_names = next(os.walk(input_path), ([], [], [])) files = [ os.path.join(input_path, file_name) for file_name in file_names ] file_report_map = defaultdict(list) plist_pltf = PlistToPlaintextFormatter(suppr_handler, skip_handler, context.severity_map, processed_path_hashes, trim_path_prefixes, src_comment_status_filter) plist_pltf.print_steps = 'print_steps' in args for file_path in files: f_change = parse_with_plt_formatter(file_path, metadata_dict, plist_pltf, file_report_map) file_change = file_change.union(f_change) report_stats = plist_pltf.write(file_report_map) sev_stats = report_stats.get('severity') for severity in sev_stats: severity_stats[severity] += sev_stats[severity] f_stats = report_stats.get('files') for file_path in f_stats: file_stats[file_path] += f_stats[file_path] rep_stats = report_stats.get('reports') report_count += rep_stats.get("report_count", 0) # Create index.html and statistics.html for the generated html files. if html_builder: html_builder.create_index_html(args.output_path) html_builder.create_statistics_html(args.output_path) print('\nTo view statistics in a browser run:\n> firefox {0}'.format( os.path.join(args.output_path, 'statistics.html'))) print('\nTo view the results in a browser run:\n> firefox {0}'.format( os.path.join(args.output_path, 'index.html'))) else: print("\n----==== Summary ====----") if file_stats: vals = [[os.path.basename(k), v] for k, v in dict(file_stats).items()] vals.sort(key=itemgetter(0)) keys = ['Filename', 'Report count'] table = twodim.to_str('table', keys, vals, 1, True) print(table) if severity_stats: vals = [[k, v] for k, v in dict(severity_stats).items()] vals.sort(key=itemgetter(0)) keys = ['Severity', 'Report count'] table = twodim.to_str('table', keys, vals, 1, True) print(table) print("----=================----") print("Total number of reports: {}".format(report_count)) print("----=================----") if file_change: changed_files = '\n'.join([' - ' + f for f in file_change]) LOG.warning( "The following source file contents changed since the " "latest analysis:\n%s\nMultiple reports were not " "shown and skipped from the statistics. Please " "analyze your project again to update the " "reports!", changed_files) os.chdir(original_cwd) if report_count != 0: sys.exit(2)
def main(args): """ Store the defect results in the specified input list as bug reports in the database. """ logger.setup_logger(args.verbose if 'verbose' in args else None) try: cmd_config.check_config_file(args) except FileNotFoundError as fnerr: LOG.error(fnerr) sys.exit(1) if not host_check.check_zlib(): raise Exception("zlib is not available on the system!") # To ensure the help message prints the default folder properly, # the 'default' for 'args.input' is a string, not a list. # But we need lists for the foreach here to work. if isinstance(args.input, str): args.input = [args.input] if 'name' not in args: LOG.debug("Generating name for analysis...") generated = __get_run_name(args.input) if generated: setattr(args, 'name', generated) else: LOG.error("No suitable name was found in the inputs for the " "analysis run. Please specify one by passing argument " "--name run_name in the invocation.") sys.exit(2) # argparse returns error code 2 for bad invocations. LOG.info("Storing analysis results for run '" + args.name + "'") if 'force' in args: LOG.info("argument --force was specified: the run with name '" + args.name + "' will be deleted.") # Setup connection to the remote server. client = libclient.setup_client(args.product_url) _, zip_file = tempfile.mkstemp('.zip') LOG.debug("Will write mass store ZIP to '%s'...", zip_file) try: LOG.debug("Assembling zip file.") try: assemble_zip(args.input, zip_file, client) except Exception as ex: print(ex) import traceback traceback.print_stack() zip_size = os.stat(zip_file).st_size LOG.debug("Assembling zip done, size is %s", sizeof_fmt(zip_size)) if zip_size > MAX_UPLOAD_SIZE: LOG.error("The result list to upload is too big (max: %s).", sizeof_fmt(MAX_UPLOAD_SIZE)) sys.exit(1) with open(zip_file, 'rb') as zf: b64zip = base64.b64encode(zf.read()).decode("utf-8") context = webserver_context.get_context() trim_path_prefixes = args.trim_path_prefix if \ 'trim_path_prefix' in args else None description = args.description if 'description' in args else None LOG.info("Storing results to the server...") client.massStoreRun(args.name, args.tag if 'tag' in args else None, str(context.version), b64zip, 'force' in args, trim_path_prefixes, description) # Storing analysis statistics if the server allows them. if client.allowsStoringAnalysisStatistics(): storing_analysis_statistics(client, args.input, args.name) LOG.info("Storage finished successfully.") except RequestFailed as reqfail: if reqfail.errorCode == ErrorCode.SOURCE_FILE: header = ['File', 'Line', 'Checker name'] table = twodim_to_str('table', header, [c.split('|') for c in reqfail.extraInfo]) LOG.warning("Setting the review statuses for some reports failed " "because of non valid source code comments: " "%s\n %s", reqfail.message, table) sys.exit(1) except Exception as ex: import traceback traceback.print_stack() LOG.info("Storage failed: %s", str(ex)) sys.exit(1) finally: os.remove(zip_file)
def main(args): """ Perform analysis on the given logfiles and store the results in a machine- readable format. """ logger.setup_logger(args.verbose if 'verbose' in args else None) # CTU loading mode is only meaningful if CTU itself is enabled. if 'ctu_ast_mode' in args and 'ctu_phases' not in args: LOG.error("Analyzer option 'ctu-ast-mode' requires CTU mode enabled") sys.exit(1) try: cmd_config.check_config_file(args) except FileNotFoundError as fnerr: LOG.error(fnerr) sys.exit(1) if not os.path.exists(args.logfile): LOG.error("The specified logfile '%s' does not exist!", args.logfile) sys.exit(1) args.output_path = os.path.abspath(args.output_path) if os.path.exists(args.output_path) and \ not os.path.isdir(args.output_path): LOG.error("The given output path is not a directory: " + args.output_path) sys.exit(1) if 'enable_all' in args: LOG.info("'--enable-all' was supplied for this analysis.") config_option_re = re.compile(r'^({}):.+=.+$'.format( '|'.join(analyzer_types.supported_analyzers))) # Check the format of analyzer options. if 'analyzer_config' in args: for config in args.analyzer_config: if not re.match(config_option_re, config): LOG.error("Analyzer option in wrong format: %s", config) sys.exit(1) # Check the format of checker options. if 'checker_config' in args: for config in args.checker_config: if not re.match(config_option_re, config): LOG.error("Checker option in wrong format: %s", config) sys.exit(1) # Process the skip list if present. skip_handler = __get_skip_handler(args) # Enable alpha uniqueing by default if ctu analysis is used. if 'none' in args.compile_uniqueing and 'ctu_phases' in args: args.compile_uniqueing = "alpha" compiler_info_file = None if 'compiler_info_file' in args: LOG.debug("Compiler info is read from: %s", args.compiler_info_file) if not os.path.exists(args.compiler_info_file): LOG.error("Compiler info file %s does not exist", args.compiler_info_file) sys.exit(1) compiler_info_file = args.compiler_info_file ctu_or_stats_enabled = False # Skip list is applied only in pre-analysis # if --ctu-collect was called explicitly. pre_analysis_skip_handler = None if 'ctu_phases' in args: ctu_collect = args.ctu_phases[0] ctu_analyze = args.ctu_phases[1] if ctu_collect and not ctu_analyze: pre_analysis_skip_handler = skip_handler if ctu_collect or ctu_analyze: ctu_or_stats_enabled = True # Skip list is applied only in pre-analysis # if --stats-collect was called explicitly. if 'stats_output' in args and args.stats_output: pre_analysis_skip_handler = skip_handler ctu_or_stats_enabled = True if 'stats_enabled' in args and args.stats_enabled: ctu_or_stats_enabled = True context = analyzer_context.get_context() analyzer_env = env.extend(context.path_env_extra, context.ld_lib_path_extra) compile_commands = load_json_or_empty(args.logfile) if compile_commands is None: sys.exit(1) # Number of all the compilation commands in the parsed log files, # logged by the logger. all_cmp_cmd_count = len(compile_commands) # We clear the output directory in the following cases. ctu_dir = os.path.join(args.output_path, 'ctu-dir') if 'ctu_phases' in args and args.ctu_phases[0] and \ os.path.isdir(ctu_dir): # Clear the CTU-dir if the user turned on the collection phase. LOG.debug("Previous CTU contents have been deleted.") shutil.rmtree(ctu_dir) if 'clean' in args and os.path.isdir(args.output_path): LOG.info("Previous analysis results in '%s' have been removed, " "overwriting with current result", args.output_path) shutil.rmtree(args.output_path) if not os.path.exists(args.output_path): os.makedirs(args.output_path) # TODO: I'm not sure that this directory should be created here. fixit_dir = os.path.join(args.output_path, 'fixit') if not os.path.exists(fixit_dir): os.makedirs(fixit_dir) LOG.debug("args: %s", str(args)) LOG.debug("Output will be stored to: '%s'", args.output_path) analyzer_clang_binary = \ context.analyzer_binaries.get( clangsa.analyzer.ClangSA.ANALYZER_NAME) analyzer_clang_version = None if analyzer_clang_binary: analyzer_clang_version = clangsa.version.get(analyzer_clang_binary, analyzer_env) actions, skipped_cmp_cmd_count = log_parser.parse_unique_log( compile_commands, args.output_path, args.compile_uniqueing, compiler_info_file, args.keep_gcc_include_fixed, args.keep_gcc_intrin, skip_handler, pre_analysis_skip_handler, ctu_or_stats_enabled, analyzer_env, analyzer_clang_version) if not actions: LOG.info("No analysis is required.\nThere were no compilation " "commands in the provided compilation database or " "all of them were skipped.") sys.exit(0) uniqued_compilation_db_file = os.path.join( args.output_path, "unique_compile_commands.json") with open(uniqued_compilation_db_file, 'w', encoding="utf-8", errors="ignore") as f: json.dump(actions, f, cls=log_parser.CompileCommandEncoder) metadata = { 'version': 2, 'tools': [{ 'name': 'codechecker', 'action_num': len(actions), 'command': sys.argv, 'version': "{0} ({1})".format(context.package_git_tag, context.package_git_hash), 'working_directory': os.getcwd(), 'output_path': args.output_path, 'result_source_files': {}, 'analyzers': {} }]} metadata_tool = metadata['tools'][0] if 'name' in args: metadata_tool['run_name'] = args.name # Update metadata dictionary with old values. metadata_file = os.path.join(args.output_path, 'metadata.json') metadata_prev = None if os.path.exists(metadata_file): metadata_prev = load_json_or_empty(metadata_file) metadata_tool['result_source_files'] = \ __get_result_source_files(metadata_prev) CompileCmdParseCount = \ collections.namedtuple('CompileCmdParseCount', 'total, analyze, skipped, removed_by_uniqueing') cmp_cmd_to_be_uniqued = all_cmp_cmd_count - skipped_cmp_cmd_count # Number of compile commands removed during uniqueing. removed_during_uniqueing = cmp_cmd_to_be_uniqued - len(actions) all_to_be_analyzed = cmp_cmd_to_be_uniqued - removed_during_uniqueing compile_cmd_count = CompileCmdParseCount( total=all_cmp_cmd_count, analyze=all_to_be_analyzed, skipped=skipped_cmp_cmd_count, removed_by_uniqueing=removed_during_uniqueing) LOG.debug_analyzer("Total number of compile commands without " "skipping or uniqueing: %d", compile_cmd_count.total) LOG.debug_analyzer("Compile commands removed by uniqueing: %d", compile_cmd_count.removed_by_uniqueing) LOG.debug_analyzer("Compile commands skipped during log processing: %d", compile_cmd_count.skipped) LOG.debug_analyzer("Compile commands forwarded for analysis: %d", compile_cmd_count.analyze) analyzer.perform_analysis(args, skip_handler, context, actions, metadata_tool, compile_cmd_count) __update_skip_file(args) __cleanup_metadata(metadata_prev, metadata) LOG.debug("Analysis metadata write to '%s'", metadata_file) with open(metadata_file, 'w', encoding="utf-8", errors="ignore") as metafile: json.dump(metadata, metafile) # WARN: store command will search for this file!!!! compile_cmd_json = os.path.join(args.output_path, 'compile_cmd.json') try: source = os.path.abspath(args.logfile) target = os.path.abspath(compile_cmd_json) if source != target: shutil.copyfile(source, target) except shutil.Error: LOG.debug("Compilation database JSON file is the same.") except Exception: LOG.debug("Copying compilation database JSON file failed.") try: # pylint: disable=no-name-in-module from codechecker_analyzer import analyzer_statistics analyzer_statistics.collect(metadata, "analyze") except Exception: pass # Generally exit status is set by sys.exit() call in CodeChecker. However, # exit code 3 has a special meaning: it returns when the underlying # analyzer tool fails. # "CodeChecker analyze" is special in the sense that it can be invoked # either top-level or through "CodeChecker check". In the latter case # "CodeChecker check" should have the same exit status. Calling sys.exit() # at this specific point is not an option, because the remaining statements # of "CodeChecker check" after the analysis wouldn't execute. for analyzer_data in metadata_tool['analyzers'].values(): if analyzer_data['analyzer_statistics']['failed'] != 0: return 3
def main(args): """ Entry point for parsing some analysis results and printing them to the stdout in a human-readable format. """ # If the given output format is not 'table', redirect logger's output to # the stderr. stream = None if 'export' in args and args.export not in [None, 'table', 'html']: stream = 'stderr' init_logger(args.verbose if 'verbose' in args else None, stream) try: cmd_config.check_config_file(args) except FileNotFoundError as fnerr: LOG.error(fnerr) sys.exit(1) export = args.export if 'export' in args else None if export == 'html' and 'output_path' not in args: LOG.error("Argument --export not allowed without argument --output " "when exporting to HTML.") sys.exit(1) if export == 'gerrit' and not gerrit.mandatory_env_var_is_set(): sys.exit(1) if export and export not in EXPORT_TYPES: LOG.error("Unknown export format: %s", export) sys.exit(1) context = analyzer_context.get_context() # To ensure the help message prints the default folder properly, # the 'default' for 'args.input' is a string, not a list. # But we need lists for the foreach here to work. if isinstance(args.input, str): args.input = [args.input] src_comment_status_filter = args.review_status suppr_handler = None if 'suppress' in args: __make_handler = False if not os.path.isfile(args.suppress): if 'create_suppress' in args: with open(args.suppress, 'w', encoding='utf-8', errors='ignore') as _: # Just create the file. __make_handler = True LOG.info( "Will write source-code suppressions to " "suppress file: %s", args.suppress) else: LOG.warning( "Suppress file '%s' given, but it does not exist" " -- will not suppress anything.", args.suppress) else: __make_handler = True if __make_handler: suppr_handler = suppress_handler.\ GenericSuppressHandler(args.suppress, 'create_suppress' in args, src_comment_status_filter) elif 'create_suppress' in args: LOG.error("Can't use '--export-source-suppress' unless '--suppress " "SUPPRESS_FILE' is also given.") sys.exit(1) output_dir_path = None output_file_path = None if 'output_path' in args: output_path = os.path.abspath(args.output_path) if export == 'html': output_dir_path = output_path else: if os.path.exists(output_path) and os.path.isdir(output_path): # For backward compatibility reason we handle the use case # when directory is provided to this command. LOG.error( "Please provide a file path instead of a directory " "for '%s' export type!", export) sys.exit(1) if export == 'baseline' and not baseline.check(output_path): LOG.error("Baseline files must have '.baseline' extensions.") sys.exit(1) output_file_path = output_path output_dir_path = os.path.dirname(output_file_path) if not os.path.exists(output_dir_path): os.makedirs(output_dir_path) def get_output_file_path(default_file_name: str) -> Optional[str]: """ Return an output file path. """ if output_file_path: return output_file_path if output_dir_path: return os.path.join(output_dir_path, default_file_name) skip_handlers = SkipListHandlers() if 'files' in args: items = [f"+{file_path}" for file_path in args.files] items.append("-*") skip_handlers.append(SkipListHandler("\n".join(items))) if 'skipfile' in args: with open(args.skipfile, 'r', encoding='utf-8', errors='ignore') as f: skip_handlers.append(SkipListHandler(f.read())) trim_path_prefixes = args.trim_path_prefix if \ 'trim_path_prefix' in args else None all_reports = [] statistics = Statistics() file_cache = {} # For memory effiency. changed_files: Set[str] = set() processed_path_hashes = set() processed_file_paths = set() print_steps = 'print_steps' in args html_builder: Optional[report_to_html.HtmlBuilder] = None if export == 'html': html_builder = report_to_html.HtmlBuilder( context.path_plist_to_html_dist, context.checker_labels) for dir_path, file_paths in report_file.analyzer_result_files(args.input): metadata = get_metadata(dir_path) for file_path in file_paths: reports = report_file.get_reports(file_path, context.checker_labels, file_cache) reports = reports_helper.skip(reports, processed_path_hashes, skip_handlers, suppr_handler, src_comment_status_filter) statistics.num_of_analyzer_result_files += 1 for report in reports: if report.changed_files: changed_files.update(report.changed_files) statistics.add_report(report) if trim_path_prefixes: report.trim_path_prefixes(trim_path_prefixes) all_reports.extend(reports) # Print reports continously. if not export: file_report_map = plaintext.get_file_report_map( reports, file_path, metadata) plaintext.convert(file_report_map, processed_file_paths, print_steps) elif export == 'html': print(f"Parsing input file '{file_path}'.") report_to_html.convert(file_path, reports, output_dir_path, html_builder) if export is None: # Plain text output statistics.write() elif export == 'html': html_builder.finish(output_dir_path, statistics) elif export == 'json': data = report_to_json.convert(all_reports) dump_json_output(data, get_output_file_path("reports.json")) elif export == 'codeclimate': data = codeclimate.convert(all_reports) dump_json_output(data, get_output_file_path("reports.json")) elif export == 'gerrit': data = gerrit.convert(all_reports) dump_json_output(data, get_output_file_path("reports.json")) elif export == 'baseline': data = baseline.convert(all_reports) output_path = get_output_file_path("reports.baseline") if output_path: baseline.write(output_path, data) reports_helper.dump_changed_files(changed_files) if statistics.num_of_reports: sys.exit(2)