def handle_list_products(args): # If the given output format is not 'table', redirect logger's output to # the stderr. stream = None if 'output_format' in args and args.output_format != 'table': stream = 'stderr' init_logger(args.verbose if 'verbose' in args else None, stream) protocol, host, port = split_server_url(args.server_url) client = setup_product_client(protocol, host, port) products = client.getProducts(None, None) if args.output_format == 'json': results = [] for product in products: results.append({product.endpoint: product}) print(CmdLineOutputEncoder().encode(results)) else: # plaintext, csv header = ['Database status', 'Endpoint', 'Name', 'Description'] rows = [] for product in products: name = convert.from_b64(product.displayedName_b64) \ if product.displayedName_b64 else '' description = convert.from_b64(product.description_b64) \ if product.description_b64 else '' if not product.accessible: db_status_msg = 'No access.' else: db_status = product.databaseStatus db_status_msg = database_status.db_status_msg.get( db_status, 'Unknown database status') rows.append((db_status_msg, product.endpoint, name, description)) print(twodim.to_str(args.output_format, header, rows))
def main(args): """ Entry point for parsing some analysis results and printing them to the stdout in a human-readable format. """ logger.setup_logger(args.verbose if 'verbose' in args else None) try: cmd_config.check_config_file(args) except FileNotFoundError as fnerr: LOG.error(fnerr) sys.exit(1) export = args.export if 'export' in args else None if export == 'html' and 'output_path' not in args: LOG.error("Argument --export not allowed without argument --output " "when exporting to HTML.") sys.exit(1) if export == 'gerrit' and not gerrit.mandatory_env_var_is_set(): sys.exit(1) context = analyzer_context.get_context() # To ensure the help message prints the default folder properly, # the 'default' for 'args.input' is a string, not a list. # But we need lists for the foreach here to work. if isinstance(args.input, str): args.input = [args.input] original_cwd = os.getcwd() src_comment_status_filter = args.review_status suppr_handler = None if 'suppress' in args: __make_handler = False if not os.path.isfile(args.suppress): if 'create_suppress' in args: with open(args.suppress, 'w', encoding='utf-8', errors='ignore') as _: # Just create the file. __make_handler = True LOG.info( "Will write source-code suppressions to " "suppress file: %s", args.suppress) else: LOG.warning( "Suppress file '%s' given, but it does not exist" " -- will not suppress anything.", args.suppress) else: __make_handler = True if __make_handler: suppr_handler = suppress_handler.\ GenericSuppressHandler(args.suppress, 'create_suppress' in args, src_comment_status_filter) elif 'create_suppress' in args: LOG.error("Can't use '--export-source-suppress' unless '--suppress " "SUPPRESS_FILE' is also given.") sys.exit(2) processed_path_hashes = set() skip_handler = None if 'skipfile' in args: with open(args.skipfile, 'r', encoding='utf-8', errors='ignore') as skip_file: skip_handler = SkipListHandler(skip_file.read()) trim_path_prefixes = args.trim_path_prefix if \ 'trim_path_prefix' in args else None if export: if export not in EXPORT_TYPES: LOG.error(f"Unknown export format: {export}") return # The HTML part will be handled separately below. if export != 'html': try: res = parse_convert_reports(args.input, export, context.severity_map, trim_path_prefixes) if 'output_path' in args: output_path = os.path.abspath(args.output_path) if not os.path.exists(output_path): os.mkdir(output_path) reports_json = os.path.join(output_path, 'reports.json') with open(reports_json, mode='w', encoding='utf-8', errors="ignore") as output_f: output_f.write(json.dumps(res)) return print(json.dumps(res)) except Exception as ex: LOG.error(ex) sys.exit(1) def trim_path_prefixes_handler(source_file): """ Callback to util.trim_path_prefixes to prevent module dependency of plist_to_html """ return util.trim_path_prefixes(source_file, trim_path_prefixes) html_builder = None def skip_html_report_data_handler(report_hash, source_file, report_line, checker_name, diag, files): """ Report handler which skips bugs which were suppressed by source code comments. This function will return a tuple. The first element will decide whether the report should be skipped or not and the second element will be a list of source code comments related to the actual report. """ files_dict = {k: v for k, v in enumerate(files)} report = Report({'check_name': checker_name}, diag['path'], files_dict, metadata=None) path_hash = get_report_path_hash(report) if path_hash in processed_path_hashes: LOG.debug("Skip report because it is a deduplication of an " "already processed report!") LOG.debug("Path hash: %s", path_hash) LOG.debug(diag) return True, [] skip, source_code_comments = skip_report(report_hash, source_file, report_line, checker_name, suppr_handler, src_comment_status_filter) if skip_handler: skip |= skip_handler.should_skip(source_file) if not skip: processed_path_hashes.add(path_hash) return skip, source_code_comments file_change = set() severity_stats = defaultdict(int) file_stats = defaultdict(int) report_count = 0 for input_path in args.input: input_path = os.path.abspath(input_path) os.chdir(original_cwd) LOG.debug("Parsing input argument: '%s'", input_path) if export == 'html': output_path = os.path.abspath(args.output_path) if not html_builder: html_builder = \ PlistToHtml.HtmlBuilder(context.path_plist_to_html_dist, context.severity_map) LOG.info("Generating html output files:") PlistToHtml.parse(input_path, output_path, context.path_plist_to_html_dist, skip_html_report_data_handler, html_builder, trim_path_prefixes_handler) continue files = [] metadata_dict = {} if os.path.isfile(input_path): files.append(input_path) elif os.path.isdir(input_path): metadata_file = os.path.join(input_path, "metadata.json") if os.path.exists(metadata_file): metadata_dict = util.load_json_or_empty(metadata_file) LOG.debug(metadata_dict) if 'working_directory' in metadata_dict: working_dir = metadata_dict['working_directory'] try: os.chdir(working_dir) except OSError as oerr: LOG.debug(oerr) LOG.error( "Working directory %s is missing.\n" "Can not parse reports safely.", working_dir) sys.exit(1) _, _, file_names = next(os.walk(input_path), ([], [], [])) files = [ os.path.join(input_path, file_name) for file_name in file_names ] file_report_map = defaultdict(list) plist_pltf = PlistToPlaintextFormatter(suppr_handler, skip_handler, context.severity_map, processed_path_hashes, trim_path_prefixes, src_comment_status_filter) plist_pltf.print_steps = 'print_steps' in args for file_path in files: f_change = parse_with_plt_formatter(file_path, metadata_dict, plist_pltf, file_report_map) file_change = file_change.union(f_change) report_stats = plist_pltf.write(file_report_map) sev_stats = report_stats.get('severity') for severity in sev_stats: severity_stats[severity] += sev_stats[severity] f_stats = report_stats.get('files') for file_path in f_stats: file_stats[file_path] += f_stats[file_path] rep_stats = report_stats.get('reports') report_count += rep_stats.get("report_count", 0) # Create index.html and statistics.html for the generated html files. if html_builder: html_builder.create_index_html(args.output_path) html_builder.create_statistics_html(args.output_path) print('\nTo view statistics in a browser run:\n> firefox {0}'.format( os.path.join(args.output_path, 'statistics.html'))) print('\nTo view the results in a browser run:\n> firefox {0}'.format( os.path.join(args.output_path, 'index.html'))) else: print("\n----==== Summary ====----") if file_stats: vals = [[os.path.basename(k), v] for k, v in dict(file_stats).items()] vals.sort(key=itemgetter(0)) keys = ['Filename', 'Report count'] table = twodim.to_str('table', keys, vals, 1, True) print(table) if severity_stats: vals = [[k, v] for k, v in dict(severity_stats).items()] vals.sort(key=itemgetter(0)) keys = ['Severity', 'Report count'] table = twodim.to_str('table', keys, vals, 1, True) print(table) print("----=================----") print("Total number of reports: {}".format(report_count)) print("----=================----") if file_change: changed_files = '\n'.join([' - ' + f for f in file_change]) LOG.warning( "The following source file contents changed since the " "latest analysis:\n%s\nMultiple reports were not " "shown and skipped from the statistics. Please " "analyze your project again to update the " "reports!", changed_files) os.chdir(original_cwd) if report_count != 0: sys.exit(2)
def main(args): """ List the checkers available in the specified (or all supported) analyzers alongside with their description or enabled status in various formats. """ # If the given output format is not 'table', redirect logger's output to # the stderr. logger.setup_logger(args.verbose if 'verbose' in args else None, None if args.output_format == 'table' else 'stderr') context = analyzer_context.get_context() working_analyzers, errored = analyzer_types.check_supported_analyzers( args.analyzers, context) analyzer_environment = env.extend(context.path_env_extra, context.ld_lib_path_extra) analyzer_config_map = analyzer_types.build_config_handlers( args, context, working_analyzers) def uglify(text): """ csv and json format output contain this non human readable header string: no CamelCase and no space. """ return text.lower().replace(' ', '_') def match_guideline(checker_name, selected_guidelines): """ Returns True if checker_name gives reports related to any of the selected guideline rule. checker_name -- A full checker name. selected_guidelines -- A list of guideline names or guideline rule IDs. """ guideline = context.guideline_map.get(checker_name, {}) guideline_set = set(guideline) for value in guideline.values(): guideline_set |= set(value) return any(g in guideline_set for g in selected_guidelines) def format_guideline(guideline): """ Convert guideline rules to human-readable format. guideline -- Dictionary in the following format: {"guideline_1": ["rule_1", "rule_2"]} """ return ' '.join('Related {} rules: {}'.format(g, ', '.join(r)) for g, r in guideline.items()) # List available checker profiles. if 'profile' in args and args.profile == 'list': if 'details' in args: header = ['Profile name', 'Description'] rows = context.profile_map.available_profiles().items() else: header = ['Profile name'] rows = [(key, "") for key in context.profile_map.available_profiles()] if args.output_format in ['csv', 'json']: header = list(map(uglify, header)) print(twodim.to_str(args.output_format, header, rows)) return # List checker config options. if 'checker_config' in args: if 'details' in args: header = ['Option', 'Description'] else: header = ['Option'] if args.output_format in ['csv', 'json']: header = list(map(uglify, header)) rows = [] for analyzer in working_analyzers: config_handler = analyzer_config_map.get(analyzer) analyzer_class = analyzer_types.supported_analyzers[analyzer] configs = analyzer_class.get_checker_config( config_handler, analyzer_environment) rows.extend( (':'.join((analyzer, c[0])), c[1]) if 'details' in args else (':'.join((analyzer, c[0])), ) for c in configs) print(twodim.to_str(args.output_format, header, rows)) return if args.guideline is not None and len(args.guideline) == 0: result = defaultdict(set) for _, guidelines in context.guideline_map.items(): for guideline, rules in guidelines.items(): result[guideline] |= set(rules) header = ['Guideline', 'Rules'] if args.output_format in ['csv', 'json']: header = list(map(uglify, header)) if args.output_format == 'json': rows = [(g, sorted(list(r))) for g, r in result.items()] else: rows = [(g, ', '.join(sorted(r))) for g, r in result.items()] if args.output_format == 'rows': for row in rows: print('Guideline: {}'.format(row[0])) print('Rules: {}'.format(row[1])) else: print(twodim.to_str(args.output_format, header, rows)) return # List available checkers. if 'details' in args: header = [ 'Enabled', 'Name', 'Analyzer', 'Severity', 'Guideline', 'Description' ] else: header = ['Name'] if args.output_format in ['csv', 'json']: header = list(map(uglify, header)) rows = [] for analyzer in working_analyzers: config_handler = analyzer_config_map.get(analyzer) analyzer_class = analyzer_types.supported_analyzers[analyzer] checkers = analyzer_class.get_analyzer_checkers( config_handler, analyzer_environment) profile_checkers = [] if 'profile' in args: if args.profile not in context.profile_map.available_profiles(): LOG.error("Checker profile '%s' does not exist!", args.profile) LOG.error("To list available profiles, use '--profile list'.") sys.exit(1) profile_checkers = [('profile:' + args.profile, True)] config_handler.initialize_checkers(context, checkers, profile_checkers) for checker_name, value in config_handler.checks().items(): state, description = value if state != CheckerState.enabled and 'profile' in args: continue if state == CheckerState.enabled and 'only_disabled' in args: continue elif state != CheckerState.enabled and 'only_enabled' in args: continue if args.output_format == 'json': state = state == CheckerState.enabled else: state = '+' if state == CheckerState.enabled else '-' if args.guideline is not None: if not match_guideline(checker_name, args.guideline): continue if 'details' in args: severity = context.severity_map.get(checker_name) guideline = context.guideline_map.get(checker_name, {}) if args.output_format != 'json': guideline = format_guideline(guideline) rows.append([ state, checker_name, analyzer, severity, guideline, description ]) else: rows.append([checker_name]) if 'show_warnings' in args: severity = context.severity_map.get('clang-diagnostic-') for warning in get_warnings(analyzer_environment): warning = 'clang-diagnostic-' + warning[2:] if 'guideline' in args: if not match_guideline(warning, args.guideline): continue guideline = context.guideline_map.get(warning, {}) if args.output_format != 'json': guideline = format_guideline(guideline) if 'details' in args: rows.append(['', warning, '-', severity, guideline, '-']) else: rows.append([warning]) if rows: print(twodim.to_str(args.output_format, header, rows)) for analyzer_binary, reason in errored: LOG.error( "Failed to get checkers for '%s'!" "The error reason was: '%s'", analyzer_binary, reason) LOG.error("Please check your installation and the " "'config/package_layout.json' file!")
def main(args): """ Store the defect results in the specified input list as bug reports in the database. """ logger.setup_logger(args.verbose if 'verbose' in args else None) try: cmd_config.check_config_file(args) except FileNotFoundError as fnerr: LOG.error(fnerr) sys.exit(1) if not host_check.check_zlib(): raise Exception("zlib is not available on the system!") # To ensure the help message prints the default folder properly, # the 'default' for 'args.input' is a string, not a list. # But we need lists for the foreach here to work. if isinstance(args.input, str): args.input = [args.input] if 'name' not in args: LOG.debug("Generating name for analysis...") generated = __get_run_name(args.input) if generated: setattr(args, 'name', generated) else: LOG.error("No suitable name was found in the inputs for the " "analysis run. Please specify one by passing argument " "--name run_name in the invocation.") sys.exit(2) # argparse returns error code 2 for bad invocations. LOG.info("Storing analysis results for run '" + args.name + "'") if 'force' in args: LOG.info("argument --force was specified: the run with name '" + args.name + "' will be deleted.") # Setup connection to the remote server. client = libclient.setup_client(args.product_url) _, zip_file = tempfile.mkstemp('.zip') LOG.debug("Will write mass store ZIP to '%s'...", zip_file) try: LOG.debug("Assembling zip file.") try: assemble_zip(args.input, zip_file, client) except Exception as ex: print(ex) import traceback traceback.print_stack() zip_size = os.stat(zip_file).st_size LOG.debug("Assembling zip done, size is %s", sizeof_fmt(zip_size)) if zip_size > MAX_UPLOAD_SIZE: LOG.error("The result list to upload is too big (max: %s).", sizeof_fmt(MAX_UPLOAD_SIZE)) sys.exit(1) with open(zip_file, 'rb') as zf: b64zip = base64.b64encode(zf.read()).decode("utf-8") context = webserver_context.get_context() trim_path_prefixes = args.trim_path_prefix if \ 'trim_path_prefix' in args else None description = args.description if 'description' in args else None LOG.info("Storing results to the server...") client.massStoreRun(args.name, args.tag if 'tag' in args else None, str(context.version), b64zip, 'force' in args, trim_path_prefixes, description) # Storing analysis statistics if the server allows them. if client.allowsStoringAnalysisStatistics(): storing_analysis_statistics(client, args.input, args.name) LOG.info("Storage finished successfully.") except RequestFailed as reqfail: if reqfail.errorCode == ErrorCode.SOURCE_FILE: header = ['File', 'Line', 'Checker name'] table = twodim.to_str('table', header, [c.split('|') for c in reqfail.extraInfo]) LOG.warning( "Setting the review statuses for some reports failed " "because of non valid source code comments: " "%s\n %s", reqfail.message, table) sys.exit(1) except Exception as ex: import traceback traceback.print_stack() LOG.info("Storage failed: %s", str(ex)) sys.exit(1) finally: os.remove(zip_file)
def main(args): """ List the analyzers' basic information supported by CodeChecker. """ # If the given output format is not 'table', redirect logger's output to # the stderr. stream = None if 'output_format' in args and args.output_format != 'table': stream = 'stderr' logger.setup_logger(args.verbose if 'verbose' in args else None, stream) context = analyzer_context.get_context() working_analyzers, errored = \ analyzer_types.check_supported_analyzers( analyzer_types.supported_analyzers, context) if args.dump_config: binary = context.analyzer_binaries.get(args.dump_config) if args.dump_config == 'clang-tidy': subprocess.call([binary, '-dump-config', '-checks=*'], encoding="utf-8", errors="ignore") elif args.dump_config == 'clangsa': ret = subprocess.call([ binary, '-cc1', '-analyzer-checker-option-help', '-analyzer-checker-option-help-alpha' ], stderr=subprocess.PIPE, encoding="utf-8", errors="ignore") if ret: # This flag is supported from Clang 9. LOG.warning("'--dump-config clangsa' is not supported yet. " "Please make sure that you are using Clang 9 or " "newer.") return analyzer_environment = env.extend(context.path_env_extra, context.ld_lib_path_extra) analyzer_config_map = analyzer_types.build_config_handlers( args, context, working_analyzers) def uglify(text): """ csv and json format output contain this non human readable header string: no CamelCase and no space. """ return text.lower().replace(' ', '_') if 'analyzer_config' in args: if 'details' in args: header = ['Option', 'Description'] else: header = ['Option'] if args.output_format in ['csv', 'json']: header = list(map(uglify, header)) analyzer = args.analyzer_config config_handler = analyzer_config_map.get(analyzer) analyzer_class = analyzer_types.supported_analyzers[analyzer] configs = analyzer_class.get_analyzer_config(config_handler, analyzer_environment) rows = [(':'.join((analyzer, c[0])), c[1]) if 'details' in args else (':'.join((analyzer, c[0])), ) for c in configs] print(twodim.to_str(args.output_format, header, rows)) return if 'details' in args: header = ['Name', 'Path', 'Version'] else: header = ['Name'] if args.output_format in ['csv', 'json']: header = list(map(uglify, header)) rows = [] for analyzer in working_analyzers: if 'details' not in args: rows.append([analyzer]) else: binary = context.analyzer_binaries.get(analyzer) try: version = subprocess.check_output([binary, '--version'], encoding="utf-8", errors="ignore") except (subprocess.CalledProcessError, OSError): version = 'ERROR' rows.append([analyzer, binary, version]) if 'all' in args: for analyzer, err_reason in errored: if 'details' not in args: rows.append([analyzer]) else: rows.append([ analyzer, context.analyzer_binaries.get(analyzer), err_reason ]) if rows: print(twodim.to_str(args.output_format, header, rows))