def printResult(getterFn, baseid, newid, suppr, output_format): report_filter = [ codeCheckerDBAccess.ttypes.ReportFilter(suppressed=suppr) ] add_filter_conditions(report_filter[0], args.filter) sort_type = None limit = codeCheckerDBAccess.constants.MAX_QUERY_SIZE offset = 0 all_results = [] results = getterFn(baseid, newid, limit, offset, sort_type, report_filter) while results: all_results.extend(results) offset += limit results = getterFn(baseid, newid, limit, offset, sort_type, report_filter) if output_format == 'json': print(CmdLineOutputEncoder().encode(all_results)) else: header = ['File', 'Checker', 'Severity', 'Msg'] rows = [] for res in all_results: bug_line = res.lastBugPosition.startLine sev = shared.ttypes.Severity._VALUES_TO_NAMES[res.severity] checked_file = res.checkedFile + ' @ ' + str(bug_line) rows.append((checked_file, res.checkerId, sev, res.checkerMsg)) print(twodim_to_str(output_format, header, rows))
def handle_list_products(args): init_logger(args.verbose if 'verbose' in args else None) protocol, host, port = split_server_url(args.server_url) client = setup_product_client(protocol, host, port) products = client.getProducts(None, None) if args.output_format == 'json': results = [] for product in products: results.append({product.endpoint: product}) print(CmdLineOutputEncoder().encode(results)) else: # plaintext, csv header = ['Database status', 'Endpoint', 'Name', 'Description'] rows = [] for product in products: name = base64.b64decode(product.displayedName_b64) \ if product.displayedName_b64 else '' description = base64.b64decode(product.description_b64) \ if product.description_b64 else '' if not product.accessible: db_status_msg = 'No access.' else: db_status = product.databaseStatus db_status_msg = database_status.db_status_msg.get( db_status, 'Unknown database status') rows.append((db_status_msg, product.endpoint, name, description)) print(twodim_to_str(args.output_format, header, rows))
def main(args): """ List the analyzers' basic information supported by CodeChecker. """ logger.setup_logger(args.verbose if 'verbose' in args else None) context = generic_package_context.get_context() working, errored = \ analyzer_types.check_supported_analyzers( analyzer_types.supported_analyzers, context) if args.dump_config: binary = context.analyzer_binaries.get(args.dump_config) if args.dump_config == 'clang-tidy': subprocess.call([binary, '-dump-config', '-checks=*']) elif args.dump_config == 'clangsa': # TODO: Not supported by ClangSA yet! LOG.warning("'--dump-config clangsa' is not supported yet.") return if args.output_format not in ['csv', 'json']: if 'details' not in args: header = ['Name'] else: header = ['Name', 'Path', 'Version'] else: if 'details' not in args: header = ['name'] else: header = ['name', 'path', 'version_string'] rows = [] for analyzer in working: if 'details' not in args: rows.append([analyzer]) else: binary = context.analyzer_binaries.get(analyzer) try: version = subprocess.check_output([binary, '--version']) except (subprocess.CalledProcessError, OSError): version = 'ERROR' rows.append([analyzer, binary, version]) if 'all' in args: for analyzer, err_reason in errored: if 'details' not in args: rows.append([analyzer]) else: rows.append([ analyzer, context.analyzer_binaries.get(analyzer), err_reason ]) if len(rows) > 0: print(output_formatters.twodim_to_str(args.output_format, header, rows))
def handle_list_runs(args): init_logger(args.verbose if 'verbose' in args else None) client = setup_client(args.product_url) runs = client.getRunData(None) if args.output_format == 'json': results = [] for run in runs: results.append({run.name: run}) print(CmdLineOutputEncoder().encode(results)) else: # plaintext, csv header = [ 'Name', 'Number of reports', 'Storage date', 'Version tag', 'Duration' ] rows = [] for run in runs: duration = str(timedelta(seconds=run.duration)) \ if run.duration > -1 else 'Not finished' rows.append((run.name, str(run.resultCount), run.runDate, run.versionTag if run.versionTag else '', duration)) print(twodim_to_str(args.output_format, header, rows))
def main(args): """ Get and print the version information from the version config file and Thrift API definition. """ context = generic_package_context.get_context() rows = [ ("Base package version", context.version), ("Package build date", context.package_build_date), ("Git commit ID (hash)", context.package_git_hash), ("Git tag information", context.package_git_tag), ("Database schema version", str(context.db_version_info)), ("Client API version (Thrift)", constants.API_VERSION) ] if args.output_format != "json": print(output_formatters.twodim_to_str(args.output_format, ["Kind", "Version"], rows)) elif args.output_format == "json": # Use a special JSON format here, instead of # [ {"kind": "something", "version": "0.0.0"}, {"kind": "foo", ... } ] # do # { "something": "0.0.0", "foo": ... } print(json.dumps(dict(rows)))
def print_version(output_format=None): """ Print web server version information in the given format. """ context = webserver_context.get_context() server_versions = [ '{0}.{1}'.format(major, minor) for major, minor in version.SUPPORTED_VERSIONS.items() ] if output_format != 'json': server_versions = ', '.join(server_versions) rows = [("Base package version", context.version), ("Package build date", context.package_build_date), ("Git commit ID (hash)", context.package_git_hash), ("Git tag information", context.package_git_tag), ("Configuration schema", str(context.product_db_version_info)), ("Database schema", str(context.run_db_version_info)), ("Server supported API (Thrift)", server_versions), ("Client API (Thrift)", version.CLIENT_API)] if output_format != "json": print( output_formatters.twodim_to_str(output_format, ["Kind", "Version"], rows)) elif output_format == "json": # Use a special JSON format here, instead of # [ {"kind": "something", "version": "0.0.0"}, {"kind": "foo", ... } ] # do # { "something": "0.0.0", "foo": ... } print(json.dumps(dict(rows)))
def handle_list_run_histories(args): init_logger(args.verbose if 'verbose' in args else None) client = setup_client(args.product_url) run_ids = None if 'names' in args: runs = get_runs(client, args.names) run_ids = [r.runId for r in runs] run_history = client.getRunHistory(run_ids, None, None, None) if args.output_format == 'json': print(CmdLineOutputEncoder().encode(run_history)) else: # plaintext, csv header = [ 'Date', 'Run name', 'Version tag', 'User', 'CodeChecker version', 'Analyzer statistics' ] rows = [] for h in run_history: analyzer_statistics = [] for analyzer in h.analyzerStatistics: stat = h.analyzerStatistics[analyzer] num_of_all_files = stat.successful + stat.failed analyzer_statistics.append(analyzer + ' (' + str(num_of_all_files) + '/' + str(stat.successful) + ')') rows.append( (h.time, h.runName, h.versionTag if h.versionTag else '', h.user, h.codeCheckerVersion if h.codeCheckerVersion else '', ', '.join(analyzer_statistics))) print(twodim_to_str(args.output_format, header, rows))
def handle_list_results(args): init_logger(args.verbose if 'verbose' in args else None) check_deprecated_arg_usage(args) client = setup_client(args.product_url) run_names = map(lambda x: x.strip(), args.name.split(':')) run_ids = [run.runId for run in get_runs(client, run_names)] if not len(run_ids): LOG.warning("No runs were found!") sys.exit(1) limit = constants.MAX_QUERY_SIZE offset = 0 report_filter = ttypes.ReportFilter() add_filter_conditions(client, report_filter, args) all_results = [] results = client.getRunResults(run_ids, limit, offset, None, report_filter, None) while results: all_results.extend(results) offset += limit results = client.getRunResults(run_ids, limit, offset, None, report_filter, None) if args.output_format == 'json': print(CmdLineOutputEncoder().encode(all_results)) else: header = [ 'File', 'Checker', 'Severity', 'Msg', 'Review status', 'Detection status' ] rows = [] for res in all_results: bug_line = res.line checked_file = res.checkedFile + ' @ ' + str(bug_line) sev = ttypes.Severity._VALUES_TO_NAMES[res.severity] rw_status = \ ttypes.ReviewStatus._VALUES_TO_NAMES[res.reviewData.status] dt_status = \ ttypes.DetectionStatus._VALUES_TO_NAMES[res.detectionStatus] rows.append((checked_file, res.checkerId, sev, res.checkerMsg, rw_status, dt_status)) print(twodim_to_str(args.output_format, header, rows))
def __instance_management(args): """Handles the instance-manager commands --list/--stop/--stop-all.""" # TODO: The server stopping and listing must be revised on its invocation # once "workspace", as a concept, is removed. # QUESTION: What is the bestest way here to identify a server for the user? if 'list' in args: instances = instance_manager.get_instances() instances_on_multiple_hosts = any(True for inst in instances if inst['hostname'] != socket.gethostname()) if not instances_on_multiple_hosts: head = ['Workspace', 'View port'] else: head = ['Workspace', 'Computer host', 'View port'] rows = [] for instance in instances: if not instances_on_multiple_hosts: rows.append((instance['workspace'], str(instance['port']))) else: rows.append((instance['workspace'], instance['hostname'] if instance['hostname'] != socket.gethostname() else '', str(instance['port']))) print("Your running CodeChecker servers:") print(output_formatters.twodim_to_str('table', head, rows)) elif 'stop' in args or 'stop_all' in args: for i in instance_manager.get_instances(): if i['hostname'] != socket.gethostname(): continue # A STOP only stops the server associated with the given workspace # and view-port. if 'stop' in args and \ not (i['port'] == args.view_port and os.path.abspath(i['workspace']) == os.path.abspath(args.config_directory)): continue try: util.kill_process_tree(i['pid']) LOG.info("Stopped CodeChecker server running on port {0} " "in workspace {1} (PID: {2})". format(i['port'], i['workspace'], i['pid'])) except Exception: # Let the exception come out if the commands fail LOG.error("Couldn't stop process PID #" + str(i['pid'])) raise
def handle_list_results(args): client = setup_client(args.host, args.port, '/') run_info = check_run_names(client, [args.name]) run_id, _ = run_info.get(args.name) limit = codeCheckerDBAccess.constants.MAX_QUERY_SIZE offset = 0 filters = [] if args.suppressed: report_filter = codeCheckerDBAccess.ttypes.ReportFilter( suppressed=True) else: report_filter = codeCheckerDBAccess.ttypes.ReportFilter( suppressed=False) add_filter_conditions(report_filter, args.filter) filters.append(report_filter) all_results = [] results = client.getRunResults(run_id, limit, offset, None, filters) while results: all_results.extend(results) offset += limit results = client.getRunResults(run_id, limit, offset, None, filters) if args.output_format == 'json': print(CmdLineOutputEncoder().encode(all_results)) else: if args.suppressed: header = ['File', 'Checker', 'Severity', 'Msg', 'Suppress comment'] else: header = ['File', 'Checker', 'Severity', 'Msg'] rows = [] for res in all_results: bug_line = res.lastBugPosition.startLine checked_file = res.checkedFile + ' @ ' + str(bug_line) sev = shared.ttypes.Severity._VALUES_TO_NAMES[res.severity] if args.suppressed: rows.append((checked_file, res.checkerId, sev, res.checkerMsg, res.suppressComment)) else: rows.append((checked_file, res.checkerId, sev, res.checkerMsg)) print(twodim_to_str(args.output_format, header, rows))
def main(args): """ List the analyzers' basic information supported by CodeChecker. """ context = generic_package_context.get_context() working, errored = \ analyzer_types.check_supported_analyzers( analyzer_types.supported_analyzers, context) if args.output_format not in ['csv', 'json']: if 'details' not in args: header = ['Name'] else: header = ['Name', 'Path', 'Version'] else: if 'details' not in args: header = ['name'] else: header = ['name', 'path', 'version_string'] rows = [] for analyzer in working: if 'details' not in args: rows.append([analyzer]) else: binary = context.analyzer_binaries.get(analyzer) try: version = subprocess.check_output([binary, '--version']) except (subprocess.CalledProcessError, OSError): version = 'ERROR' rows.append([analyzer, binary, version]) if 'all' in args: for analyzer, err_reason in errored: if 'details' not in args: rows.append([analyzer]) else: rows.append([analyzer, context.analyzer_binaries.get(analyzer), err_reason]) if len(rows) > 0: print(output_formatters.twodim_to_str(args.output_format, header, rows))
def handle_list_results(args): client = setup_client(args.product_url) run_info = check_run_names(client, [args.name]) run = run_info.get(args.name) limit = constants.MAX_QUERY_SIZE offset = 0 report_filter = ttypes.ReportFilter() add_filter_conditions(report_filter, args.filter) all_results = [] results = client.getRunResults([run.runId], limit, offset, None, report_filter, None) while results: all_results.extend(results) offset += limit results = client.getRunResults([run.runId], limit, offset, None, report_filter, None) if args.output_format == 'json': print(CmdLineOutputEncoder().encode(all_results)) else: if args.suppressed: header = ['File', 'Checker', 'Severity', 'Msg', 'Suppress comment'] else: header = ['File', 'Checker', 'Severity', 'Msg'] rows = [] for res in all_results: bug_line = res.line checked_file = res.checkedFile + ' @ ' + str(bug_line) sev = ttypes.Severity._VALUES_TO_NAMES[res.severity] if args.suppressed: rows.append((checked_file, res.checkerId, sev, res.checkerMsg, res.suppressComment)) else: rows.append( (checked_file, res.checkerId, sev, res.checkerMsg)) print(twodim_to_str(args.output_format, header, rows))
def handle_list_runs(args): client = setup_client(args.product_url) runs = client.getRunData(None) if args.output_format == 'json': results = [] for run in runs: results.append({run.name: run}) print(CmdLineOutputEncoder().encode(results)) else: # plaintext, csv header = ['Name', 'ResultCount', 'RunDate'] rows = [] for run in runs: rows.append((run.name, str(run.resultCount), run.runDate)) print(twodim_to_str(args.output_format, header, rows))
def handle_list_runs(args): init_logger(args.verbose if 'verbose' in args else None) client = setup_client(args.product_url) run_filter = None if 'names' in args: run_filter = ttypes.RunFilter() run_filter.names = args.names runs = client.getRunData(run_filter) if args.output_format == 'json': results = [] for run in runs: results.append({run.name: run}) print(CmdLineOutputEncoder().encode(results)) else: # plaintext, csv header = [ 'Name', 'Number of unresolved reports', 'Analyzer statistics', 'Storage date', 'Version tag', 'Duration', 'CodeChecker version' ] rows = [] for run in runs: duration = str(timedelta(seconds=run.duration)) \ if run.duration > -1 else 'Not finished' analyzer_statistics = [] for analyzer in run.analyzerStatistics: stat = run.analyzerStatistics[analyzer] num_of_all_files = stat.successful + stat.failed analyzer_statistics.append(analyzer + ' (' + str(num_of_all_files) + '/' + str(stat.successful) + ')') codechecker_version = run.codeCheckerVersion \ if run.codeCheckerVersion else '' rows.append((run.name, str(run.resultCount), ', '.join(analyzer_statistics), run.runDate, run.versionTag if run.versionTag else '', duration, codechecker_version)) print(twodim_to_str(args.output_format, header, rows))
def handle_list_result_types(args): client = setup_client(args.host, args.port, '/') filters = [] if args.suppressed: report_filter = codeCheckerDBAccess.ttypes.ReportFilter( suppressed=True) else: report_filter = codeCheckerDBAccess.ttypes.ReportFilter( suppressed=False) add_filter_conditions(report_filter, args.filter) filters.append(report_filter) if 'all_results' in args: items = check_run_names(client, None).items() else: items = [] run_info = check_run_names(client, args.names) for name in args.names: items.append((name, run_info.get(name))) results_collector = [] for name, run_info in items: run_id, run_date = run_info results = client.getRunResultTypes(run_id, filters) if args.output_format == 'json': for res in results: res.severity =\ shared.ttypes.Severity._VALUES_TO_NAMES[res.severity] results_collector.append({name: results}) else: # plaintext, csv print("Run '" + name + "', executed at '" + run_date + "'") rows = [] header = ['Checker', 'Severity', 'Count'] for res in results: sev = shared.ttypes.Severity._VALUES_TO_NAMES[res.severity] rows.append((res.checkerId, sev, str(res.count))) print(twodim_to_str(args.output_format, header, rows)) if args.output_format == 'json': print(CmdLineOutputEncoder().encode(results_collector))
def printReports(client, reports, output_format): if output_format == 'json': output = [] for report in reports: if type(report) is Report: output.append(report.main) else: output.append(report) print(CmdLineOutputEncoder().encode(output)) return header = ['File', 'Checker', 'Severity', 'Msg', 'Source'] rows = [] for report in reports: if type(report) is Report: bug_line = report.main['location']['line'] bug_col = report.main['location']['col'] sev = 'unknown' checked_file = report.main['location']['file_name']\ + ':' + str(bug_line) + ":" + str(bug_col) check_name = report.main['check_name'] check_msg = report.main['description'] source_line =\ getLineFromFile(report.main['location']['file_name'], bug_line) else: bug_line = report.lastBugPosition.startLine bug_col = report.lastBugPosition.startCol sev = \ shared.ttypes.Severity._VALUES_TO_NAMES[report.severity] checked_file = report.checkedFile + ':' + str(bug_line) +\ ":" + str(bug_col) source_line =\ getLineFromRemoteFile(client, report.fileId, bug_line) check_name = report.checkerId check_msg = report.checkerMsg rows.append( (checked_file, check_name, sev, check_msg, source_line)) if output_format == 'plaintext': for row in rows: print("{0}: {1} [{2}]\n{3}\n".format(row[0], row[3], row[1], row[4])) else: print(twodim_to_str(output_format, header, rows))
def handle_list_components(args): init_logger(args.verbose if 'verbose' in args else None) client = setup_client(args.product_url) components = client.getSourceComponents(None) if args.output_format == 'json': print(CmdLineOutputEncoder().encode(components)) else: # plaintext, csv header = ['Name', 'Value', 'Description'] rows = [] for res in components: for idx, value in enumerate(res.value.split('\n')): name = res.name if idx == 0 else '' description = res.description \ if idx == 0 and res.description else '' rows.append((name, value, description)) print(twodim_to_str(args.output_format, header, rows))
def handle_list_tokens(args): """ List personal access tokens of the currently logged in user. """ init_logger(args.verbose if 'verbose' in args else None) protocol, host, port = split_server_url(args.server_url) client, curr_token = setup_auth_client(protocol, host, port) tokens = client.getTokens() if args.output_format == 'json': print(CmdLineOutputEncoder().encode(tokens)) else: # plaintext, csv header = ['Token', 'Description', 'Last access'] rows = [] for res in tokens: rows.append((res.token, res.description if res.description else '', res.lastAccess)) print(twodim_to_str(args.output_format, header, rows))
def handle_list_run_histories(args): init_logger(args.verbose if 'verbose' in args else None) client = setup_client(args.product_url) run_ids = None if 'names' in args: runs = get_runs(client, args.names) run_ids = [r.runId for r in runs] run_history = client.getRunHistory(run_ids, None, None, None) if args.output_format == 'json': print(CmdLineOutputEncoder().encode(run_history)) else: # plaintext, csv header = ['Date', 'Run name', 'Version tag', 'User'] rows = [] for h in run_history: rows.append((h.time, h.runName, h.versionTag if h.versionTag else '', h.user)) print(twodim_to_str(args.output_format, header, rows))
def print_version(output_format=None): """ Print analyzer version information in the given format. """ context = analyzer_context.get_context() rows = [ ("Base package version", context.version), ("Package build date", context.package_build_date), ("Git commit ID (hash)", context.package_git_hash), ("Git tag information", context.package_git_tag) ] if output_format != "json": print(output_formatters.twodim_to_str(output_format, ["Kind", "Version"], rows)) elif output_format == "json": # Use a special JSON format here, instead of # [ {"kind": "something", "version": "0.0.0"}, {"kind": "foo", ... } ] # do # { "something": "0.0.0", "foo": ... } print(json.dumps(dict(rows)))
def handle_list_products(args): protocol, host, port = split_server_url(args.server_url) client = setup_product_client(protocol, host, port) products = client.getProducts(None, None) if args.output_format == 'json': results = [] for product in products: results.append({product.endpoint: product}) print(CmdLineOutputEncoder().encode(results)) else: # plaintext, csv header = ['Status', 'Endpoint', 'Name', 'Description'] rows = [] for product in products: name = base64.b64decode(product.displayedName_b64) \ if product.displayedName_b64 else '' description = base64.b64decode(product.description_b64) \ if product.description_b64 else '' rows.append(('Database error' if not product.connected else 'No access' if not product.accessible else '', product.endpoint, name, description)) print(twodim_to_str(args.output_format, header, rows))
def print_prod_status(prod_status): """ Print the database statuses for each of the products. """ header = ['Product endpoint', 'Database status', 'Database location', 'Schema version in the database', 'Schema version in the package'] rows = [] for k, v in prod_status.items(): db_status, schema_ver, package_ver, db_location = v db_status_msg = database_status.db_status_msg.get(db_status) if schema_ver == package_ver: schema_ver += " (up to date)" rows.append([k, db_status_msg, db_location, schema_ver, package_ver]) prod_status = output_formatters.twodim_to_str('table', header, rows, sort_by_column_number=0) LOG.info('Status of products:\n{0}'.format(prod_status))
def main(args): """ Get and print the version information from the version config file and Thrift API definition. """ logger.setup_logger(args.verbose if 'verbose' in args else None) context = generic_package_context.get_context() server_versions = [ '{0}.{1}'.format(major, minor) for major, minor in version.SUPPORTED_VERSIONS.items() ] if args.output_format != 'json': server_versions = ', '.join(server_versions) rows = [("Base package version", context.version), ("Package build date", context.package_build_date), ("Git commit ID (hash)", context.package_git_hash), ("Git tag information", context.package_git_tag), ("Configuration schema", str(context.product_db_version_info)), ("Database schema", str(context.run_db_version_info)), ("Server supported API (Thrift)", server_versions), ("Client API (Thrift)", version.CLIENT_API)] if args.output_format != "json": print( output_formatters.twodim_to_str(args.output_format, ["Kind", "Version"], rows)) elif args.output_format == "json": # Use a special JSON format here, instead of # [ {"kind": "something", "version": "0.0.0"}, {"kind": "foo", ... } ] # do # { "something": "0.0.0", "foo": ... } print(json.dumps(dict(rows)))
def handle_list_result_types(args): init_logger(args.verbose if 'verbose' in args else None) def get_statistics(client, run_ids, field, values): report_filter = ttypes.ReportFilter() report_filter.isUnique = True add_filter_conditions(report_filter, args.filter) setattr(report_filter, field, values) checkers = client.getCheckerCounts(run_ids, report_filter, None, None, 0) return dict((res.name, res.count) for res in checkers) def checker_count(checker_dict, key): return checker_dict.get(key, 0) client = setup_client(args.product_url) run_ids = None if 'all_results' not in args: run_ids = [run.runId for run in get_runs(client, args.names)] if not len(run_ids): LOG.warning("No runs were found!") sys.exit(1) all_checkers_report_filter = ttypes.ReportFilter() all_checkers_report_filter.isUnique = True add_filter_conditions(all_checkers_report_filter, args.filter) all_checkers = client.getCheckerCounts(run_ids, all_checkers_report_filter, None, None, 0) all_checkers_dict = dict((res.name, res) for res in all_checkers) unrev_checkers = get_statistics(client, run_ids, 'reviewStatus', [ttypes.ReviewStatus.UNREVIEWED]) confirmed_checkers = get_statistics(client, run_ids, 'reviewStatus', [ttypes.ReviewStatus.CONFIRMED]) false_checkers = get_statistics(client, run_ids, 'reviewStatus', [ttypes.ReviewStatus.FALSE_POSITIVE]) intentional_checkers = get_statistics(client, run_ids, 'reviewStatus', [ttypes.ReviewStatus.INTENTIONAL]) resolved_checkers = get_statistics(client, run_ids, 'detectionStatus', [ttypes.DetectionStatus.RESOLVED]) # Get severity counts report_filter = ttypes.ReportFilter() report_filter.isUnique = True add_filter_conditions(report_filter, args.filter) sev_count = client.getSeverityCounts(run_ids, report_filter, None) severities = [] severity_total = 0 for key, count in sorted(sev_count.items(), reverse=True): severities.append( dict(severity=ttypes.Severity._VALUES_TO_NAMES[key], reports=count)) severity_total += count all_results = [] total = defaultdict(int) for key, checker_data in sorted(all_checkers_dict.items(), key=lambda x: x[1].severity, reverse=True): all_results.append( dict( checker=key, severity=ttypes.Severity._VALUES_TO_NAMES[ checker_data.severity], reports=checker_data.count, unreviewed=checker_count(unrev_checkers, key), confirmed=checker_count(confirmed_checkers, key), false_positive=checker_count(false_checkers, key), intentional=checker_count(intentional_checkers, key), resolved=checker_count(resolved_checkers, key), )) total['total_reports'] += checker_data.count total['total_resolved'] += checker_count(resolved_checkers, key) total['total_unreviewed'] += checker_count(unrev_checkers, key) total['total_confirmed'] += checker_count(confirmed_checkers, key) total['total_false_positive'] += checker_count(false_checkers, key) total['total_intentional'] += checker_count(intentional_checkers, key) if args.output_format == 'json': print(CmdLineOutputEncoder().encode(all_results)) else: header = [ 'Checker', 'Severity', 'All reports', 'Resolved', 'Unreviewed', 'Confirmed', 'False positive', "Intentional" ] rows = [] for stat in all_results: rows.append( (stat['checker'], stat['severity'], str(stat['reports']), str(stat['resolved']), str(stat['unreviewed']), str(stat['confirmed']), str(stat['false_positive']), str(stat['intentional']))) rows.append( ('Total', '-', str(total['total_reports']), str(total['total_resolved']), str(total['total_unreviewed']), str(total['total_confirmed']), str(total['total_false_positive']), str(total['total_intentional']))) print( twodim_to_str(args.output_format, header, rows, separate_footer=True)) # Print severity counts header = ['Severity', 'All reports'] rows = [] for stat in severities: rows.append((stat['severity'], str(stat['reports']))) rows.append(('Total', str(severity_total))) print( twodim_to_str(args.output_format, header, rows, separate_footer=True))
def handle_list_result_types(args): def get_statistics(client, run_ids, field, values): report_filter = ttypes.ReportFilter() report_filter.isUnique = True setattr(report_filter, field, values) checkers = client.getCheckerCounts(run_ids, report_filter, None) return dict((res.name, res.count) for res in checkers) def checker_count(dict, key): return dict[key] if key in dict else 0 client = setup_client(args.product_url) run_ids = None if 'all_results' not in args: items = check_run_names(client, args.names) run_ids = map(lambda run: run.runId, items.values()) all_checkers_report_filter = ttypes.ReportFilter() all_checkers_report_filter.isUnique = True all_checkers = client.getCheckerCounts(run_ids, all_checkers_report_filter, None) all_checkers_dict = dict((res.name, res) for res in all_checkers) unrev_checkers = get_statistics(client, run_ids, 'reviewStatus', [ttypes.ReviewStatus.UNREVIEWED]) confirmed_checkers = get_statistics(client, run_ids, 'reviewStatus', [ttypes.ReviewStatus.CONFIRMED]) false_checkers = get_statistics(client, run_ids, 'reviewStatus', [ttypes.ReviewStatus.FALSE_POSITIVE]) intentional_checkers = get_statistics(client, run_ids, 'reviewStatus', [ttypes.ReviewStatus.INTENTIONAL]) resolved_checkers = get_statistics(client, run_ids, 'detectionStatus', [ttypes.DetectionStatus.RESOLVED]) all_results = [] for key, checker_data in sorted(all_checkers_dict.items(), key=lambda x: x[1].severity, reverse=True): all_results.append( dict( checker=key, severity=ttypes.Severity._VALUES_TO_NAMES[ checker_data.severity], reports=checker_data.count, unreviewed=checker_count(unrev_checkers, key), confirmed=checker_count(confirmed_checkers, key), false_positive=checker_count(false_checkers, key), intentional=checker_count(intentional_checkers, key), resolved=checker_count(resolved_checkers, key), )) if args.output_format == 'json': print(CmdLineOutputEncoder().encode(all_results)) else: header = [ 'Checker', 'Severity', 'All reports', 'Resolved', 'Unreviewed', 'Confirmed', 'False positive', "Intentional" ] rows = [] for stat in all_results: rows.append( (stat['checker'], stat['severity'], str(stat['reports']), str(stat['resolved']), str(stat['unreviewed']), str(stat['confirmed']), str(stat['false_positive']), str(stat['intentional']))) print(twodim_to_str(args.output_format, header, rows))
def print_reports(client, reports, output_format, diff_type): output_dir = args.export_dir if 'export_dir' in args else None if 'clean' in args and os.path.isdir(output_dir): print("Previous analysis results in '{0}' have been removed, " "overwriting with current results.".format(output_dir)) shutil.rmtree(output_dir) if output_format == 'json': output = [] for report in reports: if isinstance(report, Report): output.append(report.main) else: output.append(report) print(CmdLineOutputEncoder().encode(output)) return if output_format == 'html': if len(reports) == 0: print('No {0} reports was found!'.format(diff_type)) return output_dir = args.export_dir if not os.path.exists(output_dir): os.makedirs(output_dir) print("Generating HTML output files to file://{0} directory:\n". format(output_dir)) report_to_html(client, reports, output_dir) print('\nTo view the results in a browser run:\n' ' $ firefox {0}'.format(args.export_dir)) return header = ['File', 'Checker', 'Severity', 'Msg', 'Source'] rows = [] for report in reports: if type(report) is Report: bug_line = report.main['location']['line'] bug_col = report.main['location']['col'] sev = 'unknown' checked_file = report.main['location']['file_name']\ + ':' + str(bug_line) + ":" + str(bug_col) check_name = report.main['check_name'] check_msg = report.main['description'] source_line =\ get_line_from_file(report.main['location']['file_name'], bug_line) else: bug_line = report.line bug_col = report.column sev = ttypes.Severity._VALUES_TO_NAMES[report.severity] checked_file = report.checkedFile + ':' + str(bug_line) +\ ":" + str(bug_col) source_line =\ get_line_from_remote_file(client, report.fileId, bug_line) check_name = report.checkerId check_msg = report.checkerMsg rows.append( (checked_file, check_name, sev, check_msg, source_line)) if output_format == 'plaintext': for row in rows: print("{0}: {1} [{2}]\n{3}\n".format(row[0], row[3], row[1], row[4])) else: print(twodim_to_str(output_format, header, rows))
def main(args): """ Entry point for parsing some analysis results and printing them to the stdout in a human-readable format. """ logger.setup_logger(args.verbose if 'verbose' in args else None) context = generic_package_context.get_context() # To ensure the help message prints the default folder properly, # the 'default' for 'args.input' is a string, not a list. # But we need lists for the foreach here to work. if isinstance(args.input, str): args.input = [args.input] original_cwd = os.getcwd() suppress_handler = None if 'suppress' in args: __make_handler = False if not os.path.isfile(args.suppress): if 'create_suppress' in args: with open(args.suppress, 'w') as _: # Just create the file. __make_handler = True LOG.info("Will write source-code suppressions to " "suppress file.") else: LOG.warning("Suppress file '" + args.suppress + "' given, but " "it does not exist -- will not suppress anything.") else: __make_handler = True if __make_handler: suppress_handler = generic_package_suppress_handler.\ GenericSuppressHandler(args.suppress, 'create_suppress' in args) elif 'create_suppress' in args: LOG.error("Can't use '--export-source-suppress' unless '--suppress " "SUPPRESS_FILE' is also given.") sys.exit(2) skip_handler = None if 'skipfile' in args: skip_handler = SkipListHandler(args.skipfile) for input_path in args.input: input_path = os.path.abspath(input_path) os.chdir(original_cwd) LOG.debug("Parsing input argument: '" + input_path + "'") export = args.export if 'export' in args else None if export is not None and export == 'html': output_path = os.path.abspath(args.output_path) LOG.info("Generating html output files:") PlistToHtml.parse(input_path, output_path, context.path_plist_to_html_dist, 'clean' in args) continue severity_stats = Counter({}) file_stats = Counter({}) report_count = Counter({}) files = [] metadata_dict = {} if os.path.isfile(input_path): files.append(input_path) elif os.path.isdir(input_path): metadata_file = os.path.join(input_path, "metadata.json") if os.path.exists(metadata_file): with open(metadata_file, 'r') as metadata: metadata_dict = json.load(metadata) LOG.debug(metadata_dict) if 'working_directory' in metadata_dict: os.chdir(metadata_dict['working_directory']) _, _, file_names = next(os.walk(input_path), ([], [], [])) files = [ os.path.join(input_path, file_name) for file_name in file_names ] for file_path in files: report_stats = parse(file_path, context, metadata_dict, suppress_handler, skip_handler, 'print_steps' in args) severity_stats.update(Counter(report_stats.get('severity', {}))) file_stats.update(Counter(report_stats.get('files', {}))) report_count.update(Counter(report_stats.get('reports', {}))) print("\n----==== Summary ====----") if file_stats: vals = [[os.path.basename(k), v] for k, v in dict(file_stats).items()] keys = ['Filename', 'Report count'] table = twodim_to_str('table', keys, vals, 1, True) print(table) if severity_stats: vals = [[k, v] for k, v in dict(severity_stats).items()] keys = ['Severity', 'Report count'] table = twodim_to_str('table', keys, vals, 1, True) print(table) report_count = dict(report_count).get("report_count", 0) print("----=================----") print("Total number of reports: {}".format(report_count)) print("----=================----") os.chdir(original_cwd)
def main(args): """ Entry point for parsing some analysis results and printing them to the stdout in a human-readable format. """ logger.setup_logger(args.verbose if 'verbose' in args else None) context = package_context.get_context() # To ensure the help message prints the default folder properly, # the 'default' for 'args.input' is a string, not a list. # But we need lists for the foreach here to work. if isinstance(args.input, str): args.input = [args.input] original_cwd = os.getcwd() suppr_handler = None if 'suppress' in args: __make_handler = False if not os.path.isfile(args.suppress): if 'create_suppress' in args: with open(args.suppress, 'w') as _: # Just create the file. __make_handler = True LOG.info("Will write source-code suppressions to " "suppress file.") else: LOG.warning("Suppress file '" + args.suppress + "' given, but " "it does not exist -- will not suppress anything.") else: __make_handler = True if __make_handler: suppr_handler = suppress_handler.\ GenericSuppressHandler(args.suppress, 'create_suppress' in args) elif 'create_suppress' in args: LOG.error("Can't use '--export-source-suppress' unless '--suppress " "SUPPRESS_FILE' is also given.") sys.exit(2) processed_path_hashes = set() def skip_html_report_data_handler(report_hash, source_file, report_line, checker_name, diag, files): """ Report handler which skips bugs which were suppressed by source code comments. """ report = Report(None, diag['path'], files) path_hash = get_report_path_hash(report, files) if path_hash in processed_path_hashes: LOG.debug("Skip report because it is a deduplication of an " "already processed report!") LOG.debug("Path hash: %s", path_hash) LOG.debug(diag) return True skip = plist_parser.skip_report(report_hash, source_file, report_line, checker_name, suppr_handler) if not skip: processed_path_hashes.add(path_hash) return skip skip_handler = None if 'skipfile' in args: with open(args.skipfile, 'r') as skip_file: skip_handler = SkipListHandler(skip_file.read()) html_builder = None for input_path in args.input: input_path = os.path.abspath(input_path) os.chdir(original_cwd) LOG.debug("Parsing input argument: '" + input_path + "'") export = args.export if 'export' in args else None if export is not None and export == 'html': output_path = os.path.abspath(args.output_path) if not html_builder: html_builder = \ PlistToHtml.HtmlBuilder(context.path_plist_to_html_dist, context.severity_map) LOG.info("Generating html output files:") PlistToHtml.parse(input_path, output_path, context.path_plist_to_html_dist, skip_html_report_data_handler, html_builder) continue files = [] metadata_dict = {} if os.path.isfile(input_path): files.append(input_path) elif os.path.isdir(input_path): metadata_file = os.path.join(input_path, "metadata.json") if os.path.exists(metadata_file): metadata_dict = util.load_json_or_empty(metadata_file) LOG.debug(metadata_dict) if 'working_directory' in metadata_dict: working_dir = metadata_dict['working_directory'] try: os.chdir(working_dir) except OSError as oerr: LOG.debug(oerr) LOG.error( "Working directory %s is missing.\n" "Can not parse reports safely.", working_dir) sys.exit(1) _, _, file_names = next(os.walk(input_path), ([], [], [])) files = [ os.path.join(input_path, file_name) for file_name in file_names ] file_change = set() file_report_map = defaultdict(list) rh = plist_parser.PlistToPlaintextFormatter(suppr_handler, skip_handler, context.severity_map, processed_path_hashes) rh.print_steps = 'print_steps' in args for file_path in files: f_change = parse(file_path, metadata_dict, rh, file_report_map) file_change = file_change.union(f_change) report_stats = rh.write(file_report_map) severity_stats = report_stats.get('severity') file_stats = report_stats.get('files') reports_stats = report_stats.get('reports') print("\n----==== Summary ====----") if file_stats: vals = [[os.path.basename(k), v] for k, v in dict(file_stats).items()] keys = ['Filename', 'Report count'] table = twodim_to_str('table', keys, vals, 1, True) print(table) if severity_stats: vals = [[k, v] for k, v in dict(severity_stats).items()] keys = ['Severity', 'Report count'] table = twodim_to_str('table', keys, vals, 1, True) print(table) report_count = reports_stats.get("report_count", 0) print("----=================----") print("Total number of reports: {}".format(report_count)) print("----=================----") if file_change: changed_files = '\n'.join([' - ' + f for f in file_change]) LOG.warning("The following source file contents changed since the " "latest analysis:\n{0}\nMultiple reports were not " "shown and skipped from the statistics. Please " "analyze your project again to update the " "reports!".format(changed_files)) os.chdir(original_cwd) # Create index.html for the generated html files. if html_builder: html_builder.create_index_html(args.output_path)
def main(args): """ Store the defect results in the specified input list as bug reports in the database. """ logger.setup_logger(args.verbose if 'verbose' in args else None) if not host_check.check_zlib(): raise Exception("zlib is not available on the system!") # To ensure the help message prints the default folder properly, # the 'default' for 'args.input' is a string, not a list. # But we need lists for the foreach here to work. if isinstance(args.input, str): args.input = [args.input] if 'name' not in args: LOG.debug("Generating name for analysis...") generated = __get_run_name(args.input) if generated: setattr(args, 'name', generated) else: LOG.error("No suitable name was found in the inputs for the " "analysis run. Please specify one by passing argument " "--name run_name in the invocation.") sys.exit(2) # argparse returns error code 2 for bad invocations. LOG.info("Storing analysis results for run '" + args.name + "'") if 'force' in args: LOG.info("argument --force was specified: the run with name '" + args.name + "' will be deleted.") protocol, host, port, product_name = split_product_url(args.product_url) # Before any transmission happens, check if we have the PRODUCT_STORE # permission to prevent a possibly long ZIP operation only to get an # error later on. product_client = libclient.setup_product_client(protocol, host, port, product_name) product_id = product_client.getCurrentProduct().id auth_client, _ = libclient.setup_auth_client(protocol, host, port) has_perm = libclient.check_permission(auth_client, Permission.PRODUCT_STORE, {'productID': product_id}) if not has_perm: LOG.error("You are not authorised to store analysis results in " "product '{0}'".format(product_name)) sys.exit(1) # Setup connection to the remote server. client = libclient.setup_client(args.product_url, product_client=False) LOG.debug("Initializing client connecting to {0}:{1}/{2} done.".format( host, port, product_name)) _, zip_file = tempfile.mkstemp('.zip') LOG.debug("Will write mass store ZIP to '{0}'...".format(zip_file)) try: assemble_zip(args.input, zip_file, client) if os.stat(zip_file).st_size > MAX_UPLOAD_SIZE: LOG.error("The result list to upload is too big (max: {}).".format( sizeof_fmt(MAX_UPLOAD_SIZE))) sys.exit(1) with open(zip_file, 'rb') as zf: b64zip = base64.b64encode(zf.read()) context = generic_package_context.get_context() trim_path_prefixes = args.trim_path_prefix if \ 'trim_path_prefix' in args else None client.massStoreRun(args.name, args.tag if 'tag' in args else None, str(context.version), b64zip, 'force' in args, trim_path_prefixes) LOG.info("Storage finished successfully.") except RequestFailed as reqfail: if reqfail.errorCode == ErrorCode.SOURCE_FILE: header = ['File', 'Line', 'Checker name'] table = twodim_to_str('table', header, [c.split('|') for c in reqfail.extraInfo]) LOG.warning("Setting the review statuses for some reports failed " "because of non valid source code comments: " "{0}\n {1}".format(reqfail.message, table)) sys.exit(1) except Exception as ex: LOG.info("Storage failed: " + str(ex)) sys.exit(1) finally: os.remove(zip_file)
def main(args): """ List the checkers available in the specified (or all supported) analyzers alongside with their description or enabled status in various formats. """ logger.setup_logger(args.verbose if 'verbose' in args else None) # If nothing is set, list checkers for all supported analyzers. analyzers = args.analyzers \ if 'analyzers' in args \ else analyzer_types.supported_analyzers context = package_context.get_context() working, errored = analyzer_types.check_supported_analyzers( analyzers, context) analyzer_environment = analyzer_env.get_check_env( context.path_env_extra, context.ld_lib_path_extra) analyzer_config_map = analyzer_types.build_config_handlers( args, context, working) # List available checker profiles. if 'profile' in args and args.profile == 'list': if 'details' not in args: if args.output_format not in ['csv', 'json']: header = ['Profile name'] else: header = ['profile_name'] else: if args.output_format not in ['csv', 'json']: header = ['Profile name', 'Description'] else: header = ['profile_name', 'description'] rows = [] for (profile, description) in context.available_profiles.items(): if 'details' not in args: rows.append([profile]) else: rows.append([profile, description]) print(output_formatters.twodim_to_str(args.output_format, header, rows)) return # Use good looking different headers based on format. if 'details' not in args: if args.output_format not in ['csv', 'json']: header = ['Name'] else: header = ['name'] else: if args.output_format not in ['csv', 'json']: header = ['', 'Name', 'Analyzer', 'Severity', 'Description'] else: header = ['enabled', 'name', 'analyzer', 'severity', 'description'] rows = [] for analyzer in working: config_handler = analyzer_config_map.get(analyzer) analyzer_class = \ analyzer_types.supported_analyzers[analyzer] checkers = analyzer_class.get_analyzer_checkers( config_handler, analyzer_environment) default_checker_cfg = context.checker_config.get(analyzer + '_checkers') profile_checkers = None if 'profile' in args: if args.profile not in context.available_profiles: LOG.error("Checker profile '" + args.profile + "' does not exist!") LOG.error("To list available profiles, use '--profile list'.") return profile_checkers = [(args.profile, True)] config_handler.initialize_checkers(context.available_profiles, context.package_root, checkers, default_checker_cfg, profile_checkers) for checker_name, value in config_handler.checks().items(): enabled, description = value if not enabled and 'profile' in args: continue if enabled and 'only_disabled' in args: continue elif not enabled and 'only_enabled' in args: continue if args.output_format != 'json': enabled = '+' if enabled else '-' if 'details' not in args: rows.append([checker_name]) else: severity = context.severity_map.get(checker_name) rows.append( [enabled, checker_name, analyzer, severity, description]) if len(rows) > 0: print(output_formatters.twodim_to_str(args.output_format, header, rows)) for analyzer_binary, reason in errored: LOG.error("Failed to get checkers for '" + analyzer_binary + "'! The error reason was: '" + reason + "'") LOG.error("Please check your installation and the " "'config/package_layout.json' file!")