def timeit(function): """ Decorator to measure function call time. """ func_name = function.__name__ def debug_wrapper(*args, **kwargs): """ Log measured time. """ before = datetime.now() res = function(*args, **kwargs) after = datetime.now() timediff = after - before diff = timediff.microseconds/1000 LOG.debug('['+str(diff)+'ms] ' + func_name) return res def release_wrapper(*args, **kwargs): """ No logging and measuring. """ res = function(*args, **kwargs) return res if LoggerFactory.get_log_level() == logger.DEBUG: return debug_wrapper else: return release_wrapper
def start(self, db_version_info, wait_for_start=True, init=False): """ Start a PostgreSQL instance with given path, host and port. Return with process instance. """ LOG.debug('Starting/connecting to database.') if not self._is_running(): if not util.is_localhost(self.host): LOG.info('Database is not running yet.') sys.exit(1) if not self._is_database_data_exist(): if not init: # The database does not exists. LOG.error('Database data is missing!') LOG.error('Please check your configuration!') sys.exit(1) elif not self._initialize_database_data(): # The database does not exist and cannot create. LOG.error('Database data is missing and ' 'the initialization of a new failed!') LOG.error('Please check your configuration!') sys.exit(1) LOG.info('Starting database') LOG.debug('Starting database at ' + self.host + ':' + str( self.port) + ' ' + self.path) db_logfile = os.path.join(self.workspace, 'postgresql.log') \ if LoggerFactory.get_log_level() == logger.DEBUG \ else os.devnull self._db_log = open(db_logfile, 'wb') start_db = ['postgres', '-i', '-D', self.path, '-p', str(self.port), '-h', self.host] self.proc = subprocess.Popen(start_db, bufsize=-1, env=self.run_env, stdout=self._db_log, stderr=subprocess.STDOUT) add_version = False if init: self._wait_or_die() self._create_database() add_version = not self.check_db_version(db_version_info) self._create_or_update_schema() elif wait_for_start: self._wait_or_die() add_version = not self.check_db_version(db_version_info) if add_version: self._add_version(db_version_info) atexit.register(self.stop) LOG.debug('Done')
def start(self, db_version_info, wait_for_start=True, init=False): """ Start a PostgreSQL instance with given path, host and port. Return with process instance. """ LOG.debug('Starting/connecting to database.') if not self._is_running(): if not util.is_localhost(self.host): LOG.info('Database is not running yet.') sys.exit(1) if not self._is_database_data_exist(): if not init: # The database does not exists. LOG.error('Database data is missing!') LOG.error('Please check your configuration!') sys.exit(1) elif not self._initialize_database_data(): # The database does not exist and cannot create. LOG.error('Database data is missing and ' 'the initialization of a new failed!') LOG.error('Please check your configuration!') sys.exit(1) LOG.info('Starting database') LOG.debug('Starting database at ' + self.host + ':' + str( self.port) + ' ' + self.path) db_logfile = os.path.join(self.workspace, 'postgresql.log') \ if LoggerFactory.get_log_level() == logger.DEBUG \ else os.devnull self._db_log = open(db_logfile, 'wb') start_db = ['postgres', '-i', '-D', self.path, '-p', str(self.port), '-h', self.host] self.proc = subprocess.Popen(start_db, bufsize=-1, env=self.run_env, stdout=self._db_log, stderr=subprocess.STDOUT) add_version = False if init: self._wait_or_die() self._create_database() add_version = not self.check_db_version(db_version_info) self._create_or_update_schema() elif wait_for_start: self._wait_or_die() add_version = not self.check_db_version(db_version_info) if add_version: self._add_version(db_version_info) atexit.register(self.stop) LOG.debug('Done')
def handle_results(self): """ Send the plist content to the database. Server API calls should be used in one connection. - addBuildAction - addReport - needFileContent - addFileContent - finishBuildAction """ with client.get_connection() as connection: LOG.debug('Storing original build and analyzer command ' 'to the database.') _, source_file_name = ntpath.split(self.analyzed_source_file) if LoggerFactory.get_log_level() == logger.DEBUG: analyzer_cmd = ' '.join(self.analyzer_cmd) else: analyzer_cmd = '' build_cmd_hash = self.buildaction.original_command_hash analysis_id = \ connection.add_build_action(self.__run_id, build_cmd_hash, analyzer_cmd, self.buildaction.analyzer_type, source_file_name) assert self.analyzer_returncode == 0 plist_file = self.analyzer_result_file try: files, bugs = plist_parser.parse_plist(plist_file) except Exception as ex: LOG.debug(str(ex)) msg = 'Parsing the generated result file failed.' LOG.error(msg + ' ' + plist_file) connection.finish_build_action(analysis_id, msg) return 1 self.__store_bugs(files, bugs, connection, analysis_id) connection.finish_build_action(analysis_id, self.analyzer_stderr)
def handle_results(self): """ Send the plist content to the database. Server API calls should be used in one connection. - addBuildAction - addReport - needFileContent - addFileContent - finishBuildAction """ with client.get_connection() as connection: LOG.debug('Storing original build and analyzer command ' 'to the database.') _, source_file_name = ntpath.split(self.analyzed_source_file) if LoggerFactory.get_log_level() == logger.DEBUG: analyzer_cmd = ' '.join(self.analyzer_cmd) else: analyzer_cmd = '' build_cmd_hash = self.buildaction.original_command_hash analysis_id = \ connection.add_build_action(self.__run_id, build_cmd_hash, analyzer_cmd, self.buildaction.analyzer_type, source_file_name) assert self.analyzer_returncode == 0 plist_file = self.analyzer_result_file try: files, bugs = plist_parser.parse_plist(plist_file) except Exception as ex: LOG.debug(str(ex)) msg = 'Parsing the generated result file failed.' LOG.error(msg + ' ' + plist_file) connection.finish_build_action(analysis_id, msg) return 1 self.__store_bugs(files, bugs, connection, analysis_id) connection.finish_build_action(analysis_id, self.analyzer_stderr)
import zlib import os from collections import defaultdict import ntpath import codecs import shared from Authentication import constants from Authentication.ttypes import * from codechecker_lib import session_manager from codechecker_lib.logger import LoggerFactory from codechecker_lib.profiler import timeit LOG = LoggerFactory.get_new_logger('AUTH HANDLER') def conv(text): ''' Convert * to % got from clients for the database queries. ''' if text is None: return '%' return text.replace('*', '%') class ThriftAuthHandler(): ''' Handle Thrift authentication requests. '''
# License. See LICENSE.TXT for details. # ------------------------------------------------------------------------- import linecache import math import ntpath import os import sys from abc import ABCMeta from codechecker_lib import plist_parser from codechecker_lib import suppress_handler from codechecker_lib.logger import LoggerFactory from codechecker_lib.analyzers.result_handler_base import ResultHandler LOG = LoggerFactory.get_new_logger('PLIST TO STDOUT') class PlistToStdout(ResultHandler): """ Result handler for processing a plist file with the analysis results and print them to the standard output. """ __metaclass__ = ABCMeta def __init__(self, buildaction, workspace, lock): super(PlistToStdout, self).__init__(buildaction, workspace) self.__print_steps = False self.__output = sys.stdout self.__lock = lock
# The CodeChecker Infrastructure # This file is distributed under the University of Illinois Open Source # License. See LICENSE.TXT for details. # ------------------------------------------------------------------------- """ Suppress handling. """ import abc import linecache import ntpath import re from codechecker_lib.logger import LoggerFactory LOG = LoggerFactory.get_new_logger('SUPPRESS HANDLER') class SuppressHandler(object): """ Suppress handler base class. """ __metaclass__ = abc.ABCMeta __suppressfile = None @abc.abstractmethod def store_suppress_bug_id(self, bug_id, file_name, comment): """ Store the suppress bug_id. """
import multiprocessing import ntpath import os import re import shutil import signal import sys import traceback from collections import defaultdict from codechecker_lib import analyzer_env from codechecker_lib.logger import LoggerFactory from codechecker_lib.analyzers import analyzer_types LOG = LoggerFactory.get_new_logger('ANALYSIS MANAGER') def worker_result_handler(results): """ Print the analysis summary. """ successful_analysis = defaultdict(int) failed_analysis = defaultdict(int) skipped_num = 0 for res, skipped, analyzer_type in results: if skipped: skipped_num += 1 else:
import copy import json import os import shlex import subprocess import sys import time from codechecker_lib import analysis_manager from codechecker_lib import client from codechecker_lib import skiplist_handler from codechecker_lib import analyzer_env from codechecker_lib.logger import LoggerFactory from codechecker_lib.analyzers import analyzer_types LOG = LoggerFactory.get_new_logger('ANALYZER') def prepare_actions(actions, enabled_analyzers): """ Set the analyzer type for each buildaction. Multiple actions if multiple source analyzers are set. """ res = [] for ea in enabled_analyzers: for action in actions: new_action = copy.deepcopy(action) new_action.analyzer_type = ea res.append(new_action) return res
# ------------------------------------------------------------------------- import os import sys import sqlalchemy from sqlalchemy.sql import and_ from codechecker_lib import analyzer_crash_handler from codechecker_lib import analyzer_env from codechecker_lib import database_handler from codechecker_lib.logger import LoggerFactory from db_model.orm_model import BuildAction from db_model.orm_model import Run LOG = LoggerFactory.get_new_logger('DEBUG_REPORTER') # ----------------------------------------------------------------------------- def get_dump_file_name(run_id, action_id): return 'action_' + str(run_id) + '_' + str(action_id) + '_dump.log' # ----------------------------------------------------------------------------- def debug(context, connection_string, force): try: engine = database_handler.SQLServer.create_engine(connection_string) session = sqlalchemy.orm.scoped_session( sqlalchemy.orm.sessionmaker(bind=engine)) # Get latest run id.
Used to filter out or change compiler argument not supported by clang or clang-tidy. Keys are the option name, value is the number of options to skip. Possible improvements: - modular option handling system configuring possibility from config file. """ import os import re import shlex from codechecker_lib.logger import LoggerFactory LOG = LoggerFactory.get_new_logger('OPTION PARSER') # Compiler options. COMPILE_OPTION_MAP = { '-idirafter': 1, '-imacros': 1, '-include': 1, '-iprefix': 1, '-isysroot': 1, '-isystem': 1, '-iwithprefix': 1, '-iwithprefixbefore': 1, '-nostdinc': 0, '-sysroot': 1 }
import argparse from argparse import ArgumentDefaultsHelpFormatter as ADHF import os import signal import sys import shared from cmdline_client import cmd_line_client from codechecker_lib import arg_handler from codechecker_lib import util from codechecker_lib.logger import LoggerFactory from codechecker_lib.analyzers import analyzer_types LOG = LoggerFactory.get_new_logger('MAIN') analyzers = ' '.join(list(analyzer_types.supported_analyzers)) class OrderedCheckersAction(argparse.Action): """ Action to store enabled and disabled checkers and keep ordering from command line. Create separate lists based on the checker names for each analyzer. """ def __init__(self, option_strings, dest, nargs=None, **kwargs): if nargs is not None:
# ------------------------------------------------------------------------- # The CodeChecker Infrastructure # This file is distributed under the University of Illinois Open Source # License. See LICENSE.TXT for details. # ------------------------------------------------------------------------- import re from codechecker_lib.logger import LoggerFactory from codechecker_lib.analyzers import config_handler LOG = LoggerFactory.get_new_logger('CLANGSA CONFIG HANDLER') class ClangSAConfigHandler(config_handler.AnalyzerConfigHandler): """ Configuration handler for the clang static analyzer. """ def __init__(self): super(ClangSAConfigHandler, self).__init__() self.__checker_configs = [] def add_checker_config(self, config): """ Add a (checker_name, key, value) tuple to the list. """ self.__checker_configs.append(config) def get_checker_configs(self): """ Process raw config data (from the command line).
# ------------------------------------------------------------------------- # The CodeChecker Infrastructure # This file is distributed under the University of Illinois Open Source # License. See LICENSE.TXT for details. # ------------------------------------------------------------------------- """ """ import fnmatch import re from codechecker_lib.logger import LoggerFactory LOG = LoggerFactory.get_new_logger('SKIPLIST_HANDLER') class SkipListHandler(object): """ Skiplist file format: -/skip/all/source/in/directory* -/do/not/check/this.file +/dir/check.this.file -/dir/* """ def __init__(self, skip_file): """ read up the skip file """ self.__skip = []
from alembic import command, config from sqlalchemy import event from sqlalchemy.engine import Engine from sqlalchemy.engine.url import URL, make_url from sqlalchemy.sql.elements import quoted_name from codechecker_lib import host_check from codechecker_lib import pgpass from codechecker_lib import util from codechecker_lib import logger from codechecker_lib.logger import LoggerFactory from db_model.orm_model import CC_META from db_model.orm_model import CreateSession from db_model.orm_model import DBVersion LOG = LoggerFactory.get_new_logger('DB_HANDLER') class SQLServer(object): """ Abstract base class for database server handling. An SQLServer instance is responsible for the initialization, starting, and stopping the database server, and also for connection string management. SQLServer implementations are created via SQLServer.from_cmdline_args(). How to add a new database server implementation: 1, Derive from SQLServer and implement the abstract methods 2, Add/modify some command line options in CodeChecker.py 3, Modify SQLServer.from_cmdline_args() in order to create an instance of the new server type if needed
import ntpath import os import zlib from abc import ABCMeta import shared from codechecker_lib import client from codechecker_lib import plist_parser from codechecker_lib import suppress_handler from codechecker_lib import logger from codechecker_lib.logger import LoggerFactory from codechecker_lib.analyzers.result_handler_base import ResultHandler LOG = LoggerFactory.get_new_logger('PLIST TO DB') class PlistToDB(ResultHandler): """ Result handler for processing a plist file with the analysis results and stores them to the database. """ __metaclass__ = ABCMeta def __init__(self, buildaction, workspace, run_id): super(PlistToDB, self).__init__(buildaction, workspace) self.__run_id = run_id def __store_bugs(self, files, bugs, connection, analisys_id):
# ------------------------------------------------------------------------- # The CodeChecker Infrastructure # This file is distributed under the University of Illinois Open Source # License. See LICENSE.TXT for details. # ------------------------------------------------------------------------- import re import shlex import subprocess from codechecker_lib.logger import LoggerFactory from codechecker_lib.analyzers import analyzer_base LOG = LoggerFactory.get_new_logger('CLANGSA') class ClangSA(analyzer_base.SourceAnalyzer): """ Constructs clang static analyzer commands. """ def __parse_checkers(self, clangsa_output): """ Parse clang static analyzer checkers, store them to checkers. """ # Checker name and description in one line. pattern = re.compile( r'^\s\s(?P<checker_name>\S*)\s*(?P<description>.*)') checker_name = None
def main(): """ CodeChecker main command line. """ def signal_handler(sig, frame): """ Without this handler the PostgreSQL server does not terminate at signal. """ sys.exit(1) signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGTERM, signal_handler) try: parser = argparse.ArgumentParser( prog='CodeChecker', formatter_class=argparse.RawDescriptionHelpFormatter, description=''' Run the CodeChecker source analyzer framework. See the subcommands for specific features.''', epilog=''' Example usage: -------------- Analyzing a project with default settings: CodeChecker check -b "cd ~/myproject && make" -n myproject Start the viewer to see the results: CodeChecker server See the results in a web browser: localhost:8001 See results in the command line: CodeChecker cmd results -p 8001 -n myproject To analyze a small project quickcheck feature can be used. The results will be printed only to the standard output. (No database will be used) CodeChecker quickcheck -b "cd ~/myproject && make" ''') subparsers = parser.add_subparsers(help='commands') workspace_help_msg = 'Directory where the CodeChecker can' \ ' store analysis related data.' name_help_msg = 'Name of the analysis.' jobs_help_msg = 'Number of jobs.' \ 'Start multiple processes for faster analysis.' log_argument_help_msg = "Path to the log file which is created " "during the build. \nIf there is an already generated log file " "with the compilation commands\ngenerated by 'CodeChecker log' or " "'cmake -DCMAKE_EXPORT_COMPILE_COMMANDS' \n CodeChecker check can " "use it for the analysis in that case running the original build " "will \nbe left out from the analysis process (no log is needed)." suppress_help_msg = "Path to suppress file.\nSuppress file can be used" "to suppress analysis results during the analysis.\nIt is based on the" "bug identifier generated by the compiler which is experimental.\nDo" "not depend too much on this file because identifier or file format " "can be changed.\nFor other in source suppress features see the user" "guide." # -------------------------------------- # check commands check_parser = subparsers.add_parser('check', formatter_class=ADHF, help=''' \ Run the supported source code analyzers on a project.''') check_parser.add_argument('-w', '--workspace', type=str, default=util.get_default_workspace(), dest="workspace", help=workspace_help_msg) check_parser.add_argument('-n', '--name', type=str, dest="name", required=True, default=argparse.SUPPRESS, help=name_help_msg) checkgroup = check_parser.add_mutually_exclusive_group(required=True) checkgroup.add_argument('-b', '--build', type=str, dest="command", default=argparse.SUPPRESS, required=False, help='''\ Build command which is used to build the project.''') checkgroup.add_argument('-l', '--log', type=str, dest="logfile", default=argparse.SUPPRESS, required=False, help=log_argument_help_msg) check_parser.add_argument('-j', '--jobs', type=int, dest="jobs", default=1, required=False, help=jobs_help_msg) check_parser.add_argument('-u', '--suppress', type=str, dest="suppress", default=argparse.SUPPRESS, required=False, help=suppress_help_msg) check_parser.add_argument('-c', '--clean', default=argparse.SUPPRESS, action=DeprecatedOptionAction) check_parser.add_argument('--update', action=DeprecatedOptionAction, dest="update", default=False, required=False, help="Incremental parsing, update the " "results of a previous run. " "Only the files changed since the last " "build will be reanalyzed. Depends on" " the build system.") check_parser.add_argument('--force', action="store_true", dest="force", default=False, required=False, help="Delete analysis results form the " "database if a run with the " "given name already exists.") check_parser.add_argument('-s', '--skip', type=str, dest="skipfile", default=argparse.SUPPRESS, required=False, help='Path to skip file.') check_parser.add_argument('--quiet-build', action='store_true', default=False, required=False, help='Do not print out the output of the ' 'original build.') check_parser.add_argument('--add-compiler-defaults', action='store_true', default=False, required=False, help='Fetch built in compiler include' 'paths and defines ' 'and pass them to Clang. This is' 'useful when you do cross-compilation.') add_analyzer_arguments(check_parser) add_database_arguments(check_parser) add_verbose_arguments(check_parser) check_parser.set_defaults(func=arg_handler.handle_check) # -------------------------------------- # QuickCheck commands. qcheck_parser = subparsers.add_parser('quickcheck', formatter_class=ADHF, help='Run CodeChecker for a' 'project without database.') qcheckgroup = qcheck_parser.add_mutually_exclusive_group(required=True) qcheckgroup.add_argument('-b', '--build', type=str, dest="command", default=argparse.SUPPRESS, required=False, help='Build command.') qcheckgroup.add_argument('-l', '--log', type=str, dest="logfile", required=False, default=argparse.SUPPRESS, help=log_argument_help_msg) qcheck_parser.add_argument('-s', '--steps', action="store_true", dest="print_steps", help='Print steps.') qcheck_parser.add_argument('--quiet-build', action='store_true', default=False, required=False, help='Do not print out the output of the ' 'original build.') qcheck_parser.add_argument('-i', '--skip', type=str, dest="skipfile", default=argparse.SUPPRESS, required=False, help='Path to skip file.') qcheck_parser.add_argument('-j', '--jobs', type=int, dest="jobs", default=1, required=False, help=jobs_help_msg) qcheck_parser.add_argument('-u', '--suppress', type=str, dest="suppress", default=argparse.SUPPRESS, required=False, help=suppress_help_msg) qcheck_parser.add_argument('--add-compiler-defaults', action='store_true', default=False, required=False, help='Fetch built in compiler include paths' ' and defines and pass them to Clang.' 'This is useful when you' 'do cross-compilation.') add_analyzer_arguments(qcheck_parser) add_verbose_arguments(qcheck_parser) qcheck_parser.set_defaults(func=arg_handler.handle_quickcheck) # -------------------------------------- # Log commands. logging_p = subparsers.add_parser('log', formatter_class=ADHF, help='Runs the given build ' 'command. During the ' 'build the compilation ' 'commands are collected ' 'and stored into a ' 'compilation command ' 'json file ' '(no analysis is done ' 'during the build).') logging_p.add_argument('-o', '--output', type=str, dest="logfile", default=argparse.SUPPRESS, required=True, help='Path to the log file.') logging_p.add_argument('-b', '--build', type=str, dest="command", default=argparse.SUPPRESS, required=True, help='Build command.') add_verbose_arguments(logging_p) logging_p.set_defaults(func=arg_handler.handle_log) # -------------------------------------- # Checkers parser. checker_p = subparsers.add_parser('checkers', formatter_class=ADHF, help='List the available checkers ' 'for the supported analyzers ' 'and show their default status ' '(+ for being enabled, ' '- for being disabled by ' 'default).') checker_p.add_argument('--analyzers', nargs='+', dest="analyzers", required=False, help='Select which analyzer checkers ' 'should be listed.\nCurrently supported ' 'analyzers:\n' + analyzers) add_verbose_arguments(checker_p) checker_p.set_defaults(func=arg_handler.handle_list_checkers) # -------------------------------------- # Server. server_parser = subparsers.add_parser('server', formatter_class=ADHF, help='Start the codechecker ' 'web server.') server_parser.add_argument('-w', '--workspace', type=str, dest="workspace", default=util.get_default_workspace(), help=workspace_help_msg) server_parser.add_argument('-v', '--view-port', type=int, dest="view_port", default=8001, required=False, help='Port used for viewing.') server_parser.add_argument('-u', '--suppress', type=str, dest="suppress", required=False, help='Path to suppress file.') server_parser.add_argument('--not-host-only', action="store_true", dest="not_host_only", help='Viewing the results is possible not' 'only by browsers or clients' ' started locally.') server_parser.add_argument('--check-port', type=int, dest="check_port", default=None, required=False, help='Port used for checking.') server_parser.add_argument('--check-address', type=str, dest="check_address", default="localhost", required=False, help='Server address.') add_database_arguments(server_parser) add_verbose_arguments(server_parser) server_parser.set_defaults(func=arg_handler.handle_server) # -------------------------------------- # Cmd_line. cmd_line_parser = subparsers.add_parser('cmd', help='Command line client') add_verbose_arguments(cmd_line_parser) cmd_line_client.register_client_command_line(cmd_line_parser) # -------------------------------------- # Debug parser. debug_parser = subparsers.add_parser('debug', formatter_class=ADHF, help='Generate gdb debug dump ' 'files for all the failed ' 'compilation commands in ' 'the last analyzer run.\n' 'Requires a database with ' 'the failed compilation ' 'commands.') debug_parser.add_argument('-w', '--workspace', type=str, dest="workspace", default=util.get_default_workspace(), help=workspace_help_msg) debug_parser.add_argument('-f', '--force', action="store_true", dest="force", required=False, default=False, help='Overwrite already generated files.') add_database_arguments(debug_parser) add_verbose_arguments(debug_parser) debug_parser.set_defaults(func=arg_handler.handle_debug) # -------------------------------------- # Plist parser. plist_parser = subparsers.add_parser('plist', formatter_class=ADHF, help='Parse plist files in ' 'the given directory and ' 'store them to the database ' 'or print to the standard ' 'output.') plist_parser.add_argument('-w', '--workspace', type=str, dest="workspace", default=util.get_default_workspace(), help=workspace_help_msg) plist_parser.add_argument('-n', '--name', type=str, dest="name", required=True, default=argparse.SUPPRESS, help=name_help_msg) plist_parser.add_argument('-d', '--directory', type=str, dest="directory", required=True, help='Path of a directory containing plist ' ' files to parse.') plist_parser.add_argument('-j', '--jobs', type=int, dest="jobs", default=1, required=False, help=jobs_help_msg) plist_parser.add_argument('-s', '--steps', action="store_true", dest="print_steps", help='Print steps.') plist_parser.add_argument('--stdout', action="store_true", dest="stdout", required=False, default=False, help='Print results to stdout instead of ' 'storing to the database.') plist_parser.add_argument('--force', action="store_true", dest="force", default=False, required=False, help='Delete analysis results form the ' 'database if a run with the given ' 'name already exists.') add_database_arguments(plist_parser) add_verbose_arguments(plist_parser) plist_parser.set_defaults(func=arg_handler.handle_plist) # -------------------------------------- # Package version info. version_parser = subparsers.add_parser('version', help='Print package version ' 'information.') version_parser.set_defaults(func=arg_handler.handle_version_info) add_verbose_arguments(version_parser) args = parser.parse_args() LoggerFactory.set_log_level(args.verbose) args.func(args) except KeyboardInterrupt as kb_err: LOG.info(str(kb_err)) LOG.info("Interrupted by user...") sys.exit(1) except shared.ttypes.RequestFailed as thrift_ex: LOG.info("Server error.") LOG.info("Error code: " + str(thrift_ex.error_code)) LOG.info("Error message: " + str(thrift_ex.message)) sys.exit(1) # Handle all exception, but print stacktrace. It is needed for atexit. # atexit does not work correctly when an unhandled exception occurred. # So in this case, the servers left running when the script exited. except Exception: import traceback traceback.print_exc(file=sys.stdout) sys.exit(2)
# ------------------------------------------------------------------------- # The CodeChecker Infrastructure # This file is distributed under the University of Illinois Open Source # License. See LICENSE.TXT for details. # ------------------------------------------------------------------------- from datetime import datetime import cProfile import pstats import StringIO from codechecker_lib import logger from codechecker_lib.logger import LoggerFactory LOG = LoggerFactory.get_new_logger('PROFILER') def timeit(function): """ Decorator to measure function call time. """ func_name = function.__name__ def debug_wrapper(*args, **kwargs): """ Log measured time. """ before = datetime.now() res = function(*args, **kwargs) after = datetime.now()
"root", "myname" ], "groups": [ "adm", "cc-users" ] } """ import pam import grp import pwd from codechecker_lib.logger import LoggerFactory LOG = LoggerFactory.get_new_logger("PAM") def auth_user(pam_config, username, password): """ Authenticate user with PAM. """ LOG.debug('Authenticating user with PAM.') auth = pam.pam() if auth.authenticate(username, password): allowed_users = pam_config.get("users") \ or [] allowed_group = pam_config.get("groups")\
""" import datetime import glob import hashlib import ntpath import os import shutil import socket import subprocess import sys from codechecker_lib.logger import LoggerFactory # WARNING! LOG should be only used in this module. LOG = LoggerFactory.get_new_logger('UTIL') # --------------------------------------------------------------------- def get_free_port(): """ Get a free port from the OS. """ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.bind(('', 0)) free_port = s.getsockname()[1] s.close() return free_port # ---------------------------------------------------------------------
# ------------------------------------------------------------------------- # The CodeChecker Infrastructure # This file is distributed under the University of Illinois Open Source # License. See LICENSE.TXT for details. # ------------------------------------------------------------------------- """ """ import re import shlex import subprocess from codechecker_lib.logger import LoggerFactory from codechecker_lib.analyzers import analyzer_base LOG = LoggerFactory.get_new_logger('CLANG TIDY') class ClangTidy(analyzer_base.SourceAnalyzer): """ Constructs the clang tidy analyzer commands. """ def __parse_checkers(self, tidy_output): """ Parse clang tidy checkers list. Skip clang static analyzer checkers. Store them to checkers. """ for line in tidy_output.splitlines(): line = line.strip() if re.match(r'^Enabled checks:', line) or line == '':
from codechecker_lib import build_action from codechecker_lib import build_manager from codechecker_lib import client from codechecker_lib import debug_reporter from codechecker_lib import generic_package_context from codechecker_lib import generic_package_suppress_handler from codechecker_lib import host_check from codechecker_lib import log_parser from codechecker_lib import session_manager from codechecker_lib import util from codechecker_lib.logger import LoggerFactory from codechecker_lib.analyzers import analyzer_types from codechecker_lib.database_handler import SQLServer from viewer_server import client_db_access_server LOG = LoggerFactory.get_new_logger('ARG_HANDLER') def log_startserver_hint(args): db_data = "" if args.postgresql: db_data += " --postgresql" \ + " --dbname " + args.dbname \ + " --dbport " + str(args.dbport) \ + " --dbusername " + args.dbusername LOG.info("To view results run:\nCodeChecker server -w " + args.workspace + db_data) def handle_list_checkers(args):
# ------------------------------------------------------------------------- import os import sys import sqlalchemy from sqlalchemy.sql import and_ from codechecker_lib import analyzer_crash_handler from codechecker_lib import analyzer_env from codechecker_lib import database_handler from codechecker_lib.logger import LoggerFactory from db_model.orm_model import BuildAction from db_model.orm_model import Run LOG = LoggerFactory.get_new_logger('DEBUG_REPORTER') # ----------------------------------------------------------------------------- def get_dump_file_name(run_id, action_id): return 'action_' + str(run_id) + '_' + str(action_id) + '_dump.log' # ----------------------------------------------------------------------------- def debug(context, connection_string, force): try: engine = database_handler.SQLServer.create_engine(connection_string) session = sqlalchemy.orm.scoped_session( sqlalchemy.orm.sessionmaker(bind=engine)) # Get latest run id.
def __init__(self, context, analyzer_env): self._context = context self._analyzer_env = analyzer_env self._logger = LoggerFactory.get_new_logger('ANALYZER_CRASH_HANDLER')
Used to filter out or change compiler argument not supported by clang or clang-tidy. Keys are the option name, value is the number of options to skip. Possible improvements: - modular option handling system configuring possibility from config file. """ import os import re import shlex from codechecker_lib.logger import LoggerFactory LOG = LoggerFactory.get_new_logger('OPTION PARSER') # Compiler options. COMPILE_OPTION_MAP = { '-idirafter': 1, '-imacros': 1, '-include': 1, '-iprefix': 1, '-isysroot': 1, '-isystem': 1, '-iwithprefix': 1, '-iwithprefixbefore': 1, '-nostdinc': 0, '-sysroot': 1 }
# ------------------------------------------------------------------------- # The CodeChecker Infrastructure # This file is distributed under the University of Illinois Open Source # License. See LICENSE.TXT for details. # ------------------------------------------------------------------------- import argparse import json import re import shlex from codechecker_lib.logger import LoggerFactory from codechecker_lib.analyzers import config_handler LOG = LoggerFactory.get_new_logger('CLANG TIDY CONFIG') class ClangTidyConfigHandler(config_handler.AnalyzerConfigHandler): """ Configuration handler for Clang-tidy analyzer. """ def __init__(self): super(ClangTidyConfigHandler, self).__init__() def get_checker_configs(self): """ Process the raw extra analyzer arguments and get the configuration data ('-config=' argument for Clang tidy) for the checkers. Clang tidy accepts YAML or JSON formatted config, right now
import shared from codeCheckerDBAccess import codeCheckerDBAccess from codeCheckerDBAccess import constants from codeCheckerDBAccess.ttypes import * from Authentication import codeCheckerAuthentication from Authentication import constants from Authentication.ttypes import * from client_db_access_handler import ThriftRequestHandler from client_auth_handler import ThriftAuthHandler from codechecker_lib import database_handler from codechecker_lib import session_manager from codechecker_lib.logger import LoggerFactory LOG = LoggerFactory.get_new_logger('DB ACCESS') class RequestHandler(SimpleHTTPRequestHandler): """ Handle thrift and browser requests Simply modified and extended version of SimpleHTTPRequestHandler """ def __init__(self, request, client_address, server): self.sc_session = server.sc_session self.db_version_info = server.db_version_info self.manager = server.manager BaseHTTPRequestHandler.__init__(self,
# ------------------------------------------------------------------------- # The CodeChecker Infrastructure # This file is distributed under the University of Illinois Open Source # License. See LICENSE.TXT for details. # ------------------------------------------------------------------------- """""" import os from codechecker_lib.logger import LoggerFactory LOG = LoggerFactory.get_new_logger('ENV') # ------------------------------------------------------------------------------ def get_log_env(logfile, context, original_env): """ Environment for logging. With the ld logger. Keep the original environment unmodified as possible. Only environment variables required for logging are changed. """ new_env = original_env new_env[context.env_var_cc_logger_bin] = context.path_logger_bin new_env['LD_PRELOAD'] = context.logger_lib_name try: original_ld_library_path = new_env['LD_LIBRARY_PATH'] new_env['LD_LIBRARY_PATH'] = context.path_logger_lib + ':' + \ original_ld_library_path
# ------------------------------------------------------------------------- # The CodeChecker Infrastructure # This file is distributed under the University of Illinois Open Source # License. See LICENSE.TXT for details. # ------------------------------------------------------------------------- import errno import os import subprocess from codechecker_lib.logger import LoggerFactory LOG = LoggerFactory.get_new_logger('HOST CHECK') # ----------------------------------------------------------------------------- def check_zlib(): """ Check if zlib compression is available. If wrong libraries are installed on the host machine it is possible the the compression fails which is required to store data into the database. """ try: import zlib zlib.compress('Compress this') return True except Exception as ex: LOG.error(str(ex)) LOG.error('Failed to import zlib module.') return False
from codechecker_lib.logger import LoggerFactory unsupported_methods = [] try: from codechecker_lib.auth import cc_ldap except ImportError: unsupported_methods.append("ldap") try: from codechecker_lib.auth import cc_pam except ImportError: unsupported_methods.append("pam") LOG = LoggerFactory.get_new_logger("SESSION MANAGER") SESSION_COOKIE_NAME = "__ccPrivilegedAccessToken" session_lifetimes = {} # ------------------------------ # ----------- SERVER ----------- class _Session(): """A session for an authenticated, privileged client connection.""" # Create an initial salt from system environment for use with the session # permanent persistency routine. __initial_salt = hashlib.sha256(SESSION_COOKIE_NAME + "__" + str(time.time()) + "__" + os.urandom(16)).hexdigest()
# The CodeChecker Infrastructure # This file is distributed under the University of Illinois Open Source # License. See LICENSE.TXT for details. # ------------------------------------------------------------------------- import os import sys import traceback import subprocess import shlex from codechecker_lib import build_action from codechecker_lib import option_parser from codechecker_lib.logger import LoggerFactory LOG = LoggerFactory.get_new_logger('LOG PARSER') # ----------------------------------------------------------------------------- def get_compiler_includes(compiler): """ Returns a list of default includes of the given compiler. """ LOG.debug('getting include paths for ' + compiler) start_mark = "#include <...> search starts here:" end_mark = "End of search list." cmd = compiler + " -E -x c++ - -v " # what if not c++? include_paths = [] try: proc = subprocess.Popen(shlex.split(cmd),
# ------------------------------------------------------------------------- # The CodeChecker Infrastructure # This file is distributed under the University of Illinois Open Source # License. See LICENSE.TXT for details. # ------------------------------------------------------------------------- import json import os import sys from codechecker_lib import context_base from codechecker_lib import db_version from codechecker_lib.logger import LoggerFactory from codechecker_lib.analyzers import analyzer_types LOG = LoggerFactory.get_new_logger('CONTEXT') # ----------------------------------------------------------------------------- class Context(context_base.ContextBase): """ Generic package specific context. """ __instance = None logger_bin = None logger_file = None logger_compilers = None ld_preload = None __package_version = None __package_root = None
# ------------------------------------------------------------------------- # The CodeChecker Infrastructure # This file is distributed under the University of Illinois Open Source # License. See LICENSE.TXT for details. # ------------------------------------------------------------------------- """ """ import re import shlex import subprocess from codechecker_lib.logger import LoggerFactory from codechecker_lib.analyzers import analyzer_base LOG = LoggerFactory.get_new_logger('CLANG TIDY') class ClangTidy(analyzer_base.SourceAnalyzer): """ Constructs the clang tidy analyzer commands. """ def __parse_checkers(self, tidy_output): """ Parse clang tidy checkers list. Skip clang static analyzer checkers. Store them to checkers. """ for line in tidy_output.splitlines(): line = line.strip()
# ------------------------------------------------------------------------- # The CodeChecker Infrastructure # This file is distributed under the University of Illinois Open Source # License. See LICENSE.TXT for details. # ------------------------------------------------------------------------- import re import shlex import subprocess from codechecker_lib.logger import LoggerFactory from codechecker_lib.analyzers import analyzer_base LOG = LoggerFactory.get_new_logger('CLANGSA') class ClangSA(analyzer_base.SourceAnalyzer): """ Constructs clang static analyzer commands. """ def __parse_checkers(self, clangsa_output): """ Parse clang static analyzer checkers, store them to checkers. """ # Checker name and description in one line. pattern = re.compile( r'^\s\s(?P<checker_name>\S*)\s*(?P<description>.*)') checker_name = None for line in clangsa_output.splitlines():
""" import datetime import glob import hashlib import ntpath import os import shutil import socket import subprocess import sys from codechecker_lib.logger import LoggerFactory # WARNING! LOG should be only used in this module. LOG = LoggerFactory.get_new_logger('UTIL') # --------------------------------------------------------------------- def get_free_port(): """ Get a free port from the OS. """ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.bind(('', 0)) free_port = s.getsockname()[1] s.close() return free_port # ---------------------------------------------------------------------
# ------------------------------------------------------------------------- # The CodeChecker Infrastructure # This file is distributed under the University of Illinois Open Source # License. See LICENSE.TXT for details. # ------------------------------------------------------------------------- from codechecker_lib import tidy_output_converter from codechecker_lib.logger import LoggerFactory from codechecker_lib.analyzers.result_handler_plist_to_db import PlistToDB from codechecker_lib.analyzers.result_handler_plist_to_stdout import \ PlistToStdout LOG = LoggerFactory.get_new_logger('CLANG_TIDY_RESULT_HANDLER') def generate_plist_from_tidy_result(output_file, tidy_stdout): """ Generate a plist file from the clang tidy analyzer results. """ parser = tidy_output_converter.OutputParser() messages = parser.parse_messages(tidy_stdout) plist_converter = tidy_output_converter.PListConverter() plist_converter.add_messages(messages) plist_converter.write_to_file(output_file) class ClangTidyPlistToDB(PlistToDB):
from codechecker_gen.DBThriftAPI import CheckerReport from codechecker_gen.DBThriftAPI.ttypes import * from thrift.protocol import TBinaryProtocol from thrift.server import TServer from thrift.transport import TSocket from thrift.transport import TTransport from codechecker_lib import database_handler from codechecker_lib import decorators from codechecker_lib import logger from codechecker_lib.logger import LoggerFactory from db_model.orm_model import * from codechecker_lib.profiler import timeit LOG = LoggerFactory.get_new_logger('CC SERVER') if os.environ.get('CODECHECKER_ALCHEMY_LOG') is not None: import logging logging.basicConfig() logging.getLogger('sqlalchemy.engine').setLevel(logging.DEBUG) logging.getLogger('sqlalchemy.orm').setLevel(logging.DEBUG) class CheckerReportHandler(object): """ Class to handle requests from the CodeChecker script to store run information to the database. """
import shared from codeCheckerDBAccess import codeCheckerDBAccess from codeCheckerDBAccess import constants from codeCheckerDBAccess.ttypes import * from Authentication import codeCheckerAuthentication from Authentication import constants from Authentication.ttypes import * from client_db_access_handler import ThriftRequestHandler from client_auth_handler import ThriftAuthHandler from codechecker_lib import database_handler from codechecker_lib import session_manager from codechecker_lib.logger import LoggerFactory LOG = LoggerFactory.get_new_logger('DB ACCESS') class RequestHandler(SimpleHTTPRequestHandler): """ Handle thrift and browser requests Simply modified and extended version of SimpleHTTPRequestHandler """ def __init__(self, request, client_address, server): self.sc_session = server.sc_session self.db_version_info = server.db_version_info self.manager = server.manager BaseHTTPRequestHandler.__init__(self, request, client_address, server)
""" Build and log related functionality. """ import os import pickle import platform import subprocess import sys from uuid import uuid4 from distutils.spawn import find_executable from codechecker_lib import analyzer_env from codechecker_lib import host_check from codechecker_lib.logger import LoggerFactory LOG = LoggerFactory.get_new_logger('BUILD MANAGER') def execute_buildcmd(command, silent=False, env=None, cwd=None): """ Execute the the build command and continuously write the output from the process to the standard output. """ proc = subprocess.Popen(command, bufsize=-1, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=cwd, shell=True)
import copy import json import os import shlex import subprocess import sys import time from codechecker_lib import analysis_manager from codechecker_lib import client from codechecker_lib import skiplist_handler from codechecker_lib import analyzer_env from codechecker_lib.logger import LoggerFactory from codechecker_lib.analyzers import analyzer_types LOG = LoggerFactory.get_new_logger('ANALYZER') def prepare_actions(actions, enabled_analyzers): """ Set the analyzer type for each buildaction. Multiple actions if multiple source analyzers are set. """ res = [] for ea in enabled_analyzers: for action in actions: new_action = copy.deepcopy(action) new_action.analyzer_type = ea res.append(new_action) return res
import ntpath import os import sqlalchemy import zlib import shared from codeCheckerDBAccess import constants from codeCheckerDBAccess.ttypes import * from codechecker_lib.logger import LoggerFactory from codechecker_lib.profiler import timeit from db_model.orm_model import * LOG = LoggerFactory.get_new_logger('ACCESS HANDLER') def conv(text): """ Convert * to % got from clients for the database queries. """ if text is None: return '%' return text.replace('*', '%') def construct_report_filter(report_filters): """ Construct the report filter for reports and suppressed reports. """
# License. See LICENSE.TXT for details. # ------------------------------------------------------------------------- import linecache import math import ntpath import os import sys from abc import ABCMeta from codechecker_lib import plist_parser from codechecker_lib import suppress_handler from codechecker_lib.logger import LoggerFactory from codechecker_lib.analyzers.result_handler_base import ResultHandler LOG = LoggerFactory.get_new_logger('PLIST TO STDOUT') class PlistToStdout(ResultHandler): """ Result handler for processing a plist file with the analysis results and print them to the standard output. """ __metaclass__ = ABCMeta def __init__(self, buildaction, workspace, lock): super(PlistToStdout, self).__init__(buildaction, workspace) self.__print_steps = False self.__output = sys.stdout self.__lock = lock
# ------------------------------------------------------------------------- # The CodeChecker Infrastructure # This file is distributed under the University of Illinois Open Source # License. See LICENSE.TXT for details. # ------------------------------------------------------------------------- import re from codechecker_lib.logger import LoggerFactory from codechecker_lib.analyzers import config_handler LOG = LoggerFactory.get_new_logger('CLANGSA CONFIG HANDLER') class ClangSAConfigHandler(config_handler.AnalyzerConfigHandler): """ Configuration handler for the clang static analyzer. """ def __init__(self): super(ClangSAConfigHandler, self).__init__() self.__checker_configs = [] def add_checker_config(self, config): """ Add a (checker_name, key, value) tuple to the list. """ self.__checker_configs.append(config) def get_checker_configs(self): """
import sys import time import shared from DBThriftAPI import CheckerReport from DBThriftAPI.ttypes import SuppressBugData from thrift import Thrift from thrift.protocol import TBinaryProtocol from thrift.transport import TSocket from thrift.transport import TTransport from codechecker_lib import suppress_file_handler from codechecker_lib.logger import LoggerFactory from storage_server import report_server LOG = LoggerFactory.get_new_logger('CLIENT') # ----------------------------------------------------------------------------- def clean_suppress(connection, run_id): """ Clean all the suppress information from the database. """ connection.clean_suppress_data(run_id) # ----------------------------------------------------------------------------- def send_suppress(run_id, connection, file_name): """ Collect suppress information from the suppress file to be stored in the database.
# ------------------------------------------------------------------------- # The CodeChecker Infrastructure # This file is distributed under the University of Illinois Open Source # License. See LICENSE.TXT for details. # ------------------------------------------------------------------------- import abc import json import os from codechecker_lib.logger import LoggerFactory LOG = LoggerFactory.get_new_logger("CONTEXT BASE") # ----------------------------------------------------------------------------- class ContextBase(object): __metaclass__ = abc.ABCMeta _package_root = None _verbose_level = None _alchemy_log_level = None _env_path = None _env_vars = None _extra_include_paths = [] _compiler_sysroot = None _extra_system_include_paths = [] _codechecker_enable_checkers = set() _codechecker_disable_checkers = set() _codechecker_workspace = None _db_username = None _module_id = ''
# License. See LICENSE.TXT for details. # ------------------------------------------------------------------------- """ This file contains some workaround for CodeChecker to work with older clang versions. It is for demonstration purposes only. The names, hashes will change after switching to a newer clang version. """ import hashlib import linecache import json import os import re from codechecker_lib.logger import LoggerFactory LOG = LoggerFactory.get_new_logger('PLIST_HELPER') def gen_bug_hash(bug): line_content = linecache.getline(bug.file_path, bug.from_line) if line_content == '' and not os.path.isfile(bug.file_path): LOG.debug('%s does not exists!' % bug.file_path) file_name = os.path.basename(bug.file_path) l = [ file_name, bug.checker_name, bug.msg, line_content, str(bug.from_col), str(bug.until_col) ] for p in bug.paths(): l.append(str(p.start_pos.col))
# This is the new format. 123324353456463442341242342343#1 || filename || bug hash comment After removing the hash_value_type the generated format is: 123324353456463442341242342343 || filename || bug hash comment For backward compatibility the hash_value_type is an optional filed. """ import codecs import os import re from codechecker_lib.logger import LoggerFactory LOG = LoggerFactory.get_new_logger('SUPPRESS_FILE_HANDLER') COMMENT_SEPARATOR = '||' HASH_TYPE_SEPARATOR = '#' def get_suppress_data(suppress_file): """ Process a file object for suppress information. """ old_format_pattern = r"^(?P<bug_hash>[\d\w]{32})" \ r"(\#(?P<bug_hash_type>\d))?\s*\|\|\s*(?P<comment>[^\|]*)$" old_format = re.compile(old_format_pattern, re.UNICODE) new_format_pattern = r"^(?P<bug_hash>[\d\w]{32})" \
$USERDN$ will be automatically replaced by the queried user account DN. Example configuration: *(member=$USERDN$)* `groupScope` Scope of the search performed. (Valid values are: base, one, subtree) """ import ldap from contextlib import contextmanager from codechecker_lib.logger import LoggerFactory LOG = LoggerFactory.get_new_logger("LDAP") def log_ldap_error(ldap_error): """ Log the LDAP error details in debug mode. """ toprint = '' if ldap_error.message: if 'info' in ldap_error.message: toprint = toprint + ldap_error.message['info'] if 'info' in ldap_error.message and 'desc' in ldap_error.message: toprint = toprint + "; " if 'desc' in ldap_error.message: toprint = toprint + ldap_error.message['desc'] else:
# License. See LICENSE.TXT for details. # ------------------------------------------------------------------------- """ This file contains some workaround for CodeChecker to work with older clang versions. It is for demonstration purposes only. The names, hashes will change after switching to a newer clang version. """ import hashlib import linecache import json import os import re from codechecker_lib.logger import LoggerFactory LOG = LoggerFactory.get_new_logger('PLIST_HELPER') def gen_bug_hash(bug): line_content = linecache.getline(bug.file_path, bug.from_line) if line_content == '' and not os.path.isfile(bug.file_path): LOG.debug('%s does not exists!' % bug.file_path) file_name = os.path.basename(bug.file_path) l = [file_name, bug.checker_name, bug.msg, line_content, str(bug.from_col), str(bug.until_col)] for p in bug.paths(): l.append(str(p.start_pos.col)) l.append(str(p.end_pos.col)) string_to_hash = '|||'.join(l) return hashlib.md5(string_to_hash.encode()).hexdigest()
import sys import time import shared from DBThriftAPI import CheckerReport from DBThriftAPI.ttypes import SuppressBugData from thrift import Thrift from thrift.protocol import TBinaryProtocol from thrift.transport import TSocket from thrift.transport import TTransport from codechecker_lib import suppress_file_handler from codechecker_lib.logger import LoggerFactory from storage_server import report_server LOG = LoggerFactory.get_new_logger('CLIENT') # ----------------------------------------------------------------------------- def clean_suppress(connection, run_id): """ Clean all the suppress information from the database. """ connection.clean_suppress_data(run_id) # ----------------------------------------------------------------------------- def send_suppress(run_id, connection, file_name): """ Collect suppress information from the suppress file to be stored in the database.
import ntpath import os import sqlalchemy import zlib import shared from codeCheckerDBAccess import constants from codeCheckerDBAccess.ttypes import * from codechecker_lib.logger import LoggerFactory from codechecker_lib.profiler import timeit from db_model.orm_model import * LOG = LoggerFactory.get_new_logger('ACCESS HANDLER') def conv(text): """ Convert * to % got from clients for the database queries. """ if text is None: return '%' return text.replace('*', '%') def construct_report_filter(report_filters): """ Construct the report filter for reports and suppressed reports. """