def timeit(function): """ Decorator to measure function call time. """ func_name = function.__name__ def debug_wrapper(*args, **kwargs): """ Log measured time. """ before = datetime.now() res = function(*args, **kwargs) after = datetime.now() timediff = after - before diff = timediff.microseconds / 1000 LOG.debug('[' + str(diff) + 'ms] ' + func_name) return res def release_wrapper(*args, **kwargs): """ No logging and measuring. """ res = function(*args, **kwargs) return res if LoggerFactory.get_log_level() == logger.DEBUG: return debug_wrapper else: return release_wrapper
def start(self, db_version_info, wait_for_start=True, init=False): """ Start a PostgreSQL instance with given path, host and port. Return with process instance. """ LOG.debug('Starting/connecting to database.') if not self._is_running(): if not util.is_localhost(self.host): LOG.info('Database is not running yet.') sys.exit(1) if not self._is_database_data_exist(): if not init: # The database does not exists. LOG.error('Database data is missing!') LOG.error('Please check your configuration!') sys.exit(1) elif not self._initialize_database_data(): # The database does not exist and cannot create. LOG.error('Database data is missing and ' 'the initialization of a new failed!') LOG.error('Please check your configuration!') sys.exit(1) LOG.info('Starting database') LOG.debug('Starting database at ' + self.host + ':' + str(self.port) + ' ' + self.path) db_logfile = os.path.join(self.workspace, 'postgresql.log') \ if LoggerFactory.get_log_level() == logger.DEBUG \ else os.devnull self._db_log = open(db_logfile, 'wb') start_db = [ 'postgres', '-i', '-D', self.path, '-p', str(self.port), '-h', self.host ] self.proc = subprocess.Popen(start_db, bufsize=-1, env=self.run_env, stdout=self._db_log, stderr=subprocess.STDOUT) add_version = False if init: self._wait_or_die() self._create_database() add_version = not self.check_db_version(db_version_info) self._create_or_update_schema() elif wait_for_start: self._wait_or_die() add_version = not self.check_db_version(db_version_info) if add_version: self._add_version(db_version_info) atexit.register(self.stop) LOG.debug('Done')
def handle_results(self, client=None): """ Send the plist content to the database. Server API calls should be used in one connection. - addBuildAction - addReport - needFileContent - addFileContent - finishBuildAction """ if not client: LOG.error("Client needs to be set to store the " "results to the database") return LOG.debug('Storing original build and analyzer command ' 'to the database.') _, source_file_name = os.path.split(self.analyzed_source_file) if LoggerFactory.get_log_level() == logger.DEBUG: analyzer_cmd = ' '.join(self.analyzer_cmd) else: analyzer_cmd = '' build_cmd_hash = self.buildaction.original_command_hash analysis_id = \ client.addBuildAction(self.__run_id, build_cmd_hash, analyzer_cmd, self.buildaction.analyzer_type, source_file_name) plist_file = self.analyzer_result_file try: files, reports = plist_parser.parse_plist(plist_file) except Exception as ex: LOG.debug(str(ex)) msg = 'Parsing the generated result file failed.' LOG.error(msg + ' ' + plist_file) client.finishBuildAction(analysis_id, msg) return 1 try: self.__store_bugs(files, reports, client, analysis_id) except Exception as ex: LOG.error("Failed to store results") traceback.print_stack() LOG.error(ex) return 1 finally: LOG.debug("Finishing buildaction") client.finishBuildAction(analysis_id, self.analyzer_stderr) LOG.debug("Finishing buildaction done.")
def get_env_checkers_list(env_name): env_set_checkers = util.get_env_var(env_name) LOG = LoggerFactory.get_new_logger("CHECKERS") LOG.debug_analyzer('Getting checkers list from environment variable %s' % env_name) if env_set_checkers: checkers_list = env_set_checkers.split(':') LOG.debug_analyzer('Checkers list is -> ' + env_set_checkers) return sorted(checkers_list) else: LOG.debug_analyzer('No checkers list was defined.') return None
def handle_results(self): """ Send the plist content to the database. Server API calls should be used in one connection. - addBuildAction - addReport - needFileContent - addFileContent - finishBuildAction """ with client.get_connection() as connection: LOG.debug('Storing original build and analyzer command ' 'to the database.') _, source_file_name = os.path.split(self.analyzed_source_file) if LoggerFactory.get_log_level() == logger.DEBUG: analyzer_cmd = ' '.join(self.analyzer_cmd) else: analyzer_cmd = '' build_cmd_hash = self.buildaction.original_command_hash analysis_id = \ connection.add_build_action(self.__run_id, build_cmd_hash, analyzer_cmd, self.buildaction.analyzer_type, source_file_name) assert self.analyzer_returncode == 0 plist_file = self.analyzer_result_file try: files, reports = plist_parser.parse_plist(plist_file) except Exception as ex: LOG.debug(str(ex)) msg = 'Parsing the generated result file failed.' LOG.error(msg + ' ' + plist_file) connection.finish_build_action(analysis_id, msg) return 1 self.__store_bugs(files, reports, connection, analysis_id) connection.finish_build_action(analysis_id, self.analyzer_stderr)
# ------------------------------------------------------------------------- from abc import ABCMeta import os import zlib import shared from libcodechecker import client from libcodechecker import logger from libcodechecker import suppress_handler from libcodechecker.analyze import plist_parser from libcodechecker.analyze.analyzers.result_handler_base import ResultHandler from libcodechecker.logger import LoggerFactory LOG = LoggerFactory.get_new_logger('PLIST TO DB') class PlistToDB(ResultHandler): """ Result handler for processing a plist file with the analysis results and stores them to the database. """ __metaclass__ = ABCMeta def __init__(self, buildaction, workspace, run_id): super(PlistToDB, self).__init__(buildaction, workspace) self.__run_id = run_id def __store_bugs(self, files, reports, connection, analisys_id):
import random import sqlalchemy import shared from ProductManagement_v6 import ttypes from libcodechecker import util from libcodechecker.logger import LoggerFactory from libcodechecker.profiler import timeit from libcodechecker.server import permissions from libcodechecker.server.database.config_db_model import * from libcodechecker.server.database.database import SQLServer from libcodechecker.server.routing import is_valid_product_endpoint LOG = LoggerFactory.get_new_logger('PRODUCT HANDLER') class ThriftProductHandler(object): """ Connect to database and handle thrift client requests. """ def __init__(self, server, auth_session, config_session, routed_product, package_version): self.__server = server
Build and log related functionality. """ from distutils.spawn import find_executable import os import pickle import platform import subprocess import sys from uuid import uuid4 from libcodechecker import host_check from libcodechecker.logger import LoggerFactory # TODO: Cross-reference between subpacakges... from libcodechecker.analyze import analyzer_env LOG = LoggerFactory.get_new_logger('BUILD MANAGER') def execute_buildcmd(command, silent=False, env=None, cwd=None): """ Execute the the build command and continuously write the output from the process to the standard output. """ proc = subprocess.Popen(command, bufsize=-1, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=cwd, shell=True)
from libcodechecker.logger import LoggerFactory unsupported_methods = [] try: from libcodechecker.libauth import cc_ldap except ImportError: unsupported_methods.append("ldap") try: from libcodechecker.libauth import cc_pam except ImportError: unsupported_methods.append("pam") LOG = LoggerFactory.get_new_logger("SESSION MANAGER") SESSION_COOKIE_NAME = "__ccPrivilegedAccessToken" session_lifetimes = {} class _Session(object): """A session for an authenticated, privileged client connection.""" # Create an initial salt from system environment for use with the session # permanent persistency routine. __initial_salt = hashlib.sha256(SESSION_COOKIE_NAME + "__" + str(time.time()) + "__" + os.urandom(16)).hexdigest() @staticmethod def calc_persistency_hash(auth_string):
# This is the new format. 123324353456463442341242342343#1 || filename || bug hash comment After removing the hash_value_type the generated format is: 123324353456463442341242342343 || filename || bug hash comment For backward compatibility the hash_value_type is an optional filed. """ import codecs import os import re from libcodechecker.logger import LoggerFactory LOG = LoggerFactory.get_new_logger('SUPPRESS_FILE_HANDLER') COMMENT_SEPARATOR = '||' HASH_TYPE_SEPARATOR = '#' def get_suppress_data(suppress_file): """ Process a file object for suppress information. """ old_format_pattern = r"^(?P<bug_hash>[\d\w]{32})" \ r"(\#(?P<bug_hash_type>\d))?\s*\|\|\s*(?P<comment>[^\|]*)$" old_format = re.compile(old_format_pattern, re.UNICODE) new_format_pattern = r"^(?P<bug_hash>[\d\w]{32})" \
# License. See LICENSE.TXT for details. # ------------------------------------------------------------------------- """ This module is responsible for parsing clang-tidy output and generating plist for the plist_parser module. """ import copy import json import os import plistlib import re from libcodechecker.logger import LoggerFactory LOG = LoggerFactory.get_new_logger('TIDY OUTPUT CONVERTER') class Note(object): """ Represents a note and also this is the base class of Message. """ def __init__(self, path, line, column, message): self.path = path self.line = line self.column = column self.message = message def __eq__(self, other): return self.path == other.path and \ self.line == other.line and \
Util module. """ import datetime import hashlib import os import re import socket import subprocess import psutil from libcodechecker.logger import LoggerFactory # WARNING! LOG should be only used in this module. LOG = LoggerFactory.get_new_logger('UTIL') def get_free_port(): """ Get a free port from the OS. """ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.bind(('', 0)) free_port = s.getsockname()[1] s.close() return free_port def is_localhost(address): """
# License. See LICENSE.TXT for details. # ------------------------------------------------------------------------- from datetime import datetime import cProfile import pstats try: from StringIO import StringIO except ImportError: from io import BytesIO as StringIO from libcodechecker import logger from libcodechecker.logger import LoggerFactory LOG = LoggerFactory.get_new_logger('PROFILER') def timeit(function): """ Decorator to measure function call time. """ func_name = function.__name__ def debug_wrapper(*args, **kwargs): """ Log measured time. """ before = datetime.now() res = function(*args, **kwargs)
# ------------------------------------------------------------------------- # The CodeChecker Infrastructure # This file is distributed under the University of Illinois Open Source # License. See LICENSE.TXT for details. # ------------------------------------------------------------------------- """ Handle thrift requests for authentication """ import shared from Authentication.ttypes import * from libcodechecker.logger import LoggerFactory from libcodechecker.profiler import timeit LOG = LoggerFactory.get_new_logger('AUTH HANDLER') def conv(text): """ Convert * to % got from clients for the database queries. """ if text is None: return '%' return text.replace('*', '%') class ThriftAuthHandler(object): """ Handle Thrift authentication requests. """
import sqlalchemy from sqlalchemy import event from sqlalchemy.engine import Engine from sqlalchemy.engine.url import URL, make_url from sqlalchemy.sql.elements import quoted_name from libcodechecker import host_check from libcodechecker import logger from libcodechecker import pgpass from libcodechecker import util from libcodechecker.logger import LoggerFactory from libcodechecker.orm_model import CC_META from libcodechecker.orm_model import CreateSession from libcodechecker.orm_model import DBVersion LOG = LoggerFactory.get_new_logger('DB_HANDLER') class SQLServer(object): """ Abstract base class for database server handling. An SQLServer instance is responsible for the initialization, starting, and stopping the database server, and also for connection string management. SQLServer implementations are created via SQLServer.from_cmdline_args(). How to add a new database server implementation: 1, Derive from SQLServer and implement the abstract methods 2, Add/modify some command line options in CodeChecker.py 3, Modify SQLServer.from_cmdline_args() in order to create an instance of the new server type if needed
# This file is distributed under the University of Illinois Open Source # License. See LICENSE.TXT for details. # ------------------------------------------------------------------------- from abc import ABCMeta import linecache import math import os import sys from libcodechecker import suppress_handler from libcodechecker.analyze import plist_parser from libcodechecker.analyze.analyzers.result_handler_base import ResultHandler from libcodechecker.logger import LoggerFactory LOG = LoggerFactory.get_new_logger('PLIST TO STDOUT') class PlistToStdout(ResultHandler): """ Result handler for processing a plist file with the analysis results and print them to the standard output. """ __metaclass__ = ABCMeta def __init__(self, buildaction, workspace, lock): super(PlistToStdout, self).__init__(buildaction, workspace) self.__print_steps = False self.__output = sys.stdout self.__lock = lock