def timeit(function):
"""
Decorator to measure function call time.
"""
func_name = function.__name__
def debug_wrapper(*args, **kwargs):
"""
Log measured time.
"""
before = datetime.now()
res = function(*args, **kwargs)
after = datetime.now()
timediff = after - before
diff = timediff.microseconds / 1000
LOG.debug('[' + str(diff) + 'ms] ' + func_name)
return res
def release_wrapper(*args, **kwargs):
"""
No logging and measuring.
"""
res = function(*args, **kwargs)
return res
if LoggerFactory.get_log_level() == logger.DEBUG:
return debug_wrapper
else:
return release_wrapper
def start(self, db_version_info, wait_for_start=True, init=False):
"""
Start a PostgreSQL instance with given path, host and port.
Return with process instance.
"""
LOG.debug('Starting/connecting to database.')
if not self._is_running():
if not util.is_localhost(self.host):
LOG.info('Database is not running yet.')
sys.exit(1)
if not self._is_database_data_exist():
if not init:
# The database does not exists.
LOG.error('Database data is missing!')
LOG.error('Please check your configuration!')
sys.exit(1)
elif not self._initialize_database_data():
# The database does not exist and cannot create.
LOG.error('Database data is missing and '
'the initialization of a new failed!')
LOG.error('Please check your configuration!')
sys.exit(1)
LOG.info('Starting database')
LOG.debug('Starting database at ' + self.host + ':' +
str(self.port) + ' ' + self.path)
db_logfile = os.path.join(self.workspace, 'postgresql.log') \
if LoggerFactory.get_log_level() == logger.DEBUG \
else os.devnull
self._db_log = open(db_logfile, 'wb')
start_db = [
'postgres', '-i', '-D', self.path, '-p',
str(self.port), '-h', self.host
]
self.proc = subprocess.Popen(start_db,
bufsize=-1,
env=self.run_env,
stdout=self._db_log,
stderr=subprocess.STDOUT)
add_version = False
if init:
self._wait_or_die()
self._create_database()
add_version = not self.check_db_version(db_version_info)
self._create_or_update_schema()
elif wait_for_start:
self._wait_or_die()
add_version = not self.check_db_version(db_version_info)
if add_version:
self._add_version(db_version_info)
atexit.register(self.stop)
LOG.debug('Done')
def handle_results(self, client=None):
"""
Send the plist content to the database.
Server API calls should be used in one connection.
- addBuildAction
- addReport
- needFileContent
- addFileContent
- finishBuildAction
"""
if not client:
LOG.error("Client needs to be set to store the "
"results to the database")
return
LOG.debug('Storing original build and analyzer command '
'to the database.')
_, source_file_name = os.path.split(self.analyzed_source_file)
if LoggerFactory.get_log_level() == logger.DEBUG:
analyzer_cmd = ' '.join(self.analyzer_cmd)
else:
analyzer_cmd = ''
build_cmd_hash = self.buildaction.original_command_hash
analysis_id = \
client.addBuildAction(self.__run_id,
build_cmd_hash,
analyzer_cmd,
self.buildaction.analyzer_type,
source_file_name)
plist_file = self.analyzer_result_file
try:
files, reports = plist_parser.parse_plist(plist_file)
except Exception as ex:
LOG.debug(str(ex))
msg = 'Parsing the generated result file failed.'
LOG.error(msg + ' ' + plist_file)
client.finishBuildAction(analysis_id, msg)
return 1
try:
self.__store_bugs(files, reports, client, analysis_id)
except Exception as ex:
LOG.error("Failed to store results")
traceback.print_stack()
LOG.error(ex)
return 1
finally:
LOG.debug("Finishing buildaction")
client.finishBuildAction(analysis_id, self.analyzer_stderr)
LOG.debug("Finishing buildaction done.")
def get_env_checkers_list(env_name):
env_set_checkers = util.get_env_var(env_name)
LOG = LoggerFactory.get_new_logger("CHECKERS")
LOG.debug_analyzer('Getting checkers list from environment variable %s' %
env_name)
if env_set_checkers:
checkers_list = env_set_checkers.split(':')
LOG.debug_analyzer('Checkers list is -> ' + env_set_checkers)
return sorted(checkers_list)
else:
LOG.debug_analyzer('No checkers list was defined.')
return None
def handle_results(self):
"""
Send the plist content to the database.
Server API calls should be used in one connection.
- addBuildAction
- addReport
- needFileContent
- addFileContent
- finishBuildAction
"""
with client.get_connection() as connection:
LOG.debug('Storing original build and analyzer command '
'to the database.')
_, source_file_name = os.path.split(self.analyzed_source_file)
if LoggerFactory.get_log_level() == logger.DEBUG:
analyzer_cmd = ' '.join(self.analyzer_cmd)
else:
analyzer_cmd = ''
build_cmd_hash = self.buildaction.original_command_hash
analysis_id = \
connection.add_build_action(self.__run_id,
build_cmd_hash,
analyzer_cmd,
self.buildaction.analyzer_type,
source_file_name)
assert self.analyzer_returncode == 0
plist_file = self.analyzer_result_file
try:
files, reports = plist_parser.parse_plist(plist_file)
except Exception as ex:
LOG.debug(str(ex))
msg = 'Parsing the generated result file failed.'
LOG.error(msg + ' ' + plist_file)
connection.finish_build_action(analysis_id, msg)
return 1
self.__store_bugs(files, reports, connection, analysis_id)
connection.finish_build_action(analysis_id, self.analyzer_stderr)
# -------------------------------------------------------------------------
from abc import ABCMeta
import os
import zlib
import shared
from libcodechecker import client
from libcodechecker import logger
from libcodechecker import suppress_handler
from libcodechecker.analyze import plist_parser
from libcodechecker.analyze.analyzers.result_handler_base import ResultHandler
from libcodechecker.logger import LoggerFactory
LOG = LoggerFactory.get_new_logger('PLIST TO DB')
class PlistToDB(ResultHandler):
"""
Result handler for processing a plist file with the
analysis results and stores them to the database.
"""
__metaclass__ = ABCMeta
def __init__(self, buildaction, workspace, run_id):
super(PlistToDB, self).__init__(buildaction, workspace)
self.__run_id = run_id
def __store_bugs(self, files, reports, connection, analisys_id):
import random
import sqlalchemy
import shared
from ProductManagement_v6 import ttypes
from libcodechecker import util
from libcodechecker.logger import LoggerFactory
from libcodechecker.profiler import timeit
from libcodechecker.server import permissions
from libcodechecker.server.database.config_db_model import *
from libcodechecker.server.database.database import SQLServer
from libcodechecker.server.routing import is_valid_product_endpoint
LOG = LoggerFactory.get_new_logger('PRODUCT HANDLER')
class ThriftProductHandler(object):
"""
Connect to database and handle thrift client requests.
"""
def __init__(self,
server,
auth_session,
config_session,
routed_product,
package_version):
self.__server = server
Build and log related functionality.
"""
from distutils.spawn import find_executable
import os
import pickle
import platform
import subprocess
import sys
from uuid import uuid4
from libcodechecker import host_check
from libcodechecker.logger import LoggerFactory
# TODO: Cross-reference between subpacakges...
from libcodechecker.analyze import analyzer_env
LOG = LoggerFactory.get_new_logger('BUILD MANAGER')
def execute_buildcmd(command, silent=False, env=None, cwd=None):
"""
Execute the the build command and continuously write
the output from the process to the standard output.
"""
proc = subprocess.Popen(command,
bufsize=-1,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
cwd=cwd,
shell=True)
from libcodechecker.logger import LoggerFactory
unsupported_methods = []
try:
from libcodechecker.libauth import cc_ldap
except ImportError:
unsupported_methods.append("ldap")
try:
from libcodechecker.libauth import cc_pam
except ImportError:
unsupported_methods.append("pam")
LOG = LoggerFactory.get_new_logger("SESSION MANAGER")
SESSION_COOKIE_NAME = "__ccPrivilegedAccessToken"
session_lifetimes = {}
class _Session(object):
"""A session for an authenticated, privileged client connection."""
# Create an initial salt from system environment for use with the session
# permanent persistency routine.
__initial_salt = hashlib.sha256(SESSION_COOKIE_NAME + "__" +
str(time.time()) + "__" +
os.urandom(16)).hexdigest()
@staticmethod
def calc_persistency_hash(auth_string):
# This is the new format.
123324353456463442341242342343#1 || filename || bug hash comment
After removing the hash_value_type the generated format is:
123324353456463442341242342343 || filename || bug hash comment
For backward compatibility the hash_value_type is an optional filed.
"""
import codecs
import os
import re
from libcodechecker.logger import LoggerFactory
LOG = LoggerFactory.get_new_logger('SUPPRESS_FILE_HANDLER')
COMMENT_SEPARATOR = '||'
HASH_TYPE_SEPARATOR = '#'
def get_suppress_data(suppress_file):
"""
Process a file object for suppress information.
"""
old_format_pattern = r"^(?P<bug_hash>[\d\w]{32})" \
r"(\#(?P<bug_hash_type>\d))?\s*\|\|\s*(?P<comment>[^\|]*)$"
old_format = re.compile(old_format_pattern, re.UNICODE)
new_format_pattern = r"^(?P<bug_hash>[\d\w]{32})" \
# License. See LICENSE.TXT for details.
# -------------------------------------------------------------------------
"""
This module is responsible for parsing clang-tidy output and generating plist
for the plist_parser module.
"""
import copy
import json
import os
import plistlib
import re
from libcodechecker.logger import LoggerFactory
LOG = LoggerFactory.get_new_logger('TIDY OUTPUT CONVERTER')
class Note(object):
"""
Represents a note and also this is the base class of Message.
"""
def __init__(self, path, line, column, message):
self.path = path
self.line = line
self.column = column
self.message = message
def __eq__(self, other):
return self.path == other.path and \
self.line == other.line and \
Util module.
"""
import datetime
import hashlib
import os
import re
import socket
import subprocess
import psutil
from libcodechecker.logger import LoggerFactory
# WARNING! LOG should be only used in this module.
LOG = LoggerFactory.get_new_logger('UTIL')
def get_free_port():
""" Get a free port from the OS. """
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(('', 0))
free_port = s.getsockname()[1]
s.close()
return free_port
def is_localhost(address):
"""
# License. See LICENSE.TXT for details.
# -------------------------------------------------------------------------
from datetime import datetime
import cProfile
import pstats
try:
from StringIO import StringIO
except ImportError:
from io import BytesIO as StringIO
from libcodechecker import logger
from libcodechecker.logger import LoggerFactory
LOG = LoggerFactory.get_new_logger('PROFILER')
def timeit(function):
"""
Decorator to measure function call time.
"""
func_name = function.__name__
def debug_wrapper(*args, **kwargs):
"""
Log measured time.
"""
before = datetime.now()
res = function(*args, **kwargs)
# -------------------------------------------------------------------------
# The CodeChecker Infrastructure
# This file is distributed under the University of Illinois Open Source
# License. See LICENSE.TXT for details.
# -------------------------------------------------------------------------
"""
Handle thrift requests for authentication
"""
import shared
from Authentication.ttypes import *
from libcodechecker.logger import LoggerFactory
from libcodechecker.profiler import timeit
LOG = LoggerFactory.get_new_logger('AUTH HANDLER')
def conv(text):
"""
Convert * to % got from clients for the database queries.
"""
if text is None:
return '%'
return text.replace('*', '%')
class ThriftAuthHandler(object):
"""
Handle Thrift authentication requests.
"""
import sqlalchemy
from sqlalchemy import event
from sqlalchemy.engine import Engine
from sqlalchemy.engine.url import URL, make_url
from sqlalchemy.sql.elements import quoted_name
from libcodechecker import host_check
from libcodechecker import logger
from libcodechecker import pgpass
from libcodechecker import util
from libcodechecker.logger import LoggerFactory
from libcodechecker.orm_model import CC_META
from libcodechecker.orm_model import CreateSession
from libcodechecker.orm_model import DBVersion
LOG = LoggerFactory.get_new_logger('DB_HANDLER')
class SQLServer(object):
"""
Abstract base class for database server handling. An SQLServer instance is
responsible for the initialization, starting, and stopping the database
server, and also for connection string management.
SQLServer implementations are created via SQLServer.from_cmdline_args().
How to add a new database server implementation:
1, Derive from SQLServer and implement the abstract methods
2, Add/modify some command line options in CodeChecker.py
3, Modify SQLServer.from_cmdline_args() in order to create an
instance of the new server type if needed
# This file is distributed under the University of Illinois Open Source
# License. See LICENSE.TXT for details.
# -------------------------------------------------------------------------
from abc import ABCMeta
import linecache
import math
import os
import sys
from libcodechecker import suppress_handler
from libcodechecker.analyze import plist_parser
from libcodechecker.analyze.analyzers.result_handler_base import ResultHandler
from libcodechecker.logger import LoggerFactory
LOG = LoggerFactory.get_new_logger('PLIST TO STDOUT')
class PlistToStdout(ResultHandler):
"""
Result handler for processing a plist file with the
analysis results and print them to the standard output.
"""
__metaclass__ = ABCMeta
def __init__(self, buildaction, workspace, lock):
super(PlistToStdout, self).__init__(buildaction, workspace)
self.__print_steps = False
self.__output = sys.stdout
self.__lock = lock
