def get_env_checkers_list(env_name):
env_set_checkers = util.get_env_var(env_name)
LOG = LoggerFactory.get_new_logger("CHECKERS")
LOG.debug_analyzer('Getting checkers list from environment variable %s' %
env_name)
if env_set_checkers:
checkers_list = env_set_checkers.split(':')
LOG.debug_analyzer('Checkers list is -> ' + env_set_checkers)
return sorted(checkers_list)
else:
LOG.debug_analyzer('No checkers list was defined.')
return None
from thrift.protocol import TJSONProtocol
from Authentication import codeCheckerAuthentication
from Authentication.ttypes import *
from codeCheckerDBAccess import codeCheckerDBAccess
from codeCheckerDBAccess.ttypes import *
from libcodechecker import database_handler
from libcodechecker import session_manager
from libcodechecker.logger import LoggerFactory
from . import instance_manager
from client_auth_handler import ThriftAuthHandler
from client_db_access_handler import ThriftRequestHandler
LOG = LoggerFactory.get_new_logger('DB ACCESS')
class RequestHandler(SimpleHTTPRequestHandler):
"""
Handle thrift and browser requests
Simply modified and extended version of SimpleHTTPRequestHandler
"""
def __init__(self, request, client_address, server):
self.sc_session = server.sc_session
self.db_version_info = server.db_version_info
self.manager = server.manager
BaseHTTPRequestHandler.__init__(self, request, client_address, server)
from plist_to_html import PlistToHtml
from codeCheckerDBAccess_v6 import constants, ttypes
from libcodechecker import generic_package_context
from libcodechecker import suppress_handler
from libcodechecker import suppress_file_handler
from libcodechecker.analyze import plist_parser
from libcodechecker.libclient.client import handle_auth
from libcodechecker.libclient.client import setup_client
from libcodechecker.logger import LoggerFactory
from libcodechecker.output_formatters import twodim_to_str
from libcodechecker.report import Report
from libcodechecker.util import split_server_url
LOG = LoggerFactory.get_new_logger('CMD')
class CmdLineOutputEncoder(json.JSONEncoder):
def default(self, obj):
d = {}
d.update(obj.__dict__)
return d
def check_run_names(client, check_names):
"""
Check if the given names are valid runs on the server. If any of the names
is not found then the script finishes. Otherwise a dictionary returns which
maps run names to runs. The dictionary contains only the runs in
check_names or all runs if check_names is empty or None.
# -------------------------------------------------------------------------
# The CodeChecker Infrastructure
# This file is distributed under the University of Illinois Open Source
# License. See LICENSE.TXT for details.
# -------------------------------------------------------------------------
""""""
import os
from libcodechecker.logger import LoggerFactory
LOG = LoggerFactory.get_new_logger('ENV')
def get_log_env(logfile, context, original_env):
"""
Environment for logging. With the ld logger.
Keep the original environment unmodified as possible.
Only environment variables required for logging are changed.
"""
new_env = original_env
new_env[context.env_var_cc_logger_bin] = context.path_logger_bin
new_env['LD_PRELOAD'] = context.logger_lib_name
try:
original_ld_library_path = new_env['LD_LIBRARY_PATH']
new_env['LD_LIBRARY_PATH'] = context.path_logger_lib + ':' + \
original_ld_library_path
except:
# License. See LICENSE.TXT for details.
# -------------------------------------------------------------------------
import os
import re
import sys
import traceback
import subprocess
import shlex
# TODO: This is a cross-subpackage import!
from libcodechecker.log import build_action
from libcodechecker.log import option_parser
from libcodechecker.logger import LoggerFactory
LOG = LoggerFactory.get_new_logger('LOG PARSER')
# If these options are present in the original build command, they must
# be forwarded to get_compiler_includes and get_compiler_defines so the
# resulting includes point to the target that was used in the build.
COMPILE_OPTS_FWD_TO_DEFAULTS_GETTER = frozenset(['^-m(32|64)', '^-std=.*'])
def get_compiler_includes(compiler, extra_opts=None):
"""
Returns a list of default includes of the given compiler.
"""
start_mark = "#include <...> search starts here:"
end_mark = "End of search list."
if extra_opts is None:
import argparse
import json
import os
import shutil
import sys
from libcodechecker import generic_package_context
from libcodechecker import util
from libcodechecker.analyze import log_parser
from libcodechecker.analyze import analyzer
from libcodechecker.analyze.analyzers import analyzer_types
from libcodechecker.logger import add_verbose_arguments
from libcodechecker.logger import LoggerFactory
LOG = LoggerFactory.get_new_logger('ANALYZE')
class OrderedCheckersAction(argparse.Action):
"""
Action to store enabled and disabled checkers
and keep ordering from command line.
Create separate lists based on the checker names for
each analyzer.
"""
# Users can supply invocation to 'codechecker-analyze' as follows:
# -e core -d core.uninitialized -e core.uninitialized.Assign
# We must support having multiple '-e' and '-d' options and the order
# specified must be kept when the list of checkers are assembled for Clang.
import getpass
import sys
from Authentication import ttypes as AuthTypes
from thrift.Thrift import TApplicationException
from . import thrift_helper
from . import authentication_helper
from libcodechecker import session_manager
from libcodechecker.logger import LoggerFactory
import shared
LOG = LoggerFactory.get_new_logger('CLIENT')
SUPPORTED_VERSION = '5.0'
def check_api_version(client):
"""
Check if server API is supported by the client.
"""
version = client.getAPIVersion()
supp_major_version = SUPPORTED_VERSION.split('.')[0]
api_major_version = version.split('.')[0]
# There is NO compatibility between major versions.
return supp_major_version == api_major_version
from collections import defaultdict
import multiprocessing
import ntpath
import os
import re
import shutil
import signal
import sys
import traceback
from libcodechecker.analyze import analyzer_env
from libcodechecker.analyze.analyzers import analyzer_types
from libcodechecker.logger import LoggerFactory
LOG = LoggerFactory.get_new_logger('ANALYSIS MANAGER')
def worker_result_handler(results):
"""
Print the analysis summary.
"""
successful_analysis = defaultdict(int)
failed_analysis = defaultdict(int)
skipped_num = 0
for res, skipped, analyzer_type in results:
if skipped:
skipped_num += 1
else:
# License. See LICENSE.TXT for details.
# -------------------------------------------------------------------------
from datetime import datetime
import cProfile
import pstats
try:
from StringIO import StringIO
except ImportError:
from io import BytesIO as StringIO
from libcodechecker import logger
from libcodechecker.logger import LoggerFactory
LOG = LoggerFactory.get_new_logger('PROFILER')
def timeit(function):
"""
Decorator to measure function call time.
"""
func_name = function.__name__
def debug_wrapper(*args, **kwargs):
"""
Log measured time.
"""
before = datetime.now()
res = function(*args, **kwargs)
# -------------------------------------------------------------------------
# The CodeChecker Infrastructure
# This file is distributed under the University of Illinois Open Source
# License. See LICENSE.TXT for details.
# -------------------------------------------------------------------------
"""
Handle thrift requests for authentication
"""
import shared
from Authentication.ttypes import *
from libcodechecker.logger import LoggerFactory
from libcodechecker.profiler import timeit
LOG = LoggerFactory.get_new_logger('AUTH HANDLER')
def conv(text):
"""
Convert * to % got from clients for the database queries.
"""
if text is None:
return '%'
return text.replace('*', '%')
class ThriftAuthHandler(object):
"""
Handle Thrift authentication requests.
"""
import sqlalchemy
from sqlalchemy import event
from sqlalchemy.engine import Engine
from sqlalchemy.engine.url import URL, make_url
from sqlalchemy.sql.elements import quoted_name
from libcodechecker import host_check
from libcodechecker import logger
from libcodechecker import pgpass
from libcodechecker import util
from libcodechecker.logger import LoggerFactory
from libcodechecker.orm_model import CC_META
from libcodechecker.orm_model import CreateSession
from libcodechecker.orm_model import DBVersion
LOG = LoggerFactory.get_new_logger('DB_HANDLER')
class SQLServer(object):
"""
Abstract base class for database server handling. An SQLServer instance is
responsible for the initialization, starting, and stopping the database
server, and also for connection string management.
SQLServer implementations are created via SQLServer.from_cmdline_args().
How to add a new database server implementation:
1, Derive from SQLServer and implement the abstract methods
2, Add/modify some command line options in CodeChecker.py
3, Modify SQLServer.from_cmdline_args() in order to create an
instance of the new server type if needed
# This file is distributed under the University of Illinois Open Source
# License. See LICENSE.TXT for details.
# -------------------------------------------------------------------------
from abc import ABCMeta
import linecache
import math
import os
import sys
from libcodechecker import suppress_handler
from libcodechecker.analyze import plist_parser
from libcodechecker.analyze.analyzers.result_handler_base import ResultHandler
from libcodechecker.logger import LoggerFactory
LOG = LoggerFactory.get_new_logger('PLIST TO STDOUT')
class PlistToStdout(ResultHandler):
"""
Result handler for processing a plist file with the
analysis results and print them to the standard output.
"""
__metaclass__ = ABCMeta
def __init__(self, buildaction, workspace, lock):
super(PlistToStdout, self).__init__(buildaction, workspace)
self.__print_steps = False
self.__output = sys.stdout
self.__lock = lock
# The CodeChecker Infrastructure
# This file is distributed under the University of Illinois Open Source
# License. See LICENSE.TXT for details.
# -------------------------------------------------------------------------
"""
Handler for suppressing a bug.
"""
import os
from libcodechecker import suppress_file_handler
from libcodechecker import suppress_handler
from libcodechecker.logger import LoggerFactory
# Warning! this logger should only be used in this module.
LOG = LoggerFactory.get_new_logger('SUPPRESS')
class GenericSuppressHandler(suppress_handler.SuppressHandler):
def __init__(self, suppress_file, allow_write):
"""
Create a new suppress handler with a suppress_file as backend.
"""
super(GenericSuppressHandler, self).__init__()
self.__suppress_info = []
self.__allow_write = allow_write
if suppress_file:
self.suppress_file = suppress_file
self.__have_memory_backend = True
# -------------------------------------------------------------------------
# The CodeChecker Infrastructure
# This file is distributed under the University of Illinois Open Source
# License. See LICENSE.TXT for details.
# -------------------------------------------------------------------------
from libcodechecker.analyze import tidy_output_converter
from libcodechecker.analyze.analyzers.result_handler_plist_to_db \
import PlistToDB
from libcodechecker.analyze.analyzers.result_handler_plist_to_stdout \
import PlistToStdout
from libcodechecker.logger import LoggerFactory
LOG = LoggerFactory.get_new_logger('CLANG_TIDY_RESULT_HANDLER')
def generate_plist_from_tidy_result(output_file, tidy_stdout):
"""
Generate a plist file from the clang tidy analyzer results.
"""
parser = tidy_output_converter.OutputParser()
messages = parser.parse_messages(tidy_stdout)
plist_converter = tidy_output_converter.PListConverter()
plist_converter.add_messages(messages)
plist_converter.write_to_file(output_file)
class ClangTidyPlistToDB(PlistToDB):
"""
This module defines the list of permissions of a CodeChecker server, and
provides the handling of permission matching and databases.
"""
from abc import ABCMeta
from abc import abstractmethod
from sqlalchemy import and_
from shared.ttypes import Permission as PermissionEnum
from libcodechecker.logger import LoggerFactory
LOG = LoggerFactory.get_new_logger('PERMISSIONS')
config_db_model = None # Module will be loaded later...
class Permission(object):
"""
The base class for a permission declaration.
"""
__metaclass__ = ABCMeta
def __init__(self, name, default_enable=True,
inherited_from=None, managed_by=None):
"""
Creates the definition of a new permission.
import os
import socket
import sys
from thrift.transport import THttpClient
from thrift.protocol import TJSONProtocol
from thrift.protocol.TProtocol import TProtocolException
import shared
from codeCheckerDBAccess_v6 import codeCheckerDBAccess
from libcodechecker import session_manager
from libcodechecker import util
from libcodechecker.logger import LoggerFactory
LOG = LoggerFactory.get_new_logger('THRIFT HELPER')
class ThriftClientHelper(object):
def __init__(self, protocol, host, port, uri, session_token=None):
self.__host = host
self.__port = port
url = util.create_product_url(protocol, host, port, uri)
self.transport = THttpClient.THttpClient(url)
self.protocol = TJSONProtocol.TJSONProtocol(self.transport)
self.client = codeCheckerDBAccess.Client(self.protocol)
if session_token:
headers = {
'Cookie':
session_manager.SESSION_COOKIE_NAME + "=" + session_token
# This file is distributed under the University of Illinois Open Source
# License. See LICENSE.TXT for details.
# -------------------------------------------------------------------------
"""
"""
import os
import re
import shlex
import subprocess
from libcodechecker.analyze.analyzers import analyzer_base
from libcodechecker.logger import LoggerFactory
from libcodechecker.util import get_binary_in_path
LOG = LoggerFactory.get_new_logger('CLANG TIDY')
class ClangTidy(analyzer_base.SourceAnalyzer):
"""
Constructs the clang tidy analyzer commands.
"""
def __parse_checkers(self, tidy_output):
"""
Parse clang tidy checkers list.
Skip clang static analyzer checkers.
Store them to checkers.
"""
for line in tidy_output.splitlines():
line = line.strip()
if re.match(r'^Enabled checks:', line) or line == '':
Util module.
"""
import datetime
import hashlib
import os
import re
import socket
import subprocess
import psutil
from libcodechecker.logger import LoggerFactory
# WARNING! LOG should be only used in this module.
LOG = LoggerFactory.get_new_logger('UTIL')
def get_free_port():
""" Get a free port from the OS. """
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(('', 0))
free_port = s.getsockname()[1]
s.close()
return free_port
def is_localhost(address):
"""
def __init__(self, context, analyzer_env):
self._context = context
self._analyzer_env = analyzer_env
self._logger = LoggerFactory.get_new_logger('ANALYZER_CRASH_HANDLER')
# License. See LICENSE.TXT for details.
# -------------------------------------------------------------------------
"""
This module is responsible for parsing clang-tidy output and generating plist
for the plist_parser module.
"""
import copy
import json
import os
import plistlib
import re
from libcodechecker.logger import LoggerFactory
LOG = LoggerFactory.get_new_logger('TIDY OUTPUT CONVERTER')
class Note(object):
"""
Represents a note and also this is the base class of Message.
"""
def __init__(self, path, line, column, message):
self.path = path
self.line = line
self.column = column
self.message = message
def __eq__(self, other):
return self.path == other.path and \
self.line == other.line and \
# License. See LICENSE.TXT for details.
# -------------------------------------------------------------------------
"""
This module is responsible for parsing clang-tidy output and generating plist
for the plist_parser module.
"""
import copy
import json
import os
import plistlib
import re
from libcodechecker.logger import LoggerFactory
LOG = LoggerFactory.get_new_logger('TIDY_OUTPUT_HANDLER')
class Note(object):
"""
Represents a note and also this is the base class of Message.
"""
def __init__(self, path, line, column, message):
self.path = path
self.line = line
self.column = column
self.message = message
def __eq__(self, other):
return self.path == other.path and \
self.line == other.line and \
# This is the new format.
123324353456463442341242342343#1 || filename || bug hash comment
After removing the hash_value_type the generated format is:
123324353456463442341242342343 || filename || bug hash comment
For backward compatibility the hash_value_type is an optional filed.
"""
import codecs
import os
import re
from libcodechecker.logger import LoggerFactory
LOG = LoggerFactory.get_new_logger('SUPPRESS_FILE_HANDLER')
COMMENT_SEPARATOR = '||'
HASH_TYPE_SEPARATOR = '#'
def get_suppress_data(suppress_file):
"""
Process a file object for suppress information.
"""
old_format_pattern = r"^(?P<bug_hash>[\d\w]{32})" \
r"(\#(?P<bug_hash_type>\d))?\s*\|\|\s*(?P<comment>[^\|]*)$"
old_format = re.compile(old_format_pattern, re.UNICODE)
new_format_pattern = r"^(?P<bug_hash>[\d\w]{32})" \
stdout.
"""
import argparse
import os
import shutil
import tempfile
from libcodechecker import host_check
from libcodechecker import libhandlers
from libcodechecker.analyze.analyzers import analyzer_types
from libcodechecker.logger import add_verbose_arguments
from libcodechecker.logger import LoggerFactory
from libcodechecker.util import get_default_workspace
LOG = LoggerFactory.get_new_logger('CHECK')
class OrderedCheckersAction(argparse.Action):
"""
Action to store enabled and disabled checkers
and keep ordering from command line.
Create separate lists based on the checker names for
each analyzer.
"""
# Users can supply invocation to 'codechecker-analyze' as follows:
# -e core -d core.uninitialized -e core.uninitialized.Assign
# We must support having multiple '-e' and '-d' options and the order
# specified must be kept when the list of checkers are assembled for Clang.
from libcodechecker.logger import LoggerFactory
unsupported_methods = []
try:
from libcodechecker.libauth import cc_ldap
except ImportError:
unsupported_methods.append("ldap")
try:
from libcodechecker.libauth import cc_pam
except ImportError:
unsupported_methods.append("pam")
LOG = LoggerFactory.get_new_logger("SESSION MANAGER")
SESSION_COOKIE_NAME = "__ccPrivilegedAccessToken"
session_lifetimes = {}
class _Session(object):
"""A session for an authenticated, privileged client connection."""
# Create an initial salt from system environment for use with the session
# permanent persistency routine.
__initial_salt = hashlib.sha256(SESSION_COOKIE_NAME + "__" +
str(time.time()) + "__" +
os.urandom(16)).hexdigest()
@staticmethod
def calc_persistency_hash(auth_string):
The attribute of the group object which contains the name of the group.
`groupScope`
Scope of the search performed. (Valid values are: base, one, subtree)
"""
from contextlib import contextmanager
import ldap
from libcodechecker.logger import LoggerFactory
LOG = LoggerFactory.get_new_logger("LDAP")
def log_ldap_error(ldap_error):
"""
Log the LDAP error details in debug mode.
"""
toprint = ''
if ldap_error.message:
if 'info' in ldap_error.message:
toprint = toprint + ldap_error.message['info']
if 'info' in ldap_error.message and 'desc' in ldap_error.message:
toprint = toprint + "; "
if 'desc' in ldap_error.message:
toprint = toprint + ldap_error.message['desc']
else:
Build and log related functionality.
"""
from distutils.spawn import find_executable
import os
import pickle
import platform
import subprocess
import sys
from uuid import uuid4
from libcodechecker import host_check
from libcodechecker.logger import LoggerFactory
# TODO: Cross-reference between subpacakges...
from libcodechecker.analyze import analyzer_env
LOG = LoggerFactory.get_new_logger('BUILD MANAGER')
def execute_buildcmd(command, silent=False, env=None, cwd=None):
"""
Execute the the build command and continuously write
the output from the process to the standard output.
"""
proc = subprocess.Popen(command,
bufsize=-1,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
cwd=cwd,
shell=True)
# -------------------------------------------------------------------------
# The CodeChecker Infrastructure
# This file is distributed under the University of Illinois Open Source
# License. See LICENSE.TXT for details.
# -------------------------------------------------------------------------
import argparse
import json
import re
import shlex
from libcodechecker.analyze.analyzers import config_handler
from libcodechecker.logger import LoggerFactory
LOG = LoggerFactory.get_new_logger('CLANG TIDY CONFIG')
class ClangTidyConfigHandler(config_handler.AnalyzerConfigHandler):
"""
Configuration handler for Clang-tidy analyzer.
"""
def __init__(self):
super(ClangTidyConfigHandler, self).__init__()
def get_checker_configs(self):
"""
Process the raw extra analyzer arguments and get the configuration
data ('-config=' argument for Clang tidy) for the checkers.
Clang tidy accepts YAML or JSON formatted config, right now
parsing only the JSON format is supported.
import random
import sqlalchemy
import shared
from ProductManagement_v6 import ttypes
from libcodechecker import util
from libcodechecker.logger import LoggerFactory
from libcodechecker.profiler import timeit
from libcodechecker.server import permissions
from libcodechecker.server.database.config_db_model import *
from libcodechecker.server.database.database import SQLServer
from libcodechecker.server.routing import is_valid_product_endpoint
LOG = LoggerFactory.get_new_logger('PRODUCT HANDLER')
class ThriftProductHandler(object):
"""
Connect to database and handle thrift client requests.
"""
def __init__(self,
server,
auth_session,
config_session,
routed_product,
package_version):
self.__server = server
Used to filter out or change compiler argument not supported by clang
or clang-tidy.
Keys are the option name, value is the number of options to skip.
Possible improvements:
- modular option handling system configuring possibility from config file.
"""
import os
import re
import shlex
from libcodechecker.logger import LoggerFactory
LOG = LoggerFactory.get_new_logger('OPTION PARSER')
# Compiler options.
COMPILE_OPTION_MAP = {
'-idirafter': 1,
'-imacros': 1,
'-include': 1,
'-iprefix': 1,
'-isysroot': 1,
'-isystem': 1,
'-iwithprefix': 1,
'-iwithprefixbefore': 1,
'-nostdinc': 0,
'-sysroot': 1
}
# -------------------------------------------------------------------------
from abc import ABCMeta
import os
import zlib
import shared
from libcodechecker import client
from libcodechecker import logger
from libcodechecker import suppress_handler
from libcodechecker.analyze import plist_parser
from libcodechecker.analyze.analyzers.result_handler_base import ResultHandler
from libcodechecker.logger import LoggerFactory
LOG = LoggerFactory.get_new_logger('PLIST TO DB')
class PlistToDB(ResultHandler):
"""
Result handler for processing a plist file with the
analysis results and stores them to the database.
"""
__metaclass__ = ABCMeta
def __init__(self, buildaction, workspace, run_id):
super(PlistToDB, self).__init__(buildaction, workspace)
self.__run_id = run_id
def __store_bugs(self, files, reports, connection, analisys_id):
