def test_new_intercept_build(self):
"""
Test log file parsing escapes with upstream (GitHub) intercept-build.
"""
logfile = os.path.join(self.__test_files, "intercept-new.json")
# Upstream scan-build-py creates an argument vector, as opposed to a
# command string. This argument vector contains the define as it's
# element in the following format:
# -DVARIABLE=\"some value\"
# and the target triplet, e.g.:
# --target=x86_64-linux-gnu
#
# The define is passed to the analyzer properly.
build_action = log_parser.parse_log(load_json_or_empty(logfile))[0]
self.assertEqual(build_action.source, r'/tmp/a.cpp')
self.assertEqual(len(build_action.analyzer_options), 1)
self.assertTrue(len(build_action.target) > 0)
self.assertEqual(build_action.analyzer_options[0],
r'-DVARIABLE="\"some value"\"')
# Test source file with spaces.
logfile = os.path.join(self.__test_files, "intercept-new-space.json")
build_action = log_parser.parse_log(load_json_or_empty(logfile))[0]
self.assertEqual(build_action.source, r'/tmp/a\ b.cpp')
self.assertEqual(build_action.lang, 'c++')
def test_old_intercept_build(self):
"""
Test log file parsing escape behaviour with clang-5.0 intercept-build.
"""
logfile = os.path.join(self.__test_files, "intercept-old.json")
# Scan-build-py shipping with clang-5.0 makes a logfile that contains:
# -DVARIABLE=\"some value\" and --target=x86_64-linux-gnu
#
# The define is passed to the analyzer properly.
build_action = log_parser.parse_log(load_json_or_empty(logfile))[0]
self.assertEqual(build_action.source, r'/tmp/a.cpp')
self.assertEqual(len(build_action.analyzer_options), 1)
self.assertTrue(len(build_action.target) > 0)
self.assertEqual(build_action.analyzer_options[0],
r'-DVARIABLE="\"some value"\"')
# Test source file with spaces.
logfile = os.path.join(self.__test_files, "intercept-old-space.json")
build_action = log_parser.parse_log(load_json_or_empty(logfile))[0]
self.assertEqual(build_action.source, r'/tmp/a\ b.cpp')
self.assertEqual(build_action.lang, 'c++')
def test_new_ldlogger(self):
"""
Test log file parsing escape behaviour with after-#631 LD-LOGGER.
"""
logfile = os.path.join(self.__test_files, "ldlogger-new.json")
# LD-LOGGERS after http://github.com/Ericsson/codechecker/pull/631
# now properly log the multiword arguments. When these are parsed by
# the log_parser, the define's value will be passed to the analyzer.
#
# Logfile contains -DVARIABLE="some value"
# and --target=x86_64-linux-gnu.
build_action = log_parser.parse_log(load_json_or_empty(logfile))[0]
self.assertEqual(build_action.source, r'/tmp/a.cpp')
self.assertEqual(len(build_action.analyzer_options), 1)
self.assertTrue(len(build_action.target) > 0)
self.assertEqual(build_action.analyzer_options[0],
r'-DVARIABLE="\"some value"\"')
# Test source file with spaces.
logfile = os.path.join(self.__test_files, "ldlogger-new-space.json")
build_action = log_parser.parse_log(load_json_or_empty(logfile))[0]
self.assertEqual(build_action.source, r'/tmp/a\ b.cpp')
self.assertEqual(build_action.lang, 'c++')
def get_context():
LOG.debug('Loading package config.')
package_root = os.environ['CC_PACKAGE_ROOT']
pckg_config_file = os.path.join(package_root, "config", "config.json")
LOG.debug('Reading config: ' + pckg_config_file)
cfg_dict = load_json_or_empty(pckg_config_file)
if not cfg_dict:
sys.exit(1)
LOG.debug(cfg_dict)
LOG.debug('Loading layout config.')
layout_cfg_file = os.path.join(package_root, "config",
"package_layout.json")
LOG.debug(layout_cfg_file)
lcfg_dict = load_json_or_empty(layout_cfg_file)
if not lcfg_dict:
sys.exit(1)
# Merge static and runtime layout.
layout_config = lcfg_dict['static'].copy()
layout_config.update(lcfg_dict['runtime'])
LOG.debug(layout_config)
try:
return Context(package_root, layout_config, cfg_dict)
except KeyError as kerr:
LOG.error(kerr)
sys.exit(1)
def __init__(self, package_root, pckg_layout, cfg_dict):
env_vars = cfg_dict['environment_variables']
self.__checker_config = cfg_dict['checker_config']
self.__available_profiles = cfg_dict['available_profiles']
# Get the common environment variables.
self.pckg_layout = pckg_layout
self.env_vars = env_vars
self._package_root = package_root
self._severity_map = SeverityMap(
load_json_or_empty(self.checkers_severity_map_file, {}))
self.__package_version = None
self.__package_build_date = None
self.__package_git_hash = None
self.__analyzers = {}
self.logger_bin = None
self.logger_file = None
self.logger_compilers = None
# Get package specific environment variables.
self.set_env(env_vars)
self.__set_version()
self.__populate_analyzers()
def __set_version(self):
"""
Get the package version from the version config file.
"""
vfile_data = load_json_or_empty(self.version_file)
if not vfile_data:
sys.exit(1)
package_version = vfile_data['version']
package_build_date = vfile_data['package_build_date']
package_git_hash = vfile_data.get('git_hash')
package_git_tag = vfile_data.get('git_describe', {}).get('tag')
package_git_dirtytag = vfile_data.get('git_describe', {}).get('dirty')
self.__package_version = package_version['major'] + '.' + \
package_version['minor'] + '.' + \
package_version['revision']
self.__package_build_date = package_build_date
self.__package_git_hash = package_git_hash
self.__package_git_tag = package_git_tag
if (LOG.getEffectiveLevel() == logger.DEBUG or
LOG.getEffectiveLevel() ==
logger.DEBUG_ANALYZER):
self.__package_git_tag = package_git_dirtytag
def __set_version(self):
"""
Get the package version from the version config file.
"""
vfile_data = load_json_or_empty(self.version_file)
if not vfile_data:
sys.exit(1)
package_version = vfile_data['version']
package_build_date = vfile_data['package_build_date']
package_git_hash = vfile_data['git_hash']
package_git_tag = vfile_data['git_describe']['tag']
package_git_dirtytag = vfile_data['git_describe']['dirty']
product_database_version = vfile_data['product_db_version']
run_database_version = vfile_data['run_db_version']
self.__package_version = package_version['major'] + '.' + \
package_version['minor'] + '.' + \
package_version['revision']
self.__product_db_version_info = db_version.DBVersionInfo(
product_database_version['major'],
product_database_version['minor'])
self.__run_db_version_info = db_version.DBVersionInfo(
run_database_version['major'], run_database_version['minor'])
self.__package_build_date = package_build_date
self.__package_git_hash = package_git_hash
self.__package_git_tag = package_git_tag
if (LOG.getEffectiveLevel() == logger.DEBUG
or LOG.getEffectiveLevel() == logger.DEBUG_ANALYZER):
self.__package_git_tag = package_git_dirtytag
def enable_auth(workspace):
"""
Create a dummy authentication-enabled configuration and
an auth-enabled server.
Running the tests only work if the initial value (in package
server_config.json) is FALSE for authentication.enabled.
"""
server_config_filename = "server_config.json"
cc_package = codechecker_package()
original_auth_cfg = os.path.join(cc_package, 'config',
server_config_filename)
shutil.copy(original_auth_cfg, workspace)
server_cfg_file = os.path.join(workspace, server_config_filename)
scfg_dict = util.load_json_or_empty(server_cfg_file, {})
scfg_dict["authentication"]["enabled"] = True
scfg_dict["authentication"]["method_dictionary"]["enabled"] = True
scfg_dict["authentication"]["method_dictionary"]["auths"] = \
["cc:test", "john:doe", "admin:admin123"]
with open(server_cfg_file, 'w') as scfg:
json.dump(scfg_dict, scfg, indent=2, sort_keys=True)
# Create a root user.
root_file = os.path.join(workspace, 'root.user')
with open(root_file, 'w') as rootf:
rootf.write(sha256("root:root").hexdigest())
os.chmod(root_file, stat.S_IRUSR | stat.S_IWUSR)
def metadata_info(metadata_file):
check_commands = []
check_durations = []
cc_version = None
analyzer_statistics = {}
if not os.path.isfile(metadata_file):
return check_commands, check_durations, cc_version, analyzer_statistics
metadata_dict = load_json_or_empty(metadata_file, {})
if 'command' in metadata_dict:
check_commands.append(metadata_dict['command'])
if 'timestamps' in metadata_dict:
check_durations.append(
float(metadata_dict['timestamps']['end'] -
metadata_dict['timestamps']['begin']))
# Get CodeChecker version.
cc_version = metadata_dict.get('versions', {}).get('codechecker')
# Get analyzer statistics.
analyzer_statistics = metadata_dict.get('analyzer_statistics', {})
return check_commands, check_durations, cc_version, analyzer_statistics
def load_compiler_info(filename, compiler, attr):
data = load_json_or_empty(filename, {})
value = data.get(compiler)
if value is None:
LOG.error("Could not find compiler %s in file %s" %
(compiler, filename))
return value.get(attr) if isinstance(value, dict) else value
def get_instances(folder=None):
"""Returns the list of running servers for the current user."""
# This method does NOT write the descriptor file.
descriptor = __get_instance_descriptor_path(folder)
instances = load_json_or_empty(descriptor, {}, lock=True)
return [i for i in instances if __check_instance(i['hostname'], i['pid'])]
def dump_compiler_info(filename, compiler, attr, data):
all_data = dict()
if os.path.exists(filename):
all_data = load_json_or_empty(filename)
if compiler not in all_data:
all_data[compiler] = dict()
all_data[compiler].update({attr: data})
with open(filename, 'w') as f:
json.dump(all_data, f)
def test_include_rel_to_abs(self):
"""
Test working directory prepending to relative include paths.
"""
logfile = os.path.join(self.__test_files, "include.json")
build_action = log_parser.parse_log(load_json_or_empty(logfile))[0]
self.assertEqual(len(build_action.analyzer_options), 4)
self.assertEqual(build_action.analyzer_options[0], '-I')
self.assertEqual(build_action.analyzer_options[1], '/include')
self.assertEqual(build_action.analyzer_options[2], '-I/include')
self.assertEqual(build_action.analyzer_options[3], '-I/tmp')
def __init__(self):
LOG.debug("Loading clientside session config.")
# Check whether user's configuration exists.
user_home = os.path.expanduser("~")
session_cfg_file = os.path.join(user_home,
".codechecker.passwords.json")
LOG.debug(session_cfg_file)
scfg_dict = {}
if os.path.exists(session_cfg_file):
scfg_dict = load_json_or_empty(session_cfg_file, {},
"user authentication")
if os.path.exists(session_cfg_file):
check_file_owner_rw(session_cfg_file)
if not scfg_dict.get('credentials'):
scfg_dict['credentials'] = {}
self.__save = scfg_dict
self.__autologin = scfg_dict.get('client_autologin', True)
# Check and load token storage for user.
self.token_file = os.path.join(user_home, ".codechecker.session.json")
LOG.debug(self.token_file)
if os.path.exists(self.token_file):
token_dict = load_json_or_empty(self.token_file, {},
"user authentication")
check_file_owner_rw(self.token_file)
self.__tokens = token_dict.get('tokens')
else:
with open(self.token_file, 'w') as f:
json.dump({'tokens': {}}, f)
os.chmod(self.token_file, stat.S_IRUSR | stat.S_IWUSR)
self.__tokens = {}
def __init__(self, package_root, pckg_layout, cfg_dict):
# Get the common environment variables.
self.pckg_layout = pckg_layout
self._package_root = package_root
self._severity_map = SeverityMap(
load_json_or_empty(self.checkers_severity_map_file, {}))
self.__package_version = None
self.__product_db_version_info = None
self.__run_db_version_info = None
self.__package_build_date = None
self.__package_git_hash = None
self.__set_version()
def metadata_info(metadata_file):
check_commands = []
check_durations = []
if not os.path.isfile(metadata_file):
return check_commands, check_durations
metadata_dict = load_json_or_empty(metadata_file, {})
if 'command' in metadata_dict:
check_commands.append(metadata_dict['command'])
if 'timestamps' in metadata_dict:
check_durations.append(
float(metadata_dict['timestamps']['end'] -
metadata_dict['timestamps']['begin']))
return check_commands, check_durations
def __get_config_dict(self):
"""
Get server config information from the configuration file. Raise
ValueError if the configuration file is invalid.
"""
LOG.debug(self.__configuration_file)
cfg_dict = load_json_or_empty(self.__configuration_file, {},
'server configuration')
if cfg_dict != {}:
check_file_owner_rw(self.__configuration_file)
else:
# If the configuration dict is empty, it means a JSON couldn't
# have been parsed from it.
raise ValueError("Server configuration file was invalid, or "
"empty.")
return cfg_dict
def set_env(self, env_vars):
"""
Get the environment variables.
"""
self._package_root = os.environ.get(env_vars['env_package_root'])
self.codechecker_workspace = os.environ.get('codechecker_workspace')
self._severity_map = SeverityMap(
load_json_or_empty(self.checkers_severity_map_file, {}))
# Get generic package specific environment variables.
self.logger_bin = os.environ.get(env_vars['cc_logger_bin'])
self.logger_file = os.environ.get(env_vars['cc_logger_file'])
self.logger_compilers = os.environ.get(env_vars['cc_logger_compiles'])
self.ld_preload = os.environ.get(env_vars['ld_preload'])
self.ld_lib_path = env_vars['env_ld_lib_path']
def __get_run_name(input_list):
"""Create a runname for the stored analysis from the input list."""
# Try to create a name from the metada JSON(s).
names = []
for input_path in input_list:
metafile = os.path.join(input_path, "metadata.json")
if os.path.isdir(input_path) and os.path.exists(metafile):
metajson = util.load_json_or_empty(metafile)
if 'name' in metajson:
names.append(metajson['name'])
else:
names.append("unnamed result folder")
if len(names) == 1 and names[0] != "unnamed result folder":
return names[0]
elif len(names) > 1:
return "multiple projects: " + ', '.join(names)
else:
return False
def parse_log(logfilepath, parseLogOptions):
"""
logfilepath: the compile command json file which should be parsed.
"""
LOG.debug('Parsing log file: ' + logfilepath)
try:
data = load_json_or_empty(logfilepath, {})
actions = parse_compile_commands_json(data, parseLogOptions)
except (ValueError, KeyError, TypeError) as ex:
if os.stat(logfilepath).st_size == 0:
LOG.error('The compile database is empty.')
else:
LOG.error('The compile database is not valid.')
LOG.debug(traceback.format_exc())
LOG.debug(ex)
sys.exit(1)
LOG.debug('Parsing log file done.')
return actions
def enable_storage_of_analysis_statistics(workspace):
"""
Enables storing analysis statistics information for the server.
"""
server_config_filename = "server_config.json"
cc_package = codechecker_package()
original_auth_cfg = os.path.join(cc_package, 'config',
server_config_filename)
shutil.copy(original_auth_cfg, workspace)
server_cfg_file = os.path.join(workspace, server_config_filename)
scfg_dict = util.load_json_or_empty(server_cfg_file, {})
scfg_dict["store"]["analysis_statistics_dir"] = \
os.path.join(workspace, 'analysis_statistics')
with open(server_cfg_file, 'w') as scfg:
json.dump(scfg_dict, scfg, indent=2, sort_keys=True)
def test_old_ldlogger(self):
"""
Test log file parsing escape behaviour with pre-2017 Q2 LD-LOGGER.
"""
logfile = os.path.join(self.__test_files, "ldlogger-old.json")
# LD-LOGGER before http://github.com/Ericsson/codechecker/pull/631
# used an escape mechanism that, when parsed by the log parser via
# shlex, made CodeChecker parse arguments with multiword string
# literals in them be considered as "file" (instead of compile option),
# eventually ignored by the command builder, thus lessening analysis
# accuracy, as defines were lost.
#
# Logfile contains "-DVARIABLE="some value"".
#
# There is no good way to back-and-forth convert in log_parser or
# option_parser, so here we aim for a non-failing stalemate of the
# define being considered a file and ignored, for now.
build_action = log_parser.parse_log(load_json_or_empty(logfile))[0]
self.assertEqual(build_action.source, r'/tmp/a.cpp')
self.assertEqual(len(build_action.analyzer_options), 0)
def server_init_start(args):
"""
Start or manage a CodeChecker report server.
"""
if 'list' in args or 'stop' in args or 'stop_all' in args:
__instance_management(args)
sys.exit(0)
if 'reload' in args:
__reload_config(args)
sys.exit(0)
# Actual server starting from this point.
if not host_check.check_zlib():
raise Exception("zlib is not available on the system!")
# WARNING
# In case of SQLite args.dbaddress default value is used
# for which the is_localhost should return true.
if util.is_localhost(args.dbaddress) and \
not os.path.exists(args.config_directory):
os.makedirs(args.config_directory)
# Make sure the SQLite file can be created if it not exists.
if 'sqlite' in args and \
not os.path.isdir(os.path.dirname(args.sqlite)):
os.makedirs(os.path.dirname(args.sqlite))
if 'reset_root' in args:
try:
os.remove(os.path.join(args.config_directory, 'root.user'))
LOG.info("Master superuser (root) credentials invalidated and "
"deleted. New ones will be generated...")
except OSError:
# File doesn't exist.
pass
if 'force_auth' in args:
LOG.info("'--force-authentication' was passed as a command-line "
"option. The server will ask for users to authenticate!")
context = package_context.get_context()
context.codechecker_workspace = args.config_directory
context.db_username = args.dbusername
check_env = analyzer_env.get_check_env(context.path_env_extra,
context.ld_lib_path_extra)
cfg_sql_server = database.SQLServer.from_cmdline_args(
vars(args), CONFIG_META, context.config_migration_root,
interactive=True, env=check_env)
LOG.info("Checking configuration database ...")
db_status = cfg_sql_server.connect()
db_status_msg = database_status.db_status_msg.get(db_status)
LOG.info(db_status_msg)
if db_status == DBStatus.SCHEMA_MISSING:
LOG.debug("Config database schema is missing, initializing new.")
db_status = cfg_sql_server.connect(init=True)
if db_status != DBStatus.OK:
LOG.error("Config database initialization failed!")
LOG.error("Please check debug logs.")
sys.exit(1)
if db_status == DBStatus.SCHEMA_MISMATCH_NO:
LOG.debug("Configuration database schema mismatch.")
LOG.debug("No schema upgrade is possible.")
sys.exit(1)
force_upgrade = True if 'force_upgrade' in args else False
if db_status == DBStatus.SCHEMA_MISMATCH_OK:
LOG.debug("Configuration database schema mismatch.")
LOG.debug("Schema upgrade is possible.")
LOG.warning("Please note after migration only "
"newer CodeChecker versions can be used"
"to start the server")
LOG.warning("It is advised to make a full backup of your "
"configuration database")
LOG.warning(cfg_sql_server.get_db_location())
question = 'Do you want to upgrade to the new schema?' \
' Y(es)/n(o) '
if force_upgrade or util.get_user_input(question):
print("Upgrading schema ...")
ret = cfg_sql_server.upgrade()
msg = database_status.db_status_msg.get(
ret, 'Unknown database status')
print(msg)
if ret != DBStatus.OK:
LOG.error("Schema migration failed")
sys.exit(ret)
else:
LOG.info("No schema migration was done.")
sys.exit(0)
if db_status == DBStatus.MISSING:
LOG.error("Missing configuration database.")
LOG.error("Server can not be started.")
sys.exit(1)
# Configuration database setup and check is needed before database
# statuses can be checked.
try:
if args.status:
ret = __db_status_check(cfg_sql_server, context, args.status)
sys.exit(ret)
except AttributeError:
LOG.debug('Status was not in the arguments.')
try:
if args.product_to_upgrade:
ret = __db_migration(cfg_sql_server, context,
args.product_to_upgrade, force_upgrade)
sys.exit(ret)
except AttributeError:
LOG.debug('Product upgrade was not in the arguments.')
# Create the main database link from the arguments passed over the
# command line.
cfg_dir = os.path.abspath(args.config_directory)
default_product_path = os.path.join(cfg_dir, 'Default.sqlite')
create_default_product = 'sqlite' in args and \
not os.path.exists(default_product_path)
if create_default_product:
# Create a default product and add it to the configuration database.
LOG.debug("Create default product...")
LOG.debug("Configuring schema and migration...")
prod_server = database.SQLiteDatabase(
default_product_path, RUN_META,
context.run_migration_root, check_env)
LOG.debug("Checking 'Default' product database.")
db_status = prod_server.connect()
if db_status != DBStatus.MISSING:
db_status = prod_server.connect(init=True)
LOG.debug(database_status.db_status_msg.get(db_status))
if db_status != DBStatus.OK:
LOG.error("Failed to configure default product")
sys.exit(1)
product_conn_string = prod_server.get_connection_string()
server.add_initial_run_database(
cfg_sql_server, product_conn_string)
LOG.info("Product 'Default' at '{0}' created and set up."
.format(default_product_path))
prod_statuses = check_product_db_status(cfg_sql_server, context)
upgrade_available = {}
for k, v in prod_statuses.items():
db_status, _, _, _ = v
if db_status == DBStatus.SCHEMA_MISMATCH_OK or \
db_status == DBStatus.SCHEMA_MISSING:
upgrade_available[k] = v
if upgrade_available:
print_prod_status(prod_statuses)
LOG.warning("Multiple products can be upgraded, make a backup!")
__db_migration(cfg_sql_server, context, 'all', force_upgrade)
prod_statuses = check_product_db_status(cfg_sql_server, context)
print_prod_status(prod_statuses)
non_ok_db = False
for k, v in prod_statuses.items():
db_status, _, _, _ = v
if db_status != DBStatus.OK:
non_ok_db = True
break
if non_ok_db:
msg = "There are some database issues. " \
"Do you want to start the " \
"server? Y(es)/n(o) "
if not util.get_user_input(msg):
sys.exit(1)
# Start database viewer.
checker_md_docs = os.path.join(context.doc_root, 'checker_md_docs')
checker_md_docs_map = os.path.join(checker_md_docs,
'checker_doc_map.json')
checker_md_docs_map = util.load_json_or_empty(checker_md_docs_map, {})
package_data = {'www_root': context.www_root,
'doc_root': context.doc_root,
'checker_md_docs': checker_md_docs,
'checker_md_docs_map': checker_md_docs_map,
'version': context.package_git_tag}
suppr_handler = suppress_handler. \
GenericSuppressHandler(None, False)
try:
server.start_server(args.config_directory,
package_data,
args.view_port,
cfg_sql_server,
suppr_handler,
args.listen_address,
'force_auth' in args,
args.skip_db_cleanup,
context,
check_env)
except socket.error as err:
if err.errno == errno.EADDRINUSE:
LOG.error("Server can't be started, maybe the given port number "
"({}) is already used. Check the connection "
"parameters.".format(args.view_port))
sys.exit(1)
else:
raise
def main(args):
"""
Entry point for parsing some analysis results and printing them to the
stdout in a human-readable format.
"""
logger.setup_logger(args.verbose if 'verbose' in args else None)
context = package_context.get_context()
# To ensure the help message prints the default folder properly,
# the 'default' for 'args.input' is a string, not a list.
# But we need lists for the foreach here to work.
if isinstance(args.input, str):
args.input = [args.input]
original_cwd = os.getcwd()
suppr_handler = None
if 'suppress' in args:
__make_handler = False
if not os.path.isfile(args.suppress):
if 'create_suppress' in args:
with open(args.suppress, 'w') as _:
# Just create the file.
__make_handler = True
LOG.info("Will write source-code suppressions to "
"suppress file.")
else:
LOG.warning("Suppress file '" + args.suppress + "' given, but "
"it does not exist -- will not suppress anything.")
else:
__make_handler = True
if __make_handler:
suppr_handler = suppress_handler.\
GenericSuppressHandler(args.suppress,
'create_suppress' in args)
elif 'create_suppress' in args:
LOG.error("Can't use '--export-source-suppress' unless '--suppress "
"SUPPRESS_FILE' is also given.")
sys.exit(2)
processed_path_hashes = set()
def skip_html_report_data_handler(report_hash, source_file, report_line,
checker_name, diag, files):
"""
Report handler which skips bugs which were suppressed by source code
comments.
"""
report = Report(None, diag['path'], files)
path_hash = get_report_path_hash(report, files)
if path_hash in processed_path_hashes:
LOG.debug("Skip report because it is a deduplication of an "
"already processed report!")
LOG.debug("Path hash: %s", path_hash)
LOG.debug(diag)
return True
skip = plist_parser.skip_report(report_hash, source_file, report_line,
checker_name, suppr_handler)
if not skip:
processed_path_hashes.add(path_hash)
return skip
skip_handler = None
if 'skipfile' in args:
with open(args.skipfile, 'r') as skip_file:
skip_handler = SkipListHandler(skip_file.read())
html_builder = None
for input_path in args.input:
input_path = os.path.abspath(input_path)
os.chdir(original_cwd)
LOG.debug("Parsing input argument: '" + input_path + "'")
export = args.export if 'export' in args else None
if export is not None and export == 'html':
output_path = os.path.abspath(args.output_path)
if not html_builder:
html_builder = \
PlistToHtml.HtmlBuilder(context.path_plist_to_html_dist,
context.severity_map)
LOG.info("Generating html output files:")
PlistToHtml.parse(input_path, output_path,
context.path_plist_to_html_dist,
skip_html_report_data_handler, html_builder)
continue
files = []
metadata_dict = {}
if os.path.isfile(input_path):
files.append(input_path)
elif os.path.isdir(input_path):
metadata_file = os.path.join(input_path, "metadata.json")
if os.path.exists(metadata_file):
metadata_dict = util.load_json_or_empty(metadata_file)
LOG.debug(metadata_dict)
if 'working_directory' in metadata_dict:
working_dir = metadata_dict['working_directory']
try:
os.chdir(working_dir)
except OSError as oerr:
LOG.debug(oerr)
LOG.error(
"Working directory %s is missing.\n"
"Can not parse reports safely.", working_dir)
sys.exit(1)
_, _, file_names = next(os.walk(input_path), ([], [], []))
files = [
os.path.join(input_path, file_name) for file_name in file_names
]
file_change = set()
file_report_map = defaultdict(list)
rh = plist_parser.PlistToPlaintextFormatter(suppr_handler,
skip_handler,
context.severity_map,
processed_path_hashes)
rh.print_steps = 'print_steps' in args
for file_path in files:
f_change = parse(file_path, metadata_dict, rh, file_report_map)
file_change = file_change.union(f_change)
report_stats = rh.write(file_report_map)
severity_stats = report_stats.get('severity')
file_stats = report_stats.get('files')
reports_stats = report_stats.get('reports')
print("\n----==== Summary ====----")
if file_stats:
vals = [[os.path.basename(k), v]
for k, v in dict(file_stats).items()]
keys = ['Filename', 'Report count']
table = twodim_to_str('table', keys, vals, 1, True)
print(table)
if severity_stats:
vals = [[k, v] for k, v in dict(severity_stats).items()]
keys = ['Severity', 'Report count']
table = twodim_to_str('table', keys, vals, 1, True)
print(table)
report_count = reports_stats.get("report_count", 0)
print("----=================----")
print("Total number of reports: {}".format(report_count))
print("----=================----")
if file_change:
changed_files = '\n'.join([' - ' + f for f in file_change])
LOG.warning("The following source file contents changed since the "
"latest analysis:\n{0}\nMultiple reports were not "
"shown and skipped from the statistics. Please "
"analyze your project again to update the "
"reports!".format(changed_files))
os.chdir(original_cwd)
# Create index.html for the generated html files.
if html_builder:
html_builder.create_index_html(args.output_path)
def __init__(self,
configuration_file,
session_salt,
root_sha,
force_auth=False):
"""
Initialise a new Session Manager on the server.
:param configuration_file: The configuration file to read
authentication backends from.
:param session_salt: An initial salt that will be used in hashing
the session to the database.
:param root_sha: The SHA-256 hash of the root user's authentication.
:param force_auth: If True, the manager will be enabled even if the
configuration file disables authentication.
"""
self.__database_connection = None
self.__logins_since_prune = 0
self.__sessions = []
self.__session_salt = hashlib.sha1(session_salt).hexdigest()
LOG.debug(configuration_file)
scfg_dict = load_json_or_empty(configuration_file, {},
'server configuration')
if scfg_dict != {}:
check_file_owner_rw(configuration_file)
else:
# If the configuration dict is empty, it means a JSON couldn't
# have been parsed from it.
raise ValueError("Server configuration file was invalid, or "
"empty.")
# FIXME: Refactor this. This is irrelevant to authentication config,
# so it should NOT be handled by session_manager. A separate config
# handler for the server's stuff should be created, that can properly
# instantiate SessionManager with the found configuration.
self.__max_run_count = scfg_dict['max_run_count'] \
if 'max_run_count' in scfg_dict else None
self.__auth_config = scfg_dict['authentication']
if force_auth:
LOG.debug("Authentication was force-enabled.")
self.__auth_config['enabled'] = True
if 'soft_expire' in self.__auth_config:
LOG.debug("Found deprecated argument 'soft_expire' in "
"server_config.authentication.")
self.__refresh_time = self.__auth_config['refresh_time'] \
if 'refresh_time' in self.__auth_config else None
# Save the root SHA into the configuration (but only in memory!)
self.__auth_config['method_root'] = root_sha
# If no methods are configured as enabled, disable authentication.
if scfg_dict['authentication'].get('enabled'):
found_auth_method = False
if 'method_dictionary' in self.__auth_config and \
self.__auth_config['method_dictionary'].get('enabled'):
found_auth_method = True
if 'method_ldap' in self.__auth_config and \
self.__auth_config['method_ldap'].get('enabled'):
if 'ldap' not in UNSUPPORTED_METHODS:
found_auth_method = True
else:
LOG.warning("LDAP authentication was enabled but "
"prerequisites are NOT installed on the system"
"... Disabling LDAP authentication.")
self.__auth_config['method_ldap']['enabled'] = False
if 'method_pam' in self.__auth_config and \
self.__auth_config['method_pam'].get('enabled'):
if 'pam' not in UNSUPPORTED_METHODS:
found_auth_method = True
else:
LOG.warning("PAM authentication was enabled but "
"prerequisites are NOT installed on the system"
"... Disabling PAM authentication.")
self.__auth_config['method_pam']['enabled'] = False
if not found_auth_method:
if force_auth:
LOG.warning("Authentication was manually enabled, but no "
"valid authentication backends are "
"configured... The server will only allow "
"the master superuser (root) access.")
else:
LOG.warning("Authentication is enabled but no valid "
"authentication backends are configured... "
"Falling back to no authentication.")
self.__auth_config['enabled'] = False
def main(args):
"""
Perform analysis on the given logfiles and store the results in a machine-
readable format.
"""
logger.setup_logger(args.verbose if 'verbose' in args else None)
if len(args.logfile) != 1:
LOG.warning("Only one log file can be processed right now!")
sys.exit(1)
args.output_path = os.path.abspath(args.output_path)
if os.path.exists(args.output_path) and \
not os.path.isdir(args.output_path):
LOG.error("The given output path is not a directory: " +
args.output_path)
sys.exit(1)
if 'enable_all' in args:
LOG.info("'--enable-all' was supplied for this analysis.")
# We clear the output directory in the following cases.
ctu_dir = os.path.join(args.output_path, 'ctu-dir')
if 'ctu_phases' in args and args.ctu_phases[0] and \
os.path.isdir(ctu_dir):
# Clear the CTU-dir if the user turned on the collection phase.
LOG.debug("Previous CTU contents have been deleted.")
shutil.rmtree(ctu_dir)
if 'clean' in args and os.path.isdir(args.output_path):
LOG.info("Previous analysis results in '{0}' have been removed, "
"overwriting with current result".format(args.output_path))
shutil.rmtree(args.output_path)
if not os.path.exists(args.output_path):
os.makedirs(args.output_path)
LOG.debug("args: " + str(args))
LOG.debug("Output will be stored to: '" + args.output_path + "'")
# Process the skip list if present.
skip_handler = __get_skip_handler(args)
# Parse the JSON CCDBs and retrieve the compile commands.
actions = []
for log_file in args.logfile:
if not os.path.exists(log_file):
LOG.error("The specified logfile '" + log_file + "' does not "
"exist!")
continue
parseLogOptions = ParseLogOptions(args)
actions += log_parser.parse_log(log_file, parseLogOptions,
skip_handler)
if len(actions) == 0:
LOG.info("None of the specified build log files contained "
"valid compilation commands. No analysis needed...")
sys.exit(1)
context = package_context.get_context()
metadata = {
'action_num': len(actions),
'command': sys.argv,
'versions': {
'codechecker':
"{0} ({1})".format(context.package_git_tag,
context.package_git_hash)
},
'working_directory': os.getcwd(),
'output_path': args.output_path,
'result_source_files': {}
}
if 'name' in args:
metadata['name'] = args.name
# Update metadata dictionary with old values.
metadata_file = os.path.join(args.output_path, 'metadata.json')
if os.path.exists(metadata_file):
metadata_prev = load_json_or_empty(metadata_file)
metadata['result_source_files'] = \
metadata_prev['result_source_files']
analyzer.perform_analysis(args, skip_handler, context, actions, metadata)
__update_skip_file(args)
LOG.debug("Analysis metadata write to '" + metadata_file + "'")
with open(metadata_file, 'w') as metafile:
json.dump(metadata, metafile)
# WARN: store command will search for this file!!!!
compile_cmd_json = os.path.join(args.output_path, 'compile_cmd.json')
try:
source = os.path.abspath(args.logfile[0])
target = os.path.abspath(compile_cmd_json)
if source != target:
shutil.copyfile(source, target)
except shutil.Error:
LOG.debug("Compilation database JSON file is the same.")
except Exception:
LOG.debug("Copying compilation database JSON file failed.")
