def handle_check(args): """ Runs the original build and logs the buildactions. Based on the log runs the analysis. """ try: if not host_check.check_zlib(): sys.exit(1) args.workspace = os.path.abspath(args.workspace) if not os.path.isdir(args.workspace): os.mkdir(args.workspace) context = generic_package_context.get_context() context.codechecker_workspace = args.workspace context.db_username = args.dbusername log_file, set_in_cmdline = build_manager.check_log_file(args, context) if not log_file: LOG.error("Failed to generate compilation command file: " + log_file) sys.exit(1) actions = log_parser.parse_log(log_file, args.add_compiler_defaults) check_env = analyzer_env.get_check_env(context.path_env_extra, context.ld_lib_path_extra) sql_server = SQLServer.from_cmdline_args(args, context.codechecker_workspace, context.migration_root, check_env) conn_mgr = client.ConnectionManager(sql_server, 'localhost', util.get_free_port()) sql_server.start(context.db_version_info, wait_for_start=True, init=True) conn_mgr.start_report_server() LOG.debug("Checker server started.") analyzer.run_check(args, actions, context) LOG.info("Analysis has finished.") log_startserver_hint(args) except Exception as ex: LOG.error(ex) import traceback print(traceback.format_exc()) finally: if not args.keep_tmp: if log_file and not set_in_cmdline: LOG.debug('Removing temporary log file: ' + log_file) os.remove(log_file)
def server_init_start(args): """ Start or manage a CodeChecker report server. """ if 'list' in args or 'stop' in args or 'stop_all' in args: __instance_management(args) sys.exit(0) if 'reload' in args: __reload_config(args) sys.exit(0) # Actual server starting from this point. if not host_check.check_zlib(): raise Exception("zlib is not available on the system!") # WARNING # In case of SQLite args.dbaddress default value is used # for which the is_localhost should return true. if util.is_localhost(args.dbaddress) and \ not os.path.exists(args.config_directory): os.makedirs(args.config_directory) # Make sure the SQLite file can be created if it not exists. if 'sqlite' in args and \ not os.path.isdir(os.path.dirname(args.sqlite)): os.makedirs(os.path.dirname(args.sqlite)) if 'reset_root' in args: try: os.remove(os.path.join(args.config_directory, 'root.user')) LOG.info("Master superuser (root) credentials invalidated and " "deleted. New ones will be generated...") except OSError: # File doesn't exist. pass if 'force_auth' in args: LOG.info("'--force-authentication' was passed as a command-line " "option. The server will ask for users to authenticate!") context = package_context.get_context() context.codechecker_workspace = args.config_directory context.db_username = args.dbusername check_env = analyzer_env.get_check_env(context.path_env_extra, context.ld_lib_path_extra) cfg_sql_server = database.SQLServer.from_cmdline_args( vars(args), CONFIG_META, context.config_migration_root, interactive=True, env=check_env) LOG.info("Checking configuration database ...") db_status = cfg_sql_server.connect() db_status_msg = database_status.db_status_msg.get(db_status) LOG.info(db_status_msg) if db_status == DBStatus.SCHEMA_MISSING: LOG.debug("Config database schema is missing, initializing new.") db_status = cfg_sql_server.connect(init=True) if db_status != DBStatus.OK: LOG.error("Config database initialization failed!") LOG.error("Please check debug logs.") sys.exit(1) if db_status == DBStatus.SCHEMA_MISMATCH_NO: LOG.debug("Configuration database schema mismatch.") LOG.debug("No schema upgrade is possible.") sys.exit(1) force_upgrade = True if 'force_upgrade' in args else False if db_status == DBStatus.SCHEMA_MISMATCH_OK: LOG.debug("Configuration database schema mismatch.") LOG.debug("Schema upgrade is possible.") LOG.warning("Please note after migration only " "newer CodeChecker versions can be used" "to start the server") LOG.warning("It is advised to make a full backup of your " "configuration database") LOG.warning(cfg_sql_server.get_db_location()) question = 'Do you want to upgrade to the new schema?' \ ' Y(es)/n(o) ' if force_upgrade or util.get_user_input(question): print("Upgrading schema ...") ret = cfg_sql_server.upgrade() msg = database_status.db_status_msg.get( ret, 'Unknown database status') print(msg) if ret != DBStatus.OK: LOG.error("Schema migration failed") sys.exit(ret) else: LOG.info("No schema migration was done.") sys.exit(0) if db_status == DBStatus.MISSING: LOG.error("Missing configuration database.") LOG.error("Server can not be started.") sys.exit(1) # Configuration database setup and check is needed before database # statuses can be checked. try: if args.status: ret = __db_status_check(cfg_sql_server, context, args.status) sys.exit(ret) except AttributeError: LOG.debug('Status was not in the arguments.') try: if args.product_to_upgrade: ret = __db_migration(cfg_sql_server, context, args.product_to_upgrade, force_upgrade) sys.exit(ret) except AttributeError: LOG.debug('Product upgrade was not in the arguments.') # Create the main database link from the arguments passed over the # command line. cfg_dir = os.path.abspath(args.config_directory) default_product_path = os.path.join(cfg_dir, 'Default.sqlite') create_default_product = 'sqlite' in args and \ not os.path.exists(default_product_path) if create_default_product: # Create a default product and add it to the configuration database. LOG.debug("Create default product...") LOG.debug("Configuring schema and migration...") prod_server = database.SQLiteDatabase( default_product_path, RUN_META, context.run_migration_root, check_env) LOG.debug("Checking 'Default' product database.") db_status = prod_server.connect() if db_status != DBStatus.MISSING: db_status = prod_server.connect(init=True) LOG.debug(database_status.db_status_msg.get(db_status)) if db_status != DBStatus.OK: LOG.error("Failed to configure default product") sys.exit(1) product_conn_string = prod_server.get_connection_string() server.add_initial_run_database( cfg_sql_server, product_conn_string) LOG.info("Product 'Default' at '{0}' created and set up." .format(default_product_path)) prod_statuses = check_product_db_status(cfg_sql_server, context) upgrade_available = {} for k, v in prod_statuses.items(): db_status, _, _, _ = v if db_status == DBStatus.SCHEMA_MISMATCH_OK or \ db_status == DBStatus.SCHEMA_MISSING: upgrade_available[k] = v if upgrade_available: print_prod_status(prod_statuses) LOG.warning("Multiple products can be upgraded, make a backup!") __db_migration(cfg_sql_server, context, 'all', force_upgrade) prod_statuses = check_product_db_status(cfg_sql_server, context) print_prod_status(prod_statuses) non_ok_db = False for k, v in prod_statuses.items(): db_status, _, _, _ = v if db_status != DBStatus.OK: non_ok_db = True break if non_ok_db: msg = "There are some database issues. " \ "Do you want to start the " \ "server? Y(es)/n(o) " if not util.get_user_input(msg): sys.exit(1) # Start database viewer. checker_md_docs = os.path.join(context.doc_root, 'checker_md_docs') checker_md_docs_map = os.path.join(checker_md_docs, 'checker_doc_map.json') checker_md_docs_map = util.load_json_or_empty(checker_md_docs_map, {}) package_data = {'www_root': context.www_root, 'doc_root': context.doc_root, 'checker_md_docs': checker_md_docs, 'checker_md_docs_map': checker_md_docs_map, 'version': context.package_git_tag} suppr_handler = suppress_handler. \ GenericSuppressHandler(None, False) try: server.start_server(args.config_directory, package_data, args.view_port, cfg_sql_server, suppr_handler, args.listen_address, 'force_auth' in args, args.skip_db_cleanup, context, check_env) except socket.error as err: if err.errno == errno.EADDRINUSE: LOG.error("Server can't be started, maybe the given port number " "({}) is already used. Check the connection " "parameters.".format(args.view_port)) sys.exit(1) else: raise
def main(args): """ Store the defect results in the specified input list as bug reports in the database. """ logger.setup_logger(args.verbose if 'verbose' in args else None) if not host_check.check_zlib(): raise Exception("zlib is not available on the system!") # To ensure the help message prints the default folder properly, # the 'default' for 'args.input' is a string, not a list. # But we need lists for the foreach here to work. if isinstance(args.input, str): args.input = [args.input] if 'name' not in args: LOG.debug("Generating name for analysis...") generated = __get_run_name(args.input) if generated: setattr(args, 'name', generated) else: LOG.error("No suitable name was found in the inputs for the " "analysis run. Please specify one by passing argument " "--name run_name in the invocation.") sys.exit(2) # argparse returns error code 2 for bad invocations. LOG.info("Storing analysis results for run '" + args.name + "'") if 'force' in args: LOG.info("argument --force was specified: the run with name '" + args.name + "' will be deleted.") protocol, host, port, product_name = split_product_url(args.product_url) # Before any transmission happens, check if we have the PRODUCT_STORE # permission to prevent a possibly long ZIP operation only to get an # error later on. product_client = libclient.setup_product_client(protocol, host, port, product_name) product_id = product_client.getCurrentProduct().id auth_client, _ = libclient.setup_auth_client(protocol, host, port) has_perm = libclient.check_permission(auth_client, Permission.PRODUCT_STORE, {'productID': product_id}) if not has_perm: LOG.error("You are not authorised to store analysis results in " "product '{0}'".format(product_name)) sys.exit(1) # Setup connection to the remote server. client = libclient.setup_client(args.product_url, product_client=False) LOG.debug("Initializing client connecting to {0}:{1}/{2} done.".format( host, port, product_name)) _, zip_file = tempfile.mkstemp('.zip') LOG.debug("Will write mass store ZIP to '{0}'...".format(zip_file)) try: assemble_zip(args.input, zip_file, client) if os.stat(zip_file).st_size > MAX_UPLOAD_SIZE: LOG.error("The result list to upload is too big (max: {}).".format( sizeof_fmt(MAX_UPLOAD_SIZE))) sys.exit(1) with open(zip_file, 'rb') as zf: b64zip = base64.b64encode(zf.read()) context = generic_package_context.get_context() trim_path_prefixes = args.trim_path_prefix if \ 'trim_path_prefix' in args else None client.massStoreRun(args.name, args.tag if 'tag' in args else None, str(context.version), b64zip, 'force' in args, trim_path_prefixes) LOG.info("Storage finished successfully.") except RequestFailed as reqfail: if reqfail.errorCode == ErrorCode.SOURCE_FILE: header = ['File', 'Line', 'Checker name'] table = twodim_to_str('table', header, [c.split('|') for c in reqfail.extraInfo]) LOG.warning("Setting the review statuses for some reports failed " "because of non valid source code comments: " "{0}\n {1}".format(reqfail.message, table)) sys.exit(1) except Exception as ex: LOG.info("Storage failed: " + str(ex)) sys.exit(1) finally: os.remove(zip_file)
def main(args): """ Store the defect results in the specified input list as bug reports in the database. """ if not host_check.check_zlib(): raise Exception("zlib is not available on the system!") # To ensure the help message prints the default folder properly, # the 'default' for 'args.input' is a string, not a list. # But we need lists for the foreach here to work. if isinstance(args.input, str): args.input = [args.input] if 'name' not in args: LOG.debug("Generating name for analysis...") generated = __get_run_name(args.input) if generated: setattr(args, 'name', generated) else: LOG.error("No suitable name was found in the inputs for the " "analysis run. Please specify one by passing argument " "--name run_name in the invocation.") sys.exit(2) # argparse returns error code 2 for bad invocations. LOG.info("Storing analysis results for run '" + args.name + "'") if 'force' in args: LOG.info("argument --force was specified: the run with name '" + args.name + "' will be deleted.") protocol, host, port, product_name = split_product_url(args.product_url) # Before any transmission happens, check if we have the PRODUCT_STORE # permission to prevent a possibly long ZIP operation only to get an # error later on. product_client = libclient.setup_product_client(protocol, host, port, product_name) product_id = product_client.getCurrentProduct().id auth_client, _ = libclient.setup_auth_client(protocol, host, port) has_perm = libclient.check_permission( auth_client, Permission.PRODUCT_STORE, {'productID': product_id}) if not has_perm: LOG.error("You are not authorised to store analysis results in " "product '{0}'".format(product_name)) sys.exit(1) # Setup connection to the remote server. client = libclient.setup_client(args.product_url) LOG.debug("Initializing client connecting to {0}:{1}/{2} done." .format(host, port, product_name)) _, zip_file = tempfile.mkstemp('.zip') LOG.debug("Will write mass store ZIP to '{0}'...".format(zip_file)) try: assemble_zip(args.input, zip_file, client) with open(zip_file, 'rb') as zf: b64zip = base64.b64encode(zf.read()) context = generic_package_context.get_context() client.massStoreRun(args.name, args.tag if 'tag' in args else None, str(context.version), b64zip, 'force' in args) LOG.info("Storage finished successfully.") except Exception as ex: LOG.info("Storage failed: " + str(ex)) sys.exit(1) finally: os.remove(zip_file)
def handle_server(args): """ Starts the report viewer server. """ if not host_check.check_zlib(): sys.exit(1) workspace = args.workspace if (args.list or args.stop or args.stop_all) and \ not (args.list ^ args.stop ^ args.stop_all): print("CodeChecker server: error: argument -l/--list and -s/--stop" "and --stop-all are mutually exclusive.") sys.exit(2) if args.list: instances = instance_manager.list() instances_on_multiple_hosts = any(True for inst in instances if inst['hostname'] != socket.gethostname()) if not instances_on_multiple_hosts: rows = [('Workspace', 'View port')] else: rows = [('Workspace', 'Computer host', 'View port')] for instance in instance_manager.list(): if not instances_on_multiple_hosts: rows.append((instance['workspace'], str(instance['port']))) else: rows.append((instance['workspace'], instance['hostname'] if instance['hostname'] != socket.gethostname() else '', str(instance['port']))) print("Your running CodeChecker servers:") print(util.twodim_to_table(rows)) sys.exit(0) elif args.stop or args.stop_all: for i in instance_manager.list(): # A STOP only stops the server associated with the given workspace # and view-port. if i['hostname'] != socket.gethostname() or ( args.stop and not (i['port'] == args.view_port and os.path.abspath(i['workspace']) == os.path.abspath(workspace))): continue try: util.kill_process_tree(i['pid']) LOG.info("Stopped CodeChecker server running on port {0} " "in workspace {1} (PID: {2})". format(i['port'], i['workspace'], i['pid'])) except: # Let the exception come out if the commands fail LOG.error("Couldn't stop process PID #" + str(i['pid'])) raise sys.exit(0) # WARNING # In case of SQLite args.dbaddress default value is used # for which the is_localhost should return true. if util.is_localhost(args.dbaddress) and not os.path.exists(workspace): os.makedirs(workspace) suppress_handler = generic_package_suppress_handler.\ GenericSuppressHandler(None) if args.suppress is None: LOG.warning('No suppress file was given, suppressed results will ' 'be only stored in the database.') else: if not os.path.exists(args.suppress): LOG.error('Suppress file ' + args.suppress + ' not found!') sys.exit(1) context = generic_package_context.get_context() context.codechecker_workspace = workspace session_manager.SessionManager.CodeChecker_Workspace = workspace context.db_username = args.dbusername check_env = analyzer_env.get_check_env(context.path_env_extra, context.ld_lib_path_extra) sql_server = SQLServer.from_cmdline_args(args, context.migration_root, check_env) conn_mgr = client.ConnectionManager(sql_server, args.check_address, args.check_port) if args.check_port: LOG.debug('Starting CodeChecker server and database server.') sql_server.start(context.db_version_info, wait_for_start=True, init=True) conn_mgr.start_report_server() else: LOG.debug('Starting database.') sql_server.start(context.db_version_info, wait_for_start=True, init=True) # Start database viewer. db_connection_string = sql_server.get_connection_string() suppress_handler.suppress_file = args.suppress LOG.debug('Using suppress file: ' + str(suppress_handler.suppress_file)) checker_md_docs = os.path.join(context.doc_root, 'checker_md_docs') checker_md_docs_map = os.path.join(checker_md_docs, 'checker_doc_map.json') with open(checker_md_docs_map, 'r') as dFile: checker_md_docs_map = json.load(dFile) package_data = {'www_root': context.www_root, 'doc_root': context.doc_root, 'checker_md_docs': checker_md_docs, 'checker_md_docs_map': checker_md_docs_map, 'version': context.package_git_tag} try: client_db_access_server.start_server(package_data, args.view_port, db_connection_string, suppress_handler, args.not_host_only, context) except socket.error as err: if err.errno == errno.EADDRINUSE: LOG.error("Server can't be started, maybe the given port number " "({}) is already used. Check the connection " "parameters.".format(args.view_port)) sys.exit(1) else: raise
def main(args): """ Start or manage a CodeChecker report server. """ if 'list' in args or 'stop' in args or 'stop_all' in args: __instance_management(args) sys.exit(0) # Actual server starting from this point. if not host_check.check_zlib(): raise Exception("zlib is not available on the system!") # WARNING # In case of SQLite args.dbaddress default value is used # for which the is_localhost should return true. if util.is_localhost(args.dbaddress) and \ not os.path.exists(args.config_directory): os.makedirs(args.config_directory) # Make sure the SQLite file can be created if it not exists. if 'sqlite' in args and \ not os.path.isdir(os.path.dirname(args.sqlite)): os.makedirs(os.path.dirname(args.sqlite)) suppress_handler = generic_package_suppress_handler. \ GenericSuppressHandler(None, False) if 'reset_root' in args: try: os.remove(os.path.join(args.config_directory, 'root.user')) LOG.info("Master superuser (root) credentials invalidated and " "deleted. New ones will be generated...") except OSError: # File doesn't exist. pass if 'force_auth' in args: LOG.info("'--force-authentication' was passed as a command-line " "option. The server will ask for users to authenticate!") context = generic_package_context.get_context() context.codechecker_workspace = args.config_directory session_manager.SessionManager.CodeChecker_Workspace = \ args.config_directory context.db_username = args.dbusername check_env = analyzer_env.get_check_env(context.path_env_extra, context.ld_lib_path_extra) # Create the main database link from the arguments passed over the # command line. default_product_path = os.path.join(args.config_directory, 'Default.sqlite') create_default_product = 'sqlite' in args and \ not os.path.exists(args.sqlite) and \ not os.path.exists(default_product_path) sql_server = database.SQLServer.from_cmdline_args( vars(args), CONFIG_META, context.config_migration_root, interactive=True, env=check_env) LOG.debug("Connecting to product configuration database.") sql_server.connect(context.product_db_version_info, init=True) if create_default_product: # Create a default product and add it to the configuration database. LOG.debug("Create default product...") LOG.debug("Configuring schema and migration...") prod_server = database.SQLiteDatabase( default_product_path, RUN_META, context.run_migration_root, check_env) prod_server.connect(context.run_db_version_info, init=True) LOG.debug("Connecting database engine for default product") product_conn_string = prod_server.get_connection_string() LOG.debug("Default database created and connected.") server.add_initial_run_database( sql_server, product_conn_string) LOG.info("Product 'Default' at '{0}' created and set up." .format(default_product_path)) # Start database viewer. checker_md_docs = os.path.join(context.doc_root, 'checker_md_docs') checker_md_docs_map = os.path.join(checker_md_docs, 'checker_doc_map.json') with open(checker_md_docs_map, 'r') as dFile: checker_md_docs_map = json.load(dFile) package_data = {'www_root': context.www_root, 'doc_root': context.doc_root, 'checker_md_docs': checker_md_docs, 'checker_md_docs_map': checker_md_docs_map, 'version': context.package_git_tag} try: server.start_server(args.config_directory, package_data, args.view_port, sql_server, suppress_handler, args.listen_address, 'force_auth' in args, 'skip_db_cleanup' not in args, context, check_env) except socket.error as err: if err.errno == errno.EADDRINUSE: LOG.error("Server can't be started, maybe the given port number " "({}) is already used. Check the connection " "parameters.".format(args.view_port)) sys.exit(1) else: raise
def main(args): """ Store the defect results in the specified input list as bug reports in the database. """ if not host_check.check_zlib(): raise Exception("zlib is not available on the system!") # To ensure the help message prints the default folder properly, # the 'default' for 'args.input' is a string, not a list. # But we need lists for the foreach here to work. if isinstance(args.input, str): args.input = [args.input] if 'name' not in args: LOG.debug("Generating name for analysis...") generated = __get_run_name(args.input) if generated: setattr(args, 'name', generated) else: LOG.error("No suitable name was found in the inputs for the " "analysis run. Please specify one by passing argument " "--name run_name in the invocation.") sys.exit(2) # argparse returns error code 2 for bad invocations. LOG.info("Storing analysis results for run '" + args.name + "'") if args.force: LOG.info("argument --force was specified: the run with name '" + args.name + "' will be deleted.") context = generic_package_context.get_context() context.db_username = args.dbusername check_env = analyzer_env.get_check_env(context.path_env_extra, context.ld_lib_path_extra) sql_server = SQLServer.from_cmdline_args(args, context.migration_root, check_env) conn_mgr = client.ConnectionManager(sql_server, 'localhost', util.get_free_port()) sql_server.start(context.db_version_info, wait_for_start=True, init=True) conn_mgr.start_report_server() original_cwd = os.getcwd() empty_metadata = {'working_directory': original_cwd} check_commands = [] check_durations = [] skip_handlers = [] items = [] for input_path in args.input: LOG.debug("Parsing input argument: '" + input_path + "'") if os.path.isfile(input_path): if not input_path.endswith(".plist"): continue items.append((input_path, context, empty_metadata)) elif os.path.isdir(input_path): metadata_file = os.path.join(input_path, "metadata.json") if os.path.exists(metadata_file): with open(metadata_file, 'r') as metadata: metadata_dict = json.load(metadata) LOG.debug(metadata_dict) if 'command' in metadata_dict: check_commands.append(metadata_dict['command']) if 'timestamps' in metadata_dict: check_durations.append( float(metadata_dict['timestamps']['end'] - metadata_dict['timestamps']['begin'])) if 'skip_data' in metadata_dict: # Save previously stored skip data for sending to the # database, to ensure skipped headers are actually # skipped --- 'analyze' can't do this. handle, path = tempfile.mkstemp() with os.fdopen(handle, 'w') as tmpf: tmpf.write('\n'.join(metadata_dict['skip_data'])) skip_handlers.append( skiplist_handler.SkipListHandler(path)) os.remove(path) else: metadata_dict = empty_metadata _, _, files = next(os.walk(input_path), ([], [], [])) for f in files: if not f.endswith(".plist"): continue items.append((os.path.join(input_path, f), context, metadata_dict)) with client.get_connection() as connection: if len(check_commands) == 0: command = ' '.join(sys.argv) elif len(check_commands) == 1: command = ' '.join(check_commands[0]) else: command = "multiple analyze calls: " +\ '; '.join([' '.join(com) for com in check_commands]) context.run_id = connection.add_checker_run(command, args.name, context.version, args.force) # Clean previous suppress information. client.clean_suppress(connection, context.run_id) if 'suppress' in args: if not os.path.isfile(args.suppress): LOG.warning("Suppress file '" + args.suppress + "' given, but " "it does not exist -- will not suppress anything.") else: client.send_suppress(context.run_id, connection, os.path.realpath(args.suppress)) # Send previously collected skip information to the server. for skip_handler in skip_handlers: connection.add_skip_paths(context.run_id, skip_handler.get_skiplist()) pool = multiprocessing.Pool(args.jobs) try: pool.map_async(consume_plist, items, 1).get(float('inf')) pool.close() except Exception: pool.terminate() LOG.error("Storing the results failed.") raise # CodeChecker.py is the invoker, it will handle this. finally: pool.join() os.chdir(original_cwd) with client.get_connection() as connection: connection.finish_checker_run(context.run_id) if len(check_durations) > 0: connection.set_run_duration( context.run_id, # Round the duration to seconds. int(sum(check_durations)))