def parse_maven_params(confs, chain=False, scratch=False): """ Parse .ini files that contain parameters to launch a Maven build. Return a map whose keys are package names and values are config parameters. """ config = koji.read_config_files(confs) builds = {} for package in config.sections(): buildtype = 'maven' if config.has_option(package, 'type'): buildtype = config.get(package, 'type') if buildtype == 'maven': params = maven_params(config, package, chain=chain, scratch=scratch) elif buildtype == 'wrapper': params = wrapper_params(config, package, chain=chain, scratch=scratch) if len(params.get('buildrequires')) != 1: raise ValueError( "A wrapper-rpm must depend on exactly one package") else: raise ValueError("Unsupported build type: %s" % buildtype) if 'scmurl' not in params: raise ValueError("%s is missing the scmurl parameter" % package) builds[package] = params if not builds: if not isinstance(confs, (list, tuple)): confs = [confs] raise ValueError("No sections found in: %s" % ', '.join(confs)) return builds
def send_queued_msgs(cbtype, *args, **kws): msgs = getattr(context, 'protonmsg_msgs', None) if not msgs: return log = logging.getLogger('koji.plugin.protonmsg') global CONFIG if not CONFIG: CONFIG = koji.read_config_files(CONFIG_FILE) urls = CONFIG.get('broker', 'urls').split() test_mode = False if CONFIG.has_option('broker', 'test_mode'): test_mode = CONFIG.getboolean('broker', 'test_mode') if test_mode: log.debug('test mode: skipping send to urls: %r', urls) for msg in msgs: log.debug('test mode: skipped msg: %r', msg) return random.shuffle(urls) for url in urls: container = Container(TimeoutHandler(url, msgs, CONFIG)) container.run() if msgs: log.debug('could not send to %s, %s messages remaining', url, len(msgs)) else: log.debug('all messages sent to %s successfully', url) break else: log.error('could not send messages to any destinations')
def parse_maven_params(confs, chain=False, scratch=False): """ Parse .ini files that contain parameters to launch a Maven build. Return a map whose keys are package names and values are config parameters. """ config = koji.read_config_files(confs) builds = {} for package in config.sections(): buildtype = 'maven' if config.has_option(package, 'type'): buildtype = config.get(package, 'type') if buildtype == 'maven': params = maven_params(config, package, chain=chain, scratch=scratch) elif buildtype == 'wrapper': params = wrapper_params(config, package, chain=chain, scratch=scratch) if len(params.get('buildrequires')) != 1: raise ValueError("A wrapper-rpm must depend on exactly one package") else: raise ValueError("Unsupported build type: %s" % buildtype) if 'scmurl' not in params: raise ValueError("%s is missing the scmurl parameter" % package) builds[package] = params if not builds: if not isinstance(confs, (list, tuple)): confs = [confs] raise ValueError("No sections found in: %s" % ', '.join(confs)) return builds
def send_queued_msgs(cbtype, *args, **kws): msgs = getattr(context, 'protonmsg_msgs', None) if not msgs: return log = logging.getLogger('koji.plugin.protonmsg') global CONFIG if not CONFIG: CONFIG = koji.read_config_files(CONFIG_FILE) urls = CONFIG.get('broker', 'urls').split() test_mode = False if CONFIG.has_option('broker', 'test_mode'): test_mode = CONFIG.getboolean('broker', 'test_mode') if test_mode: log.debug('test mode: skipping send to urls: %r', urls) for msg in msgs: log.debug('test mode: skipped msg: %r', msg) return random.shuffle(urls) for url in urls: container = Container(TimeoutHandler(url, msgs, CONFIG)) container.run() if msgs: log.debug('could not send to %s, %s messages remaining', url, len(msgs)) else: log.debug('all messages sent to %s successfully', url) break else: log.error('could not send messages to any destinations')
def maven_import(cbtype, *args, **kws): global config if not context.opts.get('EnableMaven', False): return if kws.get('type') != 'rpm': return buildinfo = kws['build'] rpminfo = kws['rpm'] filepath = kws['filepath'] if not config: config = koji.read_config_files([(CONFIG_FILE, True)]) name_patterns = config.get('patterns', 'rpm_names').split() for pattern in name_patterns: if fnmatch.fnmatch(rpminfo['name'], pattern): break else: return tmpdir = joinpath(koji.pathinfo.work(), 'rpm2maven', koji.buildLabel(buildinfo)) try: if os.path.exists(tmpdir): rmtree(tmpdir) koji.ensuredir(tmpdir) expand_rpm(filepath, tmpdir) scan_and_import(buildinfo, rpminfo, tmpdir) finally: if os.path.exists(tmpdir): rmtree(tmpdir)
def _read_config(self): cp = koji.read_config_files(CONFIG_FILE) self.config = { 'default_mounts': [], 'safe_roots': [], 'path_subs': [], 'paths': [], 'internal_dev_setup': None, } # main options if cp.has_option('runroot', 'internal_dev_setup'): self.config['internal_dev_setup'] = cp.getboolean( 'runroot', 'internal_dev_setup') # path options if cp.has_option('paths', 'default_mounts'): self.config['default_mounts'] = cp.get('paths', 'default_mounts').split(',') if cp.has_option('paths', 'safe_roots'): self.config['safe_roots'] = cp.get('paths', 'safe_roots').split(',') if cp.has_option('paths', 'path_subs'): self.config['path_subs'] = [] for line in cp.get('paths', 'path_subs').splitlines(): line = line.strip() if not line: continue sub = line.split(',') if len(sub) != 2: raise koji.GenericError('bad runroot substitution: %s' % sub) self.config['path_subs'].append(sub) # path section are in form 'path%d' while order is important as some # paths can be mounted inside other mountpoints path_sections = [p for p in cp.sections() if re.match(r'path\d+', p)] for section_name in sorted(path_sections, key=lambda x: int(x[4:])): try: self.config['paths'].append({ 'mountpoint': cp.get(section_name, 'mountpoint'), 'path': cp.get(section_name, 'path'), 'fstype': cp.get(section_name, 'fstype'), 'options': cp.get(section_name, 'options'), }) except six.moves.configparser.NoOptionError: raise koji.GenericError( "bad config: missing options in %s section" % section_name) for path in self.config['default_mounts'] + self.config['safe_roots'] + \ [x[0] for x in self.config['path_subs']]: if not path.startswith('/'): raise koji.GenericError( "bad config: all paths (default_mounts, safe_roots, path_subs) needs to be " "absolute: %s" % path)
def read_config(): global config cp = koji.read_config_files(CONFIG_FILE) config = { 'path_filters': [], 'volume': None, } if cp.has_option('filters', 'paths'): config['path_filters'] = cp.get('filters', 'paths').split() if cp.has_option('general', 'volume'): config['volume'] = cp.get('general', 'volume').strip()
def load_config(self, environ): """Load configuration options Options are read from a kojiweb config file. To override the configuration file location, use the SetEnv Apache directive. For example: SetEnv koji.web.ConfigFile /home/developer/koji/www/conf/web.conf Backwards compatibility: - if ConfigFile is not set, opts are loaded from http config - if ConfigFile is set, then the http config must not provide Koji options - In a future version we will load the default hub config regardless - all PythonOptions (except koji.web.ConfigFile) are now deprecated and support for them will disappear in a future version of Koji """ cf = environ.get('koji.web.ConfigFile', '/etc/kojiweb/web.conf') cfdir = environ.get('koji.web.ConfigDir', '/etc/kojiweb/web.conf.d') config = koji.read_config_files([cfdir, (cf, True)]) opts = {} for name, dtype, default in self.cfgmap: key = ('web', name) if config.has_option(*key): if dtype == 'integer': opts[name] = config.getint(*key) elif dtype == 'boolean': opts[name] = config.getboolean(*key) elif dtype == 'list': opts[name] = [ x.strip() for x in config.get(*key).split(',') ] else: opts[name] = config.get(*key) else: opts[name] = default opts['Secret'] = koji.util.HiddenValue(opts['Secret']) if opts['WebAuthType'] not in (None, 'gssapi', 'ssl'): raise koji.ConfigurationError( f"Invalid value {opts['WebAuthType']} for " "WebAuthType (ssl/gssapi)") if opts['WebAuthType'] == 'gssapi': opts['WebAuthType'] = koji.AUTHTYPE_GSSAPI elif opts['WebAuthType'] == 'ssl': opts['WebAuthType'] = koji.AUTHTYPE_SSL # if there is no explicit request, use same authtype as web has elif opts['WebPrincipal']: opts['WebAuthType'] = koji.AUTHTYPE_GSSAPI elif opts['WebCert']: opts['WebAuthType'] = koji.AUTHTYPE_SSL self.options = opts return opts
def test_read_config_files(self, scp_clz, rcp_clz, cp_clz, open_mock): files = 'test1.conf' conf = koji.read_config_files(files) open_mock.assert_called_once_with(files, 'r') if six.PY2: self.assertTrue(isinstance(conf, six.moves.configparser.SafeConfigParser.__class__)) scp_clz.assert_called_once() scp_clz.return_value.readfp.assert_called_once() else: self.assertTrue(isinstance(conf, six.moves.configparser.ConfigParser.__class__)) cp_clz.assert_called_once() cp_clz.return_value.read_file.assert_called_once() open_mock.reset_mock() cp_clz.reset_mock() scp_clz.reset_mock() files = ['test1.conf', 'test2.conf'] koji.read_config_files(files) open_mock.assert_has_calls([call('test1.conf', 'r'), call('test2.conf', 'r')], any_order=True) if six.PY2: scp_clz.assert_called_once() self.assertEqual(scp_clz.return_value.readfp.call_count, 2) else: cp_clz.assert_called_once() self.assertEqual(cp_clz.return_value.read_file.call_count, 2) open_mock.reset_mock() cp_clz.reset_mock() scp_clz.reset_mock() conf = koji.read_config_files(files, raw=True) self.assertTrue(isinstance(conf, six.moves.configparser.RawConfigParser.__class__)) cp_clz.assert_not_called() scp_clz.assert_not_called() rcp_clz.assert_called_once()
def send_queued_msgs(cbtype, *args, **kws): global CONFIG msgs = getattr(context, 'protonmsg_msgs', None) if not msgs: return if not CONFIG: CONFIG = koji.read_config_files([(CONFIG_FILE, True)]) urls = CONFIG.get('broker', 'urls').split() test_mode = False if CONFIG.has_option('broker', 'test_mode'): test_mode = CONFIG.getboolean('broker', 'test_mode') db_enabled = False if CONFIG.has_option('queue', 'enabled'): db_enabled = CONFIG.getboolean('queue', 'enabled') if test_mode: LOG.debug('test mode: skipping send to urls: %r', urls) fail_chance = CONFIG.getint('broker', 'test_mode_fail', fallback=0) if fail_chance: # simulate unsent messages in test mode sent = [] unsent = [] for m in msgs: if random.randint(1, 100) <= fail_chance: unsent.append(m) else: sent.append(m) if unsent: LOG.info('simulating %i unsent messages' % len(unsent)) else: sent = msgs unsent = [] for msg in sent: LOG.debug('test mode: skipped msg: %r', msg) else: unsent = _send_msgs(urls, msgs, CONFIG) if db_enabled: if unsent: # if we still have some messages, store them and leave for another call to pick them up store_to_db(msgs) else: # otherwise we are another call - look to db if there remains something to send handle_db_msgs(urls, CONFIG) elif unsent: LOG.error('could not send %i messages. db queue disabled' % len(msgs))
def _strip_extra(buildinfo): """If extra_limit is configured, compare extra's size and drop it, if it is over""" global CONFIG if not CONFIG: CONFIG = koji.read_config_files([(CONFIG_FILE, True)]) if CONFIG.has_option('message', 'extra_limit'): extra_limit = abs(CONFIG.getint('message', 'extra_limit')) if extra_limit == 0: return buildinfo extra_size = len(json.dumps(buildinfo.get('extra', {}), default=json_serialize)) if extra_limit and extra_size > extra_limit: LOG.debug("Dropping 'extra' from build %s (length: %d > %d)" % (buildinfo['nvr'], extra_size, extra_limit)) buildinfo = buildinfo.copy() del buildinfo['extra'] return buildinfo
def saveFailedTree(buildrootID, full=False, **opts): """Create saveFailedTree task If arguments are invalid, error message is returned. Otherwise task id of newly created task is returned.""" global config, allowed_methods # let it raise errors buildrootID = int(buildrootID) full = bool(full) # read configuration only once if config is None: config = koji.read_config_files([(CONFIG_FILE, True)]) allowed_methods = config.get('permissions', 'allowed_methods').split(',') if len(allowed_methods) == 1 and allowed_methods[0] == '*': allowed_methods = '*' brinfo = kojihub.get_buildroot(buildrootID, strict=True) taskID = brinfo['task_id'] task_info = kojihub.Task(taskID).getInfo() if task_info['state'] != koji.TASK_STATES['FAILED']: raise koji.PreBuildError( "Task %s has not failed. Only failed tasks can upload their buildroots." % taskID) elif allowed_methods != '*' and task_info['method'] not in allowed_methods: raise koji.PreBuildError("Only %s tasks can upload their buildroots (Task %s is %s)." % \ (', '.join(allowed_methods), task_info['id'], task_info['method'])) elif task_info[ "owner"] != context.session.user_id and not context.session.hasPerm( 'admin'): raise koji.ActionNotAllowed( "Only owner of failed task or 'admin' can run this task.") elif not kojihub.get_host(task_info['host_id'])['enabled']: raise koji.PreBuildError("Host is disabled.") args = koji.encode_args(buildrootID, full, **opts) taskopts = { 'assign': brinfo['host_id'], } return kojihub.make_task('saveFailedTree', args, **taskopts)
def load_config(self, environ): """Load configuration options Options are read from a kojiweb config file. To override the configuration file location, use the SetEnv Apache directive. For example: SetEnv koji.web.ConfigFile /home/developer/koji/www/conf/web.conf Backwards compatibility: - if ConfigFile is not set, opts are loaded from http config - if ConfigFile is set, then the http config must not provide Koji options - In a future version we will load the default hub config regardless - all PythonOptions (except koji.web.ConfigFile) are now deprecated and support for them will disappear in a future version of Koji """ cf = environ.get('koji.web.ConfigFile', '/etc/kojiweb/web.conf') cfdir = environ.get('koji.web.ConfigDir', '/etc/kojiweb/web.conf.d') config = koji.read_config_files([cfdir, (cf, True)]) opts = {} for name, dtype, default in self.cfgmap: key = ('web', name) if config.has_option(*key): if dtype == 'integer': opts[name] = config.getint(*key) elif dtype == 'boolean': opts[name] = config.getboolean(*key) elif dtype == 'list': opts[name] = [ x.strip() for x in config.get(*key).split(',') ] else: opts[name] = config.get(*key) else: opts[name] = default opts['Secret'] = koji.util.HiddenValue(opts['Secret']) self.options = opts return opts
import sys import logging import subprocess import koji from koji.plugin import register_callback, ignore_error if '/usr/share/koji-hub' not in sys.path: sys.path.append("/usr/share/koji-hub") import kojihub from kojihub import RootExports # CONVERT TO CONFIG FILE CONFIG_FILE = '/etc/koji-hub/plugins/key_signing.conf' CONFIG = None if not CONFIG: CONFIG = koji.read_config_files([(CONFIG_FILE, True)]) passphrase = CONFIG.get('signing', 'passphrase') gpg_key_name = CONFIG.get('signing', 'gpg_key_name') gpg_key_id = CONFIG.get('signing', 'gpg_key_id') build_target = CONFIG.get('signing', 'build_target').split() testing_tag = CONFIG.get('signing', 'testing_tag') send_to_testing = CONFIG.get('signing', 'send_to_testing') sigul_config = CONFIG.get('signing', 'sigul_config') def key_signing(cbtype, *args, **kws): # Make sure this is a package build and nothing else if kws['tag']['name'] not in build_target: return
def load_config(environ): """Load configuration options Options are read from a config file. The config file location is controlled by the koji.hub.ConfigFile environment variable in the httpd config. To override this (for example): SetEnv koji.hub.ConfigFile /home/developer/koji/hub/hub.conf Backwards compatibility: - if ConfigFile is not set, opts are loaded from http config - if ConfigFile is set, then the http config must not provide Koji options - In a future version we will load the default hub config regardless - all PythonOptions (except ConfigFile) are now deprecated and support for them will disappear in a future version of Koji """ # get our config file(s) cf = environ.get('koji.hub.ConfigFile', '/etc/koji-hub/hub.conf') cfdir = environ.get('koji.hub.ConfigDir', '/etc/koji-hub/hub.conf.d') config = koji.read_config_files([cfdir, (cf, True)], raw=True) cfgmap = [ # option, type, default ['DBName', 'string', None], ['DBUser', 'string', None], ['DBHost', 'string', None], ['DBhost', 'string', None], # alias for backwards compatibility ['DBPort', 'integer', None], ['DBPass', 'string', None], ['DBConnectionString', 'string', None], ['KojiDir', 'string', None], ['AuthPrincipal', 'string', None], ['AuthKeytab', 'string', None], ['ProxyPrincipals', 'string', ''], ['HostPrincipalFormat', 'string', None], ['AllowedKrbRealms', 'string', '*'], # TODO: this option should be removed in future release ['DisableGSSAPIProxyDNFallback', 'boolean', False], ['DNUsernameComponent', 'string', 'CN'], ['ProxyDNs', 'string', ''], ['CheckClientIP', 'boolean', True], ['LoginCreatesUser', 'boolean', True], ['AllowProxyAuthType', 'boolean', False], ['KojiWebURL', 'string', 'http://localhost.localdomain/koji'], ['EmailDomain', 'string', None], ['NotifyOnSuccess', 'boolean', True], ['DisableNotifications', 'boolean', False], ['Plugins', 'string', ''], ['PluginPath', 'string', '/usr/lib/koji-hub-plugins'], ['KojiDebug', 'boolean', False], ['KojiTraceback', 'string', None], ['VerbosePolicy', 'boolean', False], ['LogLevel', 'string', 'WARNING'], [ 'LogFormat', 'string', '%(asctime)s [%(levelname)s] m=%(method)s u=%(user_name)s p=%(process)s r=%(remoteaddr)s ' '%(name)s: %(message)s' ], ['MissingPolicyOk', 'boolean', True], ['EnableMaven', 'boolean', False], ['EnableWin', 'boolean', False], ['RLIMIT_AS', 'string', None], ['RLIMIT_CORE', 'string', None], ['RLIMIT_CPU', 'string', None], ['RLIMIT_DATA', 'string', None], ['RLIMIT_FSIZE', 'string', None], ['RLIMIT_MEMLOCK', 'string', None], ['RLIMIT_NOFILE', 'string', None], ['RLIMIT_NPROC', 'string', None], ['RLIMIT_OFILE', 'string', None], ['RLIMIT_RSS', 'string', None], ['RLIMIT_STACK', 'string', None], ['MemoryWarnThreshold', 'integer', 5000], ['MaxRequestLength', 'integer', 4194304], ['LockOut', 'boolean', False], ['ServerOffline', 'boolean', False], ['OfflineMessage', 'string', None], ] opts = {} for name, dtype, default in cfgmap: key = ('hub', name) if config and config.has_option(*key): if dtype == 'integer': opts[name] = config.getint(*key) elif dtype == 'boolean': opts[name] = config.getboolean(*key) else: opts[name] = config.get(*key) continue opts[name] = default if opts['DBHost'] is None: opts['DBHost'] = opts['DBhost'] # load policies # (only from config file) if config and config.has_section('policy'): # for the moment, we simply transfer the policy conf to opts opts['policy'] = dict(config.items('policy')) else: opts['policy'] = {} for pname, text in _default_policies.items(): opts['policy'].setdefault(pname, text) # use configured KojiDir if opts.get('KojiDir') is not None: koji.BASEDIR = opts['KojiDir'] koji.pathinfo.topdir = opts['KojiDir'] return opts
# also shouldn't happen, but just in case return if not is_sidetag(tag): return # is the tag now empty? query = QueryProcessor( tables=["tag_listing"], clauses=["tag_id = %(tag_id)s", "active IS TRUE"], values={"tag_id": tag["id"]}, opts={"countOnly": True}, ) if query.execute(): return # looks like we've just untagged the last build from a side tag try: # XXX: are we double updating tag_listing? _remove_sidetag(tag) except koji.GenericError: pass # read config and register if not CONFIG: CONFIG = koji.read_config_files(CONFIG_FILE) if CONFIG.has_option("sidetag", "remove_empty") and CONFIG.getboolean( "sidetag", "remove_empty" ): handle_sidetag_untag = callback("postUntag")(handle_sidetag_untag) if CONFIG.has_option("sidetag", "allowed_suffixes"): ALLOWED_SUFFIXES = CONFIG.get("sidetag", "allowed_suffixes").split(',')
# is the tag now empty? query = QueryProcessor( tables=["tag_listing"], clauses=["tag_id = %(tag_id)s", "active IS TRUE"], values={"tag_id": tag["id"]}, opts={"countOnly": True}, ) if query.execute(): return # looks like we've just untagged the last build from a side tag try: # XXX: are we double updating tag_listing? _remove_sidetag(tag) except koji.GenericError: pass # read config and register if not CONFIG: CONFIG = koji.read_config_files(CONFIG_FILE, raw=True) if CONFIG.has_option("sidetag", "remove_empty") and CONFIG.getboolean( "sidetag", "remove_empty" ): handle_sidetag_untag = callback("postUntag")(handle_sidetag_untag) if CONFIG.has_option("sidetag", "allowed_suffixes"): ALLOWED_SUFFIXES = CONFIG.get("sidetag", "allowed_suffixes").split(',') if CONFIG.has_option("sidetag", "name_template"): NAME_TEMPLATE = CONFIG.get("sidetag", "name_template") else: NAME_TEMPLATE = '{basetag}-side-{tag_id}'