def parse_maven_params(confs, chain=False, scratch=False):
"""
Parse .ini files that contain parameters to launch a Maven build.
Return a map whose keys are package names and values are config parameters.
"""
config = koji.read_config_files(confs)
builds = {}
for package in config.sections():
buildtype = 'maven'
if config.has_option(package, 'type'):
buildtype = config.get(package, 'type')
if buildtype == 'maven':
params = maven_params(config,
package,
chain=chain,
scratch=scratch)
elif buildtype == 'wrapper':
params = wrapper_params(config,
package,
chain=chain,
scratch=scratch)
if len(params.get('buildrequires')) != 1:
raise ValueError(
"A wrapper-rpm must depend on exactly one package")
else:
raise ValueError("Unsupported build type: %s" % buildtype)
if 'scmurl' not in params:
raise ValueError("%s is missing the scmurl parameter" % package)
builds[package] = params
if not builds:
if not isinstance(confs, (list, tuple)):
confs = [confs]
raise ValueError("No sections found in: %s" % ', '.join(confs))
return builds
def send_queued_msgs(cbtype, *args, **kws):
msgs = getattr(context, 'protonmsg_msgs', None)
if not msgs:
return
log = logging.getLogger('koji.plugin.protonmsg')
global CONFIG
if not CONFIG:
CONFIG = koji.read_config_files(CONFIG_FILE)
urls = CONFIG.get('broker', 'urls').split()
test_mode = False
if CONFIG.has_option('broker', 'test_mode'):
test_mode = CONFIG.getboolean('broker', 'test_mode')
if test_mode:
log.debug('test mode: skipping send to urls: %r', urls)
for msg in msgs:
log.debug('test mode: skipped msg: %r', msg)
return
random.shuffle(urls)
for url in urls:
container = Container(TimeoutHandler(url, msgs, CONFIG))
container.run()
if msgs:
log.debug('could not send to %s, %s messages remaining', url,
len(msgs))
else:
log.debug('all messages sent to %s successfully', url)
break
else:
log.error('could not send messages to any destinations')
def parse_maven_params(confs, chain=False, scratch=False):
"""
Parse .ini files that contain parameters to launch a Maven build.
Return a map whose keys are package names and values are config parameters.
"""
config = koji.read_config_files(confs)
builds = {}
for package in config.sections():
buildtype = 'maven'
if config.has_option(package, 'type'):
buildtype = config.get(package, 'type')
if buildtype == 'maven':
params = maven_params(config, package, chain=chain, scratch=scratch)
elif buildtype == 'wrapper':
params = wrapper_params(config, package, chain=chain, scratch=scratch)
if len(params.get('buildrequires')) != 1:
raise ValueError("A wrapper-rpm must depend on exactly one package")
else:
raise ValueError("Unsupported build type: %s" % buildtype)
if 'scmurl' not in params:
raise ValueError("%s is missing the scmurl parameter" % package)
builds[package] = params
if not builds:
if not isinstance(confs, (list, tuple)):
confs = [confs]
raise ValueError("No sections found in: %s" % ', '.join(confs))
return builds
def send_queued_msgs(cbtype, *args, **kws):
msgs = getattr(context, 'protonmsg_msgs', None)
if not msgs:
return
log = logging.getLogger('koji.plugin.protonmsg')
global CONFIG
if not CONFIG:
CONFIG = koji.read_config_files(CONFIG_FILE)
urls = CONFIG.get('broker', 'urls').split()
test_mode = False
if CONFIG.has_option('broker', 'test_mode'):
test_mode = CONFIG.getboolean('broker', 'test_mode')
if test_mode:
log.debug('test mode: skipping send to urls: %r', urls)
for msg in msgs:
log.debug('test mode: skipped msg: %r', msg)
return
random.shuffle(urls)
for url in urls:
container = Container(TimeoutHandler(url, msgs, CONFIG))
container.run()
if msgs:
log.debug('could not send to %s, %s messages remaining',
url, len(msgs))
else:
log.debug('all messages sent to %s successfully', url)
break
else:
log.error('could not send messages to any destinations')
def maven_import(cbtype, *args, **kws):
global config
if not context.opts.get('EnableMaven', False):
return
if kws.get('type') != 'rpm':
return
buildinfo = kws['build']
rpminfo = kws['rpm']
filepath = kws['filepath']
if not config:
config = koji.read_config_files([(CONFIG_FILE, True)])
name_patterns = config.get('patterns', 'rpm_names').split()
for pattern in name_patterns:
if fnmatch.fnmatch(rpminfo['name'], pattern):
break
else:
return
tmpdir = joinpath(koji.pathinfo.work(), 'rpm2maven', koji.buildLabel(buildinfo))
try:
if os.path.exists(tmpdir):
rmtree(tmpdir)
koji.ensuredir(tmpdir)
expand_rpm(filepath, tmpdir)
scan_and_import(buildinfo, rpminfo, tmpdir)
finally:
if os.path.exists(tmpdir):
rmtree(tmpdir)
def _read_config(self):
cp = koji.read_config_files(CONFIG_FILE)
self.config = {
'default_mounts': [],
'safe_roots': [],
'path_subs': [],
'paths': [],
'internal_dev_setup': None,
}
# main options
if cp.has_option('runroot', 'internal_dev_setup'):
self.config['internal_dev_setup'] = cp.getboolean(
'runroot', 'internal_dev_setup')
# path options
if cp.has_option('paths', 'default_mounts'):
self.config['default_mounts'] = cp.get('paths',
'default_mounts').split(',')
if cp.has_option('paths', 'safe_roots'):
self.config['safe_roots'] = cp.get('paths',
'safe_roots').split(',')
if cp.has_option('paths', 'path_subs'):
self.config['path_subs'] = []
for line in cp.get('paths', 'path_subs').splitlines():
line = line.strip()
if not line:
continue
sub = line.split(',')
if len(sub) != 2:
raise koji.GenericError('bad runroot substitution: %s' %
sub)
self.config['path_subs'].append(sub)
# path section are in form 'path%d' while order is important as some
# paths can be mounted inside other mountpoints
path_sections = [p for p in cp.sections() if re.match(r'path\d+', p)]
for section_name in sorted(path_sections, key=lambda x: int(x[4:])):
try:
self.config['paths'].append({
'mountpoint':
cp.get(section_name, 'mountpoint'),
'path':
cp.get(section_name, 'path'),
'fstype':
cp.get(section_name, 'fstype'),
'options':
cp.get(section_name, 'options'),
})
except six.moves.configparser.NoOptionError:
raise koji.GenericError(
"bad config: missing options in %s section" % section_name)
for path in self.config['default_mounts'] + self.config['safe_roots'] + \
[x[0] for x in self.config['path_subs']]:
if not path.startswith('/'):
raise koji.GenericError(
"bad config: all paths (default_mounts, safe_roots, path_subs) needs to be "
"absolute: %s" % path)
def read_config():
global config
cp = koji.read_config_files(CONFIG_FILE)
config = {
'path_filters': [],
'volume': None,
}
if cp.has_option('filters', 'paths'):
config['path_filters'] = cp.get('filters', 'paths').split()
if cp.has_option('general', 'volume'):
config['volume'] = cp.get('general', 'volume').strip()
def load_config(self, environ):
"""Load configuration options
Options are read from a kojiweb config file. To override the
configuration file location, use the SetEnv Apache directive. For
example:
SetEnv koji.web.ConfigFile /home/developer/koji/www/conf/web.conf
Backwards compatibility:
- if ConfigFile is not set, opts are loaded from http config
- if ConfigFile is set, then the http config must not provide Koji options
- In a future version we will load the default hub config regardless
- all PythonOptions (except koji.web.ConfigFile) are now deprecated and
support for them will disappear in a future version of Koji
"""
cf = environ.get('koji.web.ConfigFile', '/etc/kojiweb/web.conf')
cfdir = environ.get('koji.web.ConfigDir', '/etc/kojiweb/web.conf.d')
config = koji.read_config_files([cfdir, (cf, True)])
opts = {}
for name, dtype, default in self.cfgmap:
key = ('web', name)
if config.has_option(*key):
if dtype == 'integer':
opts[name] = config.getint(*key)
elif dtype == 'boolean':
opts[name] = config.getboolean(*key)
elif dtype == 'list':
opts[name] = [
x.strip() for x in config.get(*key).split(',')
]
else:
opts[name] = config.get(*key)
else:
opts[name] = default
opts['Secret'] = koji.util.HiddenValue(opts['Secret'])
if opts['WebAuthType'] not in (None, 'gssapi', 'ssl'):
raise koji.ConfigurationError(
f"Invalid value {opts['WebAuthType']} for "
"WebAuthType (ssl/gssapi)")
if opts['WebAuthType'] == 'gssapi':
opts['WebAuthType'] = koji.AUTHTYPE_GSSAPI
elif opts['WebAuthType'] == 'ssl':
opts['WebAuthType'] = koji.AUTHTYPE_SSL
# if there is no explicit request, use same authtype as web has
elif opts['WebPrincipal']:
opts['WebAuthType'] = koji.AUTHTYPE_GSSAPI
elif opts['WebCert']:
opts['WebAuthType'] = koji.AUTHTYPE_SSL
self.options = opts
return opts
def test_read_config_files(self, scp_clz, rcp_clz, cp_clz, open_mock):
files = 'test1.conf'
conf = koji.read_config_files(files)
open_mock.assert_called_once_with(files, 'r')
if six.PY2:
self.assertTrue(isinstance(conf,
six.moves.configparser.SafeConfigParser.__class__))
scp_clz.assert_called_once()
scp_clz.return_value.readfp.assert_called_once()
else:
self.assertTrue(isinstance(conf,
six.moves.configparser.ConfigParser.__class__))
cp_clz.assert_called_once()
cp_clz.return_value.read_file.assert_called_once()
open_mock.reset_mock()
cp_clz.reset_mock()
scp_clz.reset_mock()
files = ['test1.conf', 'test2.conf']
koji.read_config_files(files)
open_mock.assert_has_calls([call('test1.conf', 'r'),
call('test2.conf', 'r')],
any_order=True)
if six.PY2:
scp_clz.assert_called_once()
self.assertEqual(scp_clz.return_value.readfp.call_count, 2)
else:
cp_clz.assert_called_once()
self.assertEqual(cp_clz.return_value.read_file.call_count, 2)
open_mock.reset_mock()
cp_clz.reset_mock()
scp_clz.reset_mock()
conf = koji.read_config_files(files, raw=True)
self.assertTrue(isinstance(conf,
six.moves.configparser.RawConfigParser.__class__))
cp_clz.assert_not_called()
scp_clz.assert_not_called()
rcp_clz.assert_called_once()
def send_queued_msgs(cbtype, *args, **kws):
global CONFIG
msgs = getattr(context, 'protonmsg_msgs', None)
if not msgs:
return
if not CONFIG:
CONFIG = koji.read_config_files([(CONFIG_FILE, True)])
urls = CONFIG.get('broker', 'urls').split()
test_mode = False
if CONFIG.has_option('broker', 'test_mode'):
test_mode = CONFIG.getboolean('broker', 'test_mode')
db_enabled = False
if CONFIG.has_option('queue', 'enabled'):
db_enabled = CONFIG.getboolean('queue', 'enabled')
if test_mode:
LOG.debug('test mode: skipping send to urls: %r', urls)
fail_chance = CONFIG.getint('broker', 'test_mode_fail', fallback=0)
if fail_chance:
# simulate unsent messages in test mode
sent = []
unsent = []
for m in msgs:
if random.randint(1, 100) <= fail_chance:
unsent.append(m)
else:
sent.append(m)
if unsent:
LOG.info('simulating %i unsent messages' % len(unsent))
else:
sent = msgs
unsent = []
for msg in sent:
LOG.debug('test mode: skipped msg: %r', msg)
else:
unsent = _send_msgs(urls, msgs, CONFIG)
if db_enabled:
if unsent:
# if we still have some messages, store them and leave for another call to pick them up
store_to_db(msgs)
else:
# otherwise we are another call - look to db if there remains something to send
handle_db_msgs(urls, CONFIG)
elif unsent:
LOG.error('could not send %i messages. db queue disabled' % len(msgs))
def _strip_extra(buildinfo):
"""If extra_limit is configured, compare extra's size and drop it,
if it is over"""
global CONFIG
if not CONFIG:
CONFIG = koji.read_config_files([(CONFIG_FILE, True)])
if CONFIG.has_option('message', 'extra_limit'):
extra_limit = abs(CONFIG.getint('message', 'extra_limit'))
if extra_limit == 0:
return buildinfo
extra_size = len(json.dumps(buildinfo.get('extra', {}), default=json_serialize))
if extra_limit and extra_size > extra_limit:
LOG.debug("Dropping 'extra' from build %s (length: %d > %d)" %
(buildinfo['nvr'], extra_size, extra_limit))
buildinfo = buildinfo.copy()
del buildinfo['extra']
return buildinfo
def saveFailedTree(buildrootID, full=False, **opts):
"""Create saveFailedTree task
If arguments are invalid, error message is returned. Otherwise task id of
newly created task is returned."""
global config, allowed_methods
# let it raise errors
buildrootID = int(buildrootID)
full = bool(full)
# read configuration only once
if config is None:
config = koji.read_config_files([(CONFIG_FILE, True)])
allowed_methods = config.get('permissions',
'allowed_methods').split(',')
if len(allowed_methods) == 1 and allowed_methods[0] == '*':
allowed_methods = '*'
brinfo = kojihub.get_buildroot(buildrootID, strict=True)
taskID = brinfo['task_id']
task_info = kojihub.Task(taskID).getInfo()
if task_info['state'] != koji.TASK_STATES['FAILED']:
raise koji.PreBuildError(
"Task %s has not failed. Only failed tasks can upload their buildroots."
% taskID)
elif allowed_methods != '*' and task_info['method'] not in allowed_methods:
raise koji.PreBuildError("Only %s tasks can upload their buildroots (Task %s is %s)." % \
(', '.join(allowed_methods), task_info['id'], task_info['method']))
elif task_info[
"owner"] != context.session.user_id and not context.session.hasPerm(
'admin'):
raise koji.ActionNotAllowed(
"Only owner of failed task or 'admin' can run this task.")
elif not kojihub.get_host(task_info['host_id'])['enabled']:
raise koji.PreBuildError("Host is disabled.")
args = koji.encode_args(buildrootID, full, **opts)
taskopts = {
'assign': brinfo['host_id'],
}
return kojihub.make_task('saveFailedTree', args, **taskopts)
def load_config(self, environ):
"""Load configuration options
Options are read from a kojiweb config file. To override the
configuration file location, use the SetEnv Apache directive. For
example:
SetEnv koji.web.ConfigFile /home/developer/koji/www/conf/web.conf
Backwards compatibility:
- if ConfigFile is not set, opts are loaded from http config
- if ConfigFile is set, then the http config must not provide Koji options
- In a future version we will load the default hub config regardless
- all PythonOptions (except koji.web.ConfigFile) are now deprecated and
support for them will disappear in a future version of Koji
"""
cf = environ.get('koji.web.ConfigFile', '/etc/kojiweb/web.conf')
cfdir = environ.get('koji.web.ConfigDir', '/etc/kojiweb/web.conf.d')
config = koji.read_config_files([cfdir, (cf, True)])
opts = {}
for name, dtype, default in self.cfgmap:
key = ('web', name)
if config.has_option(*key):
if dtype == 'integer':
opts[name] = config.getint(*key)
elif dtype == 'boolean':
opts[name] = config.getboolean(*key)
elif dtype == 'list':
opts[name] = [
x.strip() for x in config.get(*key).split(',')
]
else:
opts[name] = config.get(*key)
else:
opts[name] = default
opts['Secret'] = koji.util.HiddenValue(opts['Secret'])
self.options = opts
return opts
import sys
import logging
import subprocess
import koji
from koji.plugin import register_callback, ignore_error
if '/usr/share/koji-hub' not in sys.path:
sys.path.append("/usr/share/koji-hub")
import kojihub
from kojihub import RootExports
# CONVERT TO CONFIG FILE
CONFIG_FILE = '/etc/koji-hub/plugins/key_signing.conf'
CONFIG = None
if not CONFIG:
CONFIG = koji.read_config_files([(CONFIG_FILE, True)])
passphrase = CONFIG.get('signing', 'passphrase')
gpg_key_name = CONFIG.get('signing', 'gpg_key_name')
gpg_key_id = CONFIG.get('signing', 'gpg_key_id')
build_target = CONFIG.get('signing', 'build_target').split()
testing_tag = CONFIG.get('signing', 'testing_tag')
send_to_testing = CONFIG.get('signing', 'send_to_testing')
sigul_config = CONFIG.get('signing', 'sigul_config')
def key_signing(cbtype, *args, **kws):
# Make sure this is a package build and nothing else
if kws['tag']['name'] not in build_target:
return
def load_config(environ):
"""Load configuration options
Options are read from a config file. The config file location is
controlled by the koji.hub.ConfigFile environment variable
in the httpd config. To override this (for example):
SetEnv koji.hub.ConfigFile /home/developer/koji/hub/hub.conf
Backwards compatibility:
- if ConfigFile is not set, opts are loaded from http config
- if ConfigFile is set, then the http config must not provide Koji options
- In a future version we will load the default hub config regardless
- all PythonOptions (except ConfigFile) are now deprecated and support for them
will disappear in a future version of Koji
"""
# get our config file(s)
cf = environ.get('koji.hub.ConfigFile', '/etc/koji-hub/hub.conf')
cfdir = environ.get('koji.hub.ConfigDir', '/etc/koji-hub/hub.conf.d')
config = koji.read_config_files([cfdir, (cf, True)], raw=True)
cfgmap = [
# option, type, default
['DBName', 'string', None],
['DBUser', 'string', None],
['DBHost', 'string', None],
['DBhost', 'string', None], # alias for backwards compatibility
['DBPort', 'integer', None],
['DBPass', 'string', None],
['DBConnectionString', 'string', None],
['KojiDir', 'string', None],
['AuthPrincipal', 'string', None],
['AuthKeytab', 'string', None],
['ProxyPrincipals', 'string', ''],
['HostPrincipalFormat', 'string', None],
['AllowedKrbRealms', 'string', '*'],
# TODO: this option should be removed in future release
['DisableGSSAPIProxyDNFallback', 'boolean', False],
['DNUsernameComponent', 'string', 'CN'],
['ProxyDNs', 'string', ''],
['CheckClientIP', 'boolean', True],
['LoginCreatesUser', 'boolean', True],
['AllowProxyAuthType', 'boolean', False],
['KojiWebURL', 'string', 'http://localhost.localdomain/koji'],
['EmailDomain', 'string', None],
['NotifyOnSuccess', 'boolean', True],
['DisableNotifications', 'boolean', False],
['Plugins', 'string', ''],
['PluginPath', 'string', '/usr/lib/koji-hub-plugins'],
['KojiDebug', 'boolean', False],
['KojiTraceback', 'string', None],
['VerbosePolicy', 'boolean', False],
['LogLevel', 'string', 'WARNING'],
[
'LogFormat', 'string',
'%(asctime)s [%(levelname)s] m=%(method)s u=%(user_name)s p=%(process)s r=%(remoteaddr)s '
'%(name)s: %(message)s'
],
['MissingPolicyOk', 'boolean', True],
['EnableMaven', 'boolean', False],
['EnableWin', 'boolean', False],
['RLIMIT_AS', 'string', None],
['RLIMIT_CORE', 'string', None],
['RLIMIT_CPU', 'string', None],
['RLIMIT_DATA', 'string', None],
['RLIMIT_FSIZE', 'string', None],
['RLIMIT_MEMLOCK', 'string', None],
['RLIMIT_NOFILE', 'string', None],
['RLIMIT_NPROC', 'string', None],
['RLIMIT_OFILE', 'string', None],
['RLIMIT_RSS', 'string', None],
['RLIMIT_STACK', 'string', None],
['MemoryWarnThreshold', 'integer', 5000],
['MaxRequestLength', 'integer', 4194304],
['LockOut', 'boolean', False],
['ServerOffline', 'boolean', False],
['OfflineMessage', 'string', None],
]
opts = {}
for name, dtype, default in cfgmap:
key = ('hub', name)
if config and config.has_option(*key):
if dtype == 'integer':
opts[name] = config.getint(*key)
elif dtype == 'boolean':
opts[name] = config.getboolean(*key)
else:
opts[name] = config.get(*key)
continue
opts[name] = default
if opts['DBHost'] is None:
opts['DBHost'] = opts['DBhost']
# load policies
# (only from config file)
if config and config.has_section('policy'):
# for the moment, we simply transfer the policy conf to opts
opts['policy'] = dict(config.items('policy'))
else:
opts['policy'] = {}
for pname, text in _default_policies.items():
opts['policy'].setdefault(pname, text)
# use configured KojiDir
if opts.get('KojiDir') is not None:
koji.BASEDIR = opts['KojiDir']
koji.pathinfo.topdir = opts['KojiDir']
return opts
# also shouldn't happen, but just in case
return
if not is_sidetag(tag):
return
# is the tag now empty?
query = QueryProcessor(
tables=["tag_listing"],
clauses=["tag_id = %(tag_id)s", "active IS TRUE"],
values={"tag_id": tag["id"]},
opts={"countOnly": True},
)
if query.execute():
return
# looks like we've just untagged the last build from a side tag
try:
# XXX: are we double updating tag_listing?
_remove_sidetag(tag)
except koji.GenericError:
pass
# read config and register
if not CONFIG:
CONFIG = koji.read_config_files(CONFIG_FILE)
if CONFIG.has_option("sidetag", "remove_empty") and CONFIG.getboolean(
"sidetag", "remove_empty"
):
handle_sidetag_untag = callback("postUntag")(handle_sidetag_untag)
if CONFIG.has_option("sidetag", "allowed_suffixes"):
ALLOWED_SUFFIXES = CONFIG.get("sidetag", "allowed_suffixes").split(',')
# is the tag now empty?
query = QueryProcessor(
tables=["tag_listing"],
clauses=["tag_id = %(tag_id)s", "active IS TRUE"],
values={"tag_id": tag["id"]},
opts={"countOnly": True},
)
if query.execute():
return
# looks like we've just untagged the last build from a side tag
try:
# XXX: are we double updating tag_listing?
_remove_sidetag(tag)
except koji.GenericError:
pass
# read config and register
if not CONFIG:
CONFIG = koji.read_config_files(CONFIG_FILE, raw=True)
if CONFIG.has_option("sidetag", "remove_empty") and CONFIG.getboolean(
"sidetag", "remove_empty"
):
handle_sidetag_untag = callback("postUntag")(handle_sidetag_untag)
if CONFIG.has_option("sidetag", "allowed_suffixes"):
ALLOWED_SUFFIXES = CONFIG.get("sidetag", "allowed_suffixes").split(',')
if CONFIG.has_option("sidetag", "name_template"):
NAME_TEMPLATE = CONFIG.get("sidetag", "name_template")
else:
NAME_TEMPLATE = '{basetag}-side-{tag_id}'
