def setup_logging2(self, environ):
"""Adjust logging based on configuration options"""
opts = self.options
# determine log level
level = opts['LogLevel']
valid_levels = ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL')
# the config value can be a single level name or a series of
# logger:level names pairs. processed in order found
default = None
for part in level.split():
pair = part.split(':', 1)
if len(pair) == 2:
name, level = pair
else:
name = 'koji'
level = part
default = level
if level not in valid_levels:
raise koji.GenericError("Invalid log level: %s" % level)
# all our loggers start with koji
if name == '':
name = 'koji'
default = level
elif name.startswith('.'):
name = 'koji' + name
elif not name.startswith('koji'):
name = 'koji.' + name
level_code = logging.getLevelName(level)
logging.getLogger(name).setLevel(level_code)
logger = logging.getLogger("koji")
# if KojiDebug is set, force main log level to DEBUG
if opts.get('KojiDebug'):
logger.setLevel(logging.DEBUG)
elif default is None:
# LogLevel did not configure a default level
logger.setLevel(logging.WARNING)
self.formatter = HubFormatter(opts['LogFormat'])
self.formatter.environ = environ
self.log_handler.setFormatter(self.formatter)
def getTag(self, taginfo, **kw):
"""
Retrieve the given tag from koji.
Args:
taginfo (int or basestring): The tag you want info about.
strict (bool): If True, raise an Exception if epel tags are queried. Defaults to False.
Returns:
dict or None: A dictionary of tag information, or None if epel is requested and strict
is False.
Raises:
koji.GenericError: If strict is True and epel is requested.
"""
for nr in self.__tags__:
if taginfo == self.__tags__[nr][0]:
toreturn = self.__tags__[nr][1]
toreturn['id'] = nr
return toreturn
if isinstance(taginfo, int):
taginfo = "f%d" % taginfo
if taginfo.startswith("epel"):
if kw.get("strict", False):
raise koji.GenericError("Invalid tagInfo: '%s'" % taginfo)
else:
return None
return {
'maven_support': False,
'locked': False,
'name': taginfo,
'perm': None,
'id': 246,
'arches': None,
'maven_include_all': False,
'perm_id': None
}
def test_canfail_canceled(self):
"""Canceled canfail tasks should not raise exceptions"""
parent = 1
task_ids = [5, 6, 7]
canfail = [7]
for t in task_ids:
task = self.getTask(t)
task.getResult.return_value = "OK"
task.isCanceled.return_value = False
self.tasks[7].getResult.side_effect = koji.GenericError('canceled')
self.tasks[7].isCanceled.return_value = True
results = self.host_exports.taskWaitResults(parent,
task_ids,
canfail=canfail)
expect_f = {
'faultCode': koji.GenericError.faultCode,
'faultString': 'canceled'
}
expect = [[5, "OK"], [6, "OK"], [7, expect_f]]
self.assertEqual(results, expect)
self.host_exports.taskUnwait.assert_called_with(parent)
self.assertEqual(self.queries, [])
def test_BaseTaskHandler_wait_some_not_done_all_set_failany_set_failed_task(
self):
""" Tests that the wait function raises an exception when one of the subtask fails when the failany flag is set
to True.
"""
temp_path = get_tmp_dir_path('TestTask')
obj = TestTask(12345678, 'some_method', ['random_arg'], None, None,
temp_path)
makedirs(temp_path)
obj.session = Mock()
obj.session.host.taskSetWait.return_value = None
obj.session.host.taskWait.side_effect = [[[1551234], [1591234]],
[[1551234, 1591234], []]]
obj.session.getTaskResult.side_effect = koji.GenericError(
'Uh oh, we\'ve got a problem here!')
try:
obj.wait([1551234, 1591234], all=True, failany=True)
raise Exception('A GeneralError was not raised.')
except koji.GenericError as e:
self.assertEquals(e.args[0], 'Uh oh, we\'ve got a problem here!')
obj.session.host.taskSetWait.assert_called_once_with(
12345678, [1551234, 1591234])
def compose_create(self, nvr: NVR, distro: str, images: List[ImageRequest],
kojidata: ComposeRequest.Koji):
url = urllib.parse.urljoin(self.url, f"/compose")
req = urllib.request.Request(url)
cro = ComposeRequest(nvr, distro, images, kojidata)
dat = json.dumps(cro.as_dict())
raw = dat.encode('utf-8')
req = urllib.request.Request(url, raw)
req.add_header('Content-Type', 'application/json')
req.add_header('Content-Length', len(raw))
try:
with urllib.request.urlopen(req, raw) as res:
payload = res.read().decode('utf-8')
except urllib.error.HTTPError as e:
body = e.read().decode('utf-8').strip()
msg = f"Failed to create the compose request: {body}"
raise koji.GenericError(msg) from None
ps = json.loads(payload)
compose_id, koji_build_id = ps["id"], ps["koji_build_id"]
return compose_id, koji_build_id
def test_handle_save_failed_tree_errors(self, watch_tasks_mock,
activate_session_mock, stdout):
# koji save-failed-tree 123 456
arguments = [123, 456]
options = mock.MagicMock()
self.parser.parse_args.return_value = [options, arguments]
self.parser.error.side_effect = Exception()
self.session.getAPIVersion.return_value = koji.API_VERSION
self.session.listBuildroots.return_value = [{'id': 321}]
self.assertRaises(Exception, cli.handle_save_failed_tree, self.options,
self.session, self.args)
arguments = ["text"]
self.parser.parse_args.return_value = [options, arguments]
self.assertRaises(Exception, cli.handle_save_failed_tree, self.options,
self.session, self.args)
cli.logger = mock.MagicMock()
# plugin not installed
arguments = [123]
self.parser.parse_args.return_value = [options, arguments]
self.session.saveFailedTree.side_effect = koji.GenericError(
"Invalid method")
cli.handle_save_failed_tree(self.options, self.session, self.args)
actual = stdout.getvalue()
self.assertTrue(
'The save_failed_tree plugin appears to not be installed' in
actual)
# Task which is not FAILED, disabled in config, wrong owner
self.session.saveFailedTree.side_effect = koji.PreBuildError(
'placeholder')
with self.assertRaises(koji.PreBuildError) as cm:
cli.handle_save_failed_tree(self.options, self.session, self.args)
e = cm.exception
self.assertEqual(e, self.session.saveFailedTree.side_effect)
def test_handle_import_src_rpm_import_with_no_exist_build(self):
"""Test handle_import source RPM import.
No build is on the server case,
this tests are focusing on do_import() function coverage.
"""
arguments = ['/path/to/bash-4.4.12-5.fc26.src.rpm', '--src-epoch', 'None']
options = mock.MagicMock()
session = mock.MagicMock()
# No exist build test case
# import general case
session.getBuild.return_value = None
session.getRPM.return_value = None
expected = "uploading %s... done\n" % arguments[0]
expected += "importing %s... done\n" % arguments[0]
self.__do_import_test(
options, session, arguments,
rpm_header=self.srpm_header, expected=expected)
# import error case
session.importRPM.side_effect = koji.GenericError('fake-import-error')
expected = "uploading %s... done\n" % arguments[0]
expected += "importing %s... \n" % arguments[0]
expected += "Error importing: fake-import-error\n"
self.__do_import_test(
options, session, arguments,
rpm_header=self.srpm_header, expected=expected)
# import with --link (hardlink) option
session.importRPM.side_effect = None
expected = "importing %s... done\n" % arguments[0]
self.__do_import_test(
options, session, arguments + ['--link'],
upload_rpm_mock=mock.patch('koji_cli.commands.linked_upload').start(),
rpm_header=self.srpm_header, expected=expected)
session.uploadWrapper.assert_not_called()
def cleanupTask(self, task_id, wait=True):
"""Clean up after task
- kill children
- expire session
Return True if all children were successfully killed, False otherwise.
"""
pid = self.pids.get(task_id)
if not pid:
raise koji.GenericError("No pid for task %i" % task_id)
children = self._childPIDs(pid)
if children:
# send SIGINT once to let mock mock try to clean up
self._killChildren(task_id, children, sig=signal.SIGINT, pause=3.0)
if children:
self._killChildren(task_id, children)
if children:
self._killChildren(task_id, children, sig=signal.SIGKILL, timeout=3.0)
#expire the task's subsession
session_id = self.subsessions.get(task_id)
if session_id:
self.logger.info("Expiring subsession %i (task %i)" % (session_id, task_id))
try:
self.session.logoutChild(session_id)
del self.subsessions[task_id]
except:
#not much we can do about it
pass
if wait:
return self._waitTask(task_id, pid)
else:
# task has already been waited on, and we've cleaned
# up as much as we can
return True
def rmtree(path, logger=None):
"""Delete a directory tree without crossing fs boundaries"""
# implemented to avoid forming long paths
# see: https://pagure.io/koji/issue/201
logger = logger or logging.getLogger('koji')
try:
st = os.lstat(path)
except FileNotFoundError:
logger.warning("No such file/dir %s for removal" % path)
return
if not stat.S_ISDIR(st.st_mode):
raise koji.GenericError("Not a directory: %s" % path)
dev = st.st_dev
cwd = os.getcwd()
root = os.path.abspath(path)
try:
try:
os.chdir(path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
return
raise
_rmtree(dev, root)
finally:
os.chdir(cwd)
try:
os.rmdir(path)
except OSError as e:
if e.errno == errno.ENOTEMPTY:
logger.warning(
'%s path is not empty, but it may be a phantom error caused by some'
' race condition',
path,
exc_info=True)
elif e.errno != errno.ENOENT:
raise
def __init__(self, str):
super(HasTest, self).__init__(str)
try:
self.field = str.split()[1]
except IndexError:
raise koji.GenericError("Invalid or missing field in policy test")
def get_test_handler(self, str):
name = str.split(None, 1)[0]
try:
return self.tests[name](str)
except KeyError:
raise koji.GenericError("missing test handler: %s" % name)
def handler(self, host):
#note: this is a foreground task
if host['id'] != self.session.host.getID():
raise koji.GenericError("Host mismatch")
self.manager.restart_pending = True
return "graceful restart initiated"
def handler(self, *args, **opts):
raise koji.GenericError("Invalid method: %s" % self.method)
def get_tasks(session, parser, opts):
channels = opts.channels
hosts = opts.hosts
methods = opts.methods
states = getattr(opts, 'states', None)
limit = getattr(opts, 'limit', None)
offset = getattr(opts, 'offset', None)
channel_ids = []
if channels:
for channel in channels:
channel_ids.append(session.getChannel(channel, strict=True)['id'])
host_ids = []
if hosts:
for host in hosts:
host_ids.append(session.getHost(host, strict=True)['id'])
state_nums = []
for state in states:
if isinstance(state, six.integer_types):
if 0 <= state <= 5:
state_nums.append(state)
else:
raise koji.GenericError("integer state should >=0 and <=5")
elif isinstance(state, six.string_types):
state_nums.append(koji.TASK_STATES[state])
else:
raise koji.GenericError("unacceptable state type")
options = {}
if state_nums:
options['state'] = state_nums
options['parent'] = None
options['decode'] = True
queryOpts = {}
if limit:
queryOpts['limit'] = limit
if offset:
queryOpts['offset'] = offset
queryOpts['order'] = '-id'
cvs = []
with session.multicall() as m:
if channel_ids:
for channel_id in channel_ids:
if methods:
for method in methods:
cvs.append((m.listTasks(
dict(options, method=method,
channel_id=channel_id), queryOpts)))
else:
cvs.append(
m.listTasks(dict(options, channel_id=channel_id),
queryOpts))
elif host_ids:
for host_id in host_ids:
if methods:
for method in methods:
cvs.append(
m.listTasks(
dict(options, method=method, host_id=host_id),
queryOpts))
else:
cvs.append(
m.listTasks(dict(options, host_id=host_id), queryOpts))
else:
if methods:
for method in methods:
cvs.append(
m.listTasks(dict(options, method=method), queryOpts))
else:
cvs.append(m.listTasks(options, queryOpts))
tasks = sum([cv.result for cv in cvs], [])
print(tasks)
if not tasks:
raise koji.GenericError("no tasks to replicate.")
else:
logger.debug("to replicate tasks:\n%s",
"\n".join([str(t['id']) for t in tasks]))
return tasks
def do_mounts(self, rootdir, mounts):
if not mounts:
return
self.logger.info('New runroot')
self.logger.info("Runroot mounts: %s" % mounts)
fn = '%s/tmp/runroot_mounts' % rootdir
fslog = file(fn, 'a')
logfile = "%s/do_mounts.log" % self.workdir
uploadpath = self.getUploadDir()
error = None
for dev, path, type, opts in mounts:
if not path.startswith('/'):
raise koji.GenericError("invalid mount point: %s" % path)
mpoint = "%s%s" % (rootdir, path)
if opts is None:
opts = []
else:
opts = opts.split(',')
if 'bind' in opts:
#make sure dir exists
if not os.path.isdir(dev):
error = koji.GenericError(
"No such directory or mount: %s" % dev)
break
type = 'none'
if path is None:
#shorthand for "same path"
path = dev
if 'bg' in opts:
error = koji.GenericError(
"bad config: background mount not allowed")
break
opts = ','.join(opts)
cmd = ['mount', '-t', type, '-o', opts, dev, mpoint]
self.logger.info("Mount command: %r" % cmd)
koji.ensuredir(mpoint)
if compat_mode:
status = log_output(cmd[0],
cmd,
logfile,
uploadpath,
logerror=True,
append=True)
else:
status = log_output(self.session,
cmd[0],
cmd,
logfile,
uploadpath,
logerror=True,
append=True)
if not _isSuccess(status):
error = koji.GenericError("Unable to mount %s: %s" \
% (mpoint, _parseStatus(status, cmd)))
break
fslog.write("%s\n" % mpoint)
fslog.flush()
fslog.close()
if error is not None:
self.undo_mounts(rootdir, fatal=False)
raise error
def test_bad_tag(self):
self.get_tag.side_effect = koji.GenericError("FOO")
with self.assertRaises(koji.GenericError):
kojihub.delete_tag('badtag')
self.assertEqual(self.updates, [])
self.context.session.assertPerm.assert_called_with('admin')
def listTagged(self, tag, *args, **kwargs):
raise koji.GenericError("foo")
def updateTasks(self):
"""Read and process task statuses from server
The processing we do is:
1) clean up after tasks that are not longer active:
* kill off processes
* retire buildroots
* remove buildroots
- with some possible exceptions
2) wake waiting tasks if appropriate
"""
tasks = {}
stale = []
task_load = 0.0
if self.pids:
self.logger.info("pids: %r" % self.pids)
for task in self.session.host.getHostTasks():
self.logger.info("open task: %r" % task)
# the tasks returned are those that are open and locked
# by this host.
id = task['id']
if id not in self.pids:
#We don't have a process for this
#Expected to happen after a restart, otherwise this is an error
stale.append(id)
continue
tasks[id] = task
if task.get('alert', False):
#wake up the process
self.logger.info("Waking up task: %r" % task)
os.kill(self.pids[id], signal.SIGUSR2)
if not task['waiting']:
task_load += task['weight']
self.logger.debug("Task Load: %s" % task_load)
self.task_load = task_load
self.tasks = tasks
self.logger.debug("Current tasks: %r" % self.tasks)
if len(stale) > 0:
#A stale task is one which is opened to us, but we know nothing
#about). This will happen after a daemon restart, for example.
self.logger.info("freeing stale tasks: %r" % stale)
self.session.host.freeTasks(stale)
for id, pid in self.pids.items():
if self._waitTask(id, pid):
# the subprocess handles most everything, we just need to clear things out
if self.cleanupTask(id, wait=False):
del self.pids[id]
if id in self.tasks:
del self.tasks[id]
for id, pid in self.pids.items():
if id not in tasks:
# expected to happen when:
# - we are in the narrow gap between the time the task
# records its result and the time the process actually
# exits.
# - task is canceled
# - task is forcibly reassigned/unassigned
tinfo = self.session.getTaskInfo(id)
if tinfo is None:
raise koji.GenericError("Invalid task %r (pid %r)" % (id, pid))
elif tinfo['state'] == koji.TASK_STATES['CANCELED']:
self.logger.info("Killing canceled task %r (pid %r)" % (id, pid))
if self.cleanupTask(id):
del self.pids[id]
elif tinfo['host_id'] != self.host_id:
self.logger.info("Killing reassigned task %r (pid %r)" % (id, pid))
if self.cleanupTask(id):
del self.pids[id]
else:
self.logger.info("Lingering task %r (pid %r)" % (id, pid))
def load_config(self, environ):
"""Load configuration options
Options are read from a config file.
Backwards compatibility:
- if ConfigFile is not set, opts are loaded from http config
- if ConfigFile is set, then the http config must not provide Koji options
- In a future version we will load the default hub config regardless
- all PythonOptions (except koji.web.ConfigFile) are now deprecated and
support for them will disappear in a future version of Koji
"""
modpy_opts = environ.get('modpy.opts', {})
if 'modpy.opts' in environ:
cf = modpy_opts.get('koji.web.ConfigFile', None)
cfdir = modpy_opts.get('koji.web.ConfigDir', None)
# to aid in the transition from PythonOptions to web.conf, we do
# not check the config file by default, it must be configured
if not cf and not cfdir:
self.logger.warn(
'Warning: configuring Koji via PythonOptions is deprecated. Use web.conf'
)
else:
cf = environ.get('koji.web.ConfigFile', '/etc/kojiweb/web.conf')
cfdir = environ.get('koji.web.ConfigDir',
'/etc/kojiweb/web.conf.d')
if cfdir:
configs = koji.config_directory_contents(cfdir)
else:
configs = []
if cf and os.path.isfile(cf):
configs.append(cf)
if configs:
config = RawConfigParser()
config.read(configs)
elif modpy_opts:
# presumably we are configured by modpy options
config = None
else:
raise koji.GenericError("Configuration missing")
opts = {}
for name, dtype, default in self.cfgmap:
if config:
key = ('web', name)
if config.has_option(*key):
if dtype == 'integer':
opts[name] = config.getint(*key)
elif dtype == 'boolean':
opts[name] = config.getboolean(*key)
elif dtype == 'list':
opts[name] = [
x.strip() for x in config.get(*key).split(',')
]
else:
opts[name] = config.get(*key)
else:
opts[name] = default
else:
if modpy_opts.get(name, None) is not None:
if dtype == 'integer':
opts[name] = int(modpy_opts.get(name))
elif dtype == 'boolean':
opts[name] = modpy_opts.get(name).lower() in ('yes',
'on',
'true',
'1')
else:
opts[name] = modpy_opts.get(name)
else:
opts[name] = default
if 'modpy.conf' in environ:
debug = environ['modpy.conf'].get('PythonDebug', '0').lower()
opts['PythonDebug'] = (debug in ['yes', 'on', 'true', '1'])
opts['Secret'] = koji.util.HiddenValue(opts['Secret'])
self.options = opts
return opts
def _list_tasks(options, session):
"Retrieve a list of tasks"
callopts = {
'decode': True,
}
if not getattr(options, 'all', False):
callopts['state'] = [
koji.TASK_STATES[s] for s in ('FREE', 'OPEN', 'ASSIGNED')
]
if getattr(options, 'after', False):
callopts['startedAfter'] = options.after
if getattr(options, 'before', False):
callopts['startedBefore'] = options.before
if getattr(options, 'mine', False):
if getattr(options, 'user', None):
raise koji.GenericError(
"Can't specify 'mine' and 'user' in same time")
user = session.getLoggedInUser()
if not user:
print("Unable to determine user")
sys.exit(1)
callopts['owner'] = user['id']
if getattr(options, 'user', None):
user = session.getUser(options.user)
if not user:
print("No such user: %s" % options.user)
sys.exit(1)
callopts['owner'] = user['id']
if getattr(options, 'arch', None):
callopts['arch'] = parse_arches(options.arch, to_list=True)
if getattr(options, 'method', None):
callopts['method'] = options.method
if getattr(options, 'channel', None):
chan = session.getChannel(options.channel)
if not chan:
print("No such channel: %s" % options.channel)
sys.exit(1)
callopts['channel_id'] = chan['id']
if getattr(options, 'host', None):
host = session.getHost(options.host)
if not host:
print("No such host: %s" % options.host)
sys.exit(1)
callopts['host_id'] = host['id']
qopts = {'order': 'priority,create_time'}
tasklist = session.listTasks(callopts, qopts)
tasks = dict([(x['id'], x) for x in tasklist])
# thread the tasks
for t in tasklist:
if t['parent'] is not None:
parent = tasks.get(t['parent'])
if parent:
parent.setdefault('children', [])
parent['children'].append(t)
t['sub'] = True
return tasklist
def test_koji_cg_koji_import(self, tag_loader, tagger, cl_session):
""" Tests whether build is still tagged even if there's an exception in CGImport """
cl_session.return_value.CGImport = Mock(
side_effect=koji.GenericError("Build already exists asdv"))
self.cg.koji_import()
tagger.assert_called()
def createSideTag(basetag, debuginfo=False, suffix=None):
"""Create a side tag.
:param basetag: name or ID of base tag
:type basetag: str or int
:param debuginfo: should buildroot repos contain debuginfo?
:type debuginfo: bool
:param suffix: suffix which will be appended to generated sidetag name
List of allowed suffixes needs to be defined in config.
:type suffix: str
:returns dict: sidetag name + id
"""
if suffix and suffix not in ALLOWED_SUFFIXES:
raise koji.GenericError("%s suffix is not allowed for sidetag" % suffix)
# Any logged-in user is able to request creation of side tags,
# as long the request meets the policy.
context.session.assertLogin()
user = get_user(context.session.user_id, strict=True)
basetag = get_tag(basetag, strict=True)
query = QueryProcessor(
tables=["tag_extra"],
clauses=["key='sidetag_user_id'", "value=%(user_id)s", "active IS TRUE"],
columns=["COUNT(*)"],
aliases=["user_tags"],
values={"user_id": str(user["id"])},
)
user_tags = query.executeOne()
if user_tags is None:
# should not ever happen
raise koji.GenericError("Unknown db error")
# Policy is a very flexible mechanism, that can restrict for which
# tags sidetags can be created, or which users can create sidetags etc.
assert_policy(
"sidetag", {"tag": basetag["id"], "number_of_tags": user_tags["user_tags"]}
)
# ugly, it will waste one number in tag_id_seq, but result will match with
# id assigned by _create_tag
tag_id = nextval("tag_id_seq") + 1
sidetag_name = "%s-side-%s" % (basetag["name"], tag_id)
if suffix:
sidetag_name += '-%s' % suffix
extra = {
"sidetag": True,
"sidetag_user": user["name"],
"sidetag_user_id": user["id"],
}
if debuginfo:
extra['with_debuginfo'] = True
sidetag_id = _create_tag(
sidetag_name,
parent=basetag["id"],
arches=basetag["arches"],
extra=extra,
)
_create_build_target(sidetag_name, sidetag_id, sidetag_id)
return {"name": sidetag_name, "id": sidetag_id}
def get(self, name):
func = self.funcs.get(name, None)
if func is None:
raise koji.GenericError("Invalid method: %s" % name)
return func
def getProductListings(productLabel, buildInfo):
"""
Get a map of which variants of the given product included packages built
by the given build, and which arches each variant included.
"""
compose_dbh = Products.compose_get_dbh()
#XXX - need access to hub kojihub functions
conf = koji.read_config('brew')
hub = conf['server']
session = koji.ClientSession(hub, {})
build = session.getBuild(buildInfo, strict=True)
sys.stderr.write("%r" % build)
sys.stderr.flush()
rpms = session.listRPMs(buildID=build['id'])
if not rpms:
raise koji.GenericError("Could not find any RPMs for build: %s" %
buildInfo)
# sort rpms, so first part of list consists of sorted 'normal' rpms and
# second part are sorted debuginfos
debuginfos = [x for x in rpms if '-debuginfo' in x['nvr']]
base_rpms = [x for x in rpms if '-debuginfo' not in x['nvr']]
rpms = sorted(base_rpms, key=lambda x: x['nvr']) + sorted(
debuginfos, key=lambda x: x['nvr'])
srpm = "%(package_name)s-%(version)s-%(release)s.src.rpm" % build
prodinfo = Products.get_product_info(compose_dbh, productLabel)
if not prodinfo:
# no product with the given label exists
raise koji.GenericError("Could not find a product with label: %s" %
productLabel)
version, variants = prodinfo
listings = {}
match_version = Products.get_match_versions(compose_dbh, productLabel)
for variant in variants:
if variant == None:
# dict keys must be a string
variant = ''
treelist = Products.precalc_treelist(compose_dbh, productLabel,
version, variant)
if not treelist:
continue
overrides = Products.get_overrides(compose_dbh, productLabel, version,
variant)
cache_map = {}
for rpm in rpms:
if rpm['name'] in match_version:
rpm_version = rpm['version']
else:
rpm_version = None
# without debuginfos
rpms_nondebug = [
rpm for rpm in rpms if not koji.is_debuginfo(rpm['name'])
]
d = {}
all_archs = set([rpm['arch'] for rpm in rpms_nondebug])
for arch in all_archs:
d[arch] = Products.dest_get_archs(
compose_dbh,
treelist,
arch,
[rpm['name'] for rpm in rpms_nondebug if rpm['arch'] == arch],
cache_map.get(srpm, {}).get(arch, {}),
rpm_version,
overrides,
)
for rpm in rpms_nondebug:
dest_archs = d[rpm['arch']].get(rpm['name'], {}).keys()
if rpm['arch'] != 'src':
cache_map.setdefault(srpm, {})
cache_map[srpm].setdefault(rpm['arch'], {})
for x in dest_archs:
cache_map[srpm][rpm['arch']][x] = 1
for dest_arch in dest_archs:
listings.setdefault(variant,
{}).setdefault(rpm['nvr'], {}).setdefault(
rpm['arch'], []).append(dest_arch)
# debuginfo only
rpms_debug = [rpm for rpm in rpms if koji.is_debuginfo(rpm['name'])]
d = {}
all_archs = set([rpm['arch'] for rpm in rpms_debug])
for arch in all_archs:
d[arch] = Products.dest_get_archs(
compose_dbh,
treelist,
arch,
[rpm['name'] for rpm in rpms_debug if rpm['arch'] == arch],
cache_map.get(srpm, {}).get(arch, {}),
rpm_version,
overrides,
)
for rpm in rpms_debug:
dest_archs = d[rpm['arch']].get(rpm['name'], {}).keys()
if rpm['arch'] != 'src':
cache_map.setdefault(srpm, {})
cache_map[srpm].setdefault(rpm['arch'], {})
for x in dest_archs:
cache_map[srpm][rpm['arch']][x] = 1
for dest_arch in dest_archs:
listings.setdefault(variant,
{}).setdefault(rpm['nvr'], {}).setdefault(
rpm['arch'], []).append(dest_arch)
for variant in listings.keys():
nvrs = listings[variant].keys()
#BREW-260: Read allow_src_only flag for the product/version
allow_src_only = Products.get_srconly_flag(compose_dbh,
productLabel, version)
if len(nvrs) == 1:
maps = listings[variant][nvrs[0]].keys()
#BREW-260: check for allow_src_only flag added
if len(maps) == 1 and maps[0] == 'src' and not allow_src_only:
del listings[variant]
return listings
def test_handle_write_signed_rpm(self, activate_session_mock, stdout):
"""Test handle_write_signed_rpm function"""
fake_sigkey = '64dab85d'
arguments = [fake_sigkey]
options = mock.MagicMock()
session = mock.MagicMock()
def get_expect_data(rpm_data):
expected = ''
calls = []
for i, data in enumerate(rpm_data):
nvra = "%(name)s-%(version)s-%(release)s.%(arch)s" % data
expected += "[%d/%d] %s" % (i + 1, len(rpm_data), nvra) + "\n"
calls.append(call(data['id'], fake_sigkey))
return expected, calls
# Case 1, specifies N-V-R-A or N-V-R format RPM
# result: write sigkey to specified RPMs
rpm_data = [GET_RPM_RESULTS[0], GET_RPM_RESULTS[3]]
session.getRPM.side_effect = [
rpm_data[0], # bash-4.4.12-5.fc26.src
koji.GenericError # bash-4.4.12-5.fc26
]
session.getBuild.return_value = {
'package_name': 'bash',
'id': 1,
'version': '4.4.12',
'nvr': 'bash-4.4.12-5.fc26',
'name': 'bash',
'release': '5.fc26'
}
session.listRPMs.return_value = [rpm_data[1]] # bash-4.4.12-5.fc26
args = arguments + ['bash-4.4.12-5.fc26.src', 'bash-4.4.12-5.fc26']
expect_msg, expect_calls = get_expect_data(rpm_data)
handle_write_signed_rpm(options, session, args)
self.assert_console_message(stdout, expect_msg)
session.writeSignedRPM.assert_has_calls(expect_calls)
session.queryRPMSigs.assert_not_called()
# Case 2, with --all option
# result: write sigkey to all RPMS
session.queryRPMSigs.return_value = QUERY_RPM_RESULTS
session.getRPM.side_effect = GET_RPM_RESULTS
expect_msg, expect_calls = get_expect_data(GET_RPM_RESULTS)
handle_write_signed_rpm(options, session, arguments + ['--all'])
self.assert_console_message(stdout, expect_msg)
session.writeSignedRPM.assert_has_calls(expect_calls)
session.queryRPMSigs.assert_called_with(sigkey=fake_sigkey)
session.queryRPMSigs.reset_mock()
# Case 3, with --buildid option
# result: write sigkey to specified build id RPM
rpm_data = [
GET_RPM_RESULTS[0], # build_id = 1
GET_RPM_RESULTS[3]
] # build_id = 1
session.listRPMs.return_value = rpm_data
expect_msg, expect_calls = get_expect_data(rpm_data)
handle_write_signed_rpm(options, session,
arguments + ['--buildid', '1'])
self.assert_console_message(stdout, expect_msg)
session.listRPMs.assert_called_with(1)
session.queryRPMSigs.assert_not_called()
session.writeSignedRPM.assert_has_calls(expect_calls)
session.listRPM.reset_mock()
session.writeSignedRPM.reset_mock()
# Case 4, RPM not exist
# result: raise koji.GenericError
session.getRPM.side_effect = koji.GenericError('fake-get-rpm-error')
session.getBuild.return_value = None
args = arguments + ['gawk-4.1.4-3.fc26.x86_64']
with self.assertRaises(koji.GenericError) as cm:
handle_write_signed_rpm(options, session, args)
self.assertEqual(str(cm.exception),
'No such rpm or build: %s' % args[1])
session.listRPM.assert_not_called()
session.queryRPMSigs.assert_not_called()
session.writeSignedRPM.assert_not_called()
def getTag(self, taginfo, **kw):
"""
Retrieve the given tag from koji.
Args:
taginfo (int or str): The tag you want info about.
strict (bool): If True, raise an Exception if epel tags are queried. Defaults to False.
Returns:
dict or None: A dictionary of tag information, or None if epel is requested and strict
is False.
Raises:
koji.GenericError: If strict is True and epel is requested.
"""
if isinstance(taginfo, int):
taginfo = "f%d" % taginfo
if taginfo.startswith("epel"):
if kw.get("strict", False):
raise koji.GenericError("Invalid tagInfo: '%s'" % taginfo)
else:
return None
# These tags needs to be created
if taginfo in [
"f32-build-side-1234-signing-pending",
"f32-build-side-1234-testing-pending"
]:
return None
# emulate a side-tag response
if taginfo in self._side_tag_ids_names:
for sidetag in self.__side_tags__:
if taginfo in (sidetag['id'], sidetag['name']):
return {
'maven_support': False,
'locked': False,
'name': sidetag['name'],
'extra': {
'sidetag_user': '******',
'sidetag': True
},
'perm': None,
'perm_id': None,
'arches': None,
'maven_include_all': False,
'id': sidetag['id']
}
if kw.get('strict'):
raise koji.GenericError("Invalid tagInfo: '%s'" % taginfo)
else:
return None
return {
'maven_support': False,
'locked': False,
'name': taginfo,
'extra': {},
'perm': None,
'id': 246,
'arches': None,
'maven_include_all': False,
'perm_id': None
}
def listTagged(self, tag, *args, **kwargs):
raise koji.GenericError(f"Invalid tagInfo: {tag!r}")
def wait(self,
subtasks=None,
all=False,
failany=False,
canfail=None,
timeout=None):
"""Wait on subtasks
subtasks is a list of integers (or an integer). If more than one subtask
is specified, then the default behavior is to return when any of those
tasks complete. However, if all is set to True, then it waits for all of
them to complete.
If all and failany are both set to True, then each finished task will
be checked for failure, and a failure will cause all of the unfinished
tasks to be cancelled.
If canfail is given a list of task ids, then those tasks can fail
without affecting the other tasks.
If timeout is specified, then subtasks will be failed and an exception
raised when the timeout is exceeded.
special values:
subtasks = None specify all subtasks
Implementation notes:
The build daemon forks all tasks as separate processes. This function
uses signal.pause to sleep. The main process watches subtasks in
the database and will send the subprocess corresponding to the
subtask a SIGUSR2 to wake it up when subtasks complete.
"""
if canfail is None:
checked = set()
else:
# canfail task are marked as checked
checked = set(canfail)
if isinstance(subtasks, int):
# allow single integer w/o enclosing list
subtasks = [subtasks]
self.session.host.taskSetWait(self.id, subtasks)
self.logger.debug("Waiting on %r" % subtasks)
start = time.time()
while True:
finished, unfinished = self.session.host.taskWait(self.id)
if len(unfinished) == 0:
#all done
break
elif len(finished) > 0:
if all:
if failany:
# we care only about tasks which are not correctly
# finished and in same time not in canfail list
for task in set(finished) - checked:
try:
self.session.getTaskResult(task)
checked.add(task)
except (koji.GenericError,
six.moves.xmlrpc_client.Fault):
self.logger.info(
"task %s failed or was canceled, cancelling unfinished tasks"
% task)
self.session.cancelTaskChildren(self.id)
# reraise the original error now, rather than waiting for
# an error in taskWaitResults()
raise
else:
# at least one done
break
if timeout:
# sleep until timeout is up (or let main process wake us up)
remain = start + timeout - time.time()
if remain > 0:
self.logger.debug("Sleeping for %.1fs", remain)
time.sleep(remain)
# check if we're timed out
duration = time.time() - start
if duration > timeout:
self.logger.info('Subtasks timed out')
self.session.cancelTaskChildren(self.id)
raise koji.GenericError('Subtasks timed out after %.1f '
'seconds' % duration)
else:
# signal handler set by TaskManager.forkTask
self.logger.debug("Pausing...")
signal.pause()
# main process will wake us up with SIGUSR2
self.logger.debug("...waking up")
self.logger.debug("Finished waiting")
if all:
finished = subtasks
return dict(
self.session.host.taskWaitResults(self.id,
finished,
canfail=canfail))
def mock_getTaskResult(task_id):
if task_id == 4:
raise koji.GenericError()
def mock_get_rpm(rpmID, strict=False):
if strict:
raise koji.GenericError('msg')
else:
return None
