def getArchList(self, build_tag, extra=None):
"""Copied from build task"""
# get list of arches to build for
buildconfig = self.session.getBuildConfig(build_tag, event=self.event_id)
arches = buildconfig['arches']
if not arches:
# XXX - need to handle this better
raise koji.BuildError("No arches for tag %(name)s [%(id)s]" % buildconfig)
tag_archlist = [koji.canonArch(a) for a in arches.split()]
self.logger.debug('arches: %s', arches)
if extra:
self.logger.debug('Got extra arches: %s', extra)
arches = "%s %s" % (arches, extra)
archlist = arches.split()
self.logger.debug('base archlist: %r' % archlist)
override = self.opts.get('arch_override')
if (self.opts.get('isolated') or self.opts.get('scratch')) and override:
# only honor override for scratch builds
self.logger.debug('arch override: %s', override)
archlist = override.split()
elif override:
raise koji.BuildError("arch-override is only allowed for isolated or scratch builds")
archdict = {}
for a in archlist:
# Filter based on canonical arches for tag
# This prevents building for an arch that we can't handle
if a == 'noarch' or koji.canonArch(a) in tag_archlist:
archdict[a] = 1
if not archdict:
raise koji.BuildError("No matching arches were found")
return list(archdict.keys())
def getArchList(self, build_tag, extra=None):
"""Copied from build task"""
# get list of arches to build for
buildconfig = self.session.getBuildConfig(build_tag, event=self.event_id)
arches = buildconfig['arches']
if not arches:
# XXX - need to handle this better
raise koji.BuildError("No arches for tag %(name)s [%(id)s]" % buildconfig)
tag_archlist = [koji.canonArch(a) for a in arches.split()]
self.logger.debug('arches: %s', arches)
if extra:
self.logger.debug('Got extra arches: %s', extra)
arches = "%s %s" % (arches, extra)
archlist = arches.split()
self.logger.debug('base archlist: %r', archlist)
override = self.opts.get('arch_override')
if (self.opts.get('isolated') or self.opts.get('scratch')) and override:
# only honor override for scratch builds
self.logger.debug('arch override: %s', override)
archlist = override.split()
elif override:
raise koji.BuildError("arch-override is only allowed for isolated or scratch builds")
archdict = {}
for a in archlist:
# Filter based on canonical arches for tag
# This prevents building for an arch that we can't handle
if a == 'noarch' or koji.canonArch(a) in tag_archlist:
archdict[a] = 1
if not archdict:
raise koji.BuildError("No matching arches were found")
return list(archdict.keys())
def test_exclusive_arch(self):
tag_arches = [koji.canonArch(a) for a in self.getBuildConfig()['arches'].split()]
# random choice involved, so we repeat this a few times
for i in range(20):
self.readSRPMHeader.return_value = FakeHeader(
buildarchs=['noarch'], exclusivearch=['noarch', 'armv7hl'], excludearch=[])
result = self.handler.choose_taskarch('noarch', 'srpm', 'build_tag')
self.assertNotEqual(result, 'noarch')
self.assertEqual(result, koji.canonArch('armv7hl'))
self.assertIn(result, tag_arches)
def test_exclusive_arch(self):
tag_arches = [koji.canonArch(a) for a in self.getBuildConfig()['arches'].split()]
# random choice involved, so we repeat this a few times
for i in range(20):
self.readSRPMHeader.return_value = FakeHeader(
buildarchs=['noarch'], exclusivearch=['noarch', 'armv7hl'], excludearch=[])
result = self.handler.choose_taskarch('noarch', 'srpm', 'build_tag')
self.assertNotEqual(result, 'noarch')
self.assertEqual(result, koji.canonArch('armv7hl'))
self.assertIn(result, tag_arches)
def get_srpm_arches(koji_session, all_arches, nvra, arch_override=None,
build_arches=None):
"""
Compute architectures that should be used for a build. Computation is based on the one
in Koji (kojid/getArchList).
:param koji_session: Koji session to be used for the query
:param all_arches: List of all arches obtained from `get_koji_arches`
:param nvra: NVRA dict of the SRPM
:param arch_override: User specified arch override
:param build_arches: List of allowed arches for building. Taken from config by default
:return: Set of architectures that can be passed to `koji_scratch_build`. May be
empty, in which case no build should be submitted.
"""
archlist = all_arches
tag_archlist = {koji.canonArch(a) for a in archlist}
headers = koji_session.getRPMHeaders(
rpmID=nvra,
headers=['BUILDARCHS', 'EXCLUDEARCH', 'EXCLUSIVEARCH'],
)
if not headers:
return None
buildarchs = headers.get('BUILDARCHS', [])
exclusivearch = headers.get('EXCLUSIVEARCH', [])
excludearch = headers.get('EXCLUDEARCH', [])
if buildarchs:
archlist = buildarchs
if exclusivearch:
archlist = [arch for arch in archlist if arch in exclusivearch]
if excludearch:
archlist = [arch for arch in archlist if arch not in excludearch]
if ('noarch' not in excludearch and
('noarch' in buildarchs or 'noarch' in exclusivearch)):
archlist.append('noarch')
if arch_override:
# we also allow inverse overrides
if arch_override.startswith('^'):
excluded = {koji.canonArch(arch) for arch in arch_override[1:].split()}
archlist = [arch for arch in archlist if koji.canonArch(arch) not in excluded]
else:
archlist = arch_override.split()
if not build_arches:
build_arches = get_config('koji_config').get('build_arches')
build_arches = {koji.canonArch(arch) for arch in build_arches}
allowed_arches = tag_archlist & build_arches
arches = set()
for arch in archlist:
if arch == 'noarch' or koji.canonArch(arch) in allowed_arches:
arches.add(arch)
return arches
def get_srpm_arches(koji_session,
all_arches,
nvra,
arch_override=None,
build_arches=None):
# compute arches the same way as koji
# see kojid/getArchList
archlist = all_arches
tag_archlist = {koji.canonArch(a) for a in archlist}
headers = koji_session.getRPMHeaders(
rpmID=nvra,
headers=['BUILDARCHS', 'EXCLUDEARCH', 'EXCLUSIVEARCH'],
)
if not headers:
return None
buildarchs = headers.get('BUILDARCHS', [])
exclusivearch = headers.get('EXCLUSIVEARCH', [])
excludearch = headers.get('EXCLUDEARCH', [])
if buildarchs:
archlist = buildarchs
if exclusivearch:
archlist = [arch for arch in archlist if arch in exclusivearch]
if excludearch:
archlist = [arch for arch in archlist if arch not in excludearch]
if ('noarch' not in excludearch
and ('noarch' in buildarchs or 'noarch' in exclusivearch)):
archlist.append('noarch')
if arch_override:
# we also allow inverse overrides
if arch_override.startswith('^'):
excluded = {
koji.canonArch(arch)
for arch in arch_override[1:].split()
}
archlist = [
arch for arch in archlist
if koji.canonArch(arch) not in excluded
]
else:
archlist = arch_override.split()
if not build_arches:
build_arches = get_config('koji_config').get('build_arches')
build_arches = {koji.canonArch(arch) for arch in build_arches}
allowed_arches = tag_archlist & build_arches
arches = set()
for arch in archlist:
if arch == 'noarch' or koji.canonArch(arch) in allowed_arches:
arches.add(arch)
return arches
def find_arch(self, arch, host, tag, preferred_arch=None):
"""
For noarch tasks, find a canonical arch that is supported by both the host and tag.
If the arch is anything other than noarch, return it unmodified.
If preferred_arch is set, try to get it, but not fail on that
"""
if arch != "noarch":
return arch
# We need a concrete arch. Pick one that:
# a) this host can handle
# b) the build tag can support
# c) is canonical
host_arches = host['arches']
if not host_arches:
raise koji.BuildError("No arch list for this host: %s" %
host['name'])
tag_arches = tag['arches']
if not tag_arches:
raise koji.BuildError("No arch list for tag: %s" % tag['name'])
# index canonical host arches
host_arches = set([koji.canonArch(a) for a in host_arches.split()])
# index canonical tag arches
tag_arches = set([koji.canonArch(a) for a in tag_arches.split()])
# find the intersection of host and tag arches
common_arches = list(host_arches & tag_arches)
if common_arches:
if preferred_arch and preferred_arch in common_arches:
self.logger.info(
'Valid arches: %s, using preferred: %s' %
(' '.join(sorted(common_arches)), preferred_arch))
return preferred_arch
# pick one of the common arches randomly
# need to re-seed the prng or we'll get the same arch every time,
# because we just forked from a common parent
random.seed()
arch = random.choice(common_arches)
self.logger.info('Valid arches: %s, using: %s' %
(' '.join(sorted(common_arches)), arch))
return arch
else:
# no overlap
raise koji.BuildError(
"host %s (%s) does not support any arches"
" of tag %s (%s)" %
(host['name'], ', '.join(sorted(host_arches)), tag['name'],
', '.join(sorted(tag_arches))))
def test_literal_arch(self):
self.options.literal_task_arches = 'ARCH'
tag_arches = [
koji.canonArch(a) for a in self.getBuildConfig()['arches'].split()
]
result = self.handler.choose_taskarch('ARCH', 'srpm', 'build_tag')
self.assertEqual(result, 'ARCH')
def runroot(tagInfo, arch, command, channel=None, **opts):
""" Create a runroot task """
context.session.assertPerm('runroot')
taskopts = {
'priority': 15,
'arch': arch,
}
taskopts['channel'] = channel or 'runroot'
if arch == 'noarch':
#not all arches can generate a proper buildroot for all tags
tag = kojihub.get_tag(tagInfo)
if not tag['arches']:
raise koji.GenericError, 'no arches defined for tag %s' % tag['name']
#get all known arches for the system
fullarches = kojihub.get_all_arches()
tagarches = tag['arches'].split()
# If our tag can't do all arches, then we need to
# specify one of the arches it can do.
if set(fullarches) - set(tagarches):
chanarches = get_channel_arches(taskopts['channel'])
choices = [x for x in tagarches if x in chanarches]
if not choices:
raise koji.GenericError, 'no common arches for tag/channel: %s/%s' \
% (tagInfo, taskopts['channel'])
taskopts['arch'] = koji.canonArch(random.choice(choices))
args = koji.encode_args(tagInfo, arch, command,**opts)
return kojihub.make_task('runroot', args, **taskopts)
def runroot(tagInfo, arch, command, channel=None, **opts):
""" Create a runroot task """
context.session.assertPerm('runroot')
taskopts = {
'priority': 15,
'arch': arch,
}
taskopts['channel'] = channel or 'runroot'
if arch == 'noarch':
#not all arches can generate a proper buildroot for all tags
tag = kojihub.get_tag(tagInfo)
if not tag['arches']:
raise koji.GenericError('no arches defined for tag %s' %
tag['name'])
#get all known arches for the system
fullarches = kojihub.get_all_arches()
tagarches = tag['arches'].split()
# If our tag can't do all arches, then we need to
# specify one of the arches it can do.
if set(fullarches) - set(tagarches):
chanarches = get_channel_arches(taskopts['channel'])
choices = [x for x in tagarches if x in chanarches]
if not choices:
raise koji.GenericError('no common arches for tag/channel: %s/%s' \
% (tagInfo, taskopts['channel']))
taskopts['arch'] = koji.canonArch(random.choice(choices))
args = koji.encode_args(tagInfo, arch, command, **opts)
return kojihub.make_task('runroot', args, **taskopts)
def test_too_exclusive(self):
tag_arches = [koji.canonArch(a) for a in self.getBuildConfig()['arches'].split()]
# random choice involved, so we repeat this a few times
for i in range(20):
self.readSRPMHeader.return_value = FakeHeader(
buildarchs=['noarch'], exclusivearch=['missing_arch'], excludearch=[])
with self.assertRaises(koji.BuildError):
result = self.handler.choose_taskarch('noarch', 'srpm', 'build_tag')
def get_channel_arches(channel):
"""determine arches available in channel"""
chan = context.handlers.call('getChannel', channel, strict=True)
ret = {}
for host in context.handlers.call('listHosts', channelID=chan['id'], enabled=True):
for a in host['arches'].split():
ret[koji.canonArch(a)] = 1
return ret
def test_all_excluded(self):
tag_arches = [koji.canonArch(a) for a in self.getBuildConfig()['arches'].split()]
# random choice involved, so we repeat this a few times
for i in range(20):
self.readSRPMHeader.return_value = FakeHeader(
buildarchs=['noarch'], exclusivearch=[], excludearch=tag_arches)
with self.assertRaises(koji.BuildError):
result = self.handler.choose_taskarch('noarch', 'srpm', 'build_tag')
def get_channel_arches(channel):
"""determine arches available in channel"""
chan = context.handlers.call('getChannel', channel, strict=True)
ret = {}
for host in context.handlers.call('listHosts', channelID=chan['id'], enabled=True):
for a in host['arches'].split():
ret[koji.canonArch(a)] = 1
return ret
def test_excluded_irrelevant(self):
tag_arches = [
koji.canonArch(a) for a in self.getBuildConfig()['arches'].split()
]
self.readSRPMHeader.return_value = FakeHeader(
buildarchs=['noarch'],
exclusivearch=[],
excludearch=['nosucharch'])
result = self.handler.choose_taskarch('noarch', 'srpm', 'build_tag')
self.assertEqual(result, 'noarch')
def find_arch(self, arch, host, tag):
"""
For noarch tasks, find a canonical arch that is supported by both the host and tag.
If the arch is anything other than noarch, return it unmodified.
"""
if arch != "noarch":
return arch
# We need a concrete arch. Pick one that:
# a) this host can handle
# b) the build tag can support
# c) is canonical
host_arches = host['arches']
if not host_arches:
raise koji.BuildError("No arch list for this host: %s" %
host['name'])
tag_arches = tag['arches']
if not tag_arches:
raise koji.BuildError("No arch list for tag: %s" % tag['name'])
# index canonical host arches
host_arches = set([koji.canonArch(a) for a in host_arches.split()])
# index canonical tag arches
tag_arches = set([koji.canonArch(a) for a in tag_arches.split()])
# find the intersection of host and tag arches
common_arches = list(host_arches & tag_arches)
if common_arches:
# pick one of the common arches randomly
# need to re-seed the prng or we'll get the same arch every time,
# because we just forked from a common parent
random.seed()
arch = random.choice(common_arches)
self.logger.info('Valid arches: %s, using: %s' %
(' '.join(sorted(common_arches)), arch))
return arch
else:
# no overlap
raise koji.BuildError("host %s (%s) does not support any arches"
" of tag %s (%s)" %
(host['name'],
', '.join(sorted(host_arches)),
tag['name'],
', '.join(sorted(tag_arches))))
def takeTask(self,task):
"""Attempt to open the specified task
Returns True if successful, False otherwise
"""
self.logger.info("Attempting to take task %s" % task['id'])
if task['method'] in ('buildArch', 'buildSRPMFromSCM', 'buildMaven') and \
task['arch'] == 'noarch':
task_info = self.session.getTaskInfo(task['id'], request=True)
if task['method'] == 'buildMaven':
tag = task_info['request'][1]
else:
tag_id = task_info['request'][1]
tag = self.session.getTag(tag_id)
if tag and tag['arches']:
tag_arches = [koji.canonArch(a) for a in tag['arches'].split()]
host_arches = self.hostdata['arches'].split()
if not set(tag_arches).intersection(host_arches):
self.logger.info('Skipping task %s (%s) because tag arches (%s) and ' \
'host arches (%s) are disjoint' % \
(task['id'], task['method'],
', '.join(tag_arches), ', '.join(host_arches)))
return False
data = self.session.host.openTask(task['id'])
if data is None:
self.logger.warn("Could not open")
return False
if not data.has_key('request') or data['request'] is None:
self.logger.warn("Task '%s' has no request" % task['id'])
return False
id = data['id']
request = data['request']
self.tasks[id] = data
params, method = xmlrpclib.loads(request)
if self.handlers.has_key(method):
handlerClass = self.handlers[method]
else:
raise koji.GenericError, "No handler found for method '%s'" % method
handler = handlerClass(id,method,params,self.session,self.options)
# set weight
self.session.host.setTaskWeight(id,handler.weight())
if handler.Foreground:
self.logger.info("running task in foreground")
handler.setManager(self)
self.runTask(handler)
else:
pid, session_id = self.forkTask(handler)
self.pids[id] = pid
self.subsessions[id] = session_id
return True
def takeTask(self,task):
"""Attempt to open the specified task
Returns True if successful, False otherwise
"""
self.logger.info("Attempting to take task %s" % task['id'])
if task['method'] in ('buildArch', 'buildSRPMFromSCM', 'buildMaven') and \
task['arch'] == 'noarch':
task_info = self.session.getTaskInfo(task['id'], request=True)
if task['method'] == 'buildMaven':
tag = task_info['request'][1]
else:
tag_id = task_info['request'][1]
tag = self.session.getTag(tag_id)
if tag and tag['arches']:
tag_arches = [koji.canonArch(a) for a in tag['arches'].split()]
host_arches = self.hostdata['arches'].split()
if not set(tag_arches).intersection(host_arches):
self.logger.info('Skipping task %s (%s) because tag arches (%s) and ' \
'host arches (%s) are disjoint' % \
(task['id'], task['method'],
', '.join(tag_arches), ', '.join(host_arches)))
return False
data = self.session.host.openTask(task['id'])
if data is None:
self.logger.warn("Could not open")
return False
if not data.has_key('request') or data['request'] is None:
self.logger.warn("Task '%s' has no request" % task['id'])
return False
id = data['id']
request = data['request']
self.tasks[id] = data
params, method = xmlrpclib.loads(request)
if self.handlers.has_key(method):
handlerClass = self.handlers[method]
else:
raise koji.GenericError, "No handler found for method '%s'" % method
handler = handlerClass(id,method,params,self.session,self.options)
# set weight
self.session.host.setTaskWeight(id,handler.weight())
if handler.Foreground:
self.logger.info("running task in foreground")
handler.setManager(self)
self.runTask(handler)
else:
pid, session_id = self.forkTask(handler)
self.pids[id] = pid
self.subsessions[id] = session_id
return True
def test_literal_arch(self):
self.options.literal_task_arches = 'ARCH'
tag_arches = [koji.canonArch(a) for a in self.getBuildConfig()['arches'].split()]
result = self.handler.choose_taskarch('ARCH', 'srpm', 'build_tag')
self.assertEqual(result, 'ARCH')
def arches_for_config(buildconfig: Dict):
archstr = buildconfig["arches"]
if not archstr:
name = buildconfig["name"]
raise koji.BuildError(f"Missing arches for tag '%{name}'")
return set(koji.canonArch(a) for a in archstr.split())
def test_binary_arches(self):
for arch in [
'i386', 'i686', 'x86_64', 'ppc', 'ppc64le', 's390', 's390x'
]:
result = self.handler.choose_taskarch(arch, 'srpm', 'build_tag')
self.assertEqual(result, koji.canonArch(arch))
def setUp(self):
self.topdir = tempfile.mkdtemp()
self.rinfo = {
'create_event': 2915,
'create_ts': 1487256924.72718,
'creation_time': '2017-02-16 14:55:24.727181',
'id': 47,
'state': 0, # INIT
'tag_id': 2,
'tag_name': 'my-tag'}
self.arch = 'x86_64'
# set up a fake koji topdir
# koji.pathinfo._topdir = self.topdir
mock.patch('koji.pathinfo._topdir', new=self.topdir).start()
repodir = koji.pathinfo.distrepo(self.rinfo['id'], self.rinfo['tag_name'])
archdir = "%s/%s" % (repodir, koji.canonArch(self.arch))
os.makedirs(archdir)
self.uploadpath = 'UNITTEST'
workdir = koji.pathinfo.work()
uploaddir = "%s/%s" % (workdir, self.uploadpath)
os.makedirs(uploaddir)
# place some test files
self.files = ['drpms/foo.drpm', 'repodata/repomd.xml']
self.expected = ['x86_64/drpms/foo.drpm', 'x86_64/repodata/repomd.xml']
for fn in self.files:
path = os.path.join(uploaddir, fn)
koji.ensuredir(os.path.dirname(path))
with open(path, 'w') as fo:
fo.write('%s' % os.path.basename(fn))
# generate pkglist file
self.files.append('pkglist')
plist = os.path.join(uploaddir, 'pkglist')
nvrs = ['aaa-1.0-2', 'bbb-3.0-5', 'ccc-8.0-13','ddd-21.0-34']
self.rpms = {}
self.builds ={}
self.key = '4c8da725'
with open(plist, 'w') as f_pkglist:
for nvr in nvrs:
binfo = koji.parse_NVR(nvr)
rpminfo = binfo.copy()
rpminfo['arch'] = 'x86_64'
builddir = koji.pathinfo.build(binfo)
relpath = koji.pathinfo.signed(rpminfo, self.key)
path = os.path.join(builddir, relpath)
koji.ensuredir(os.path.dirname(path))
basename = os.path.basename(path)
with open(path, 'w') as fo:
fo.write('%s' % basename)
f_pkglist.write(path)
f_pkglist.write('\n')
self.expected.append('x86_64/Packages/%s/%s' % (basename[0], basename))
build_id = len(self.builds) + 10000
rpm_id = len(self.rpms) + 20000
binfo['id'] = build_id
rpminfo['build_id'] = build_id
rpminfo['id'] = rpm_id
rpminfo['sigkey'] = self.key
rpminfo['size'] = 1024
rpminfo['payloadhash'] = 'helloworld'
self.builds[build_id] = binfo
self.rpms[rpm_id] = rpminfo
# write kojipkgs
kojipkgs = {}
for rpminfo in self.rpms.values():
bnp = '%(name)s-%(version)s-%(release)s.%(arch)s.rpm' % rpminfo
kojipkgs[bnp] = rpminfo
with open("%s/kojipkgs" % uploaddir, "w") as fp:
json.dump(kojipkgs, fp, indent=4)
self.files.append('kojipkgs')
# write manifest
with open("%s/repo_manifest" % uploaddir, "w") as fp:
json.dump(self.files, fp, indent=4)
# mocks
self.repo_info = mock.patch('kojihub.repo_info').start()
self.repo_info.return_value = self.rinfo.copy()
self.get_rpm = mock.patch('kojihub.get_rpm').start()
self.get_build = mock.patch('kojihub.get_build').start()
self.get_rpm.side_effect = self.our_get_rpm
self.get_build.side_effect = self.our_get_build
def get_srpm_arches(koji_session,
all_arches,
nvra,
arch_override=None,
build_arches=None):
"""
Compute architectures that should be used for a build. Computation is based on the one
in Koji (kojid/getArchList).
:param koji_session: Koji session to be used for the query
:param all_arches: List of all arches obtained from `get_koji_arches`
:param nvra: NVRA dict of the SRPM
:param arch_override: User specified arch override
:param build_arches: List of allowed arches for building. Taken from config by default
:return: Set of architectures that can be passed to `koji_scratch_build`. May be
empty, in which case no build should be submitted.
"""
archlist = all_arches
tag_archlist = {koji.canonArch(a) for a in archlist}
headers = koji_session.getRPMHeaders(
rpmID=nvra,
headers=['BUILDARCHS', 'EXCLUDEARCH', 'EXCLUSIVEARCH'],
)
if not headers:
return None
buildarchs = headers.get('BUILDARCHS', [])
exclusivearch = headers.get('EXCLUSIVEARCH', [])
excludearch = headers.get('EXCLUDEARCH', [])
if buildarchs:
archlist = buildarchs
if exclusivearch:
archlist = [arch for arch in archlist if arch in exclusivearch]
if excludearch:
archlist = [arch for arch in archlist if arch not in excludearch]
if ('noarch' not in excludearch
and ('noarch' in buildarchs or 'noarch' in exclusivearch)):
archlist.append('noarch')
if arch_override:
# we also allow inverse overrides
if arch_override.startswith('^'):
excluded = {
koji.canonArch(arch)
for arch in arch_override[1:].split()
}
archlist = [
arch for arch in archlist
if koji.canonArch(arch) not in excluded
]
else:
archlist = arch_override.split()
if not build_arches:
build_arches = get_config('koji_config').get('build_arches')
build_arches = {koji.canonArch(arch) for arch in build_arches}
allowed_arches = tag_archlist & build_arches
arches = set()
for arch in archlist:
if arch == 'noarch' or koji.canonArch(arch) in allowed_arches:
arches.add(arch)
return arches
def setUp(self):
self.topdir = tempfile.mkdtemp()
self.rinfo = {
'create_event': 2915,
'create_ts': 1487256924.72718,
'creation_time': '2017-02-16 14:55:24.727181',
'id': 47,
'state': 0, # INIT
'tag_id': 2,
'tag_name': 'my-tag'
}
self.arch = 'x86_64'
# set up a fake koji topdir
# koji.pathinfo._topdir = self.topdir
mock.patch('koji.pathinfo._topdir', new=self.topdir).start()
repodir = koji.pathinfo.distrepo(self.rinfo['id'],
self.rinfo['tag_name'])
archdir = "%s/%s" % (repodir, koji.canonArch(self.arch))
os.makedirs(archdir)
self.uploadpath = 'UNITTEST'
workdir = koji.pathinfo.work()
uploaddir = "%s/%s" % (workdir, self.uploadpath)
os.makedirs(uploaddir)
# place some test files
self.files = ['drpms/foo.drpm', 'repodata/repomd.xml']
self.expected = ['x86_64/drpms/foo.drpm', 'x86_64/repodata/repomd.xml']
for fn in self.files:
path = os.path.join(uploaddir, fn)
koji.ensuredir(os.path.dirname(path))
with open(path, 'w') as fo:
fo.write('%s' % os.path.basename(fn))
# generate pkglist file
self.files.append('pkglist')
plist = os.path.join(uploaddir, 'pkglist')
nvrs = ['aaa-1.0-2', 'bbb-3.0-5', 'ccc-8.0-13', 'ddd-21.0-34']
self.rpms = {}
self.builds = {}
self.key = '4c8da725'
with open(plist, 'w') as f_pkglist:
for nvr in nvrs:
binfo = koji.parse_NVR(nvr)
rpminfo = binfo.copy()
rpminfo['arch'] = 'x86_64'
builddir = koji.pathinfo.build(binfo)
relpath = koji.pathinfo.signed(rpminfo, self.key)
path = os.path.join(builddir, relpath)
koji.ensuredir(os.path.dirname(path))
basename = os.path.basename(path)
with open(path, 'w') as fo:
fo.write('%s' % basename)
f_pkglist.write(path)
f_pkglist.write('\n')
self.expected.append('x86_64/Packages/%s/%s' %
(basename[0], basename))
build_id = len(self.builds) + 10000
rpm_id = len(self.rpms) + 20000
binfo['id'] = build_id
rpminfo['build_id'] = build_id
rpminfo['id'] = rpm_id
rpminfo['sigkey'] = self.key
rpminfo['size'] = 1024
rpminfo['payloadhash'] = 'helloworld'
self.builds[build_id] = binfo
self.rpms[rpm_id] = rpminfo
# write kojipkgs
kojipkgs = {}
for rpminfo in self.rpms.values():
bnp = '%(name)s-%(version)s-%(release)s.%(arch)s.rpm' % rpminfo
kojipkgs[bnp] = rpminfo
with open("%s/kojipkgs" % uploaddir, "w") as fp:
json.dump(kojipkgs, fp, indent=4)
self.files.append('kojipkgs')
# write manifest
with open("%s/repo_manifest" % uploaddir, "w") as fp:
json.dump(self.files, fp, indent=4)
# mocks
self.repo_info = mock.patch('kojihub.repo_info').start()
self.repo_info.return_value = self.rinfo.copy()
self.get_rpm = mock.patch('kojihub.get_rpm').start()
self.get_build = mock.patch('kojihub.get_build').start()
self.get_rpm.side_effect = self.our_get_rpm
self.get_build.side_effect = self.our_get_build
def handle_kiwi_build(goptions, session, args):
"[build] Run a command in a buildroot"
usage = "usage: %prog kiwi-build [options] <target> <description_scm> <description_path>"
usage += "\n(Specify the --help global option for a list of other help options)"
parser = OptionParser(usage=usage)
parser.add_option("--scratch",
action="store_true",
default=False,
help="Perform a scratch build")
parser.add_option(
"--repo",
action="append",
help="Specify a repo that will override the repo used to install "
"RPMs in the image. May be used multiple times. The "
"build tag repo associated with the target is the default.")
parser.add_option("--noprogress",
action="store_true",
help="Do not display progress of the upload")
parser.add_option("--kiwi-profile",
action="store",
default=None,
help="Select profile from description file")
parser.add_option("--can-fail",
action="store",
dest="optional_arches",
metavar="ARCH1,ARCH2,...",
default="",
help="List of archs which are not blocking for build "
"(separated by commas.")
parser.add_option("--arch",
action="append",
dest="arches",
default=[],
help="Limit arches to this subset")
parser.add_option("--nowait",
action="store_false",
dest="wait",
default=True)
parser.add_option(
"--wait",
action="store_true",
help="Wait on the image creation, even if running in the background")
(options, args) = parser.parse_args(args)
if len(args) != 3:
parser.error("Incorrect number of arguments")
assert False # pragma: no cover
target, scm, path = args
activate_session(session, goptions)
kwargs = {
'scratch':
options.scratch,
'optional_arches': [
canonArch(arch) for arch in options.optional_arches.split(',')
if arch
],
'profile':
options.kiwi_profile,
}
arches = []
if options.arches:
arches = [canonArch(arch) for arch in options.arches]
task_id = session.kiwiBuild(target=target,
arches=arches,
desc_url=scm,
desc_path=path,
**kwargs)
if not goptions.quiet:
print("Created task: %d" % task_id)
print("Task info: %s/taskinfo?taskID=%s" % (goptions.weburl, task_id))
if options.wait or (options.wait is None and not _running_in_bg()):
session.logout()
return watch_tasks(session, [task_id],
quiet=goptions.quiet,
poll_interval=goptions.poll_interval,
topurl=goptions.topurl)
def test_excluded_irrelevant(self):
tag_arches = [koji.canonArch(a) for a in self.getBuildConfig()['arches'].split()]
self.readSRPMHeader.return_value = FakeHeader(
buildarchs=['noarch'], exclusivearch=[], excludearch=['nosucharch'])
result = self.handler.choose_taskarch('noarch', 'srpm', 'build_tag')
self.assertEqual(result, 'noarch')
def setUp(self):
self.topdir = tempfile.mkdtemp()
self.rinfo = {
'create_event': 2915,
'create_ts': 1487256924.72718,
'creation_time': '2017-02-16 14:55:24.727181',
'id': 47,
'state': 1,
'tag_id': 2,
'tag_name': 'my-tag'
}
self.arch = 'x86_64'
# set up a fake koji topdir
# koji.pathinfo._topdir = self.topdir
mock.patch('koji.pathinfo._topdir', new=self.topdir).start()
repodir = koji.pathinfo.distrepo(self.rinfo['id'],
self.rinfo['tag_name'])
archdir = "%s/%s" % (repodir, koji.canonArch(self.arch))
os.makedirs(archdir)
self.uploadpath = 'UNITTEST'
workdir = koji.pathinfo.work()
uploaddir = "%s/%s" % (workdir, self.uploadpath)
os.makedirs(uploaddir)
# place some test files
self.files = ['foo.drpm', 'repomd.xml']
self.expected = ['x86_64/drpms/foo.drpm', 'x86_64/repodata/repomd.xml']
for fn in self.files:
path = os.path.join(uploaddir, fn)
koji.ensuredir(os.path.dirname(path))
with open(path, 'w') as fo:
fo.write('%s' % fn)
# generate pkglist file and sigmap
self.files.append('pkglist')
plist = os.path.join(uploaddir, 'pkglist')
nvrs = ['aaa-1.0-2', 'bbb-3.0-5', 'ccc-8.0-13', 'ddd-21.0-34']
self.sigmap = []
self.rpms = {}
self.builds = {}
self.key = '4c8da725'
with open(plist, 'w') as f_pkglist:
for nvr in nvrs:
binfo = koji.parse_NVR(nvr)
rpminfo = binfo.copy()
rpminfo['arch'] = 'x86_64'
builddir = koji.pathinfo.build(binfo)
relpath = koji.pathinfo.signed(rpminfo, self.key)
path = os.path.join(builddir, relpath)
koji.ensuredir(os.path.dirname(path))
basename = os.path.basename(path)
with open(path, 'w') as fo:
fo.write('%s' % basename)
f_pkglist.write(path)
f_pkglist.write('\n')
self.expected.append('x86_64/%s/%s' % (basename[0], basename))
build_id = len(self.builds) + 10000
rpm_id = len(self.rpms) + 20000
binfo['id'] = build_id
rpminfo['build_id'] = build_id
rpminfo['id'] = rpm_id
self.builds[build_id] = binfo
self.rpms[rpm_id] = rpminfo
self.sigmap.append([rpm_id, self.key])
# mocks
self.repo_info = mock.patch('kojihub.repo_info').start()
self.repo_info.return_value = self.rinfo.copy()
self.get_rpm = mock.patch('kojihub.get_rpm').start()
self.get_build = mock.patch('kojihub.get_build').start()
self.get_rpm.side_effect = self.our_get_rpm
self.get_build.side_effect = self.our_get_build
def handler(self, root, arch, command, keep=False, packages=[], mounts=[], repo_id=None, skip_setarch=False, weight=None, upload_logs=None, new_chroot=False):
"""Create a buildroot and run a command (as root) inside of it
Command may be a string or a list.
Returns a message indicating success if the command was successful, and
raises an error otherwise. Command output will be available in
runroot.log in the task output directory on the hub.
skip_setarch is a rough approximation of an old hack
the keep option is not used. keeping for compatibility for now...
upload_logs is list of absolute paths which will be uploaded for
archiving on hub. It always consists of /tmp/runroot.log, but can be
used for additional logs (pungi.log, etc.)
"""
if weight is not None:
weight = max(weight, 0.5)
self.session.host.setTaskWeight(self.id, weight)
#noarch is funny
if arch == "noarch":
#we need a buildroot arch. Pick one that:
# a) this host can handle
# b) the build tag can support
# c) is canonical
host_arches = self.session.host.getHost()['arches']
if not host_arches:
raise koji.BuildError("No arch list for this host")
tag_arches = self.session.getBuildConfig(root)['arches']
if not tag_arches:
raise koji.BuildError("No arch list for tag: %s" % root)
#index canonical host arches
host_arches = dict([(koji.canonArch(a), 1) for a in host_arches.split()])
#pick the first suitable match from tag's archlist
for br_arch in tag_arches.split():
br_arch = koji.canonArch(br_arch)
if br_arch in host_arches:
#we're done
break
else:
#no overlap
raise koji.BuildError("host does not match tag arches: %s (%s)" % (root, tag_arches))
else:
br_arch = arch
if repo_id:
repo_info = self.session.repoInfo(repo_id, strict=True)
if repo_info['tag_name'] != root:
raise koji.BuildError("build tag (%s) does not match repo tag (%s)" % (root, repo_info['tag_name']))
if repo_info['state'] not in (koji.REPO_STATES['READY'], koji.REPO_STATES['EXPIRED']):
raise koji.BuildError("repos in the %s state may not be used by runroot" % koji.REPO_STATES[repo_info['state']])
else:
repo_info = self.session.getRepo(root)
if not repo_info:
#wait for it
task_id = self.session.host.subtask(method='waitrepo',
arglist=[root, None, None],
parent=self.id)
repo_info = self.wait(task_id)[task_id]
broot = BuildRoot(self.session, self.options, root, br_arch, self.id, repo_id=repo_info['id'], setup_dns=True)
broot.workdir = self.workdir
broot.init()
rootdir = broot.rootdir()
#workaround for rpm oddness
os.system('rm -f "%s"/var/lib/rpm/__db.*' % rootdir)
#update buildroot state (so that updateBuildRootList() will work)
self.session.host.setBuildRootState(broot.id, 'BUILDING')
try:
if packages:
#pkglog = '%s/%s' % (broot.resultdir(), 'packages.log')
pkgcmd = ['--install'] + packages
status = broot.mock(pkgcmd)
self.session.host.updateBuildRootList(broot.id, broot.getPackageList())
if not isSuccess(status):
raise koji.BuildrootError(parseStatus(status, pkgcmd))
if isinstance(command, str):
cmdstr = command
else:
#we were passed an arglist
#we still have to run this through the shell (for redirection)
#but we can preserve the list structure precisely with careful escaping
cmdstr = ' '.join(["'%s'" % arg.replace("'", r"'\''") for arg in command])
# A nasty hack to put command output into its own file until mock can be
# patched to do something more reasonable than stuff everything into build.log
cmdargs = ['/bin/sh', '-c', "{ %s; } < /dev/null 2>&1 | /usr/bin/tee /builddir/runroot.log; exit ${PIPESTATUS[0]}" % cmdstr]
# always mount /mnt/redhat (read-only)
# always mount /mnt/iso (read-only)
# also need /dev bind mount
self.do_mounts(rootdir, [self._get_path_params(x) for x in self.config['default_mounts']])
self.do_extra_mounts(rootdir, mounts)
mock_cmd = ['chroot']
if new_chroot:
mock_cmd.append('--new-chroot')
if skip_setarch:
#we can't really skip it, but we can set it to the current one instead of of the chroot one
myarch = platform.uname()[5]
mock_cmd.extend(['--arch', myarch])
mock_cmd.append('--')
mock_cmd.extend(cmdargs)
rv = broot.mock(mock_cmd)
log_paths = ['/builddir/runroot.log']
if upload_logs is not None:
log_paths += upload_logs
for log_path in log_paths:
self.uploadFile(rootdir + log_path)
finally:
# mock should umount its mounts, but it will not handle ours
self.undo_mounts(rootdir, fatal=False)
broot.expire()
if isinstance(command, str):
cmdlist = command.split()
else:
cmdlist = command
cmdlist = [param for param in cmdlist if '=' not in param]
if cmdlist:
cmd = os.path.basename(cmdlist[0])
else:
cmd = '(none)'
if isSuccess(rv):
return '%s completed successfully' % cmd
else:
raise koji.BuildrootError(parseStatus(rv, cmd))
def test_binary_arches(self):
for arch in ['i386', 'i686', 'x86_64', 'ppc', 'ppc64le', 's390',
's390x']:
result = self.handler.choose_taskarch(arch, 'srpm', 'build_tag')
self.assertEqual(result, koji.canonArch(arch))
def handler(
self,
root,
arch,
command,
keep=False,
packages=[],
mounts=[],
repo_id=None,
skip_setarch=False,
weight=None,
upload_logs=None,
):
"""Create a buildroot and run a command (as root) inside of it
Command may be a string or a list.
Returns a message indicating success if the command was successful, and
raises an error otherwise. Command output will be available in
runroot.log in the task output directory on the hub.
skip_setarch is a rough approximation of an old hack
the keep option is not used. keeping for compatibility for now...
upload_logs is list of absolute paths which will be uploaded for
archiving on hub. It always consists of /tmp/runroot.log, but can be
used for additional logs (pungi.log, etc.)
"""
if weight is not None:
weight = max(weight, 0.5)
self.session.host.setTaskWeight(self.id, weight)
# noarch is funny
if arch == "noarch":
# we need a buildroot arch. Pick one that:
# a) this host can handle
# b) the build tag can support
# c) is canonical
host_arches = self.session.host.getHost()["arches"]
if not host_arches:
raise koji.BuildError, "No arch list for this host"
tag_arches = self.session.getBuildConfig(root)["arches"]
if not tag_arches:
raise koji.BuildError, "No arch list for tag: %s" % root
# index canonical host arches
host_arches = dict([(koji.canonArch(a), 1) for a in host_arches.split()])
# pick the first suitable match from tag's archlist
for br_arch in tag_arches.split():
br_arch = koji.canonArch(br_arch)
if host_arches.has_key(br_arch):
# we're done
break
else:
# no overlap
raise koji.BuildError, "host does not match tag arches: %s (%s)" % (root, tag_arches)
else:
br_arch = arch
if repo_id:
repo_info = self.session.repoInfo(repo_id, strict=True)
if repo_info["tag_name"] != root:
raise koji.BuildError, "build tag (%s) does not match repo tag (%s)" % (root, repo_info["tag_name"])
if repo_info["state"] not in (koji.REPO_STATES["READY"], koji.REPO_STATES["EXPIRED"]):
raise koji.BuildError, "repos in the %s state may not be used by runroot" % koji.REPO_STATES[
repo_info["state"]
]
else:
repo_info = self.session.getRepo(root)
if not repo_info:
# wait for it
task_id = self.session.host.subtask(method="waitrepo", arglist=[root, None, None], parent=self.id)
repo_info = self.wait(task_id)[task_id]
if compat_mode:
broot = BuildRoot(root, br_arch, self.id, repo_id=repo_info["id"])
else:
broot = BuildRoot(self.session, self.options, root, br_arch, self.id, repo_id=repo_info["id"])
broot.workdir = self.workdir
broot.init()
rootdir = broot.rootdir()
# workaround for rpm oddness
os.system('rm -f "%s"/var/lib/rpm/__db.*' % rootdir)
# update buildroot state (so that updateBuildRootList() will work)
self.session.host.setBuildRootState(broot.id, "BUILDING")
try:
if packages:
# pkglog = '%s/%s' % (broot.resultdir(), 'packages.log')
pkgcmd = ["--install"] + packages
status = broot.mock(pkgcmd)
self.session.host.updateBuildRootList(broot.id, broot.getPackageList())
if not _isSuccess(status):
raise koji.BuildrootError, _parseStatus(status, pkgcmd)
if isinstance(command, str):
cmdstr = command
else:
# we were passed an arglist
# we still have to run this through the shell (for redirection)
# but we can preserve the list structure precisely with careful escaping
cmdstr = " ".join(["'%s'" % arg.replace("'", r"'\''") for arg in command])
# A nasty hack to put command output into its own file until mock can be
# patched to do something more reasonable than stuff everything into build.log
cmdargs = [
"/bin/sh",
"-c",
"{ %s; } < /dev/null 2>&1 | /usr/bin/tee /tmp/runroot.log; exit ${PIPESTATUS[0]}" % cmdstr,
]
# always mount /mnt/redhat (read-only)
# always mount /mnt/iso (read-only)
# also need /dev bind mount
self.do_mounts(rootdir, [self._get_path_params(x) for x in self.config["default_mounts"]])
self.do_extra_mounts(rootdir, mounts)
mock_cmd = ["chroot"]
if skip_setarch:
# we can't really skip it, but we can set it to the current one instead of of the chroot one
myarch = platform.uname()[5]
mock_cmd.extend(["--arch", myarch])
mock_cmd.append("--")
mock_cmd.extend(cmdargs)
rv = broot.mock(mock_cmd)
log_paths = ["/tmp/runroot.log"]
if upload_logs is not None:
log_paths += upload_logs
for log_path in log_paths:
self.uploadFile(rootdir + log_path)
finally:
# mock should umount its mounts, but it will not handle ours
self.undo_mounts(rootdir, fatal=False)
broot.expire()
if isinstance(command, str):
cmdlist = command.split()
else:
cmdlist = command
cmdlist = [param for param in cmdlist if "=" not in param]
if cmdlist:
cmd = os.path.basename(cmdlist[0])
else:
cmd = "(none)"
if _isSuccess(rv):
return "%s completed successfully" % cmd
else:
raise koji.BuildrootError, _parseStatus(rv, cmd)
def _get_rpms_in_external_repo(repo_url, arches, cache_dir_name):
"""
Get the available RPMs in the external repo for the provided arches.
:param str repo_url: the URL of the external repo with the "$arch" variable included
:param list arches: the list of arches to query the external repo for
:param str cache_dir_name: the cache directory name under f"{conf.cache_dir}/dnf"
:return: a set of the RPM NEVRAs
:rtype: set
:raise RuntimeError: if the cache is not writeable or the external repo couldn't be loaded
:raises ValueError: if there is no "$arch" variable in repo URL
"""
if "$arch" not in repo_url:
raise ValueError(
"The external repo {} does not contain the $arch variable".format(
repo_url))
base = dnf.Base()
try:
dnf_conf = base.conf
# Expire the metadata right away so that when a repo is loaded, it will always check to
# see if the external repo has been updated
dnf_conf.metadata_expire = 0
cache_location = os.path.join(conf.cache_dir, "dnf", cache_dir_name)
try:
# exist_ok=True can't be used in Python 2
os.makedirs(cache_location, mode=0o0770)
except OSError as e:
# Don't fail if the directories already exist
if e.errno != errno.EEXIST:
log.exception("Failed to create the cache directory %s",
cache_location)
raise RuntimeError("The MBS cache is not writeable.")
# Tell DNF to use the cache directory
dnf_conf.cachedir = cache_location
# Don't skip repos that can't be synchronized
dnf_conf.skip_if_unavailable = False
dnf_conf.timeout = conf.dnf_timeout
# Get rid of everything to be sure it's a blank slate. This doesn't delete the cached repo
# data.
base.reset(repos=True, goal=True, sack=True)
# Add a separate repo for each architecture
for arch in arches:
# Convert arch to canon_arch. This handles cases where Koji "i686" arch is mapped to
# "i386" when generating RPM repository.
canon_arch = koji.canonArch(arch)
repo_name = "repo_{}".format(canon_arch)
repo_arch_url = repo_url.replace("$arch", canon_arch)
base.repos.add_new_repo(
repo_name,
dnf_conf,
baseurl=[repo_arch_url],
minrate=conf.dnf_minrate,
)
try:
# Load the repos in parallel
base.update_cache()
except dnf.exceptions.RepoError:
msg = "Failed to load the external repos"
log.exception(msg)
raise RuntimeError(msg)
# dnf will not always raise an error on repo failures, so we check explicitly
for repo_name in base.repos:
if not base.repos[repo_name].metadata:
msg = "Failed to load metadata for repo %s" % repo_name
log.exception(msg)
raise RuntimeError(msg)
base.fill_sack(load_system_repo=False)
# Return all the available RPMs
nevras = set()
for rpm in base.sack.query().available():
rpm_dict = {
"arch": rpm.arch,
"epoch": rpm.epoch,
"name": rpm.name,
"release": rpm.release,
"version": rpm.version,
}
nevra = kobo.rpmlib.make_nvra(rpm_dict, force_epoch=True)
nevras.add(nevra)
finally:
base.close()
return nevras
