def getArchList(self, build_tag, extra=None):
        """Copied from build task"""
        # get list of arches to build for
        buildconfig = self.session.getBuildConfig(build_tag, event=self.event_id)
        arches = buildconfig['arches']
        if not arches:
            # XXX - need to handle this better
            raise koji.BuildError("No arches for tag %(name)s [%(id)s]" % buildconfig)
        tag_archlist = [koji.canonArch(a) for a in arches.split()]
        self.logger.debug('arches: %s', arches)
        if extra:
            self.logger.debug('Got extra arches: %s', extra)
            arches = "%s %s" % (arches, extra)
        archlist = arches.split()
        self.logger.debug('base archlist: %r' % archlist)

        override = self.opts.get('arch_override')
        if (self.opts.get('isolated') or self.opts.get('scratch')) and override:
            # only honor override for scratch builds
            self.logger.debug('arch override: %s', override)
            archlist = override.split()
        elif override:
            raise koji.BuildError("arch-override is only allowed for isolated or scratch builds")
        archdict = {}
        for a in archlist:
            # Filter based on canonical arches for tag
            # This prevents building for an arch that we can't handle
            if a == 'noarch' or koji.canonArch(a) in tag_archlist:
                archdict[a] = 1
        if not archdict:
            raise koji.BuildError("No matching arches were found")
        return list(archdict.keys())
コード例 #2
0
    def getArchList(self, build_tag, extra=None):
        """Copied from build task"""
        # get list of arches to build for
        buildconfig = self.session.getBuildConfig(build_tag, event=self.event_id)
        arches = buildconfig['arches']
        if not arches:
            # XXX - need to handle this better
            raise koji.BuildError("No arches for tag %(name)s [%(id)s]" % buildconfig)
        tag_archlist = [koji.canonArch(a) for a in arches.split()]
        self.logger.debug('arches: %s', arches)
        if extra:
            self.logger.debug('Got extra arches: %s', extra)
            arches = "%s %s" % (arches, extra)
        archlist = arches.split()
        self.logger.debug('base archlist: %r', archlist)

        override = self.opts.get('arch_override')
        if (self.opts.get('isolated') or self.opts.get('scratch')) and override:
            # only honor override for scratch builds
            self.logger.debug('arch override: %s', override)
            archlist = override.split()
        elif override:
            raise koji.BuildError("arch-override is only allowed for isolated or scratch builds")
        archdict = {}
        for a in archlist:
            # Filter based on canonical arches for tag
            # This prevents building for an arch that we can't handle
            if a == 'noarch' or koji.canonArch(a) in tag_archlist:
                archdict[a] = 1
        if not archdict:
            raise koji.BuildError("No matching arches were found")
        return list(archdict.keys())
コード例 #3
0
 def test_exclusive_arch(self):
     tag_arches = [koji.canonArch(a) for a in self.getBuildConfig()['arches'].split()]
     # random choice involved, so we repeat this a few times
     for i in range(20):
         self.readSRPMHeader.return_value = FakeHeader(
                 buildarchs=['noarch'], exclusivearch=['noarch', 'armv7hl'], excludearch=[])
         result = self.handler.choose_taskarch('noarch', 'srpm', 'build_tag')
         self.assertNotEqual(result, 'noarch')
         self.assertEqual(result, koji.canonArch('armv7hl'))
         self.assertIn(result, tag_arches)
コード例 #4
0
 def test_exclusive_arch(self):
     tag_arches = [koji.canonArch(a) for a in self.getBuildConfig()['arches'].split()]
     # random choice involved, so we repeat this a few times
     for i in range(20):
         self.readSRPMHeader.return_value = FakeHeader(
                 buildarchs=['noarch'], exclusivearch=['noarch', 'armv7hl'], excludearch=[])
         result = self.handler.choose_taskarch('noarch', 'srpm', 'build_tag')
         self.assertNotEqual(result, 'noarch')
         self.assertEqual(result, koji.canonArch('armv7hl'))
         self.assertIn(result, tag_arches)
コード例 #5
0
ファイル: koji_util.py プロジェクト: msimacek/koschei
def get_srpm_arches(koji_session, all_arches, nvra, arch_override=None,
                    build_arches=None):
    """
    Compute architectures that should be used for a build. Computation is based on the one
    in Koji (kojid/getArchList).

    :param koji_session: Koji session to be used for the query
    :param all_arches: List of all arches obtained from `get_koji_arches`
    :param nvra: NVRA dict of the SRPM
    :param arch_override: User specified arch override
    :param build_arches: List of allowed arches for building. Taken from config by default
    :return: Set of architectures that can be passed to `koji_scratch_build`. May be
             empty, in which case no build should be submitted.
    """
    archlist = all_arches
    tag_archlist = {koji.canonArch(a) for a in archlist}
    headers = koji_session.getRPMHeaders(
        rpmID=nvra,
        headers=['BUILDARCHS', 'EXCLUDEARCH', 'EXCLUSIVEARCH'],
    )
    if not headers:
        return None
    buildarchs = headers.get('BUILDARCHS', [])
    exclusivearch = headers.get('EXCLUSIVEARCH', [])
    excludearch = headers.get('EXCLUDEARCH', [])
    if buildarchs:
        archlist = buildarchs
    if exclusivearch:
        archlist = [arch for arch in archlist if arch in exclusivearch]
    if excludearch:
        archlist = [arch for arch in archlist if arch not in excludearch]

    if ('noarch' not in excludearch and
            ('noarch' in buildarchs or 'noarch' in exclusivearch)):
        archlist.append('noarch')

    if arch_override:
        # we also allow inverse overrides
        if arch_override.startswith('^'):
            excluded = {koji.canonArch(arch) for arch in arch_override[1:].split()}
            archlist = [arch for arch in archlist if koji.canonArch(arch) not in excluded]
        else:
            archlist = arch_override.split()

    if not build_arches:
        build_arches = get_config('koji_config').get('build_arches')
    build_arches = {koji.canonArch(arch) for arch in build_arches}
    allowed_arches = tag_archlist & build_arches

    arches = set()
    for arch in archlist:
        if arch == 'noarch' or koji.canonArch(arch) in allowed_arches:
            arches.add(arch)

    return arches
コード例 #6
0
def get_srpm_arches(koji_session,
                    all_arches,
                    nvra,
                    arch_override=None,
                    build_arches=None):
    # compute arches the same way as koji
    # see kojid/getArchList
    archlist = all_arches
    tag_archlist = {koji.canonArch(a) for a in archlist}
    headers = koji_session.getRPMHeaders(
        rpmID=nvra,
        headers=['BUILDARCHS', 'EXCLUDEARCH', 'EXCLUSIVEARCH'],
    )
    if not headers:
        return None
    buildarchs = headers.get('BUILDARCHS', [])
    exclusivearch = headers.get('EXCLUSIVEARCH', [])
    excludearch = headers.get('EXCLUDEARCH', [])
    if buildarchs:
        archlist = buildarchs
    if exclusivearch:
        archlist = [arch for arch in archlist if arch in exclusivearch]
    if excludearch:
        archlist = [arch for arch in archlist if arch not in excludearch]

    if ('noarch' not in excludearch
            and ('noarch' in buildarchs or 'noarch' in exclusivearch)):
        archlist.append('noarch')

    if arch_override:
        # we also allow inverse overrides
        if arch_override.startswith('^'):
            excluded = {
                koji.canonArch(arch)
                for arch in arch_override[1:].split()
            }
            archlist = [
                arch for arch in archlist
                if koji.canonArch(arch) not in excluded
            ]
        else:
            archlist = arch_override.split()

    if not build_arches:
        build_arches = get_config('koji_config').get('build_arches')
    build_arches = {koji.canonArch(arch) for arch in build_arches}
    allowed_arches = tag_archlist & build_arches

    arches = set()
    for arch in archlist:
        if arch == 'noarch' or koji.canonArch(arch) in allowed_arches:
            arches.add(arch)

    return arches
コード例 #7
0
ファイル: tasks.py プロジェクト: kszakharov/koji
    def find_arch(self, arch, host, tag, preferred_arch=None):
        """
        For noarch tasks, find a canonical arch that is supported by both the host and tag.
        If the arch is anything other than noarch, return it unmodified.

        If preferred_arch is set, try to get it, but not fail on that
        """
        if arch != "noarch":
            return arch

        # We need a concrete arch. Pick one that:
        #  a) this host can handle
        #  b) the build tag can support
        #  c) is canonical
        host_arches = host['arches']
        if not host_arches:
            raise koji.BuildError("No arch list for this host: %s" %
                                  host['name'])
        tag_arches = tag['arches']
        if not tag_arches:
            raise koji.BuildError("No arch list for tag: %s" % tag['name'])
        # index canonical host arches
        host_arches = set([koji.canonArch(a) for a in host_arches.split()])
        # index canonical tag arches
        tag_arches = set([koji.canonArch(a) for a in tag_arches.split()])
        # find the intersection of host and tag arches
        common_arches = list(host_arches & tag_arches)
        if common_arches:
            if preferred_arch and preferred_arch in common_arches:
                self.logger.info(
                    'Valid arches: %s, using preferred: %s' %
                    (' '.join(sorted(common_arches)), preferred_arch))
                return preferred_arch
            # pick one of the common arches randomly
            # need to re-seed the prng or we'll get the same arch every time,
            # because we just forked from a common parent
            random.seed()
            arch = random.choice(common_arches)
            self.logger.info('Valid arches: %s, using: %s' %
                             (' '.join(sorted(common_arches)), arch))
            return arch
        else:
            # no overlap
            raise koji.BuildError(
                "host %s (%s) does not support any arches"
                " of tag %s (%s)" %
                (host['name'], ', '.join(sorted(host_arches)), tag['name'],
                 ', '.join(sorted(tag_arches))))
コード例 #8
0
 def test_literal_arch(self):
     self.options.literal_task_arches = 'ARCH'
     tag_arches = [
         koji.canonArch(a) for a in self.getBuildConfig()['arches'].split()
     ]
     result = self.handler.choose_taskarch('ARCH', 'srpm', 'build_tag')
     self.assertEqual(result, 'ARCH')
コード例 #9
0
ファイル: runroot_hub.py プロジェクト: mizdebsk/koji
def runroot(tagInfo, arch, command, channel=None, **opts):
    """ Create a runroot task """
    context.session.assertPerm('runroot')
    taskopts = {
        'priority': 15,
        'arch': arch,
    }

    taskopts['channel'] = channel or 'runroot'

    if arch == 'noarch':
        #not all arches can generate a proper buildroot for all tags
        tag = kojihub.get_tag(tagInfo)
        if not tag['arches']:
            raise koji.GenericError, 'no arches defined for tag %s' % tag['name']

        #get all known arches for the system
        fullarches = kojihub.get_all_arches()

        tagarches = tag['arches'].split()

        # If our tag can't do all arches, then we need to
        # specify one of the arches it can do.
        if set(fullarches) - set(tagarches):
            chanarches = get_channel_arches(taskopts['channel'])
            choices = [x for x in tagarches if x in chanarches]
            if not choices:
                raise koji.GenericError, 'no common arches for tag/channel: %s/%s' \
                            % (tagInfo, taskopts['channel'])
            taskopts['arch'] = koji.canonArch(random.choice(choices))

    args = koji.encode_args(tagInfo, arch, command,**opts)
    return kojihub.make_task('runroot', args, **taskopts)
コード例 #10
0
def runroot(tagInfo, arch, command, channel=None, **opts):
    """ Create a runroot task """
    context.session.assertPerm('runroot')
    taskopts = {
        'priority': 15,
        'arch': arch,
    }

    taskopts['channel'] = channel or 'runroot'

    if arch == 'noarch':
        #not all arches can generate a proper buildroot for all tags
        tag = kojihub.get_tag(tagInfo)
        if not tag['arches']:
            raise koji.GenericError('no arches defined for tag %s' %
                                    tag['name'])

        #get all known arches for the system
        fullarches = kojihub.get_all_arches()

        tagarches = tag['arches'].split()

        # If our tag can't do all arches, then we need to
        # specify one of the arches it can do.
        if set(fullarches) - set(tagarches):
            chanarches = get_channel_arches(taskopts['channel'])
            choices = [x for x in tagarches if x in chanarches]
            if not choices:
                raise koji.GenericError('no common arches for tag/channel: %s/%s' \
                            % (tagInfo, taskopts['channel']))
            taskopts['arch'] = koji.canonArch(random.choice(choices))

    args = koji.encode_args(tagInfo, arch, command, **opts)
    return kojihub.make_task('runroot', args, **taskopts)
コード例 #11
0
 def test_too_exclusive(self):
     tag_arches = [koji.canonArch(a) for a in self.getBuildConfig()['arches'].split()]
     # random choice involved, so we repeat this a few times
     for i in range(20):
         self.readSRPMHeader.return_value = FakeHeader(
                 buildarchs=['noarch'], exclusivearch=['missing_arch'], excludearch=[])
         with self.assertRaises(koji.BuildError):
             result = self.handler.choose_taskarch('noarch', 'srpm', 'build_tag')
コード例 #12
0
def get_channel_arches(channel):
    """determine arches available in channel"""
    chan = context.handlers.call('getChannel', channel, strict=True)
    ret = {}
    for host in context.handlers.call('listHosts', channelID=chan['id'], enabled=True):
        for a in host['arches'].split():
            ret[koji.canonArch(a)] = 1
    return ret
コード例 #13
0
 def test_all_excluded(self):
     tag_arches = [koji.canonArch(a) for a in self.getBuildConfig()['arches'].split()]
     # random choice involved, so we repeat this a few times
     for i in range(20):
         self.readSRPMHeader.return_value = FakeHeader(
                 buildarchs=['noarch'], exclusivearch=[], excludearch=tag_arches)
         with self.assertRaises(koji.BuildError):
             result = self.handler.choose_taskarch('noarch', 'srpm', 'build_tag')
コード例 #14
0
ファイル: runroot_hub.py プロジェクト: mizdebsk/koji
def get_channel_arches(channel):
    """determine arches available in channel"""
    chan = context.handlers.call('getChannel', channel, strict=True)
    ret = {}
    for host in context.handlers.call('listHosts', channelID=chan['id'], enabled=True):
        for a in host['arches'].split():
            ret[koji.canonArch(a)] = 1
    return ret
コード例 #15
0
 def test_excluded_irrelevant(self):
     tag_arches = [
         koji.canonArch(a) for a in self.getBuildConfig()['arches'].split()
     ]
     self.readSRPMHeader.return_value = FakeHeader(
         buildarchs=['noarch'],
         exclusivearch=[],
         excludearch=['nosucharch'])
     result = self.handler.choose_taskarch('noarch', 'srpm', 'build_tag')
     self.assertEqual(result, 'noarch')
コード例 #16
0
ファイル: tasks.py プロジェクト: mikem23/koji-playground
    def find_arch(self, arch, host, tag):
        """
        For noarch tasks, find a canonical arch that is supported by both the host and tag.
        If the arch is anything other than noarch, return it unmodified.
        """
        if arch != "noarch":
            return arch

        # We need a concrete arch. Pick one that:
        #  a) this host can handle
        #  b) the build tag can support
        #  c) is canonical
        host_arches = host['arches']
        if not host_arches:
            raise koji.BuildError("No arch list for this host: %s" %
                                  host['name'])
        tag_arches = tag['arches']
        if not tag_arches:
            raise koji.BuildError("No arch list for tag: %s" % tag['name'])
        # index canonical host arches
        host_arches = set([koji.canonArch(a) for a in host_arches.split()])
        # index canonical tag arches
        tag_arches = set([koji.canonArch(a) for a in tag_arches.split()])
        # find the intersection of host and tag arches
        common_arches = list(host_arches & tag_arches)
        if common_arches:
            # pick one of the common arches randomly
            # need to re-seed the prng or we'll get the same arch every time,
            # because we just forked from a common parent
            random.seed()
            arch = random.choice(common_arches)
            self.logger.info('Valid arches: %s, using: %s' %
                             (' '.join(sorted(common_arches)), arch))
            return arch
        else:
            # no overlap
            raise koji.BuildError("host %s (%s) does not support any arches"
                                  " of tag %s (%s)" %
                                  (host['name'],
                                   ', '.join(sorted(host_arches)),
                                   tag['name'],
                                   ', '.join(sorted(tag_arches))))
コード例 #17
0
ファイル: daemon.py プロジェクト: ccoss/koji
    def takeTask(self,task):
        """Attempt to open the specified task

        Returns True if successful, False otherwise
        """
        self.logger.info("Attempting to take task %s" % task['id'])
        if task['method'] in ('buildArch', 'buildSRPMFromSCM', 'buildMaven') and \
               task['arch'] == 'noarch':
            task_info = self.session.getTaskInfo(task['id'], request=True)
            if task['method'] == 'buildMaven':
                tag = task_info['request'][1]
            else:
                tag_id = task_info['request'][1]
                tag = self.session.getTag(tag_id)
            if tag and tag['arches']:
                tag_arches = [koji.canonArch(a) for a in tag['arches'].split()]
                host_arches = self.hostdata['arches'].split()
                if not set(tag_arches).intersection(host_arches):
                    self.logger.info('Skipping task %s (%s) because tag arches (%s) and ' \
                                     'host arches (%s) are disjoint' % \
                                     (task['id'], task['method'],
                                      ', '.join(tag_arches), ', '.join(host_arches)))
                    return False
        data = self.session.host.openTask(task['id'])
        if data is None:
            self.logger.warn("Could not open")
            return False
        if not data.has_key('request') or data['request'] is None:
            self.logger.warn("Task '%s' has no request" % task['id'])
            return False
        id = data['id']
        request = data['request']
        self.tasks[id] = data
        params, method = xmlrpclib.loads(request)
        if self.handlers.has_key(method):
            handlerClass = self.handlers[method]
        else:
            raise koji.GenericError, "No handler found for method '%s'" % method
        handler = handlerClass(id,method,params,self.session,self.options)
        # set weight
        self.session.host.setTaskWeight(id,handler.weight())
        if handler.Foreground:
            self.logger.info("running task in foreground")
            handler.setManager(self)
            self.runTask(handler)
        else:
            pid, session_id = self.forkTask(handler)
            self.pids[id] = pid
            self.subsessions[id] = session_id
        return True
コード例 #18
0
ファイル: daemon.py プロジェクト: msharbiani1/koji
    def takeTask(self,task):
        """Attempt to open the specified task

        Returns True if successful, False otherwise
        """
        self.logger.info("Attempting to take task %s" % task['id'])
        if task['method'] in ('buildArch', 'buildSRPMFromSCM', 'buildMaven') and \
               task['arch'] == 'noarch':
            task_info = self.session.getTaskInfo(task['id'], request=True)
            if task['method'] == 'buildMaven':
                tag = task_info['request'][1]
            else:
                tag_id = task_info['request'][1]
                tag = self.session.getTag(tag_id)
            if tag and tag['arches']:
                tag_arches = [koji.canonArch(a) for a in tag['arches'].split()]
                host_arches = self.hostdata['arches'].split()
                if not set(tag_arches).intersection(host_arches):
                    self.logger.info('Skipping task %s (%s) because tag arches (%s) and ' \
                                     'host arches (%s) are disjoint' % \
                                     (task['id'], task['method'],
                                      ', '.join(tag_arches), ', '.join(host_arches)))
                    return False
        data = self.session.host.openTask(task['id'])
        if data is None:
            self.logger.warn("Could not open")
            return False
        if not data.has_key('request') or data['request'] is None:
            self.logger.warn("Task '%s' has no request" % task['id'])
            return False
        id = data['id']
        request = data['request']
        self.tasks[id] = data
        params, method = xmlrpclib.loads(request)
        if self.handlers.has_key(method):
            handlerClass = self.handlers[method]
        else:
            raise koji.GenericError, "No handler found for method '%s'" % method
        handler = handlerClass(id,method,params,self.session,self.options)
        # set weight
        self.session.host.setTaskWeight(id,handler.weight())
        if handler.Foreground:
            self.logger.info("running task in foreground")
            handler.setManager(self)
            self.runTask(handler)
        else:
            pid, session_id = self.forkTask(handler)
            self.pids[id] = pid
            self.subsessions[id] = session_id
        return True
コード例 #19
0
 def test_literal_arch(self):
     self.options.literal_task_arches = 'ARCH'
     tag_arches = [koji.canonArch(a) for a in self.getBuildConfig()['arches'].split()]
     result = self.handler.choose_taskarch('ARCH', 'srpm', 'build_tag')
     self.assertEqual(result, 'ARCH')
コード例 #20
0
ファイル: osbuild.py プロジェクト: larskarlitski/koji-osbuild
 def arches_for_config(buildconfig: Dict):
     archstr = buildconfig["arches"]
     if not archstr:
         name = buildconfig["name"]
         raise koji.BuildError(f"Missing arches for tag '%{name}'")
     return set(koji.canonArch(a) for a in archstr.split())
コード例 #21
0
 def test_binary_arches(self):
     for arch in [
             'i386', 'i686', 'x86_64', 'ppc', 'ppc64le', 's390', 's390x'
     ]:
         result = self.handler.choose_taskarch(arch, 'srpm', 'build_tag')
         self.assertEqual(result, koji.canonArch(arch))
コード例 #22
0
    def setUp(self):
        self.topdir = tempfile.mkdtemp()
        self.rinfo = {
            'create_event': 2915,
            'create_ts': 1487256924.72718,
            'creation_time': '2017-02-16 14:55:24.727181',
            'id': 47,
            'state': 0,  # INIT
            'tag_id': 2,
            'tag_name': 'my-tag'}
        self.arch = 'x86_64'

        # set up a fake koji topdir
        # koji.pathinfo._topdir = self.topdir
        mock.patch('koji.pathinfo._topdir', new=self.topdir).start()
        repodir = koji.pathinfo.distrepo(self.rinfo['id'], self.rinfo['tag_name'])
        archdir = "%s/%s" % (repodir, koji.canonArch(self.arch))
        os.makedirs(archdir)
        self.uploadpath = 'UNITTEST'
        workdir = koji.pathinfo.work()
        uploaddir = "%s/%s" % (workdir, self.uploadpath)
        os.makedirs(uploaddir)

        # place some test files
        self.files = ['drpms/foo.drpm', 'repodata/repomd.xml']
        self.expected = ['x86_64/drpms/foo.drpm', 'x86_64/repodata/repomd.xml']
        for fn in self.files:
            path = os.path.join(uploaddir, fn)
            koji.ensuredir(os.path.dirname(path))
            with open(path, 'w') as fo:
                fo.write('%s' % os.path.basename(fn))

        # generate pkglist file
        self.files.append('pkglist')
        plist = os.path.join(uploaddir, 'pkglist')
        nvrs = ['aaa-1.0-2', 'bbb-3.0-5', 'ccc-8.0-13','ddd-21.0-34']
        self.rpms = {}
        self.builds ={}
        self.key = '4c8da725'
        with open(plist, 'w') as f_pkglist:
            for nvr in nvrs:
                binfo = koji.parse_NVR(nvr)
                rpminfo = binfo.copy()
                rpminfo['arch'] = 'x86_64'
                builddir = koji.pathinfo.build(binfo)
                relpath = koji.pathinfo.signed(rpminfo, self.key)
                path = os.path.join(builddir, relpath)
                koji.ensuredir(os.path.dirname(path))
                basename = os.path.basename(path)
                with open(path, 'w') as fo:
                    fo.write('%s' % basename)
                f_pkglist.write(path)
                f_pkglist.write('\n')
                self.expected.append('x86_64/Packages/%s/%s' % (basename[0], basename))
                build_id = len(self.builds) + 10000
                rpm_id = len(self.rpms) + 20000
                binfo['id'] = build_id
                rpminfo['build_id'] = build_id
                rpminfo['id'] = rpm_id
                rpminfo['sigkey'] = self.key
                rpminfo['size'] = 1024
                rpminfo['payloadhash'] = 'helloworld'
                self.builds[build_id] = binfo
                self.rpms[rpm_id] = rpminfo

        # write kojipkgs
        kojipkgs = {}
        for rpminfo in self.rpms.values():
            bnp = '%(name)s-%(version)s-%(release)s.%(arch)s.rpm' % rpminfo
            kojipkgs[bnp] = rpminfo
        with open("%s/kojipkgs" % uploaddir, "w") as fp:
            json.dump(kojipkgs, fp, indent=4)
        self.files.append('kojipkgs')

        # write manifest
        with open("%s/repo_manifest" % uploaddir, "w") as fp:
            json.dump(self.files, fp, indent=4)

        # mocks
        self.repo_info = mock.patch('kojihub.repo_info').start()
        self.repo_info.return_value = self.rinfo.copy()
        self.get_rpm = mock.patch('kojihub.get_rpm').start()
        self.get_build = mock.patch('kojihub.get_build').start()
        self.get_rpm.side_effect = self.our_get_rpm
        self.get_build.side_effect = self.our_get_build
コード例 #23
0
ファイル: koji_util.py プロジェクト: sturivny/koschei
def get_srpm_arches(koji_session,
                    all_arches,
                    nvra,
                    arch_override=None,
                    build_arches=None):
    """
    Compute architectures that should be used for a build. Computation is based on the one
    in Koji (kojid/getArchList).

    :param koji_session: Koji session to be used for the query
    :param all_arches: List of all arches obtained from `get_koji_arches`
    :param nvra: NVRA dict of the SRPM
    :param arch_override: User specified arch override
    :param build_arches: List of allowed arches for building. Taken from config by default
    :return: Set of architectures that can be passed to `koji_scratch_build`. May be
             empty, in which case no build should be submitted.
    """
    archlist = all_arches
    tag_archlist = {koji.canonArch(a) for a in archlist}
    headers = koji_session.getRPMHeaders(
        rpmID=nvra,
        headers=['BUILDARCHS', 'EXCLUDEARCH', 'EXCLUSIVEARCH'],
    )
    if not headers:
        return None
    buildarchs = headers.get('BUILDARCHS', [])
    exclusivearch = headers.get('EXCLUSIVEARCH', [])
    excludearch = headers.get('EXCLUDEARCH', [])
    if buildarchs:
        archlist = buildarchs
    if exclusivearch:
        archlist = [arch for arch in archlist if arch in exclusivearch]
    if excludearch:
        archlist = [arch for arch in archlist if arch not in excludearch]

    if ('noarch' not in excludearch
            and ('noarch' in buildarchs or 'noarch' in exclusivearch)):
        archlist.append('noarch')

    if arch_override:
        # we also allow inverse overrides
        if arch_override.startswith('^'):
            excluded = {
                koji.canonArch(arch)
                for arch in arch_override[1:].split()
            }
            archlist = [
                arch for arch in archlist
                if koji.canonArch(arch) not in excluded
            ]
        else:
            archlist = arch_override.split()

    if not build_arches:
        build_arches = get_config('koji_config').get('build_arches')
    build_arches = {koji.canonArch(arch) for arch in build_arches}
    allowed_arches = tag_archlist & build_arches

    arches = set()
    for arch in archlist:
        if arch == 'noarch' or koji.canonArch(arch) in allowed_arches:
            arches.add(arch)

    return arches
コード例 #24
0
    def setUp(self):
        self.topdir = tempfile.mkdtemp()
        self.rinfo = {
            'create_event': 2915,
            'create_ts': 1487256924.72718,
            'creation_time': '2017-02-16 14:55:24.727181',
            'id': 47,
            'state': 0,  # INIT
            'tag_id': 2,
            'tag_name': 'my-tag'
        }
        self.arch = 'x86_64'

        # set up a fake koji topdir
        # koji.pathinfo._topdir = self.topdir
        mock.patch('koji.pathinfo._topdir', new=self.topdir).start()
        repodir = koji.pathinfo.distrepo(self.rinfo['id'],
                                         self.rinfo['tag_name'])
        archdir = "%s/%s" % (repodir, koji.canonArch(self.arch))
        os.makedirs(archdir)
        self.uploadpath = 'UNITTEST'
        workdir = koji.pathinfo.work()
        uploaddir = "%s/%s" % (workdir, self.uploadpath)
        os.makedirs(uploaddir)

        # place some test files
        self.files = ['drpms/foo.drpm', 'repodata/repomd.xml']
        self.expected = ['x86_64/drpms/foo.drpm', 'x86_64/repodata/repomd.xml']
        for fn in self.files:
            path = os.path.join(uploaddir, fn)
            koji.ensuredir(os.path.dirname(path))
            with open(path, 'w') as fo:
                fo.write('%s' % os.path.basename(fn))

        # generate pkglist file
        self.files.append('pkglist')
        plist = os.path.join(uploaddir, 'pkglist')
        nvrs = ['aaa-1.0-2', 'bbb-3.0-5', 'ccc-8.0-13', 'ddd-21.0-34']
        self.rpms = {}
        self.builds = {}
        self.key = '4c8da725'
        with open(plist, 'w') as f_pkglist:
            for nvr in nvrs:
                binfo = koji.parse_NVR(nvr)
                rpminfo = binfo.copy()
                rpminfo['arch'] = 'x86_64'
                builddir = koji.pathinfo.build(binfo)
                relpath = koji.pathinfo.signed(rpminfo, self.key)
                path = os.path.join(builddir, relpath)
                koji.ensuredir(os.path.dirname(path))
                basename = os.path.basename(path)
                with open(path, 'w') as fo:
                    fo.write('%s' % basename)
                f_pkglist.write(path)
                f_pkglist.write('\n')
                self.expected.append('x86_64/Packages/%s/%s' %
                                     (basename[0], basename))
                build_id = len(self.builds) + 10000
                rpm_id = len(self.rpms) + 20000
                binfo['id'] = build_id
                rpminfo['build_id'] = build_id
                rpminfo['id'] = rpm_id
                rpminfo['sigkey'] = self.key
                rpminfo['size'] = 1024
                rpminfo['payloadhash'] = 'helloworld'
                self.builds[build_id] = binfo
                self.rpms[rpm_id] = rpminfo

        # write kojipkgs
        kojipkgs = {}
        for rpminfo in self.rpms.values():
            bnp = '%(name)s-%(version)s-%(release)s.%(arch)s.rpm' % rpminfo
            kojipkgs[bnp] = rpminfo
        with open("%s/kojipkgs" % uploaddir, "w") as fp:
            json.dump(kojipkgs, fp, indent=4)
        self.files.append('kojipkgs')

        # write manifest
        with open("%s/repo_manifest" % uploaddir, "w") as fp:
            json.dump(self.files, fp, indent=4)

        # mocks
        self.repo_info = mock.patch('kojihub.repo_info').start()
        self.repo_info.return_value = self.rinfo.copy()
        self.get_rpm = mock.patch('kojihub.get_rpm').start()
        self.get_build = mock.patch('kojihub.get_build').start()
        self.get_rpm.side_effect = self.our_get_rpm
        self.get_build.side_effect = self.our_get_build
コード例 #25
0
def handle_kiwi_build(goptions, session, args):
    "[build] Run a command in a buildroot"
    usage = "usage: %prog kiwi-build [options] <target> <description_scm> <description_path>"
    usage += "\n(Specify the --help global option for a list of other help options)"
    parser = OptionParser(usage=usage)
    parser.add_option("--scratch",
                      action="store_true",
                      default=False,
                      help="Perform a scratch build")
    parser.add_option(
        "--repo",
        action="append",
        help="Specify a repo that will override the repo used to install "
        "RPMs in the image. May be used multiple times. The "
        "build tag repo associated with the target is the default.")
    parser.add_option("--noprogress",
                      action="store_true",
                      help="Do not display progress of the upload")
    parser.add_option("--kiwi-profile",
                      action="store",
                      default=None,
                      help="Select profile from description file")
    parser.add_option("--can-fail",
                      action="store",
                      dest="optional_arches",
                      metavar="ARCH1,ARCH2,...",
                      default="",
                      help="List of archs which are not blocking for build "
                      "(separated by commas.")
    parser.add_option("--arch",
                      action="append",
                      dest="arches",
                      default=[],
                      help="Limit arches to this subset")
    parser.add_option("--nowait",
                      action="store_false",
                      dest="wait",
                      default=True)
    parser.add_option(
        "--wait",
        action="store_true",
        help="Wait on the image creation, even if running in the background")
    (options, args) = parser.parse_args(args)

    if len(args) != 3:
        parser.error("Incorrect number of arguments")
        assert False  # pragma: no cover
    target, scm, path = args

    activate_session(session, goptions)

    kwargs = {
        'scratch':
        options.scratch,
        'optional_arches': [
            canonArch(arch) for arch in options.optional_arches.split(',')
            if arch
        ],
        'profile':
        options.kiwi_profile,
    }

    arches = []
    if options.arches:
        arches = [canonArch(arch) for arch in options.arches]

    task_id = session.kiwiBuild(target=target,
                                arches=arches,
                                desc_url=scm,
                                desc_path=path,
                                **kwargs)

    if not goptions.quiet:
        print("Created task: %d" % task_id)
        print("Task info: %s/taskinfo?taskID=%s" % (goptions.weburl, task_id))
    if options.wait or (options.wait is None and not _running_in_bg()):
        session.logout()
        return watch_tasks(session, [task_id],
                           quiet=goptions.quiet,
                           poll_interval=goptions.poll_interval,
                           topurl=goptions.topurl)
コード例 #26
0
 def test_excluded_irrelevant(self):
     tag_arches = [koji.canonArch(a) for a in self.getBuildConfig()['arches'].split()]
     self.readSRPMHeader.return_value = FakeHeader(
             buildarchs=['noarch'], exclusivearch=[], excludearch=['nosucharch'])
     result = self.handler.choose_taskarch('noarch', 'srpm', 'build_tag')
     self.assertEqual(result, 'noarch')
コード例 #27
0
    def setUp(self):
        self.topdir = tempfile.mkdtemp()
        self.rinfo = {
            'create_event': 2915,
            'create_ts': 1487256924.72718,
            'creation_time': '2017-02-16 14:55:24.727181',
            'id': 47,
            'state': 1,
            'tag_id': 2,
            'tag_name': 'my-tag'
        }
        self.arch = 'x86_64'

        # set up a fake koji topdir
        # koji.pathinfo._topdir = self.topdir
        mock.patch('koji.pathinfo._topdir', new=self.topdir).start()
        repodir = koji.pathinfo.distrepo(self.rinfo['id'],
                                         self.rinfo['tag_name'])
        archdir = "%s/%s" % (repodir, koji.canonArch(self.arch))
        os.makedirs(archdir)
        self.uploadpath = 'UNITTEST'
        workdir = koji.pathinfo.work()
        uploaddir = "%s/%s" % (workdir, self.uploadpath)
        os.makedirs(uploaddir)

        # place some test files
        self.files = ['foo.drpm', 'repomd.xml']
        self.expected = ['x86_64/drpms/foo.drpm', 'x86_64/repodata/repomd.xml']
        for fn in self.files:
            path = os.path.join(uploaddir, fn)
            koji.ensuredir(os.path.dirname(path))
            with open(path, 'w') as fo:
                fo.write('%s' % fn)

        # generate pkglist file and sigmap
        self.files.append('pkglist')
        plist = os.path.join(uploaddir, 'pkglist')
        nvrs = ['aaa-1.0-2', 'bbb-3.0-5', 'ccc-8.0-13', 'ddd-21.0-34']
        self.sigmap = []
        self.rpms = {}
        self.builds = {}
        self.key = '4c8da725'
        with open(plist, 'w') as f_pkglist:
            for nvr in nvrs:
                binfo = koji.parse_NVR(nvr)
                rpminfo = binfo.copy()
                rpminfo['arch'] = 'x86_64'
                builddir = koji.pathinfo.build(binfo)
                relpath = koji.pathinfo.signed(rpminfo, self.key)
                path = os.path.join(builddir, relpath)
                koji.ensuredir(os.path.dirname(path))
                basename = os.path.basename(path)
                with open(path, 'w') as fo:
                    fo.write('%s' % basename)
                f_pkglist.write(path)
                f_pkglist.write('\n')
                self.expected.append('x86_64/%s/%s' % (basename[0], basename))
                build_id = len(self.builds) + 10000
                rpm_id = len(self.rpms) + 20000
                binfo['id'] = build_id
                rpminfo['build_id'] = build_id
                rpminfo['id'] = rpm_id
                self.builds[build_id] = binfo
                self.rpms[rpm_id] = rpminfo
                self.sigmap.append([rpm_id, self.key])

        # mocks
        self.repo_info = mock.patch('kojihub.repo_info').start()
        self.repo_info.return_value = self.rinfo.copy()
        self.get_rpm = mock.patch('kojihub.get_rpm').start()
        self.get_build = mock.patch('kojihub.get_build').start()
        self.get_rpm.side_effect = self.our_get_rpm
        self.get_build.side_effect = self.our_get_build
コード例 #28
0
    def handler(self, root, arch, command, keep=False, packages=[], mounts=[], repo_id=None, skip_setarch=False, weight=None, upload_logs=None, new_chroot=False):
        """Create a buildroot and run a command (as root) inside of it

        Command may be a string or a list.

        Returns a message indicating success if the command was successful, and
        raises an error otherwise.  Command output will be available in
        runroot.log in the task output directory on the hub.

        skip_setarch is a rough approximation of an old hack

        the keep option is not used. keeping for compatibility for now...

        upload_logs is list of absolute paths which will be uploaded for
        archiving on hub. It always consists of /tmp/runroot.log, but can be
        used for additional logs (pungi.log, etc.)
        """
        if weight is not None:
            weight = max(weight, 0.5)
            self.session.host.setTaskWeight(self.id, weight)
        #noarch is funny
        if arch == "noarch":
            #we need a buildroot arch. Pick one that:
            #  a) this host can handle
            #  b) the build tag can support
            #  c) is canonical
            host_arches = self.session.host.getHost()['arches']
            if not host_arches:
                raise koji.BuildError("No arch list for this host")
            tag_arches = self.session.getBuildConfig(root)['arches']
            if not tag_arches:
                raise koji.BuildError("No arch list for tag: %s" % root)
            #index canonical host arches
            host_arches = dict([(koji.canonArch(a), 1) for a in host_arches.split()])
            #pick the first suitable match from tag's archlist
            for br_arch in tag_arches.split():
                br_arch = koji.canonArch(br_arch)
                if br_arch in host_arches:
                    #we're done
                    break
            else:
                #no overlap
                raise koji.BuildError("host does not match tag arches: %s (%s)" % (root, tag_arches))
        else:
            br_arch = arch
        if repo_id:
            repo_info = self.session.repoInfo(repo_id, strict=True)
            if repo_info['tag_name'] != root:
                raise koji.BuildError("build tag (%s) does not match repo tag (%s)" % (root, repo_info['tag_name']))
            if repo_info['state'] not in (koji.REPO_STATES['READY'], koji.REPO_STATES['EXPIRED']):
                raise koji.BuildError("repos in the %s state may not be used by runroot" % koji.REPO_STATES[repo_info['state']])
        else:
            repo_info = self.session.getRepo(root)
        if not repo_info:
            #wait for it
            task_id = self.session.host.subtask(method='waitrepo',
                                           arglist=[root, None, None],
                                           parent=self.id)
            repo_info = self.wait(task_id)[task_id]
        broot = BuildRoot(self.session, self.options, root, br_arch, self.id, repo_id=repo_info['id'], setup_dns=True)
        broot.workdir = self.workdir
        broot.init()
        rootdir = broot.rootdir()
        #workaround for rpm oddness
        os.system('rm -f "%s"/var/lib/rpm/__db.*' % rootdir)
        #update buildroot state (so that updateBuildRootList() will work)
        self.session.host.setBuildRootState(broot.id, 'BUILDING')
        try:
            if packages:
                #pkglog = '%s/%s' % (broot.resultdir(), 'packages.log')
                pkgcmd = ['--install'] + packages
                status = broot.mock(pkgcmd)
                self.session.host.updateBuildRootList(broot.id, broot.getPackageList())
                if not isSuccess(status):
                    raise koji.BuildrootError(parseStatus(status, pkgcmd))

            if isinstance(command, str):
                cmdstr = command
            else:
                #we were passed an arglist
                #we still have to run this through the shell (for redirection)
                #but we can preserve the list structure precisely with careful escaping
                cmdstr = ' '.join(["'%s'" % arg.replace("'", r"'\''") for arg in command])
            # A nasty hack to put command output into its own file until mock can be
            # patched to do something more reasonable than stuff everything into build.log
            cmdargs = ['/bin/sh', '-c', "{ %s; } < /dev/null 2>&1 | /usr/bin/tee /builddir/runroot.log; exit ${PIPESTATUS[0]}" % cmdstr]

            # always mount /mnt/redhat (read-only)
            # always mount /mnt/iso (read-only)
            # also need /dev bind mount
            self.do_mounts(rootdir, [self._get_path_params(x) for x in self.config['default_mounts']])
            self.do_extra_mounts(rootdir, mounts)
            mock_cmd = ['chroot']
            if new_chroot:
                mock_cmd.append('--new-chroot')
            if skip_setarch:
                #we can't really skip it, but we can set it to the current one instead of of the chroot one
                myarch = platform.uname()[5]
                mock_cmd.extend(['--arch', myarch])
            mock_cmd.append('--')
            mock_cmd.extend(cmdargs)
            rv = broot.mock(mock_cmd)
            log_paths = ['/builddir/runroot.log']
            if upload_logs is not None:
                log_paths += upload_logs
            for log_path in log_paths:
                self.uploadFile(rootdir + log_path)
        finally:
            # mock should umount its mounts, but it will not handle ours
            self.undo_mounts(rootdir, fatal=False)
            broot.expire()
        if isinstance(command, str):
            cmdlist = command.split()
        else:
            cmdlist = command
        cmdlist = [param for param in cmdlist if '=' not in param]
        if cmdlist:
            cmd = os.path.basename(cmdlist[0])
        else:
            cmd = '(none)'
        if isSuccess(rv):
            return '%s completed successfully' % cmd
        else:
            raise koji.BuildrootError(parseStatus(rv, cmd))
コード例 #29
0
 def test_binary_arches(self):
     for arch in ['i386', 'i686', 'x86_64', 'ppc', 'ppc64le', 's390',
                 's390x']:
         result = self.handler.choose_taskarch(arch, 'srpm', 'build_tag')
         self.assertEqual(result, koji.canonArch(arch))
コード例 #30
0
ファイル: runroot.py プロジェクト: imcleod/koji
    def handler(
        self,
        root,
        arch,
        command,
        keep=False,
        packages=[],
        mounts=[],
        repo_id=None,
        skip_setarch=False,
        weight=None,
        upload_logs=None,
    ):
        """Create a buildroot and run a command (as root) inside of it

        Command may be a string or a list.

        Returns a message indicating success if the command was successful, and
        raises an error otherwise.  Command output will be available in
        runroot.log in the task output directory on the hub.

        skip_setarch is a rough approximation of an old hack

        the keep option is not used. keeping for compatibility for now...

        upload_logs is list of absolute paths which will be uploaded for
        archiving on hub. It always consists of /tmp/runroot.log, but can be
        used for additional logs (pungi.log, etc.)
        """
        if weight is not None:
            weight = max(weight, 0.5)
            self.session.host.setTaskWeight(self.id, weight)
        # noarch is funny
        if arch == "noarch":
            # we need a buildroot arch. Pick one that:
            #  a) this host can handle
            #  b) the build tag can support
            #  c) is canonical
            host_arches = self.session.host.getHost()["arches"]
            if not host_arches:
                raise koji.BuildError, "No arch list for this host"
            tag_arches = self.session.getBuildConfig(root)["arches"]
            if not tag_arches:
                raise koji.BuildError, "No arch list for tag: %s" % root
            # index canonical host arches
            host_arches = dict([(koji.canonArch(a), 1) for a in host_arches.split()])
            # pick the first suitable match from tag's archlist
            for br_arch in tag_arches.split():
                br_arch = koji.canonArch(br_arch)
                if host_arches.has_key(br_arch):
                    # we're done
                    break
            else:
                # no overlap
                raise koji.BuildError, "host does not match tag arches: %s (%s)" % (root, tag_arches)
        else:
            br_arch = arch
        if repo_id:
            repo_info = self.session.repoInfo(repo_id, strict=True)
            if repo_info["tag_name"] != root:
                raise koji.BuildError, "build tag (%s) does not match repo tag (%s)" % (root, repo_info["tag_name"])
            if repo_info["state"] not in (koji.REPO_STATES["READY"], koji.REPO_STATES["EXPIRED"]):
                raise koji.BuildError, "repos in the %s state may not be used by runroot" % koji.REPO_STATES[
                    repo_info["state"]
                ]
        else:
            repo_info = self.session.getRepo(root)
        if not repo_info:
            # wait for it
            task_id = self.session.host.subtask(method="waitrepo", arglist=[root, None, None], parent=self.id)
            repo_info = self.wait(task_id)[task_id]
        if compat_mode:
            broot = BuildRoot(root, br_arch, self.id, repo_id=repo_info["id"])
        else:
            broot = BuildRoot(self.session, self.options, root, br_arch, self.id, repo_id=repo_info["id"])
            broot.workdir = self.workdir
        broot.init()
        rootdir = broot.rootdir()
        # workaround for rpm oddness
        os.system('rm -f "%s"/var/lib/rpm/__db.*' % rootdir)
        # update buildroot state (so that updateBuildRootList() will work)
        self.session.host.setBuildRootState(broot.id, "BUILDING")
        try:
            if packages:
                # pkglog = '%s/%s' % (broot.resultdir(), 'packages.log')
                pkgcmd = ["--install"] + packages
                status = broot.mock(pkgcmd)
                self.session.host.updateBuildRootList(broot.id, broot.getPackageList())
                if not _isSuccess(status):
                    raise koji.BuildrootError, _parseStatus(status, pkgcmd)

            if isinstance(command, str):
                cmdstr = command
            else:
                # we were passed an arglist
                # we still have to run this through the shell (for redirection)
                # but we can preserve the list structure precisely with careful escaping
                cmdstr = " ".join(["'%s'" % arg.replace("'", r"'\''") for arg in command])
            # A nasty hack to put command output into its own file until mock can be
            # patched to do something more reasonable than stuff everything into build.log
            cmdargs = [
                "/bin/sh",
                "-c",
                "{ %s; } < /dev/null 2>&1 | /usr/bin/tee /tmp/runroot.log; exit ${PIPESTATUS[0]}" % cmdstr,
            ]

            # always mount /mnt/redhat (read-only)
            # always mount /mnt/iso (read-only)
            # also need /dev bind mount
            self.do_mounts(rootdir, [self._get_path_params(x) for x in self.config["default_mounts"]])
            self.do_extra_mounts(rootdir, mounts)
            mock_cmd = ["chroot"]
            if skip_setarch:
                # we can't really skip it, but we can set it to the current one instead of of the chroot one
                myarch = platform.uname()[5]
                mock_cmd.extend(["--arch", myarch])
            mock_cmd.append("--")
            mock_cmd.extend(cmdargs)
            rv = broot.mock(mock_cmd)
            log_paths = ["/tmp/runroot.log"]
            if upload_logs is not None:
                log_paths += upload_logs
            for log_path in log_paths:
                self.uploadFile(rootdir + log_path)
        finally:
            # mock should umount its mounts, but it will not handle ours
            self.undo_mounts(rootdir, fatal=False)
            broot.expire()
        if isinstance(command, str):
            cmdlist = command.split()
        else:
            cmdlist = command
        cmdlist = [param for param in cmdlist if "=" not in param]
        if cmdlist:
            cmd = os.path.basename(cmdlist[0])
        else:
            cmd = "(none)"
        if _isSuccess(rv):
            return "%s completed successfully" % cmd
        else:
            raise koji.BuildrootError, _parseStatus(rv, cmd)
コード例 #31
0
def _get_rpms_in_external_repo(repo_url, arches, cache_dir_name):
    """
    Get the available RPMs in the external repo for the provided arches.

    :param str repo_url: the URL of the external repo with the "$arch" variable included
    :param list arches: the list of arches to query the external repo for
    :param str cache_dir_name: the cache directory name under f"{conf.cache_dir}/dnf"
    :return: a set of the RPM NEVRAs
    :rtype: set
    :raise RuntimeError: if the cache is not writeable or the external repo couldn't be loaded
    :raises ValueError: if there is no "$arch" variable in repo URL
    """
    if "$arch" not in repo_url:
        raise ValueError(
            "The external repo {} does not contain the $arch variable".format(
                repo_url))

    base = dnf.Base()
    try:
        dnf_conf = base.conf
        # Expire the metadata right away so that when a repo is loaded, it will always check to
        # see if the external repo has been updated
        dnf_conf.metadata_expire = 0

        cache_location = os.path.join(conf.cache_dir, "dnf", cache_dir_name)
        try:
            # exist_ok=True can't be used in Python 2
            os.makedirs(cache_location, mode=0o0770)
        except OSError as e:
            # Don't fail if the directories already exist
            if e.errno != errno.EEXIST:
                log.exception("Failed to create the cache directory %s",
                              cache_location)
                raise RuntimeError("The MBS cache is not writeable.")

        # Tell DNF to use the cache directory
        dnf_conf.cachedir = cache_location
        # Don't skip repos that can't be synchronized
        dnf_conf.skip_if_unavailable = False
        dnf_conf.timeout = conf.dnf_timeout
        # Get rid of everything to be sure it's a blank slate. This doesn't delete the cached repo
        # data.
        base.reset(repos=True, goal=True, sack=True)

        # Add a separate repo for each architecture
        for arch in arches:
            # Convert arch to canon_arch. This handles cases where Koji "i686" arch is mapped to
            # "i386" when generating RPM repository.
            canon_arch = koji.canonArch(arch)
            repo_name = "repo_{}".format(canon_arch)
            repo_arch_url = repo_url.replace("$arch", canon_arch)
            base.repos.add_new_repo(
                repo_name,
                dnf_conf,
                baseurl=[repo_arch_url],
                minrate=conf.dnf_minrate,
            )

        try:
            # Load the repos in parallel
            base.update_cache()
        except dnf.exceptions.RepoError:
            msg = "Failed to load the external repos"
            log.exception(msg)
            raise RuntimeError(msg)

        # dnf will not always raise an error on repo failures, so we check explicitly
        for repo_name in base.repos:
            if not base.repos[repo_name].metadata:
                msg = "Failed to load metadata for repo %s" % repo_name
                log.exception(msg)
                raise RuntimeError(msg)

        base.fill_sack(load_system_repo=False)

        # Return all the available RPMs
        nevras = set()
        for rpm in base.sack.query().available():
            rpm_dict = {
                "arch": rpm.arch,
                "epoch": rpm.epoch,
                "name": rpm.name,
                "release": rpm.release,
                "version": rpm.version,
            }
            nevra = kobo.rpmlib.make_nvra(rpm_dict, force_epoch=True)
            nevras.add(nevra)
    finally:
        base.close()

    return nevras