def initialize(self, conf): self._kojihub_url = conf.kojihub_url self._kojiroot_url = conf.kojiroot_url opts = dict() if conf.request_timeout is not None: opts['timeout'] = conf.request_timeout self._session = koji.ClientSession(self._kojihub_url, opts=opts) if not self._kojihub_url.endswith('/'): self._kojihub_url += '/' if not self._kojiroot_url.endswith('/'): self._kojiroot_url += '/'
def __init__(self, arch=None): if arch: self.kojihub = \ 'http://{arch}.koji.fedoraproject.org/kojihub'.format( arch=arch) else: self.kojihub = 'https://koji.fedoraproject.org/kojihub' self.serverca = os.path.expanduser('~/.fedora-server-ca.cert') self.clientca = os.path.expanduser('~/.fedora-upload-ca.cert') self.clientcert = os.path.expanduser('~/.fedora.cert') self.kojisession = koji.ClientSession(self.kojihub) self.kojisession.ssl_login(self.clientcert, self.clientca, self.serverca)
def setUp(self): self.ksession = koji.ClientSession('http://koji.example.com/kojihub') self.ksession.logout = mock.MagicMock() self.do_fake_login() # mocks self.ksession._callMethod = mock.MagicMock() self.ksession.retries = 1 self.rsession = mock.patch('requests.Session').start() if six.PY2: self.file_mock = mock.patch('__builtin__.open').start() else: self.file_mock = mock.patch('builtins.open').start() self.getsize_mock = mock.patch('os.path.getsize').start()
def session_maker(self): _log.info("Creating a new Koji session to %s", self.server) with _koji_session_lock: koji_session = koji.ClientSession(self.server, self.krb_sessionopts) result = koji_session.krb_login( principal=self.krb_principal, keytab=self.krb_keytab, ccache=self.krb_ccache, proxyuser=self.krb_proxyuser, ) if not result: _log.error("Koji kerberos authentication failed") return koji_session
def get_session(config, login=True): """Create and return a koji.ClientSession object :param config: the config object returned from :meth:`init_config`. :type config: :class:`Config` :param bool login: whether to log into the session. To login if True is passed, otherwise not to log into session. :return: the Koji session object. :rtype: :class:`koji.ClientSession` """ koji_config = munch.Munch( koji.read_config(profile_name=config.koji_profile, user_config=config.koji_config)) # Timeout after 10 minutes. The default is 12 hours. koji_config["timeout"] = 60 * 10 address = koji_config.server log.info("Connecting to koji %r.", address) koji_session = koji.ClientSession(address, opts=koji_config) if not login: return koji_session authtype = koji_config.authtype log.info("Authenticate session with %r.", authtype) if authtype == "kerberos": try: import krbV # We want to create a context per thread to avoid Kerberos cache corruption ctx = krbV.Context() except ImportError: # If no krbV, we can assume GSSAPI auth is available ctx = None keytab = getattr(config, "krb_keytab", None) principal = getattr(config, "krb_principal", None) if not keytab and principal: raise ValueError( "The Kerberos keytab and principal aren't set for Koji authentication") log.debug(" keytab: %r, principal: %r" % (keytab, principal)) # We want to use the thread keyring for the ccache to ensure we have one cache per # thread to avoid Kerberos cache corruption ccache = "KEYRING:thread:mbs" koji_session.krb_login(principal=principal, keytab=keytab, ctx=ctx, ccache=ccache) elif authtype == "ssl": koji_session.ssl_login( os.path.expanduser(koji_config.cert), None, os.path.expanduser(koji_config.serverca) ) else: raise ValueError("Unrecognized koji authtype %r" % authtype) return koji_session
def get_koji_session(self): """ Return an authenticated koji session. Returns: koji.ClientSession: An intialized authenticated koji client. """ config = configparser.ConfigParser() if os.path.exists(os.path.join(os.path.expanduser('~'), '.koji', 'config')): config.readfp(open(os.path.join(os.path.expanduser('~'), '.koji', 'config'))) else: config.readfp(open('/etc/koji.conf')) session = koji.ClientSession(config.get('koji', 'server')) return session
def get_builds_tags(build_nvrs, session=None): """Get tags of multiple Koji/Brew builds :param builds_nvrs: list of build nvr strings or numbers. :param session: instance of :class:`koji.ClientSession` :return: a list of Koji/Brew tag list """ if not session: session = koji.ClientSession(constants.BREW_HUB) tasks = [] with session.multicall(strict=True) as m: for nvr in build_nvrs: tasks.append(m.listTags(build=nvr)) return [task.result for task in tasks]
def get_cbs_tag_list(nvr): """ Return all the CBS tags for this build. :param nvr: a build Name-Version-Release that has been built. :returns: ``list`` of ``str``, eg ["storage7-ceph-jewel-candidate"] """ import koji conf = koji.read_config('cbs') hub = conf['server'] print('searching %s for %s tags' % (hub, nvr)) session = koji.ClientSession(hub, {}) tags = session.listTags(nvr) return [tag['name'] for tag in tags]
def get_koji_session(self, ssl=False): """ Get a koji session with or without SSL access""" if self._kojisession_with_ssl and ssl: return self._kojisession_with_ssl hub = self.config.get('koji', 'hub') kojisession = koji.ClientSession(hub) if ssl: clientcert = self.config.get('koji', 'clientcert') clientca = self.config.get('koji', 'clientca') serverca = self.config.get('koji', 'serverca') kojisession.ssl_login(clientcert, clientca, serverca) self._kojisession_with_ssl = kojisession return kojisession
def __new_session(self): server = self.config['server'] opts = { 'anon_retry': True, 'max_retries': 1000, 'offline_retry': True, 'offline_retry_interval': 120, 'timeout': 3600, } opts.update(self.config.get('session_opts', {})) session = koji.ClientSession(server, opts) if not self.__anonymous: getattr(session, self.config['login_method'])(**self.config['login_args']) return session
def _listMeadTagArtifacts(self, kojiUrl, downloadRootUrl, tagName, gavPatterns): """ Loads maven artifacts from koji (brew/mead). :param kojiUrl: Koji/Brew/Mead URL :param downloadRootUrl: Download root URL of the artifacts :param tagName: Koji/Brew/Mead tag name :returns: Dictionary where index is MavenArtifact object and value is ArtifactSpec with its repo root URL. """ import koji kojiSession = koji.ClientSession(kojiUrl) logging.debug("Getting latest maven artifacts from tag %s.", tagName) kojiArtifacts = kojiSession.getLatestMavenArchives(tagName) filenameDict = {} for artifact in kojiArtifacts: groupId = artifact['group_id'] artifactId = artifact['artifact_id'] version = artifact['version'] gavUrl = "%s%s/%s/%s/maven/" % (maven_repo_util.slashAtTheEnd(downloadRootUrl), artifact['build_name'], artifact['build_version'], artifact['build_release']) gavu = (groupId, artifactId, version, gavUrl) filename = artifact['filename'] filenameDict.setdefault(gavu, []).append(filename) gavuExtClass = {} # { (g,a,v,url): {ext: set([class])} } suffixes = {} # { (g,a,v,url): suffix } for gavu in filenameDict: artifactId = gavu[1] version = gavu[2] filenames = filenameDict[gavu] (extsAndClass, suffix) = self._getExtensionsAndClassifiers(artifactId, version, filenames) if extsAndClass: gavuExtClass[gavu] = {} self._updateExtensionsAndClassifiers(gavuExtClass[gavu], extsAndClass) if suffix is not None: suffixes[gavu] = suffix artifacts = {} for gavu in gavuExtClass: self._addArtifact(artifacts, gavu[0], gavu[1], gavu[2], gavuExtClass[gavu], suffixes.get(gavu), gavu[3]) if gavPatterns: logging.debug("Filtering artifacts contained in the tag by GAV patterns list.") return self._filterArtifactsByPatterns(artifacts, gavPatterns, None)
def __init__(self, dbpath, koji_url='http://koji.fedoraproject.org/kojihub'): self.dbpath = dbpath self.create_index() self.koji_client = koji.ClientSession( 'http://koji.fedoraproject.org/kojihub') self.koji_client.opts['anon_retry'] = True self.koji_client.opts['offline_retry'] = True self.updated_packages = {} self.new_packages = {} self.found_packages = {} self.sconn_needs_reload = False
def get_latest_repo(tag): """Find latest repo ID and its URL for tag name. \b Example of output: 756330 https://kojipkgs.fedoraproject.org/repos/f27-build/756330 """ ks = koji.ClientSession(KOJIHUB) pathinfo = koji.PathInfo(topdir="") repo = ks.getRepo(tag, state=koji.REPO_READY) repo_id = repo["id"] path = pathinfo.repo(repo_id, tag) click.echo("{} {}".format(repo_id, urljoin(KOJIPKGS, path)))
def load_installed(self, releasever): # Load from dnf rpmdb all installed packages with dnf.Base() as self.yb: cachedir = getCacheDir() self.yb.conf.cachedir = cachedir self.yb.conf.releasever = dnf.rpm.detect_releasever( self.yb.conf.installroot) self.yb.read_all_repos() self.yb.fill_sack() self.q = self.yb.sack.query() self.q = self.q.installed() self.installed_packages = self.q # Send it to all bodhi_workers for i in range(self.bodhi_workers_count): self.bodhi_workers_queue.put([ 'set_installed_packages', ['bodhi_worker' + str(i), self.installed_packages] ]) # Wait for Bodhi workers to finish self.bodhi_workers_queue.join() # Send installed packages to GUI main_thread_call(self.main_thread.set_installed_packages, self.installed_packages) # Grab it from Koji session = koji.ClientSession('http://koji.fedoraproject.org/kojihub') tagged_packages = session.listTagged('f' + str(releasever) + '-updates-testing') # Prepare packages into dictionary to find it faster pkgs_dict = {} for pkg in self.installed_packages: pkgs_dict[pkg.name] = pkg # See packages for choosen release pkgsForBodhi = [] for tagged_pkg in tagged_packages: if tagged_pkg['package_name'] in pkgs_dict.keys(): pkgsForBodhi.append(pkgs_dict[tagged_pkg['package_name']]) # Send these packages to BodhiWorker queue main_thread_call(self.main_thread.set_num_of_pkgs_to_process, len(pkgsForBodhi)) for pkg in pkgsForBodhi: self.bodhi_workers_queue.put(['package_update', pkg])
def _mirror_all_rpms(self): print('Mirroring RPMs for %r: all packages in %s tagged %s' % (self, self.koji_profile, self.tag)) hub_url = koji_config.get(self.koji_profile, 'server') koji_session = koji.ClientSession(hub_url) if self.downgradeable: result = koji_session.listTaggedRPMS(self.tag, inherit=True) else: result = koji_session.getLatestRPMS(self.tag) if not result: raise ValueError('No packages in tag %s' % self.tag) rpms, builds = result self.rpm_names.update(rpm['name'] for rpm in rpms if rpm['name'] not in self.excluded_rpms) self._mirror_rpms_for_build(self.koji_profile, builds, rpms)
def _build_client(self): """ Method to set up the KojiClient object used in this instance of KojiWrapperBase. It will call all needed methods to get the config set up for the user. """ if self.url: return koji.ClientSession(self.url) else: _configuration = koji.read_config(self.profile, user_config=self.user_config) """ NOTE: This check is here because if the user does not have and koji config files, read_config will 'helpfully' return you a useless default config. The required baseurl ('server' in _configuration) has a default, so we cannot check that. However, topurl defaults to None, so we currently use this to devine if the returned config is the useless default. """ if not _configuration.get('topurl'): raise koji.ConfigurationError("no configuration for profile \ name: {0}".format(self.profile)) return koji.ClientSession(_configuration.get('server'), opts=_configuration)
def test_operator_metadata_latest_build(self): brew = koji.ClientSession(BREW_HUB) build = brew.getLatestBuilds( "rhaos-4.2-rhel-7-candidate", package="cluster-logging-operator-container")[0] _, out, _ = run_doozer([ "--group=openshift-4.2", "--images=cluster-logging-operator", "operator-metadata:latest-build", "--stream=dev", "--nvr", build["nvr"], ]) self.assertIn("cluster-logging-operator-metadata-container-", out)
def test_krbv_old_requests_kerberos(self, _serverPrincipal_mock, krbV_mock): self.assertIsNotNone(koji.krbV) ctx = koji.krbV.default_context.return_value ctx.mk_req = mock.MagicMock() ac = mock.MagicMock() ctx.mk_req.return_value = (ac, six.b('req')) ac.rd_priv = mock.MagicMock(return_value='session-id session-key') session = koji.ClientSession('whatever') session._callMethod = mock.MagicMock( return_value=(base64.encodestring(six.b('a')), base64.encodestring(six.b('b')), [0, 1, 2, 3])) rv = session.krb_login(principal='*****@*****.**', keytab='/path/to/keytab') self.assertTrue(rv)
def koji_builds_in_tag(url, tag): """ Return the list of koji builds in a tag. """ import koji log.info("Listing rpms in koji(%s) tag %s" % (url, tag)) session = koji.ClientSession(url) rpms, builds = session.listTaggedRPMS(tag) # Extract some srpm-level info from the build attach it to each rpm builds = {build['build_id']: build for build in builds} for rpm in rpms: idx = rpm['build_id'] rpm['srpm_name'] = builds[idx]['name'] rpm['srpm_nevra'] = builds[idx]['nvr'] return rpms
def __init__(self, tasker, workflow, target, hub, root): """ constructor :param tasker: DockerTasker instance :param workflow: DockerBuildWorkflow instance :param target: string, koji target to use as a source :param hub: string, koji hub (xmlrpc) :param root: string, koji root (storage) """ # call parent constructor super(KojiPlugin, self).__init__(tasker, workflow) self.target = target self.xmlrpc = koji.ClientSession(hub) self.pathinfo = koji.PathInfo(topdir=root)
def get_koji_session(self) -> koji.ClientSession: """ Return an authenticated koji session. Returns: An initialized authenticated koji client. """ config = configparser.ConfigParser() koji_conf = os.path.join(os.path.expanduser('~'), '.koji', 'config') if not os.path.exists(koji_conf): koji_conf = '/etc/koji.conf' with open(koji_conf) as fh: config.read_file(fh) session = koji.ClientSession(config.get('koji', 'server')) return session
def create_koji_session(hub_url, auth_info=None): """ Creates and returns a Koji session. If auth_info is provided, the session will be authenticated. :param hub_url: str, Koji hub URL :param auth_info: dict, authentication parameters used for koji_login :return: koji.ClientSession instance """ session = koji.ClientSession(hub_url, opts={'krb_rdns': False}) if auth_info is not None: koji_login(session, **auth_info) return session
def get_buildsys(which): """Get a koji build system session for either the source or the destination. Caches the sessions so future calls are cheap. Destination sessions are authenticated, source sessions are not. :param which: Session to select, source or destination :returns: Koji session object, or None on error """ if 'main' not in c: logger.critical('DistroBaker is not configured, aborting.') return None if which != 'source' and which != 'destination': logger.error('Cannot get "%s" build system.', which) return None if not hasattr(get_buildsys, which): logger.debug( 'Initializing the %s koji instance with the "%s" profile.', which, c['main'][which]['profile']) try: bsys = koji.read_config(profile_name=c['main'][which]['profile']) bsys = koji.ClientSession(bsys['server'], opts=bsys) except Exception: logger.exception( 'Failed initializing the %s koji instance with the "%s" profile, skipping.', which, c['main'][which]['profile']) return None logger.debug('The %s koji instance initialized.', which) if which == 'destination': logger.debug('Authenticating with the destination koji instance.') try: bsys.gssapi_login() except Exception: logger.exception( 'Failed authenticating against the destination koji instance, skipping.' ) return None logger.debug( 'Successfully authenticated with the destination koji instance.' ) if which == 'source': get_buildsys.source = bsys else: get_buildsys.destination = bsys else: logger.debug( 'The %s koji instance is already initialized, fetching from cache.', which) return vars(get_buildsys)[which]
def get_koji_session(self, login=True): """ Return an authenticated koji session """ import koji from iniparse.compat import ConfigParser config = ConfigParser() if exists(join(expanduser('~'), '.koji', 'config')): config.readfp(open(join(expanduser('~'), '.koji', 'config'))) else: config.readfp(open('/etc/koji.conf')) cert = expanduser(config.get('koji', 'cert')) ca = expanduser(config.get('koji', 'ca')) serverca = expanduser(config.get('koji', 'serverca')) session = koji.ClientSession(config.get('koji', 'server')) if login: session.ssl_login(cert=cert, ca=ca, serverca=serverca) return session
def koji_login(config): """ Login to Koji and return the session """ koji_options = { 'krb_rdns': False, 'max_retries': 30, 'retry_interval': 10, 'offline_retry': True, 'offline_retry_interval': 10, 'anon_retry': True, } koji_client = koji.ClientSession(_koji_hub, koji_options) if not koji_client.krb_login(**get_krb_conf(config)): log.error('Koji krb_login failed') return koji_client
def setup_kojiclient(profile): """Setup koji client session """ opts = koji.read_config(profile) for key, value in six.iteritems(opts): opts[key] = os.path.expanduser(value) if isinstance(value, str) \ else value kojiclient = koji.ClientSession(opts['server'], opts=opts) # FIXME: who cares about authentication??? # Moreover kerberos authentication can be an annoyance if opts['authtype'] == "ssl" or \ (os.path.isfile(opts['cert']) and opts['authtype'] is None): kojiclient.ssl_login(opts['cert'], None, opts['serverca']) else: print('SKIP authentication') return kojiclient
def __init__(self, logfile): self.logger = logging.getLogger("Splitter") self.logger.setLevel(logging.DEBUG) kobo.log.add_rotating_file_logger(self.logger, logfile) kobo.log.add_stderr_logger(self.logger, log_level=logging.INFO) info_log = CONFIG["info_log"] % dict( date=time.strftime("%Y-%m-%d-%H%M")) kobo.log.add_file_logger(self.logger, info_log, log_level=logging.INFO) self.mash_logger = logging.getLogger("Mash") self.mash_logger.setLevel(logging.DEBUG) mash_log = CONFIG["mash_log"] % dict( date=time.strftime("%Y-%m-%d-%H%M")) kobo.log.add_file_logger(self.mash_logger, mash_log, log_level=logging.DEBUG, format="%(message)s") self.koji_session = koji.ClientSession(CONFIG["koji_url"])
def get_build_objects(ids_or_nvrs, session=None): """Get information of multiple Koji/Brew builds :param ids_or_nvrs: list of build nvr strings or numbers. :param session: instance of :class:`koji.ClientSession` :return: a list Koji/Brew build objects """ logger.debug( "Fetching build info for {} from Koji/Brew...".format(ids_or_nvrs)) if not session: session = koji.ClientSession(constants.BREW_HUB) # Use Koji multicall interface to boost performance. See https://pagure.io/koji/pull-request/957 tasks = [] with session.multicall(strict=True) as m: for b in ids_or_nvrs: tasks.append(m.getBuild(b)) return [task.result for task in tasks]
def get_koji_archives(tag, package): """ Get the list of archives to download from koji given a release tag and a package name. """ client = koji.ClientSession("https://koji.fedoraproject.org/kojihub") builds = client.listTagged(tag, latest=True, package=package, type="image") images = client.listArchives(buildID=builds[0]["id"], type="image") pi = koji.PathInfo(topdir=DOWNLOAD_URL) urls = [] for image in images: if ".tar.xz" in image["filename"]: urls.append(f"{pi.imagebuild(builds[0])}/{image['filename']}") return urls
def login(self) -> None: """ Authenticates to a koji-hub instance. Raises an AuthError in case of failure. """ self.session = koji.ClientSession(self.opts['server'], self.opts) try: self.session.ssl_login(self.opts['cert'], None, self.opts['serverca']) except requests.exceptions.RequestException as e: raise errors.KojiError(str(e)) except koji.AuthError as e: raise errors.AuthError(str(e), data={ 'ca_cert': self.opts['serverca'], 'cert': self.opts['cert'] })