def initialize(self, conf):
self._kojihub_url = conf.kojihub_url
self._kojiroot_url = conf.kojiroot_url
opts = dict()
if conf.request_timeout is not None:
opts['timeout'] = conf.request_timeout
self._session = koji.ClientSession(self._kojihub_url, opts=opts)
if not self._kojihub_url.endswith('/'):
self._kojihub_url += '/'
if not self._kojiroot_url.endswith('/'):
self._kojiroot_url += '/'
def __init__(self, arch=None):
if arch:
self.kojihub = \
'http://{arch}.koji.fedoraproject.org/kojihub'.format(
arch=arch)
else:
self.kojihub = 'https://koji.fedoraproject.org/kojihub'
self.serverca = os.path.expanduser('~/.fedora-server-ca.cert')
self.clientca = os.path.expanduser('~/.fedora-upload-ca.cert')
self.clientcert = os.path.expanduser('~/.fedora.cert')
self.kojisession = koji.ClientSession(self.kojihub)
self.kojisession.ssl_login(self.clientcert, self.clientca,
self.serverca)
def setUp(self):
self.ksession = koji.ClientSession('http://koji.example.com/kojihub')
self.ksession.logout = mock.MagicMock()
self.do_fake_login()
# mocks
self.ksession._callMethod = mock.MagicMock()
self.ksession.retries = 1
self.rsession = mock.patch('requests.Session').start()
if six.PY2:
self.file_mock = mock.patch('__builtin__.open').start()
else:
self.file_mock = mock.patch('builtins.open').start()
self.getsize_mock = mock.patch('os.path.getsize').start()
def session_maker(self):
_log.info("Creating a new Koji session to %s", self.server)
with _koji_session_lock:
koji_session = koji.ClientSession(self.server,
self.krb_sessionopts)
result = koji_session.krb_login(
principal=self.krb_principal,
keytab=self.krb_keytab,
ccache=self.krb_ccache,
proxyuser=self.krb_proxyuser,
)
if not result:
_log.error("Koji kerberos authentication failed")
return koji_session
def get_session(config, login=True):
"""Create and return a koji.ClientSession object
:param config: the config object returned from :meth:`init_config`.
:type config: :class:`Config`
:param bool login: whether to log into the session. To login if True
is passed, otherwise not to log into session.
:return: the Koji session object.
:rtype: :class:`koji.ClientSession`
"""
koji_config = munch.Munch(
koji.read_config(profile_name=config.koji_profile, user_config=config.koji_config))
# Timeout after 10 minutes. The default is 12 hours.
koji_config["timeout"] = 60 * 10
address = koji_config.server
log.info("Connecting to koji %r.", address)
koji_session = koji.ClientSession(address, opts=koji_config)
if not login:
return koji_session
authtype = koji_config.authtype
log.info("Authenticate session with %r.", authtype)
if authtype == "kerberos":
try:
import krbV
# We want to create a context per thread to avoid Kerberos cache corruption
ctx = krbV.Context()
except ImportError:
# If no krbV, we can assume GSSAPI auth is available
ctx = None
keytab = getattr(config, "krb_keytab", None)
principal = getattr(config, "krb_principal", None)
if not keytab and principal:
raise ValueError(
"The Kerberos keytab and principal aren't set for Koji authentication")
log.debug(" keytab: %r, principal: %r" % (keytab, principal))
# We want to use the thread keyring for the ccache to ensure we have one cache per
# thread to avoid Kerberos cache corruption
ccache = "KEYRING:thread:mbs"
koji_session.krb_login(principal=principal, keytab=keytab, ctx=ctx, ccache=ccache)
elif authtype == "ssl":
koji_session.ssl_login(
os.path.expanduser(koji_config.cert), None, os.path.expanduser(koji_config.serverca)
)
else:
raise ValueError("Unrecognized koji authtype %r" % authtype)
return koji_session
def get_koji_session(self):
"""
Return an authenticated koji session.
Returns:
koji.ClientSession: An intialized authenticated koji client.
"""
config = configparser.ConfigParser()
if os.path.exists(os.path.join(os.path.expanduser('~'), '.koji', 'config')):
config.readfp(open(os.path.join(os.path.expanduser('~'), '.koji', 'config')))
else:
config.readfp(open('/etc/koji.conf'))
session = koji.ClientSession(config.get('koji', 'server'))
return session
def get_builds_tags(build_nvrs, session=None):
"""Get tags of multiple Koji/Brew builds
:param builds_nvrs: list of build nvr strings or numbers.
:param session: instance of :class:`koji.ClientSession`
:return: a list of Koji/Brew tag list
"""
if not session:
session = koji.ClientSession(constants.BREW_HUB)
tasks = []
with session.multicall(strict=True) as m:
for nvr in build_nvrs:
tasks.append(m.listTags(build=nvr))
return [task.result for task in tasks]
def get_cbs_tag_list(nvr):
"""
Return all the CBS tags for this build.
:param nvr: a build Name-Version-Release that has been built.
:returns: ``list`` of ``str``, eg ["storage7-ceph-jewel-candidate"]
"""
import koji
conf = koji.read_config('cbs')
hub = conf['server']
print('searching %s for %s tags' % (hub, nvr))
session = koji.ClientSession(hub, {})
tags = session.listTags(nvr)
return [tag['name'] for tag in tags]
def get_koji_session(self, ssl=False):
""" Get a koji session with or without SSL access"""
if self._kojisession_with_ssl and ssl:
return self._kojisession_with_ssl
hub = self.config.get('koji', 'hub')
kojisession = koji.ClientSession(hub)
if ssl:
clientcert = self.config.get('koji', 'clientcert')
clientca = self.config.get('koji', 'clientca')
serverca = self.config.get('koji', 'serverca')
kojisession.ssl_login(clientcert, clientca, serverca)
self._kojisession_with_ssl = kojisession
return kojisession
def __new_session(self):
server = self.config['server']
opts = {
'anon_retry': True,
'max_retries': 1000,
'offline_retry': True,
'offline_retry_interval': 120,
'timeout': 3600,
}
opts.update(self.config.get('session_opts', {}))
session = koji.ClientSession(server, opts)
if not self.__anonymous:
getattr(session, self.config['login_method'])(**self.config['login_args'])
return session
def _listMeadTagArtifacts(self, kojiUrl, downloadRootUrl, tagName, gavPatterns):
"""
Loads maven artifacts from koji (brew/mead).
:param kojiUrl: Koji/Brew/Mead URL
:param downloadRootUrl: Download root URL of the artifacts
:param tagName: Koji/Brew/Mead tag name
:returns: Dictionary where index is MavenArtifact object and value is ArtifactSpec with its
repo root URL.
"""
import koji
kojiSession = koji.ClientSession(kojiUrl)
logging.debug("Getting latest maven artifacts from tag %s.", tagName)
kojiArtifacts = kojiSession.getLatestMavenArchives(tagName)
filenameDict = {}
for artifact in kojiArtifacts:
groupId = artifact['group_id']
artifactId = artifact['artifact_id']
version = artifact['version']
gavUrl = "%s%s/%s/%s/maven/" % (maven_repo_util.slashAtTheEnd(downloadRootUrl), artifact['build_name'],
artifact['build_version'], artifact['build_release'])
gavu = (groupId, artifactId, version, gavUrl)
filename = artifact['filename']
filenameDict.setdefault(gavu, []).append(filename)
gavuExtClass = {} # { (g,a,v,url): {ext: set([class])} }
suffixes = {} # { (g,a,v,url): suffix }
for gavu in filenameDict:
artifactId = gavu[1]
version = gavu[2]
filenames = filenameDict[gavu]
(extsAndClass, suffix) = self._getExtensionsAndClassifiers(artifactId, version, filenames)
if extsAndClass:
gavuExtClass[gavu] = {}
self._updateExtensionsAndClassifiers(gavuExtClass[gavu], extsAndClass)
if suffix is not None:
suffixes[gavu] = suffix
artifacts = {}
for gavu in gavuExtClass:
self._addArtifact(artifacts, gavu[0], gavu[1], gavu[2], gavuExtClass[gavu], suffixes.get(gavu), gavu[3])
if gavPatterns:
logging.debug("Filtering artifacts contained in the tag by GAV patterns list.")
return self._filterArtifactsByPatterns(artifacts, gavPatterns, None)
def __init__(self,
dbpath,
koji_url='http://koji.fedoraproject.org/kojihub'):
self.dbpath = dbpath
self.create_index()
self.koji_client = koji.ClientSession(
'http://koji.fedoraproject.org/kojihub')
self.koji_client.opts['anon_retry'] = True
self.koji_client.opts['offline_retry'] = True
self.updated_packages = {}
self.new_packages = {}
self.found_packages = {}
self.sconn_needs_reload = False
def get_latest_repo(tag):
"""Find latest repo ID and its URL for tag name.
\b
Example of output:
756330 https://kojipkgs.fedoraproject.org/repos/f27-build/756330
"""
ks = koji.ClientSession(KOJIHUB)
pathinfo = koji.PathInfo(topdir="")
repo = ks.getRepo(tag, state=koji.REPO_READY)
repo_id = repo["id"]
path = pathinfo.repo(repo_id, tag)
click.echo("{} {}".format(repo_id, urljoin(KOJIPKGS, path)))
def load_installed(self, releasever):
# Load from dnf rpmdb all installed packages
with dnf.Base() as self.yb:
cachedir = getCacheDir()
self.yb.conf.cachedir = cachedir
self.yb.conf.releasever = dnf.rpm.detect_releasever(
self.yb.conf.installroot)
self.yb.read_all_repos()
self.yb.fill_sack()
self.q = self.yb.sack.query()
self.q = self.q.installed()
self.installed_packages = self.q
# Send it to all bodhi_workers
for i in range(self.bodhi_workers_count):
self.bodhi_workers_queue.put([
'set_installed_packages',
['bodhi_worker' + str(i), self.installed_packages]
])
# Wait for Bodhi workers to finish
self.bodhi_workers_queue.join()
# Send installed packages to GUI
main_thread_call(self.main_thread.set_installed_packages,
self.installed_packages)
# Grab it from Koji
session = koji.ClientSession('http://koji.fedoraproject.org/kojihub')
tagged_packages = session.listTagged('f' + str(releasever) +
'-updates-testing')
# Prepare packages into dictionary to find it faster
pkgs_dict = {}
for pkg in self.installed_packages:
pkgs_dict[pkg.name] = pkg
# See packages for choosen release
pkgsForBodhi = []
for tagged_pkg in tagged_packages:
if tagged_pkg['package_name'] in pkgs_dict.keys():
pkgsForBodhi.append(pkgs_dict[tagged_pkg['package_name']])
# Send these packages to BodhiWorker queue
main_thread_call(self.main_thread.set_num_of_pkgs_to_process,
len(pkgsForBodhi))
for pkg in pkgsForBodhi:
self.bodhi_workers_queue.put(['package_update', pkg])
def _mirror_all_rpms(self):
print('Mirroring RPMs for %r: all packages in %s tagged %s' %
(self, self.koji_profile, self.tag))
hub_url = koji_config.get(self.koji_profile, 'server')
koji_session = koji.ClientSession(hub_url)
if self.downgradeable:
result = koji_session.listTaggedRPMS(self.tag, inherit=True)
else:
result = koji_session.getLatestRPMS(self.tag)
if not result:
raise ValueError('No packages in tag %s' % self.tag)
rpms, builds = result
self.rpm_names.update(rpm['name'] for rpm in rpms
if rpm['name'] not in self.excluded_rpms)
self._mirror_rpms_for_build(self.koji_profile, builds, rpms)
def _build_client(self):
"""
Method to set up the KojiClient object used in this instance of
KojiWrapperBase. It will call all needed methods to get the config set
up for the user.
"""
if self.url:
return koji.ClientSession(self.url)
else:
_configuration = koji.read_config(self.profile,
user_config=self.user_config)
"""
NOTE: This check is here because if the user does not have and koji
config files, read_config will 'helpfully' return you a useless
default config. The required baseurl ('server' in _configuration)
has a default, so we cannot check that. However, topurl defaults
to None, so we currently use this to devine if the returned config
is the useless default.
"""
if not _configuration.get('topurl'):
raise koji.ConfigurationError("no configuration for profile \
name: {0}".format(self.profile))
return koji.ClientSession(_configuration.get('server'),
opts=_configuration)
def test_operator_metadata_latest_build(self):
brew = koji.ClientSession(BREW_HUB)
build = brew.getLatestBuilds(
"rhaos-4.2-rhel-7-candidate",
package="cluster-logging-operator-container")[0]
_, out, _ = run_doozer([
"--group=openshift-4.2",
"--images=cluster-logging-operator",
"operator-metadata:latest-build",
"--stream=dev",
"--nvr",
build["nvr"],
])
self.assertIn("cluster-logging-operator-metadata-container-", out)
def test_krbv_old_requests_kerberos(self, _serverPrincipal_mock,
krbV_mock):
self.assertIsNotNone(koji.krbV)
ctx = koji.krbV.default_context.return_value
ctx.mk_req = mock.MagicMock()
ac = mock.MagicMock()
ctx.mk_req.return_value = (ac, six.b('req'))
ac.rd_priv = mock.MagicMock(return_value='session-id session-key')
session = koji.ClientSession('whatever')
session._callMethod = mock.MagicMock(
return_value=(base64.encodestring(six.b('a')),
base64.encodestring(six.b('b')), [0, 1, 2, 3]))
rv = session.krb_login(principal='*****@*****.**',
keytab='/path/to/keytab')
self.assertTrue(rv)
def koji_builds_in_tag(url, tag):
""" Return the list of koji builds in a tag. """
import koji
log.info("Listing rpms in koji(%s) tag %s" % (url, tag))
session = koji.ClientSession(url)
rpms, builds = session.listTaggedRPMS(tag)
# Extract some srpm-level info from the build attach it to each rpm
builds = {build['build_id']: build for build in builds}
for rpm in rpms:
idx = rpm['build_id']
rpm['srpm_name'] = builds[idx]['name']
rpm['srpm_nevra'] = builds[idx]['nvr']
return rpms
def __init__(self, tasker, workflow, target, hub, root):
"""
constructor
:param tasker: DockerTasker instance
:param workflow: DockerBuildWorkflow instance
:param target: string, koji target to use as a source
:param hub: string, koji hub (xmlrpc)
:param root: string, koji root (storage)
"""
# call parent constructor
super(KojiPlugin, self).__init__(tasker, workflow)
self.target = target
self.xmlrpc = koji.ClientSession(hub)
self.pathinfo = koji.PathInfo(topdir=root)
def get_koji_session(self) -> koji.ClientSession:
"""
Return an authenticated koji session.
Returns:
An initialized authenticated koji client.
"""
config = configparser.ConfigParser()
koji_conf = os.path.join(os.path.expanduser('~'), '.koji', 'config')
if not os.path.exists(koji_conf):
koji_conf = '/etc/koji.conf'
with open(koji_conf) as fh:
config.read_file(fh)
session = koji.ClientSession(config.get('koji', 'server'))
return session
def create_koji_session(hub_url, auth_info=None):
"""
Creates and returns a Koji session. If auth_info
is provided, the session will be authenticated.
:param hub_url: str, Koji hub URL
:param auth_info: dict, authentication parameters used for koji_login
:return: koji.ClientSession instance
"""
session = koji.ClientSession(hub_url, opts={'krb_rdns': False})
if auth_info is not None:
koji_login(session, **auth_info)
return session
def get_buildsys(which):
"""Get a koji build system session for either the source or the
destination. Caches the sessions so future calls are cheap.
Destination sessions are authenticated, source sessions are not.
:param which: Session to select, source or destination
:returns: Koji session object, or None on error
"""
if 'main' not in c:
logger.critical('DistroBaker is not configured, aborting.')
return None
if which != 'source' and which != 'destination':
logger.error('Cannot get "%s" build system.', which)
return None
if not hasattr(get_buildsys, which):
logger.debug(
'Initializing the %s koji instance with the "%s" profile.', which,
c['main'][which]['profile'])
try:
bsys = koji.read_config(profile_name=c['main'][which]['profile'])
bsys = koji.ClientSession(bsys['server'], opts=bsys)
except Exception:
logger.exception(
'Failed initializing the %s koji instance with the "%s" profile, skipping.',
which, c['main'][which]['profile'])
return None
logger.debug('The %s koji instance initialized.', which)
if which == 'destination':
logger.debug('Authenticating with the destination koji instance.')
try:
bsys.gssapi_login()
except Exception:
logger.exception(
'Failed authenticating against the destination koji instance, skipping.'
)
return None
logger.debug(
'Successfully authenticated with the destination koji instance.'
)
if which == 'source':
get_buildsys.source = bsys
else:
get_buildsys.destination = bsys
else:
logger.debug(
'The %s koji instance is already initialized, fetching from cache.',
which)
return vars(get_buildsys)[which]
def get_koji_session(self, login=True):
""" Return an authenticated koji session """
import koji
from iniparse.compat import ConfigParser
config = ConfigParser()
if exists(join(expanduser('~'), '.koji', 'config')):
config.readfp(open(join(expanduser('~'), '.koji', 'config')))
else:
config.readfp(open('/etc/koji.conf'))
cert = expanduser(config.get('koji', 'cert'))
ca = expanduser(config.get('koji', 'ca'))
serverca = expanduser(config.get('koji', 'serverca'))
session = koji.ClientSession(config.get('koji', 'server'))
if login:
session.ssl_login(cert=cert, ca=ca, serverca=serverca)
return session
def koji_login(config):
""" Login to Koji and return the session """
koji_options = {
'krb_rdns': False,
'max_retries': 30,
'retry_interval': 10,
'offline_retry': True,
'offline_retry_interval': 10,
'anon_retry': True,
}
koji_client = koji.ClientSession(_koji_hub, koji_options)
if not koji_client.krb_login(**get_krb_conf(config)):
log.error('Koji krb_login failed')
return koji_client
def setup_kojiclient(profile):
"""Setup koji client session
"""
opts = koji.read_config(profile)
for key, value in six.iteritems(opts):
opts[key] = os.path.expanduser(value) if isinstance(value, str) \
else value
kojiclient = koji.ClientSession(opts['server'], opts=opts)
# FIXME: who cares about authentication???
# Moreover kerberos authentication can be an annoyance
if opts['authtype'] == "ssl" or \
(os.path.isfile(opts['cert']) and opts['authtype'] is None):
kojiclient.ssl_login(opts['cert'], None, opts['serverca'])
else:
print('SKIP authentication')
return kojiclient
def __init__(self, logfile):
self.logger = logging.getLogger("Splitter")
self.logger.setLevel(logging.DEBUG)
kobo.log.add_rotating_file_logger(self.logger, logfile)
kobo.log.add_stderr_logger(self.logger, log_level=logging.INFO)
info_log = CONFIG["info_log"] % dict(
date=time.strftime("%Y-%m-%d-%H%M"))
kobo.log.add_file_logger(self.logger, info_log, log_level=logging.INFO)
self.mash_logger = logging.getLogger("Mash")
self.mash_logger.setLevel(logging.DEBUG)
mash_log = CONFIG["mash_log"] % dict(
date=time.strftime("%Y-%m-%d-%H%M"))
kobo.log.add_file_logger(self.mash_logger,
mash_log,
log_level=logging.DEBUG,
format="%(message)s")
self.koji_session = koji.ClientSession(CONFIG["koji_url"])
def get_build_objects(ids_or_nvrs, session=None):
"""Get information of multiple Koji/Brew builds
:param ids_or_nvrs: list of build nvr strings or numbers.
:param session: instance of :class:`koji.ClientSession`
:return: a list Koji/Brew build objects
"""
logger.debug(
"Fetching build info for {} from Koji/Brew...".format(ids_or_nvrs))
if not session:
session = koji.ClientSession(constants.BREW_HUB)
# Use Koji multicall interface to boost performance. See https://pagure.io/koji/pull-request/957
tasks = []
with session.multicall(strict=True) as m:
for b in ids_or_nvrs:
tasks.append(m.getBuild(b))
return [task.result for task in tasks]
def get_koji_archives(tag, package):
"""
Get the list of archives to download from koji
given a release tag and a package name.
"""
client = koji.ClientSession("https://koji.fedoraproject.org/kojihub")
builds = client.listTagged(tag, latest=True, package=package, type="image")
images = client.listArchives(buildID=builds[0]["id"], type="image")
pi = koji.PathInfo(topdir=DOWNLOAD_URL)
urls = []
for image in images:
if ".tar.xz" in image["filename"]:
urls.append(f"{pi.imagebuild(builds[0])}/{image['filename']}")
return urls
def login(self) -> None:
"""
Authenticates to a koji-hub instance.
Raises an AuthError in case of failure.
"""
self.session = koji.ClientSession(self.opts['server'], self.opts)
try:
self.session.ssl_login(self.opts['cert'], None,
self.opts['serverca'])
except requests.exceptions.RequestException as e:
raise errors.KojiError(str(e))
except koji.AuthError as e:
raise errors.AuthError(str(e),
data={
'ca_cert': self.opts['serverca'],
'cert': self.opts['cert']
})
