def create_session(cls, profile='koji'):
"""Creates new Koji session and immediately logs in to a Koji hub.
Args:
profile (str): Koji profile to use.
Returns:
koji.ClientSession: Newly created session instance.
Raises:
RebaseHelperError: If login failed.
"""
config = koji.read_config(profile)
session = koji.ClientSession(config['server'], opts=config)
try:
session.gssapi_login()
except Exception: # pylint: disable=broad-except
pass
else:
return session
# fall back to kerberos login (doesn't work with python3)
exc = (koji.AuthError,
koji.krbV.Krb5Error) if koji.krbV else koji.AuthError
try:
session.krb_login()
except exc as e:
raise RebaseHelperError('Login failed: {}'.format(
six.text_type(e)))
else:
return session
def create_session(cls, login=False, profile='koji'):
"""Creates new Koji session and immediately logs in to a Koji hub.
Args:
login (bool): Whether to perform a login.
profile (str): Koji profile to use.
Returns:
koji.ClientSession: Newly created session instance.
Raises:
RebaseHelperError: If login failed.
"""
config = koji.read_config(profile)
session = koji.ClientSession(config['server'], opts=config)
if not login:
return session
try:
session.gssapi_login()
except Exception: # pylint: disable=broad-except
pass
else:
return session
# fall back to kerberos login (doesn't work with python3)
exc = (koji.AuthError, koji.krbV.Krb5Error) if koji.krbV else koji.AuthError
try:
session.krb_login()
except exc as e:
raise RebaseHelperError('Login failed: {}'.format(str(e)))
else:
return session
def get_koji_session():
"""
Get a koji session for accessing kojihub functions.
"""
conf = koji.read_config('brew')
hub = conf['server']
return koji.ClientSession(hub, {})
def tag_build(nvr, dest_tags):
"""Tag build with specific tags
Calling Koji API to tag build might fail, however successful tagged tag will
be returned and to log the failed tag operation.
:param str nvr: build NVR.
:param dest_tags: tag names.
:type dest_tags: list[str]
:return: tag names which are tagged to build successfully.
:rtype: list[str]
"""
tagged_tags = []
koji_config = koji.read_config(conf.koji_profile)
koji_session = koji.ClientSession(koji_config['server'])
koji_session.krb_login()
for tag in dest_tags:
try:
if conf.dry_run:
logger.info("DRY-RUN: koji_session.tagBuild('%s', '%s')", tag,
nvr)
else:
koji_session.tagBuild(tag, nvr)
except Exception:
logger.exception('Failed to tag %s to build %s', tag, nvr)
else:
tagged_tags.append(tag)
koji_session.logout()
return tagged_tags
def from_config_profile(cls, profile_name: str, **kwargs) -> "Service":
"""Constructs new instance from local configuration profile.
Keyword arguments:
profile_name: Name of the profile to use.
"""
return cls(configuration=koji.read_config(profile_name), **kwargs)
def setup_kojiclient(profile):
"""Setup koji client session
"""
opts = koji.read_config(profile)
for k, v in opts.iteritems():
opts[k] = os.path.expanduser(v) if type(v) is str else v
kojiclient = koji.ClientSession(opts['server'], opts=opts)
kojiclient.ssl_login(opts['cert'], None, opts['serverca'])
return kojiclient
def __init__(self, profile: str = "koji"):
"""
:param profile: name of the koji profile to load from local
configuration locations
"""
conf = read_config(profile)
server = conf["server"]
super().__init__(server, opts=conf)
def make_koji_session():
koji_config = koji.read_config(conf.koji_profile)
session_opts = koji.grab_session_options(koji_config)
koji_session = koji.ClientSession(koji_config['server'], opts=session_opts)
login_koji(koji_session, koji_config)
yield koji_session
koji_session.logout()
def setup_kojiclient(profile):
"""Setup koji client session
"""
opts = koji.read_config(profile)
for k, v in opts.iteritems():
opts[k] = os.path.expanduser(v) if type(v) is str else v
kojiclient = koji.ClientSession(opts['server'], opts=opts)
kojiclient.ssl_login(opts['cert'], None, opts['serverca'])
return kojiclient
def get_koji_session(options):
koji_config = munch.Munch(koji.read_config(
profile_name=options.koji_profile,
))
#koji_module = koji.get_profile_module(
# options.koji_profile,
# config=koji_config,
# )
address = koji_config.server
return koji.ClientSession(address, opts=koji_config)
def get_koji_pathinfo(profile):
"""
Return a Koji PathInfo object for our profile.
:param str profile: profile name, like "koji" or "cbs"
:returns: koji.PathInfo
"""
conf = koji.read_config(profile)
top = conf['topurl']
pathinfo = koji.PathInfo(topdir=top)
return pathinfo
def build_comp(comp, ref, ns='rpms', dry_run=False):
logging.info('Processing build for {}/{}.'.format(ns, comp))
if ns == 'rpms':
try:
buildconf = koji.read_config(
profile_name=c['main']['build']['profile'])
except Exception as e:
logging.error(
'Failed initializing koji with the {} profile while building {}/{}, skipping.'
.format(c['main']['build']['profile'], ns, comp))
logging.error('EXCEPTION: ' + str(e))
return None
buildsys = koji.ClientSession(buildconf['server'], opts=buildconf)
try:
buildsys.gssapi_login()
except Exception as e:
logging.error(
'Failed authenticating against koji while building {}/{}, skipping.'
.format(ns, comp))
logging.error('EXCEPTION: ' + str(e))
return None
try:
if not dry_run:
task = buildsys.build(
'{}/{}/{}#{}'.format(c['main']['build']['prefix'], ns,
comp, ref),
c['main']['build']['target'],
{'scratch': c['main']['build']['scratch']})
logging.info(
'Build submitted for {}/{}; task {}; SCMURL: {}/{}/{}#{}.'.
format(ns, comp, task, c['main']['build']['prefix'], ns,
comp, ref))
return task
else:
logging.info(
'Running in the dry mode, not submitting any builds for {}/{} ({}/{}/{}#{}).'
.format(ns, comp, c['main']['build']['prefix'], ns, comp,
ref))
return 0
except Exception as e:
logging.error(
'Failed submitting build for {}/{} ({}/{}/{}#{}).'.format(
ns, comp, c['main']['build']['prefix'], ns, comp, ref))
logging.error('EXCEPTION: ' + str(e))
return None
elif ns == 'modules':
logging.critical(
'Cannot build {}/{}; module building not implemented.'.format(
ns, comp))
return None
else:
logging.critical('Cannot build {}/{}; unknown namespace.'.format(
ns, comp))
return None
def __init__(self, profile=None, dry_run=False):
self._config = koji.read_config(profile or 'koji')
self.dry_run = dry_run
# In case we run in DRY_RUN mode, we need to initialize
# _FAKE_TASK_ID to the id of last ODCS builds to have the IDs
# increasing and unique even between Freshmaker restarts.
if self.dry_run:
KojiService._FAKE_TASK_ID = \
ArtifactBuild.get_lowest_build_id(db.session) - 1
if KojiService._FAKE_TASK_ID >= 0:
KojiService._FAKE_TASK_ID = -1
def get_session(config, login=True):
"""Create and return a koji.ClientSession object
:param config: the config object returned from :meth:`init_config`.
:type config: :class:`Config`
:param bool login: whether to log into the session. To login if True
is passed, otherwise not to log into session.
:return: the Koji session object.
:rtype: :class:`koji.ClientSession`
"""
koji_config = munch.Munch(
koji.read_config(profile_name=config.koji_profile, user_config=config.koji_config))
# Timeout after 10 minutes. The default is 12 hours.
koji_config["timeout"] = 60 * 10
address = koji_config.server
log.info("Connecting to koji %r.", address)
koji_session = koji.ClientSession(address, opts=koji_config)
if not login:
return koji_session
authtype = koji_config.authtype
log.info("Authenticate session with %r.", authtype)
if authtype == "kerberos":
try:
import krbV
# We want to create a context per thread to avoid Kerberos cache corruption
ctx = krbV.Context()
except ImportError:
# If no krbV, we can assume GSSAPI auth is available
ctx = None
keytab = getattr(config, "krb_keytab", None)
principal = getattr(config, "krb_principal", None)
if not keytab and principal:
raise ValueError(
"The Kerberos keytab and principal aren't set for Koji authentication")
log.debug(" keytab: %r, principal: %r" % (keytab, principal))
# We want to use the thread keyring for the ccache to ensure we have one cache per
# thread to avoid Kerberos cache corruption
ccache = "KEYRING:thread:mbs"
koji_session.krb_login(principal=principal, keytab=keytab, ctx=ctx, ccache=ccache)
elif authtype == "ssl":
koji_session.ssl_login(
os.path.expanduser(koji_config.cert), None, os.path.expanduser(koji_config.serverca)
)
else:
raise ValueError("Unrecognized koji authtype %r" % authtype)
return koji_session
def get_cbs_tag_list(nvr):
"""
Return all the CBS tags for this build.
:param nvr: a build Name-Version-Release that has been built.
:returns: ``list`` of ``str``, eg ["storage7-ceph-jewel-candidate"]
"""
import koji
conf = koji.read_config('cbs')
hub = conf['server']
print('searching %s for %s tags' % (hub, nvr))
session = koji.ClientSession(hub, {})
tags = session.listTags(nvr)
return [tag['name'] for tag in tags]
def get_buildsys(which):
"""Get a koji build system session for either the source or the
destination. Caches the sessions so future calls are cheap.
Destination sessions are authenticated, source sessions are not.
:param which: Session to select, source or destination
:returns: Koji session object, or None on error
"""
if 'main' not in c:
logger.critical('DistroBaker is not configured, aborting.')
return None
if which != 'source' and which != 'destination':
logger.error('Cannot get "%s" build system.', which)
return None
if not hasattr(get_buildsys, which):
logger.debug(
'Initializing the %s koji instance with the "%s" profile.', which,
c['main'][which]['profile'])
try:
bsys = koji.read_config(profile_name=c['main'][which]['profile'])
bsys = koji.ClientSession(bsys['server'], opts=bsys)
except Exception:
logger.exception(
'Failed initializing the %s koji instance with the "%s" profile, skipping.',
which, c['main'][which]['profile'])
return None
logger.debug('The %s koji instance initialized.', which)
if which == 'destination':
logger.debug('Authenticating with the destination koji instance.')
try:
bsys.gssapi_login()
except Exception:
logger.exception(
'Failed authenticating against the destination koji instance, skipping.'
)
return None
logger.debug(
'Successfully authenticated with the destination koji instance.'
)
if which == 'source':
get_buildsys.source = bsys
else:
get_buildsys.destination = bsys
else:
logger.debug(
'The %s koji instance is already initialized, fetching from cache.',
which)
return vars(get_buildsys)[which]
def setup_kojiclient(profile):
"""Setup koji client session
"""
opts = koji.read_config(profile)
for key, value in six.iteritems(opts):
opts[key] = os.path.expanduser(value) if isinstance(value, str) \
else value
kojiclient = koji.ClientSession(opts['server'], opts=opts)
# FIXME: who cares about authentication???
# Moreover kerberos authentication can be an annoyance
if opts['authtype'] == "ssl" or \
(os.path.isfile(opts['cert']) and opts['authtype'] is None):
kojiclient.ssl_login(opts['cert'], None, opts['serverca'])
else:
print('SKIP authentication')
return kojiclient
def load_anon_kojisession(self):
"""Initiate a koji session."""
logger = logging.getLogger("load_anon_kojisession")
koji_config = koji.read_config(self.kojiprofile)
logger.debug('Initiating a brew session to %s',
os.path.basename(koji_config['server']))
# Build session options used to create instance of ClientSession
session_opts = koji.grab_session_options(koji_config)
try:
session = koji.ClientSession(koji_config['server'], session_opts)
except Exception:
raise Exception('Could not initiate brew session')
else:
return session
def get_koji_session():
# Return an unauthenticated koji session
try:
conf = koji.read_config(PROFILE)
except koji.ConfigurationError as e:
if 'no configuration for profile name' in str(e):
print('You are missing the brewkoji RPM. Please install it.')
print('It is available in the RCMTOOLS composes.')
print('http://download.devel.redhat.com/rel-eng/RCMTOOLS/')
raise
hub = conf['server']
opts = {'krbservice': conf['krbservice']}
session = koji.ClientSession(hub, opts)
# KojiOptions = namedtuple('Options', ['authtype', 'debug'])
# options = KojiOptions(authtype='')
# Can I simply pass in the conf dict like this?
activate_session(session, conf)
return session
def __init__(self, output, base):
self.output = output
self.base = base
self.base_update_info = {}
self.update_info = {}
self.build_id_to_nvr = {}
self.module_nvr_short_to_nvr = {}
self.image_packages = set()
if self.base:
self._load_build_ids(self.base)
self._load_update_info(self.base)
koji_config_file = os.path.join(os.path.dirname(__file__),
'../koji.conf')
options = koji.read_config(profile_name='fedora',
user_config=koji_config_file)
session_opts = koji.grab_session_options(options)
self.koji_session = koji.ClientSession(options['server'], session_opts)
def get_cbs_build(srpm):
"""
Return the Name-Version-Release if a build already exists in CBS for this
SRPM, or None if it does not exist.
:param version: a SRPM file name,
eg. ceph-ansible-3.0.0-0.1.rc10.1.el7.src.rpm
:returns: ``str``, eg. "ceph-ansible-3.0.0-0.1.rc10.1.el7"
if the completed build exists in CBS.
``None`` if the completed build does not exist in CBS.
"""
nvr = srpm_nvr(srpm)
import koji # oh yeah
conf = koji.read_config('cbs')
hub = conf['server']
print('searching %s for %s' % (hub, nvr))
session = koji.ClientSession(hub, {})
build = session.getBuild(nvr)
if build is None:
return None
if koji.BUILD_STATES[build['state']] == 'COMPLETE':
return nvr
return None
def new_build(profile='cbs', scratch=True):
# Very ugly: some utilities are only available in koji CLI
# and not in the koji module
if not KOJI_AVAILABLE:
raise exception.ModuleNotAvailable(module='koji')
if not RPM_AVAILABLE:
raise exception.ModuleNotAvailable(module='rpm')
import imp
kojibin = find_executable('koji')
kojicli = imp.load_source('kojicli', kojibin)
from kojicli import _unique_path, _progress_callback, watch_tasks
build_target = guess_build()
if not build_target:
log.warn("failed to identify build tag from branch")
return
options = koji.read_config(profile)
opts = KojiOpts(**options)
# Note: required to make watch_tasks work
kojicli.options, weburl = opts, opts.weburl
kojiclient = koji.ClientSession(opts.server)
kojiclient.ssl_login(opts.cert, None, opts.serverca)
retrieve_sources()
srpm = create_srpm()
opts = {'scratch': scratch}
if not srpm:
log.warn('No srpm available')
return
serverdir = _unique_path('cli-build')
kojiclient.uploadWrapper(srpm, serverdir, callback=_progress_callback)
source = "%s/%s" % (serverdir, os.path.basename(srpm))
task_id = kojiclient.build(source, build_target, opts)
print "Created task:", task_id
print "Task info: {}/taskinfo?taskID={}".format(weburl, task_id)
kojiclient.logout()
watch_tasks(kojiclient, [task_id])
def _build_client(self):
"""
Method to set up the KojiClient object used in this instance of
KojiWrapperBase. It will call all needed methods to get the config set
up for the user.
"""
if self.url:
return koji.ClientSession(self.url)
else:
_configuration = koji.read_config(self.profile,
user_config=self.user_config)
"""
NOTE: This check is here because if the user does not have and koji
config files, read_config will 'helpfully' return you a useless
default config. The required baseurl ('server' in _configuration)
has a default, so we cannot check that. However, topurl defaults
to None, so we currently use this to devine if the returned config
is the useless default.
"""
if not _configuration.get('topurl'):
raise koji.ConfigurationError("no configuration for profile \
name: {0}".format(self.profile))
return koji.ClientSession(_configuration.get('server'),
opts=_configuration)
def get_koji_pathinfo():
# Return a Koji PathInfo object for our profile.
conf = koji.read_config(PROFILE)
top = conf['topurl'] # or 'topdir' here for NFS access
pathinfo = koji.PathInfo(topdir=top)
return pathinfo
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import gitlab
import koji
from cryptography.fernet import Fernet
from koji_cli.lib import activate_session
from distrobuild.lookaside import Lookaside
from distrobuild.mbs import MBSClient
from distrobuild.settings import settings
gl = gitlab.Gitlab(f"https://{settings.gitlab_host}",
private_token=settings.gitlab_api_key)
koji_config = koji.read_config("koji")
koji_session = koji.ClientSession(koji_config["server"], koji_config)
activate_session(koji_session, koji_config)
mbs_client = MBSClient(settings.mbs_url)
message_cipher = Fernet(settings.message_secret)
lookaside_session = Lookaside(settings.storage_addr)
def getProductListings(productLabel, buildInfo):
"""
Get a map of which variants of the given product included packages built
by the given build, and which arches each variant included.
"""
compose_dbh = Products.compose_get_dbh()
#XXX - need access to hub kojihub functions
conf = koji.read_config('brew')
hub = conf['server']
session = koji.ClientSession(hub, {})
build = session.getBuild(buildInfo, strict=True)
sys.stderr.write("%r" % build)
sys.stderr.flush()
rpms = session.listRPMs(buildID=build['id'])
if not rpms:
raise koji.GenericError("Could not find any RPMs for build: %s" %
buildInfo)
# sort rpms, so first part of list consists of sorted 'normal' rpms and
# second part are sorted debuginfos
debuginfos = [x for x in rpms if '-debuginfo' in x['nvr']]
base_rpms = [x for x in rpms if '-debuginfo' not in x['nvr']]
rpms = sorted(base_rpms, key=lambda x: x['nvr']) + sorted(
debuginfos, key=lambda x: x['nvr'])
srpm = "%(package_name)s-%(version)s-%(release)s.src.rpm" % build
prodinfo = Products.get_product_info(compose_dbh, productLabel)
if not prodinfo:
# no product with the given label exists
raise koji.GenericError("Could not find a product with label: %s" %
productLabel)
version, variants = prodinfo
listings = {}
match_version = Products.get_match_versions(compose_dbh, productLabel)
for variant in variants:
if variant == None:
# dict keys must be a string
variant = ''
treelist = Products.precalc_treelist(compose_dbh, productLabel,
version, variant)
if not treelist:
continue
overrides = Products.get_overrides(compose_dbh, productLabel, version,
variant)
cache_map = {}
for rpm in rpms:
if rpm['name'] in match_version:
rpm_version = rpm['version']
else:
rpm_version = None
# without debuginfos
rpms_nondebug = [
rpm for rpm in rpms if not koji.is_debuginfo(rpm['name'])
]
d = {}
all_archs = set([rpm['arch'] for rpm in rpms_nondebug])
for arch in all_archs:
d[arch] = Products.dest_get_archs(
compose_dbh,
treelist,
arch,
[rpm['name'] for rpm in rpms_nondebug if rpm['arch'] == arch],
cache_map.get(srpm, {}).get(arch, {}),
rpm_version,
overrides,
)
for rpm in rpms_nondebug:
dest_archs = d[rpm['arch']].get(rpm['name'], {}).keys()
if rpm['arch'] != 'src':
cache_map.setdefault(srpm, {})
cache_map[srpm].setdefault(rpm['arch'], {})
for x in dest_archs:
cache_map[srpm][rpm['arch']][x] = 1
for dest_arch in dest_archs:
listings.setdefault(variant,
{}).setdefault(rpm['nvr'], {}).setdefault(
rpm['arch'], []).append(dest_arch)
# debuginfo only
rpms_debug = [rpm for rpm in rpms if koji.is_debuginfo(rpm['name'])]
d = {}
all_archs = set([rpm['arch'] for rpm in rpms_debug])
for arch in all_archs:
d[arch] = Products.dest_get_archs(
compose_dbh,
treelist,
arch,
[rpm['name'] for rpm in rpms_debug if rpm['arch'] == arch],
cache_map.get(srpm, {}).get(arch, {}),
rpm_version,
overrides,
)
for rpm in rpms_debug:
dest_archs = d[rpm['arch']].get(rpm['name'], {}).keys()
if rpm['arch'] != 'src':
cache_map.setdefault(srpm, {})
cache_map[srpm].setdefault(rpm['arch'], {})
for x in dest_archs:
cache_map[srpm][rpm['arch']][x] = 1
for dest_arch in dest_archs:
listings.setdefault(variant,
{}).setdefault(rpm['nvr'], {}).setdefault(
rpm['arch'], []).append(dest_arch)
for variant in listings.keys():
nvrs = listings[variant].keys()
#BREW-260: Read allow_src_only flag for the product/version
allow_src_only = Products.get_srconly_flag(compose_dbh,
productLabel, version)
if len(nvrs) == 1:
maps = listings[variant][nvrs[0]].keys()
#BREW-260: check for allow_src_only flag added
if len(maps) == 1 and maps[0] == 'src' and not allow_src_only:
del listings[variant]
return listings
