def test_compare_nvr(self):
first = {'name': 'a', 'version': '1', 'release': '1', 'epoch': '1'}
second = {'name': 'a', 'version': '1', 'release': '1', 'epoch': '1'}
self.assertEqual(compare_nvr(first, second), 0)
second['version'] = '0'
self.assertEqual(compare_nvr(first, second), 1)
second['version'] = 0
self.assertEqual(compare_nvr(first, second), 1)
second['version'] = 2
self.assertEqual(compare_nvr(first, second), -1)
second['version'] = 1
second['release'] = 0
self.assertEqual(compare_nvr(first, second), 1)
second['release'] = 2
self.assertEqual(compare_nvr(first, second), -1)
second['release'] = 1
second['epoch'] = 0
self.assertEqual(compare_nvr(first, second), 1)
second['epoch'] = 2
self.assertEqual(compare_nvr(first, second), -1)
first = {'name': 'a', 'version': '1', 'release': '1', 'epoch': None}
second = {'name': 'a', 'version': '1', 'release': '1', 'epoch': '1'}
self.assertEqual(compare_nvr(first, second), -1)
# missing epoch
first = {'name': 'a', 'version': '1', 'release': '1'}
self.assertEqual(compare_nvr(first, second), -1)
def test_compare_nvr(self):
first = {'name': 'a', 'version': '1', 'release': '1', 'epoch': '1'}
second = {'name': 'a', 'version': '1', 'release': '1', 'epoch': '1'}
self.assertEqual(compare_nvr(first, second), 0)
second['version'] = '0'
self.assertEqual(compare_nvr(first, second), 1)
second['version'] = 0
self.assertEqual(compare_nvr(first, second), 1)
second['version'] = 2
self.assertEqual(compare_nvr(first, second), -1)
second['version'] = 1
second['release'] = 0
self.assertEqual(compare_nvr(first, second), 1)
second['release'] = 2
self.assertEqual(compare_nvr(first, second), -1)
second['release'] = 1
second['epoch'] = 0
self.assertEqual(compare_nvr(first, second), 1)
second['epoch'] = 2
self.assertEqual(compare_nvr(first, second), -1)
first = {'name': 'a', 'version': '1', 'release': '1', 'epoch': None}
second = {'name': 'a', 'version': '1', 'release': '1', 'epoch': '1'}
self.assertEqual(compare_nvr(first, second), -1)
# missing epoch
first = {'name': 'a', 'version': '1', 'release': '1'}
self.assertEqual(compare_nvr(first, second), -1)
def get_brew_builds(koji_proxy, brew_tags, latest=False, inherit=False):
"""
"""
brew_builds_by_name = {}
brew_builds = set()
if not brew_tags:
brew_tags = []
if isinstance(brew_tags, basestring):
brew_tags = [brew_tags]
for brew_tag in brew_tags:
for build in koji_proxy.listTagged(brew_tag,
latest=latest,
inherit=inherit):
if latest:
previous_build = brew_builds_by_name.get(build["name"])
if not previous_build or compare_nvr(previous_build,
build) == 1:
brew_builds_by_name[build["name"]] = build
brew_builds.remove(previous_build)
brew_builds.add(build["nvr"])
else:
brew_builds.add(build["nvr"])
return brew_builds
def is_embargoed(an_nvr):
# .p1 or inclusion in the embargoed_tag_nvrs indicates this rpm is embargoed OR *was* embargoed.
# We can ignore it if it has already shipped.
parsed_test_nvr = parse_nvr(an_nvr)
if released_nvr is None or compare_nvr(
parsed_test_nvr,
released_nvr) > 0: # If this nvr hasn't shipped
if '.p1' in an_nvr or an_nvr in embargoed_tag_nvrs: # It's embargoed!
return True
return False
def is_embargoed(an_nvre):
# .p1 or inclusion in the embargoed_tag_nvrs indicates this rpm is embargoed OR *was* embargoed.
# We can ignore it if it has already shipped.
test_nvre_obj = parse_nvr(an_nvre)
if released_nvre_obj is None or compare_nvr(
test_nvre_obj,
released_nvre_obj) > 0: # If this nvr hasn't shipped
if '.p1' in an_nvre or strip_epoch(
an_nvre) in embargoed_tag_nvrs: # It's embargoed!
return True
return False
def from_tags(config, brew_tag, embargoed_brew_tag, embargoed_nvr,
signing_advisory_id, signing_advisory_mode, poll_for, event,
include_embargoed):
"""
The repositories are filled with RPMs derived from the list of
brew tags. If the RPMs are not signed and a repo should contain signed content,
the specified advisory will be used for signing the RPMs (requires
automatic sign on attach).
If you specify --embargoed-brew-tag, plashet will treat any nvr found in this tag as if it is
embargoed (unless has already shipped). This is useful since the .p1 convention cannot be used
on RPMs not built by ART.
\b
--brew-tag <tag> <product_version> example: --brew-tag rhaos-4.5-rhel-8-candidate OSE-4.5-RHEL-8 --brew-tag .. ..
"""
koji_proxy = KojiWrapper(
koji.ClientSession(config.brew_url,
opts={
'krbservice': 'brewhub',
'serverca': '/etc/pki/brew/legacy.crt'
}))
koji_proxy.gssapi_login()
errata_proxy = xmlrpclib.ServerProxy(config.errata_xmlrpc_url)
if event:
event = int(event)
event_info = koji_proxy.getEvent(event)
else:
# If none was specified, lock in an event so that there are no race conditions with
# packages changing while we run.
event_info = koji_proxy.getLastEvent()
event = event_info['id']
# Gather up all nvrs tagged in the embargoed brew tags into a set.
embargoed_tag_nvrs = set()
embargoed_tag_nvrs.update(embargoed_nvr)
for ebt in embargoed_brew_tag:
for build in koji_proxy.listTagged(ebt,
latest=False,
inherit=False,
event=event,
type='rpm'):
embargoed_tag_nvrs.add(build['nvr'])
logger.info(
'Will treat the following nvrs as potentially embargoed: {}'.format(
embargoed_tag_nvrs))
actual_embargoed_nvrs = list() # A list of nvrs detected as embargoed
desired_nvrs = set()
nvr_product_version = {}
for tag, product_version in brew_tag:
released_package_nvrs = {
} # maps released package names to the most recently released package (parsed) nvr
if tag.endswith('-candidate'):
"""
So here's the thing. If you ship a version of a package 1.16.6 via errata tool,
it will prevent you from shipping an older version of that package (e.g. 1.16.2) or even
attaching it to an errata. This prevents us from auto-signing the older package. Since it
is just invalid, we need to find the latest version of packages which have shipped
and make sure plashet filters out anything that is older before signing/building.
Without this filtering, the error from errata tool looks like:
b'{"error":"Unable to add build \'cri-o-1.16.6-2.rhaos4.3.git4936f44.el7\' which is older than cri-o-1.16.6-16.dev.rhaos4.3.git4936f44.el7"}'
"""
released_tag = tag[:tag.index('-candidate')]
for build in koji_proxy.listTagged(released_tag,
latest=True,
inherit=True,
event=event,
type='rpm'):
package_name = build['package_name']
released_package_nvrs[package_name] = parse_nvr(build['nvr'])
for build in koji_proxy.listTagged(tag,
latest=True,
inherit=False,
event=event,
type='rpm'):
package_name = build['package_name']
nvr = build['nvr']
parsed_nvr = parse_nvr(nvr)
released_nvr = None # if the package has shipped before, the parsed nvr of the most recently shipped
if package_name in released_package_nvrs:
released_nvr = released_package_nvrs[package_name]
if package_name in config.exclude_package:
logger.info(
f'Skipping tagged but command line excluded package: {nvr}'
)
continue
if config.include_package and package_name not in config.include_package:
logger.info(
f'Skipping tagged but not command line included package: {nvr}'
)
continue
if '.p1' in nvr or nvr in embargoed_tag_nvrs:
# p1 or inclusion in the embargoed_tag_nvrs indicates this rpm is embargoed OR *was* embargoed.
# We can ignore it if it has already shipped.
if released_nvr is None or compare_nvr(
parsed_nvr,
released_nvr) > 0: # Is our nvr > last shipped?
# Our embargoed build has not been shipped.
actual_embargoed_nvrs.append(
nvr
) # Record that at the time of build, this was considered embargoed
if not include_embargoed:
# We are being asked to build a plashet without embargoed RPMs. We need to find a stand-in.
# Search through the tag's package history to find the last build that was NOT embargoed.
unembargoed_nvr = None
for build in koji_proxy.listTagged(
tag,
package=package_name,
inherit=True,
event=event,
type='rpm'):
test_nvr = build['nvr']
parsed_test_nvr = parse_nvr(test_nvr)
if released_nvr is None or compare_nvr(
parsed_test_nvr, released_nvr
) > 0: # If this nvr hasn't shipped
if '.p1' in test_nvr or test_nvr in embargoed_tag_nvrs: # Looks like this one is embargoed too
continue
unembargoed_nvr = test_nvr
break
if unembargoed_nvr is None:
raise IOError(
f'Unable to build unembargoed plashet. Lastest build of {package_name} ({nvr}) is embargoed but unable to find unembargoed version in history'
)
plashet_concerns.append(
f'Swapping embargoed nvr {nvr} for unembargoed nvr {unembargoed_nvr}.'
)
logger.info(plashet_concerns[-1])
nvr = unembargoed_nvr
else:
logger.info(
f'NVR {nvr} was potentially embargoed, but has already shipped'
)
if released_nvr:
if compare_nvr(
parsed_nvr, released_nvr
) < 0: # if the current nvr is less than the released NVR
msg = f'Skipping tagged {nvr} because it is older than a released version: {released_nvr}'
plashet_concerns.append(msg)
logger.error(msg)
continue
logger.info(f'{tag} contains package: {nvr}')
desired_nvrs.add(nvr)
nvr_product_version[nvr] = product_version
if config.include_package and len(
config.include_package) != len(desired_nvrs):
raise IOError(
f'Did not find all command line included packages {config.include_package}; only found {desired_nvrs}'
)
if signing_advisory_id and signing_advisory_mode == 'clean':
# Remove all builds attached to advisory before attempting signing
update_advisory_builds(config, errata_proxy, signing_advisory_id, [],
nvr_product_version)
# Did any of the archs require signed content?
possible_signing_needed = signed_desired(config)
if possible_signing_needed:
logger.info(f'At least one architecture requires signed nvrs')
if signing_advisory_id:
nvrs_for_advisory = []
for nvr in desired_nvrs:
if not is_signed(config, nvr):
logger.info(
f'Found an unsigned nvr (will attempt to signed): {nvr}'
)
nvrs_for_advisory.append(nvr)
logger.info(
f'Updating advisory to get nvrs signed: {signing_advisory_id}')
update_advisory_builds(config, errata_proxy, signing_advisory_id,
nvrs_for_advisory, nvr_product_version)
else:
logger.warning(
'No signing advisory specified; will simply poll and hope')
# Whether we've attached to advisory or no, wait until signing require is met
# or throw exception on timeout.
logger.info('Waiting for all nvrs to be signed..')
for nvr in desired_nvrs:
poll_for -= assert_signed(config, nvr)
if signing_advisory_id and signing_advisory_mode == 'clean':
# Seems that everything is signed; remove builds from the advisory.
update_advisory_builds(config, errata_proxy, signing_advisory_id, [],
nvr_product_version)
extra_embargo_info = { # Data related to embargos that will be written into the plashet.yml
'embargoed_permitted': include_embargoed, # Whether we included or excluded these nvrs in the plashet
'detected_as_embargoed': actual_embargoed_nvrs,
}
extra_data = { # Data that will be included in the plashet.yml after assembly.
'embargo_info': extra_embargo_info
}
assemble_repo(config, desired_nvrs, event_info, extra_data=extra_data)
def from_tags(config, brew_tag, embargoed_brew_tag, embargoed_nvr,
signing_advisory_id, signing_advisory_mode, poll_for,
include_previous_for, include_previous, event, include_embargoed,
inherit):
"""
The repositories are filled with RPMs derived from the list of
brew tags. If the RPMs are not signed and a repo should contain signed content,
the specified advisory will be used for signing the RPMs (requires
automatic sign on attach).
If you specify --embargoed-brew-tag, plashet will treat any nvr found in this tag as if it is
embargoed (unless has already shipped). This is useful since the .p1 convention cannot be used
on RPMs not built by ART.
\b
--brew-tag <tag> <product_version> example: --brew-tag rhaos-4.5-rhel-8-candidate OSE-4.5-RHEL-8 --brew-tag .. ..
"""
koji_proxy = KojiWrapper(
koji.ClientSession(config.brew_url,
opts={
'krbservice': 'brewhub',
'serverca': '/etc/pki/brew/legacy.crt'
}))
koji_proxy.gssapi_login()
errata_proxy = xmlrpclib.ServerProxy(config.errata_xmlrpc_url)
if event:
event = int(event)
event_info = koji_proxy.getEvent(event)
else:
# If none was specified, lock in an event so that there are no race conditions with
# packages changing while we run.
event_info = koji_proxy.getLastEvent()
event = event_info['id']
# Gather up all nvrs tagged in the embargoed brew tags into a set.
embargoed_tag_nvrs = set()
embargoed_tag_nvrs.update(embargoed_nvr)
for ebt in embargoed_brew_tag:
for build in koji_proxy.listTagged(ebt,
latest=False,
inherit=False,
event=event,
type='rpm'):
embargoed_tag_nvrs.add(to_nvre(build))
logger.info(
'Will treat the following nvrs as potentially embargoed: {}'.format(
embargoed_tag_nvrs))
actual_embargoed_nvres = list() # A list of nvres detected as embargoed
desired_nvres = set()
historical_nvres = set()
nvr_product_version = {}
for tag, product_version in brew_tag:
released_package_nvre_obj = {
} # maps released package names to the most recently released package nvr object (e.g { 'name': ..., }
if tag.endswith('-candidate'):
"""
So here's the thing. If you ship a version of a package 1.16.6 via errata tool,
it will prevent you from shipping an older version of that package (e.g. 1.16.2) or even
attaching it to an errata. This prevents us from auto-signing the older package. Since it
is just invalid, we need to find the latest version of packages which have shipped
and make sure plashet filters out anything that is older before signing/building.
Without this filtering, the error from errata tool looks like:
b'{"error":"Unable to add build \'cri-o-1.16.6-2.rhaos4.3.git4936f44.el7\' which is older than cri-o-1.16.6-16.dev.rhaos4.3.git4936f44.el7"}'
"""
released_tag = tag[:tag.index('-candidate')]
for build in koji_proxy.listTagged(released_tag,
latest=True,
inherit=True,
event=event,
type='rpm'):
package_name = build['package_name']
released_package_nvre_obj[package_name] = parse_nvr(
to_nvre(build))
for build in koji_proxy.listTagged(tag,
latest=True,
inherit=inherit,
event=event,
type='rpm'):
package_name = build['package_name']
nvre = to_nvre(build)
nvre_obj = parse_nvr(nvre)
released_nvre_obj = None # if the package has shipped before, the parsed nvr of the most recently shipped
if package_name in released_package_nvre_obj:
released_nvre_obj = released_package_nvre_obj[package_name]
def is_embargoed(an_nvre):
# .p1 or inclusion in the embargoed_tag_nvrs indicates this rpm is embargoed OR *was* embargoed.
# We can ignore it if it has already shipped.
test_nvre_obj = parse_nvr(an_nvre)
if released_nvre_obj is None or compare_nvr(
test_nvre_obj,
released_nvre_obj) > 0: # If this nvr hasn't shipped
if '.p1' in an_nvre or strip_epoch(
an_nvre) in embargoed_tag_nvrs: # It's embargoed!
return True
return False
if package_name in config.exclude_package:
logger.info(
f'Skipping tagged but command line excluded package: {nvre}'
)
continue
if config.include_package and package_name not in config.include_package:
logger.info(
f'Skipping tagged but not command line included package: {nvre}'
)
continue
if is_embargoed(nvre):
# An embargoed build has not been shipped.
actual_embargoed_nvres.append(
nvre
) # Record that at the time of build, this was considered embargoed
if not include_embargoed:
# We are being asked to build a plashet without embargoed RPMs. We need to find a stand-in.
# Search through the tag's package history to find the last build that was NOT embargoed.
unembargoed_nvre = None
for build in koji_proxy.listTagged(tag,
package=package_name,
inherit=True,
event=event,
type='rpm'):
test_nvre = to_nvre(build)
if is_embargoed(test_nvre):
continue
unembargoed_nvre = test_nvre
break
if unembargoed_nvre is None:
raise IOError(
f'Unable to build unembargoed plashet. Lastest build of {package_name} ({nvre}) is embargoed but unable to find unembargoed version in history'
)
plashet_concerns.append(
f'Swapping embargoed nvr {nvre} for unembargoed nvr {unembargoed_nvre}.'
)
logger.info(plashet_concerns[-1])
nvre = unembargoed_nvre
if released_nvre_obj:
if compare_nvr(
nvre_obj, released_nvre_obj
) < 0: # if the current nvr is less than the released NVR
msg = f'Skipping tagged {nvre} because it is older than a released version: {released_nvre_obj}'
plashet_concerns.append(msg)
logger.error(msg)
continue
logger.info(f'{tag} contains package: {nvre}')
desired_nvres.add(nvre)
nvr_product_version[strip_epoch(nvre)] = product_version
if package_name.startswith(
tuple(include_previous_for)) or include_previous:
# The user has asked for non-latest entry for this package to be included in the plashet.
# we can try to find this by looking at the packages full history in this tag. Listing is
# newest -> oldest tagging event for this tag/package combination.
tag_history = koji_proxy.listTagged(tag,
package=package_name,
inherit=True,
event=event,
type='rpm')
tracking = False # There may have been embargo shenanigans above; so we can't assume [0] is our target nvr
for htag in tag_history:
history_nvre = to_nvre(htag)
if history_nvre == nvre:
# We've found the target NVR in the list. Everything that follows can be considered for history.
tracking = True
continue
if not tracking:
# Haven't found our target NVR yet; so we can't consider this entry for history.
continue
history_nvre_obj = parse_nvr(history_nvre)
if compare_nvr(history_nvre_obj, nvre_obj) > 0:
# Is our historical nvr > target for inclusion in plashet? If it is, a user of the plashet would
# pull in the historical nvr with a yum install. We can't allow that. Just give up -- this is
# not in line with the use case of history.
plashet_concerns.append(
f'Unable to include previous for {package_name} because history {history_nvre} is newer than latest tagged {nvre}'
)
break
if include_embargoed is False and is_embargoed(
history_nvre):
# smh.. history is still under embargo. What you are guys doing?!
plashet_concerns.append(
f'Unable to include previous for {package_name} because history {history_nvre} is under embargo'
)
break
historical_nvres.add(history_nvre)
nvr_product_version[strip_epoch(
history_nvre)] = product_version
break
if config.include_package and len(
config.include_package) != len(desired_nvres):
raise IOError(
f'Did not find all command line included packages {config.include_package}; only found {desired_nvres}'
)
# Did any of the archs require signed content?
possible_signing_needed = signed_desired(config)
if possible_signing_needed:
logger.info(f'At least one architecture requires signed nvres')
# Each set must be attached separately because you cannot attach two nvres of the same
# package to an errata at the same time.
for set_name, nvre_set in {
'latest_tagged': desired_nvres,
'previous_tagged': historical_nvres
}.items():
if not nvre_set:
logger.info(f'NVRE set {set_name} is empty; nothing to sign')
continue
if signing_advisory_id:
# Remove all builds attached to advisory before attempting signing
update_advisory_builds(config, errata_proxy,
signing_advisory_id, [],
nvr_product_version)
nvres_for_advisory = []
for nvre in nvre_set:
if not is_signed(config, nvre):
logger.info(
f'Found an unsigned nvr in nvre set {set_name} (will attempt to sign): {nvre}'
)
nvres_for_advisory.append(nvre)
logger.info(
f'Updating advisory to get nvre set {set_name} signed: {signing_advisory_id}'
)
update_advisory_builds(config, errata_proxy,
signing_advisory_id, nvres_for_advisory,
nvr_product_version)
else:
logger.warning(
f'No signing advisory specified; will simply poll and hope for nvre set {set_name}'
)
# Whether we've attached to advisory or no, wait until signing require is met
# or throw exception on timeout.
logger.info(
f'Waiting for all nvres in set {set_name} to be signed..')
for nvre in desired_nvres:
poll_for -= assert_signed(config, nvre)
if signing_advisory_id and signing_advisory_mode == 'clean':
# Seems that everything is signed; remove builds from the advisory.
update_advisory_builds(config, errata_proxy, signing_advisory_id, [],
nvr_product_version)
extra_embargo_info = { # Data related to embargoes that will be written into the plashet.yml
'embargoed_permitted': include_embargoed, # Whether we included or excluded these nvrs in the plashet
'detected_as_embargoed': actual_embargoed_nvres,
}
extra_data = { # Data that will be included in the plashet.yml after assembly.
'embargo_info': extra_embargo_info,
'included_previous_nvrs': list(historical_nvres),
}
all_nvres = set()
all_nvres.update(desired_nvres)
all_nvres.update(historical_nvres)
assemble_repo(config, all_nvres, event_info, extra_data=extra_data)
