def test_compare_nvr(self): first = {'name': 'a', 'version': '1', 'release': '1', 'epoch': '1'} second = {'name': 'a', 'version': '1', 'release': '1', 'epoch': '1'} self.assertEqual(compare_nvr(first, second), 0) second['version'] = '0' self.assertEqual(compare_nvr(first, second), 1) second['version'] = 0 self.assertEqual(compare_nvr(first, second), 1) second['version'] = 2 self.assertEqual(compare_nvr(first, second), -1) second['version'] = 1 second['release'] = 0 self.assertEqual(compare_nvr(first, second), 1) second['release'] = 2 self.assertEqual(compare_nvr(first, second), -1) second['release'] = 1 second['epoch'] = 0 self.assertEqual(compare_nvr(first, second), 1) second['epoch'] = 2 self.assertEqual(compare_nvr(first, second), -1) first = {'name': 'a', 'version': '1', 'release': '1', 'epoch': None} second = {'name': 'a', 'version': '1', 'release': '1', 'epoch': '1'} self.assertEqual(compare_nvr(first, second), -1) # missing epoch first = {'name': 'a', 'version': '1', 'release': '1'} self.assertEqual(compare_nvr(first, second), -1)
def get_brew_builds(koji_proxy, brew_tags, latest=False, inherit=False): """ """ brew_builds_by_name = {} brew_builds = set() if not brew_tags: brew_tags = [] if isinstance(brew_tags, basestring): brew_tags = [brew_tags] for brew_tag in brew_tags: for build in koji_proxy.listTagged(brew_tag, latest=latest, inherit=inherit): if latest: previous_build = brew_builds_by_name.get(build["name"]) if not previous_build or compare_nvr(previous_build, build) == 1: brew_builds_by_name[build["name"]] = build brew_builds.remove(previous_build) brew_builds.add(build["nvr"]) else: brew_builds.add(build["nvr"]) return brew_builds
def is_embargoed(an_nvr): # .p1 or inclusion in the embargoed_tag_nvrs indicates this rpm is embargoed OR *was* embargoed. # We can ignore it if it has already shipped. parsed_test_nvr = parse_nvr(an_nvr) if released_nvr is None or compare_nvr( parsed_test_nvr, released_nvr) > 0: # If this nvr hasn't shipped if '.p1' in an_nvr or an_nvr in embargoed_tag_nvrs: # It's embargoed! return True return False
def is_embargoed(an_nvre): # .p1 or inclusion in the embargoed_tag_nvrs indicates this rpm is embargoed OR *was* embargoed. # We can ignore it if it has already shipped. test_nvre_obj = parse_nvr(an_nvre) if released_nvre_obj is None or compare_nvr( test_nvre_obj, released_nvre_obj) > 0: # If this nvr hasn't shipped if '.p1' in an_nvre or strip_epoch( an_nvre) in embargoed_tag_nvrs: # It's embargoed! return True return False
def from_tags(config, brew_tag, embargoed_brew_tag, embargoed_nvr, signing_advisory_id, signing_advisory_mode, poll_for, event, include_embargoed): """ The repositories are filled with RPMs derived from the list of brew tags. If the RPMs are not signed and a repo should contain signed content, the specified advisory will be used for signing the RPMs (requires automatic sign on attach). If you specify --embargoed-brew-tag, plashet will treat any nvr found in this tag as if it is embargoed (unless has already shipped). This is useful since the .p1 convention cannot be used on RPMs not built by ART. \b --brew-tag <tag> <product_version> example: --brew-tag rhaos-4.5-rhel-8-candidate OSE-4.5-RHEL-8 --brew-tag .. .. """ koji_proxy = KojiWrapper( koji.ClientSession(config.brew_url, opts={ 'krbservice': 'brewhub', 'serverca': '/etc/pki/brew/legacy.crt' })) koji_proxy.gssapi_login() errata_proxy = xmlrpclib.ServerProxy(config.errata_xmlrpc_url) if event: event = int(event) event_info = koji_proxy.getEvent(event) else: # If none was specified, lock in an event so that there are no race conditions with # packages changing while we run. event_info = koji_proxy.getLastEvent() event = event_info['id'] # Gather up all nvrs tagged in the embargoed brew tags into a set. embargoed_tag_nvrs = set() embargoed_tag_nvrs.update(embargoed_nvr) for ebt in embargoed_brew_tag: for build in koji_proxy.listTagged(ebt, latest=False, inherit=False, event=event, type='rpm'): embargoed_tag_nvrs.add(build['nvr']) logger.info( 'Will treat the following nvrs as potentially embargoed: {}'.format( embargoed_tag_nvrs)) actual_embargoed_nvrs = list() # A list of nvrs detected as embargoed desired_nvrs = set() nvr_product_version = {} for tag, product_version in brew_tag: released_package_nvrs = { } # maps released package names to the most recently released package (parsed) nvr if tag.endswith('-candidate'): """ So here's the thing. If you ship a version of a package 1.16.6 via errata tool, it will prevent you from shipping an older version of that package (e.g. 1.16.2) or even attaching it to an errata. This prevents us from auto-signing the older package. Since it is just invalid, we need to find the latest version of packages which have shipped and make sure plashet filters out anything that is older before signing/building. Without this filtering, the error from errata tool looks like: b'{"error":"Unable to add build \'cri-o-1.16.6-2.rhaos4.3.git4936f44.el7\' which is older than cri-o-1.16.6-16.dev.rhaos4.3.git4936f44.el7"}' """ released_tag = tag[:tag.index('-candidate')] for build in koji_proxy.listTagged(released_tag, latest=True, inherit=True, event=event, type='rpm'): package_name = build['package_name'] released_package_nvrs[package_name] = parse_nvr(build['nvr']) for build in koji_proxy.listTagged(tag, latest=True, inherit=False, event=event, type='rpm'): package_name = build['package_name'] nvr = build['nvr'] parsed_nvr = parse_nvr(nvr) released_nvr = None # if the package has shipped before, the parsed nvr of the most recently shipped if package_name in released_package_nvrs: released_nvr = released_package_nvrs[package_name] if package_name in config.exclude_package: logger.info( f'Skipping tagged but command line excluded package: {nvr}' ) continue if config.include_package and package_name not in config.include_package: logger.info( f'Skipping tagged but not command line included package: {nvr}' ) continue if '.p1' in nvr or nvr in embargoed_tag_nvrs: # p1 or inclusion in the embargoed_tag_nvrs indicates this rpm is embargoed OR *was* embargoed. # We can ignore it if it has already shipped. if released_nvr is None or compare_nvr( parsed_nvr, released_nvr) > 0: # Is our nvr > last shipped? # Our embargoed build has not been shipped. actual_embargoed_nvrs.append( nvr ) # Record that at the time of build, this was considered embargoed if not include_embargoed: # We are being asked to build a plashet without embargoed RPMs. We need to find a stand-in. # Search through the tag's package history to find the last build that was NOT embargoed. unembargoed_nvr = None for build in koji_proxy.listTagged( tag, package=package_name, inherit=True, event=event, type='rpm'): test_nvr = build['nvr'] parsed_test_nvr = parse_nvr(test_nvr) if released_nvr is None or compare_nvr( parsed_test_nvr, released_nvr ) > 0: # If this nvr hasn't shipped if '.p1' in test_nvr or test_nvr in embargoed_tag_nvrs: # Looks like this one is embargoed too continue unembargoed_nvr = test_nvr break if unembargoed_nvr is None: raise IOError( f'Unable to build unembargoed plashet. Lastest build of {package_name} ({nvr}) is embargoed but unable to find unembargoed version in history' ) plashet_concerns.append( f'Swapping embargoed nvr {nvr} for unembargoed nvr {unembargoed_nvr}.' ) logger.info(plashet_concerns[-1]) nvr = unembargoed_nvr else: logger.info( f'NVR {nvr} was potentially embargoed, but has already shipped' ) if released_nvr: if compare_nvr( parsed_nvr, released_nvr ) < 0: # if the current nvr is less than the released NVR msg = f'Skipping tagged {nvr} because it is older than a released version: {released_nvr}' plashet_concerns.append(msg) logger.error(msg) continue logger.info(f'{tag} contains package: {nvr}') desired_nvrs.add(nvr) nvr_product_version[nvr] = product_version if config.include_package and len( config.include_package) != len(desired_nvrs): raise IOError( f'Did not find all command line included packages {config.include_package}; only found {desired_nvrs}' ) if signing_advisory_id and signing_advisory_mode == 'clean': # Remove all builds attached to advisory before attempting signing update_advisory_builds(config, errata_proxy, signing_advisory_id, [], nvr_product_version) # Did any of the archs require signed content? possible_signing_needed = signed_desired(config) if possible_signing_needed: logger.info(f'At least one architecture requires signed nvrs') if signing_advisory_id: nvrs_for_advisory = [] for nvr in desired_nvrs: if not is_signed(config, nvr): logger.info( f'Found an unsigned nvr (will attempt to signed): {nvr}' ) nvrs_for_advisory.append(nvr) logger.info( f'Updating advisory to get nvrs signed: {signing_advisory_id}') update_advisory_builds(config, errata_proxy, signing_advisory_id, nvrs_for_advisory, nvr_product_version) else: logger.warning( 'No signing advisory specified; will simply poll and hope') # Whether we've attached to advisory or no, wait until signing require is met # or throw exception on timeout. logger.info('Waiting for all nvrs to be signed..') for nvr in desired_nvrs: poll_for -= assert_signed(config, nvr) if signing_advisory_id and signing_advisory_mode == 'clean': # Seems that everything is signed; remove builds from the advisory. update_advisory_builds(config, errata_proxy, signing_advisory_id, [], nvr_product_version) extra_embargo_info = { # Data related to embargos that will be written into the plashet.yml 'embargoed_permitted': include_embargoed, # Whether we included or excluded these nvrs in the plashet 'detected_as_embargoed': actual_embargoed_nvrs, } extra_data = { # Data that will be included in the plashet.yml after assembly. 'embargo_info': extra_embargo_info } assemble_repo(config, desired_nvrs, event_info, extra_data=extra_data)
def from_tags(config, brew_tag, embargoed_brew_tag, embargoed_nvr, signing_advisory_id, signing_advisory_mode, poll_for, include_previous_for, include_previous, event, include_embargoed, inherit): """ The repositories are filled with RPMs derived from the list of brew tags. If the RPMs are not signed and a repo should contain signed content, the specified advisory will be used for signing the RPMs (requires automatic sign on attach). If you specify --embargoed-brew-tag, plashet will treat any nvr found in this tag as if it is embargoed (unless has already shipped). This is useful since the .p1 convention cannot be used on RPMs not built by ART. \b --brew-tag <tag> <product_version> example: --brew-tag rhaos-4.5-rhel-8-candidate OSE-4.5-RHEL-8 --brew-tag .. .. """ koji_proxy = KojiWrapper( koji.ClientSession(config.brew_url, opts={ 'krbservice': 'brewhub', 'serverca': '/etc/pki/brew/legacy.crt' })) koji_proxy.gssapi_login() errata_proxy = xmlrpclib.ServerProxy(config.errata_xmlrpc_url) if event: event = int(event) event_info = koji_proxy.getEvent(event) else: # If none was specified, lock in an event so that there are no race conditions with # packages changing while we run. event_info = koji_proxy.getLastEvent() event = event_info['id'] # Gather up all nvrs tagged in the embargoed brew tags into a set. embargoed_tag_nvrs = set() embargoed_tag_nvrs.update(embargoed_nvr) for ebt in embargoed_brew_tag: for build in koji_proxy.listTagged(ebt, latest=False, inherit=False, event=event, type='rpm'): embargoed_tag_nvrs.add(to_nvre(build)) logger.info( 'Will treat the following nvrs as potentially embargoed: {}'.format( embargoed_tag_nvrs)) actual_embargoed_nvres = list() # A list of nvres detected as embargoed desired_nvres = set() historical_nvres = set() nvr_product_version = {} for tag, product_version in brew_tag: released_package_nvre_obj = { } # maps released package names to the most recently released package nvr object (e.g { 'name': ..., } if tag.endswith('-candidate'): """ So here's the thing. If you ship a version of a package 1.16.6 via errata tool, it will prevent you from shipping an older version of that package (e.g. 1.16.2) or even attaching it to an errata. This prevents us from auto-signing the older package. Since it is just invalid, we need to find the latest version of packages which have shipped and make sure plashet filters out anything that is older before signing/building. Without this filtering, the error from errata tool looks like: b'{"error":"Unable to add build \'cri-o-1.16.6-2.rhaos4.3.git4936f44.el7\' which is older than cri-o-1.16.6-16.dev.rhaos4.3.git4936f44.el7"}' """ released_tag = tag[:tag.index('-candidate')] for build in koji_proxy.listTagged(released_tag, latest=True, inherit=True, event=event, type='rpm'): package_name = build['package_name'] released_package_nvre_obj[package_name] = parse_nvr( to_nvre(build)) for build in koji_proxy.listTagged(tag, latest=True, inherit=inherit, event=event, type='rpm'): package_name = build['package_name'] nvre = to_nvre(build) nvre_obj = parse_nvr(nvre) released_nvre_obj = None # if the package has shipped before, the parsed nvr of the most recently shipped if package_name in released_package_nvre_obj: released_nvre_obj = released_package_nvre_obj[package_name] def is_embargoed(an_nvre): # .p1 or inclusion in the embargoed_tag_nvrs indicates this rpm is embargoed OR *was* embargoed. # We can ignore it if it has already shipped. test_nvre_obj = parse_nvr(an_nvre) if released_nvre_obj is None or compare_nvr( test_nvre_obj, released_nvre_obj) > 0: # If this nvr hasn't shipped if '.p1' in an_nvre or strip_epoch( an_nvre) in embargoed_tag_nvrs: # It's embargoed! return True return False if package_name in config.exclude_package: logger.info( f'Skipping tagged but command line excluded package: {nvre}' ) continue if config.include_package and package_name not in config.include_package: logger.info( f'Skipping tagged but not command line included package: {nvre}' ) continue if is_embargoed(nvre): # An embargoed build has not been shipped. actual_embargoed_nvres.append( nvre ) # Record that at the time of build, this was considered embargoed if not include_embargoed: # We are being asked to build a plashet without embargoed RPMs. We need to find a stand-in. # Search through the tag's package history to find the last build that was NOT embargoed. unembargoed_nvre = None for build in koji_proxy.listTagged(tag, package=package_name, inherit=True, event=event, type='rpm'): test_nvre = to_nvre(build) if is_embargoed(test_nvre): continue unembargoed_nvre = test_nvre break if unembargoed_nvre is None: raise IOError( f'Unable to build unembargoed plashet. Lastest build of {package_name} ({nvre}) is embargoed but unable to find unembargoed version in history' ) plashet_concerns.append( f'Swapping embargoed nvr {nvre} for unembargoed nvr {unembargoed_nvre}.' ) logger.info(plashet_concerns[-1]) nvre = unembargoed_nvre if released_nvre_obj: if compare_nvr( nvre_obj, released_nvre_obj ) < 0: # if the current nvr is less than the released NVR msg = f'Skipping tagged {nvre} because it is older than a released version: {released_nvre_obj}' plashet_concerns.append(msg) logger.error(msg) continue logger.info(f'{tag} contains package: {nvre}') desired_nvres.add(nvre) nvr_product_version[strip_epoch(nvre)] = product_version if package_name.startswith( tuple(include_previous_for)) or include_previous: # The user has asked for non-latest entry for this package to be included in the plashet. # we can try to find this by looking at the packages full history in this tag. Listing is # newest -> oldest tagging event for this tag/package combination. tag_history = koji_proxy.listTagged(tag, package=package_name, inherit=True, event=event, type='rpm') tracking = False # There may have been embargo shenanigans above; so we can't assume [0] is our target nvr for htag in tag_history: history_nvre = to_nvre(htag) if history_nvre == nvre: # We've found the target NVR in the list. Everything that follows can be considered for history. tracking = True continue if not tracking: # Haven't found our target NVR yet; so we can't consider this entry for history. continue history_nvre_obj = parse_nvr(history_nvre) if compare_nvr(history_nvre_obj, nvre_obj) > 0: # Is our historical nvr > target for inclusion in plashet? If it is, a user of the plashet would # pull in the historical nvr with a yum install. We can't allow that. Just give up -- this is # not in line with the use case of history. plashet_concerns.append( f'Unable to include previous for {package_name} because history {history_nvre} is newer than latest tagged {nvre}' ) break if include_embargoed is False and is_embargoed( history_nvre): # smh.. history is still under embargo. What you are guys doing?! plashet_concerns.append( f'Unable to include previous for {package_name} because history {history_nvre} is under embargo' ) break historical_nvres.add(history_nvre) nvr_product_version[strip_epoch( history_nvre)] = product_version break if config.include_package and len( config.include_package) != len(desired_nvres): raise IOError( f'Did not find all command line included packages {config.include_package}; only found {desired_nvres}' ) # Did any of the archs require signed content? possible_signing_needed = signed_desired(config) if possible_signing_needed: logger.info(f'At least one architecture requires signed nvres') # Each set must be attached separately because you cannot attach two nvres of the same # package to an errata at the same time. for set_name, nvre_set in { 'latest_tagged': desired_nvres, 'previous_tagged': historical_nvres }.items(): if not nvre_set: logger.info(f'NVRE set {set_name} is empty; nothing to sign') continue if signing_advisory_id: # Remove all builds attached to advisory before attempting signing update_advisory_builds(config, errata_proxy, signing_advisory_id, [], nvr_product_version) nvres_for_advisory = [] for nvre in nvre_set: if not is_signed(config, nvre): logger.info( f'Found an unsigned nvr in nvre set {set_name} (will attempt to sign): {nvre}' ) nvres_for_advisory.append(nvre) logger.info( f'Updating advisory to get nvre set {set_name} signed: {signing_advisory_id}' ) update_advisory_builds(config, errata_proxy, signing_advisory_id, nvres_for_advisory, nvr_product_version) else: logger.warning( f'No signing advisory specified; will simply poll and hope for nvre set {set_name}' ) # Whether we've attached to advisory or no, wait until signing require is met # or throw exception on timeout. logger.info( f'Waiting for all nvres in set {set_name} to be signed..') for nvre in desired_nvres: poll_for -= assert_signed(config, nvre) if signing_advisory_id and signing_advisory_mode == 'clean': # Seems that everything is signed; remove builds from the advisory. update_advisory_builds(config, errata_proxy, signing_advisory_id, [], nvr_product_version) extra_embargo_info = { # Data related to embargoes that will be written into the plashet.yml 'embargoed_permitted': include_embargoed, # Whether we included or excluded these nvrs in the plashet 'detected_as_embargoed': actual_embargoed_nvres, } extra_data = { # Data that will be included in the plashet.yml after assembly. 'embargo_info': extra_embargo_info, 'included_previous_nvrs': list(historical_nvres), } all_nvres = set() all_nvres.update(desired_nvres) all_nvres.update(historical_nvres) assemble_repo(config, all_nvres, event_info, extra_data=extra_data)