def get_denylisted_srpms(self):
src_config = get_source_container(self.workflow, fallback={})
denylist_srpms = src_config.get('denylist_srpms')
if not denylist_srpms:
self.log.debug('denylist_srpms is not defined in reactor_config_map')
return []
denylist_url = denylist_srpms['denylist_url']
denylist_key = denylist_srpms['denylist_key']
req_session = get_retrying_requests_session()
response = req_session.get(denylist_url)
response.raise_for_status()
response_json = response.json()
if denylist_key not in response_json:
self.log.debug('deny list json : %s', response_json)
raise RuntimeError('Denylist key: {} missing in denylist json from : {}'.
format(denylist_key, denylist_url))
deny_list = response_json[denylist_key]
if not isinstance(deny_list, list):
self.log.error('Wrong denylist: %s', repr(deny_list))
raise RuntimeError('Denylist value in key: {} is not list: {}'.
format(denylist_key, type(deny_list)))
wrong_types = [pkg for pkg in deny_list if not isinstance(pkg, str)]
if wrong_types:
self.log.error('Wrong types in denylist, should be str: %s', repr(deny_list))
raise RuntimeError('Values in denylist has to be all strings')
self.log.debug('denylisted srpms: %s', deny_list)
return deny_list
def exclude_files_from_remote_sources(self, remote_sources_map,
remote_sources_dir):
"""
:param remote_sources_map: dict, keys are filenames of sources from cachito,
values are url with json from cachito
:param remote_sources_dir: str, dir with downloaded sources from cachito
"""
src_config = get_source_container(self.workflow, fallback={})
denylist_sources_url = src_config.get('denylist_sources')
if not denylist_sources_url:
self.log.debug('no "denylist_sources" defined, not excluding any '
'files from remote sources')
return
request_session = get_retrying_requests_session()
denylist_sources = self._get_denylist_sources(request_session,
denylist_sources_url)
# key: full path to source archive, value: cachito json
full_remote_sources_map = self._create_full_remote_sources_map(
request_session, remote_sources_map, remote_sources_dir)
for remote_archive, remote_json in full_remote_sources_map.items():
unpack_dir = remote_archive + '_unpacked'
with tarfile.open(remote_archive) as tf:
tf.extractall(unpack_dir)
delete_app = self._check_if_package_excluded(
remote_json['packages'], denylist_sources, remote_archive)
# if any package in cachito json matched excluded entry,
# remove 'app' from sources, except 'app/vendor' when exists
if delete_app and os.path.exists(os.path.join(unpack_dir, 'app')):
self._delete_app_directory(remote_sources_dir, unpack_dir,
remote_archive)
# search for excluded matches
matches = self._get_excluded_matches(unpack_dir, denylist_sources)
self._remove_excluded_matches(matches)
# delete former archive
os.unlink(remote_archive)
# re-create new archive without excluded content
with tarfile.open(remote_archive, "w:gz") as tar:
for add_file in os.listdir(unpack_dir):
tar.add(os.path.join(unpack_dir, add_file),
arcname=add_file)
# cleanup unpacked dir
shutil.rmtree(unpack_dir)
def get_srpm_urls(self, sigkeys=None, insecure=False):
"""Fetch SRPM download URLs for each image generated by a build
Build each possible SRPM URL and check if the URL is available,
respecting the signing intent preference order.
:param sigkeys: list, strings for keys which signed the srpms to be fetched
:return: list, strings with URLs pointing to SRPM files
"""
if not sigkeys:
sigkeys = ['']
self.log.debug('get srpm_urls: %s', self.koji_build_id)
archives = self.session.listArchives(self.koji_build_id, type='image')
self.log.debug('archives: %s', archives)
rpms = [rpm for archive in archives
for rpm in self.session.listRPMs(imageID=archive['id'])]
src_config = get_source_container(self.workflow, fallback={})
blacklist_srpms = src_config.get('blacklist_srpms', [])
srpm_build_paths = {}
for rpm in rpms:
rpm_id = rpm['id']
self.log.debug('Resolving SRPM for RPM ID: %s', rpm_id)
if rpm['external_repo_name'] != 'INTERNAL':
msg = ('RPM comes from an external repo (RPM ID: {}). '
'External RPMs are currently not supported.').format(rpm_id)
raise RuntimeError(msg)
rpm_hdr = self.session.getRPMHeaders(rpm_id, headers=['SOURCERPM'])
if 'SOURCERPM' not in rpm_hdr:
raise RuntimeError('Missing SOURCERPM header (RPM ID: {})'.format(rpm_id))
srpm_name = rpm_hdr['SOURCERPM'].rsplit('-', 2)[0]
if any(black == srpm_name for black in blacklist_srpms):
self.log.debug('skipping blacklisted srpm %s', rpm_hdr['SOURCERPM'])
continue
srpm_filename = rpm_hdr['SOURCERPM']
if srpm_filename in srpm_build_paths:
continue
rpm_build = self.session.getBuild(rpm['build_id'], strict=True)
base_url = self.pathinfo.build(rpm_build)
srpm_build_paths[srpm_filename] = base_url
srpm_urls = []
missing_srpms = []
req_session = get_retrying_requests_session()
for srpm_filename, base_url in srpm_build_paths.items():
for sigkey in sigkeys:
# koji uses lowercase for paths. We make sure the sigkey is in lower case
url_candidate = self.assemble_srpm_url(base_url, srpm_filename, sigkey.lower())
request = req_session.head(url_candidate, verify=not insecure)
if request.ok:
srpm_urls.append({'url': url_candidate})
self.log.debug('%s is available for signing key "%s"', srpm_filename, sigkey)
break
else:
self.log.error('%s not found for the given signing intent: %s"', srpm_filename,
self.signing_intent)
missing_srpms.append(srpm_filename)
if missing_srpms:
raise RuntimeError('Could not find files signed by any of {} for these SRPMS: {}'
.format(sigkeys, missing_srpms))
return srpm_urls
def run(self):
# Only run if the build was successful
if self.workflow.build_process_failed:
self.log.info("Not running for failed build")
return []
# Work out the name of the image to pull
if not self.workflow.tag_conf.unique_images:
raise ValueError(
"no unique image set, impossible to verify media types")
image = self.workflow.tag_conf.unique_images[0]
registries = deepcopy(get_registries(self.workflow, {}))
media_in_registry = {}
expect_list_only = self.get_manifest_list_only_expectation()
for registry_name, registry in registries.items():
expected_media_types = set(registry.get('expected_media_types',
[]))
media_types = set()
if expect_list_only:
expected_media_types = set(
[MEDIA_TYPE_DOCKER_V2_MANIFEST_LIST])
media_in_registry[registry_name] = {
'expected': expected_media_types
}
pullspec = image.copy()
pullspec.registry = registry_name
insecure = registry.get('insecure', False)
secret = registry.get('secret', None)
kwargs = {}
if PLUGIN_FETCH_SOURCES_KEY in self.workflow.prebuild_results:
# For source containers, limit the versions we ask
# about (and, if necessary, the expected media types).
# This can help to avoid issues with tooling that is
# unable to deal with the number of layers in these
# images.
src_config = get_source_container(self.workflow, fallback={})
limit_media_types = src_config.get('limit_media_types')
if limit_media_types is not None:
short_name = {
v: k
for k, v in ManifestDigest.content_type.items()
}
versions = tuple(short_name[mt]
for mt in limit_media_types)
kwargs['versions'] = versions
if expected_media_types:
expected_media_types.intersection_update(
set(limit_media_types))
digests = get_manifest_digests(pullspec,
registry_name,
insecure,
secret,
require_digest=False,
**kwargs)
if digests:
if digests.v2_list:
media_types.add(MEDIA_TYPE_DOCKER_V2_MANIFEST_LIST)
if digests.v2:
media_types.add(MEDIA_TYPE_DOCKER_V2_SCHEMA2)
if digests.v1:
media_types.add(MEDIA_TYPE_DOCKER_V2_SCHEMA1)
if digests.oci:
media_types.add(MEDIA_TYPE_OCI_V1)
if digests.oci_index:
media_types.add(MEDIA_TYPE_OCI_V1_INDEX)
media_in_registry[registry_name]['found'] = media_types
should_raise = False
all_found = set()
for registry_name, manifests in media_in_registry.items():
all_found.update(manifests['found'])
if manifests['expected'] - manifests['found']:
should_raise = True
self.log.error(
"expected media types %s not in available media types %s,"
" for registry %s",
sorted(manifests['expected'] - manifests['found']),
sorted(manifests['found']), registry_name)
if should_raise:
raise KeyError("expected media types were not found")
if expect_list_only:
return [MEDIA_TYPE_DOCKER_V2_MANIFEST_LIST]
return sorted(all_found)
