def detect_embargoes_in_tags(runtime: Runtime, kind: str,
included_tags: List[str],
excluded_tags: List[str],
event_id: Optional[int]):
""" Finds embargoes in builds with given tags
:param runtime: the runtime
:param included_tags: list of koji tags that the returned builds must have
:param excluded_tags: list of koji tags that the returned builds must not have
:return: list of Brew build dicts that have embargoed fixes
"""
brew_session = runtime.build_retrying_koji_client()
runtime.logger.info(f"Fetching builds from Brew tags {included_tags}...")
build_type = None if kind == "all" else kind
latest_build_lists = brew.get_latest_builds([(tag, None)
for tag in included_tags],
build_type, event_id,
brew_session)
included_builds = [
b for builds in latest_build_lists if builds for b in builds
] # flatten latest_build_lists
runtime.logger.info(
f"Found {len(included_builds)} builds from Brew tags {included_tags}.")
if included_builds and excluded_tags: # if we have tags to exclude, get all builds with excluded_tags then exclude them
runtime.logger.info(
f"Fetching builds from Brew tags {excluded_tags}...")
excluded_build_lists = brew.get_tagged_builds(
[(tag, None) for tag in excluded_tags], build_type, event_id,
brew_session)
excluded_build_ids = {
b["id"]
for builds in excluded_build_lists if builds for b in builds
}
builds = [
b for b in included_builds if b["id"] not in excluded_build_ids
]
runtime.logger.info(
f"Excluded {len(included_builds) - len(builds)} builds that are also tagged into {excluded_tags}."
)
included_builds = builds
# Builds may have duplicate entries if we query from multiple tags. Don't worry, BuildStatusDetector is smart.
runtime.logger.info(
f"Detecting embargoes for {len(included_builds)} builds...")
detector = bs_detector.BuildStatusDetector(runtime, runtime.logger)
embargoed_build_ids = detector.find_embargoed_builds(
included_builds, runtime.get_candidate_brew_tags())
embargoed_builds = [
b for b in included_builds if b["id"] in embargoed_build_ids
]
return embargoed_builds
def detect_in_nvr(runtime: Runtime, nvrs, as_yaml, as_json):
""" Check whether one or more Brew builds have embargoed fixes.
If neither --yaml nor --json is given, this program will exit with status 0 if true, or with status 2 if not.
Errors are signaled by a non-zero status that is not 2.
Example:
$ doozer --group=openshift-4.6 detect-embargo nvr openshift-4.6.0-202007110420.p1.git.0.4de1d1d.el8 openshift-clients-4.6.0-202007152229.p0.git.3651.9a1d25f.el8
"""
if as_yaml and as_json:
raise click.BadParameter("Must use one of --yaml or --json.")
runtime.initialize(clone_distgits=False)
embargoed_builds = detect_embargoes_in_nvrs(runtime, nvrs)
print_result_and_exit(embargoed_builds, None, None, as_yaml, as_json)
def detect_in_tag(runtime: Runtime, kind, tags, excluded_tags, event_id,
as_yaml, as_json):
""" Check whether one or more Brew tags have builds that include embargoed fixes.
If neither --yaml nor --json is given, this program will exit with status 0 if true, or with status 2 if not.
Errors are signaled by a non-zero status that is not 2.
Example:
$ doozer --group=openshift-4.6 detect-embargo tag rhaos-4.6-rhel-8-candidate rhaos-4.6-rhel-7-candidate --exclude rhaos-4.6-rhel-8 --exclude rhaos-4.6-rhel-7
"""
if as_yaml and as_json:
raise click.BadParameter("Must use one of --yaml or --json.")
runtime.initialize(clone_distgits=False)
embargoed_builds = detect_embargoes_in_tags(runtime, kind, tags,
excluded_tags, event_id)
print_result_and_exit(embargoed_builds, None, None, as_yaml, as_json)
def detect_embargoes_in_releases(runtime: Runtime, pullspecs: List[str]):
""" Finds embargoes in given release payloads
:param runtime: the runtime
:param nvrs: list of release pullspecs
:return: list of Brew build dicts that have embargoed fixes
"""
runtime.logger.info(
f"Fetching component pullspecs from {len(pullspecs)} release payloads..."
)
ignore_rhcos_tags = rhcos.get_container_names(runtime)
jobs = runtime.parallel_exec(
lambda pullspec, _: get_image_pullspecs_from_release_payload(
pullspec, ignore_rhcos_tags), pullspecs,
min(len(pullspecs),
multiprocessing.cpu_count() * 4, 32))
pullspec_lists = jobs.get()
embargoed_releases = []
embargoed_pullspecs = []
embargoed_builds = []
for index, image_pullspecs in enumerate(pullspec_lists):
p, b = detect_embargoes_in_pullspecs(runtime, list(image_pullspecs))
if p: # release has embargoes
embargoed_releases.append(pullspecs[index])
embargoed_pullspecs += p
embargoed_builds += b
return embargoed_releases, embargoed_pullspecs, embargoed_builds
def detect_embargoes_in_pullspecs(runtime: Runtime, pullspecs: List[str]):
""" Finds embargoes in given image pullspecs
:param runtime: the runtime
:param nvrs: list of image pullspecs
:return: list of Brew build dicts that have embargoed fixes
"""
runtime.logger.info(
f"Fetching manifests for {len(pullspecs)} pullspecs...")
jobs = runtime.parallel_exec(
lambda pullspec, _: get_nvr_by_pullspec(pullspec), pullspecs,
min(len(pullspecs),
multiprocessing.cpu_count() * 4, 32))
nvrs = jobs.get()
suspect_nvrs = []
suspect_pullspecs = []
for index, nvr in enumerate(nvrs):
n, v, r = nvr
if not n or not v or not r:
runtime.logger.warning(
f"Assuming {pullspecs[index]} is not embargoed because it doesn't have valid NVR labels."
)
continue
suspect_nvrs.append(f"{n}-{v}-{r}")
suspect_pullspecs.append(pullspecs[index])
embargoed_builds = detect_embargoes_in_nvrs(runtime, suspect_nvrs)
embargoed_build_nvrs = {b["nvr"] for b in embargoed_builds}
embargoed_pullspecs = [
pullspec for index, pullspec in enumerate(suspect_pullspecs)
if suspect_nvrs[index] in embargoed_build_nvrs
]
return embargoed_pullspecs, embargoed_builds
def __init__(self, runtime: Runtime, brew_session: ClientSession):
self.runtime = runtime
self.brew_session = brew_session
if runtime.mode != 'both':
raise ValueError('Runtime must be initialized with "both"')
self.assembly_rhcos_config = assembly_rhcos_config(
self.runtime.releases_config, self.runtime.assembly)
self.assembly_type: AssemblyTypes = assembly_type(
self.runtime.releases_config, self.runtime.assembly)
self._rpm_build_cache: Dict[int, Dict[str, Optional[Dict]]] = {
} # Dict[rhel_ver] -> Dict[distgit_key] -> Optional[BuildDict]
self._permits = assembly_permits(self.runtime.releases_config,
self.runtime.assembly)
# If an image component has a latest build, an ImageInspector associated with the image.
self._release_image_inspectors: Dict[
str, Optional[BrewBuildImageInspector]] = dict()
for image_meta in runtime.get_for_release_image_metas():
latest_build_obj = image_meta.get_latest_build(default=None)
if latest_build_obj:
self._release_image_inspectors[
image_meta.distgit_key] = BrewBuildImageInspector(
self.runtime, latest_build_obj['nvr'])
else:
self._release_image_inspectors[image_meta.distgit_key] = None
def detect_embargoes_in_nvrs(runtime: Runtime, nvrs: List[str]):
""" Finds embargoes in given NVRs
:param runtime: the runtime
:param nvrs: list of build NVRs
:return: list of Brew build dicts that have embargoed fixes
"""
runtime.logger.info(f"Fetching {len(nvrs)} builds from Brew...")
brew_session = runtime.build_retrying_koji_client()
builds = brew.get_build_objects(nvrs, brew_session)
for i, b in enumerate(builds):
if not b:
raise DoozerFatalError(f"Unable to get {nvrs[i]} from Brew.")
runtime.logger.info(f"Detecting embargoes for {len(nvrs)} builds...")
detector = bs_detector.BuildStatusDetector(runtime, runtime.logger)
embargoed_build_ids = detector.find_embargoed_builds(
builds, runtime.get_candidate_brew_tags())
embargoed_builds = [b for b in builds if b["id"] in embargoed_build_ids]
return embargoed_builds
def __init__(self,
runtime: Runtime,
brew_session: ClientSession = None,
lite: bool = False):
"""
:param runtime: Doozer runtime
:param brew_session: Brew session object to use for communicating with Brew
:param lite: Create a lite version without the ability to inspect Images; can be used to check AssemblyIssues,
fetch rhcos_builds and other defined methods
"""
self.runtime = runtime
self.brew_session = brew_session
if not lite and runtime.mode != 'both':
raise ValueError('Runtime must be initialized with "both"')
self.assembly_rhcos_config = assembly_rhcos_config(
self.runtime.releases_config, self.runtime.assembly)
self.assembly_type: AssemblyTypes = assembly_type(
self.runtime.releases_config, self.runtime.assembly)
self._rpm_build_cache: Dict[int, Dict[str, Optional[Dict]]] = {
} # Dict[rhel_ver] -> Dict[distgit_key] -> Optional[BuildDict]
self._permits = assembly_permits(self.runtime.releases_config,
self.runtime.assembly)
if lite:
return
# If an image component has a latest build, an ImageInspector associated with the image.
self._release_image_inspectors: Dict[
str, Optional[BrewBuildImageInspector]] = dict()
for image_meta in runtime.get_for_release_image_metas():
latest_build_obj = image_meta.get_latest_build(default=None)
if latest_build_obj:
self._release_image_inspectors[
image_meta.distgit_key] = BrewBuildImageInspector(
self.runtime, latest_build_obj['nvr'])
else:
self._release_image_inspectors[image_meta.distgit_key] = None