def _get_shipped_images(runtime, brew_session):
# retrieve all image builds ever shipped for this version (potential operands)
tag = f"{runtime.branch}-container-released"
tags = {tag, tag.replace('-rhel-7-', '-rhel-8-')} # may be one or two depending
released = brew.get_tagged_builds(tags, build_type='image', event=None, session=brew_session)
released = brew.get_build_objects([b['build_id'] for b in released], session=brew_session)
return [b for b in released if _is_image(b)] # filter out source images
def get_golang_container_nvrs(nvrs, logger):
all_build_objs = brew.get_build_objects(
['{}-{}-{}'.format(*n) for n in nvrs])
go_container_nvrs = {}
for build in all_build_objs:
go_version = 'N/A'
nvr = (build['name'], build['version'], build['release'])
name = nvr[0]
if 'golang-builder' in name or 'go-toolset' in name:
go_version = golang_builder_version(nvr, logger)
go_container_nvrs[name] = {'nvr': nvr, 'go': go_version}
continue
try:
parents = build['extra']['image']['parent_image_builds']
except KeyError:
logger.debug(f'Could not find parent build image for {nvr}')
continue
for p, pinfo in parents.items():
if 'builder' in p or 'go-toolset' in p:
go_version = pinfo.get('nvr')
go_container_nvrs[name] = {'nvr': nvr, 'go': go_version}
if not go_version or go_version == 'N/A':
logger.debug(f'Could not find parent Go builder image for {nvr}')
return go_container_nvrs
def _fetch_nvrps_by_nvr_or_id(ids_or_nvrs,
tag_pv_map,
ignore_product_version=False):
session = koji.ClientSession(constants.BREW_HUB)
builds = brew.get_build_objects(ids_or_nvrs, session)
nonexistent_builds = list(
filter(lambda b: b[1] is None, zip(ids_or_nvrs, builds)))
if nonexistent_builds:
raise ValueError(
f"The following builds are not found in Brew: {' '.join(map(lambda b: b[0],nonexistent_builds))}"
)
nvrps = []
if ignore_product_version:
for build in builds:
nvrps.append(
(build["name"], build["version"], build["release"], None))
return nvrps
for build, tags in zip(builds, brew.get_builds_tags(builds, session)):
tag_names = {tag["name"] for tag in tags}
product_versions = [
pv for tag, pv in tag_pv_map.items() if tag in tag_names
]
if not product_versions:
raise ValueError(
f"Build {build['nvr']} doesn't have any of the following whitelisted tags: {list(tag_pv_map.keys())}"
)
for pv in product_versions:
nvrps.append(
(build["name"], build["version"], build["release"], pv))
return nvrps
def get_parent_build_ids(builds):
parents = []
for b in builds:
if b.get("extra") is None:
b = brew.get_build_objects([b["id"]])[0]
parent = b["extra"]["image"].get("parent_build_id")
parents.append(parent)
return parents
def _fetch_builds_by_nvr_or_id(ids_or_nvrs, tag_pv_map):
session = koji.ClientSession(constants.BREW_HUB)
builds = brew.get_build_objects(ids_or_nvrs, session)
nvrps = []
for index, tags in enumerate(brew.get_builds_tags(builds, session)):
build = builds[index] # type: dict
tag_names = {tag["name"] for tag in tags}
for tag, prod_version in tag_pv_map.items():
if tag in tag_names:
nvrps.append((build["name"], build["version"],
build["release"], prod_version))
return nvrps
def _get_attached_image_builds(brew_session, advisories):
# get all attached image builds
build_nvrs = []
try:
for advisory in advisories:
green_print(f"Retrieving builds from advisory {advisory}")
advisory = Erratum(errata_id=advisory)
for build_list in advisory.errata_builds.values(): # one per product version
build_nvrs.extend(build_list)
except GSSError:
exit_unauthenticated()
green_print(f"Found {len(build_nvrs)} builds")
return [build for build in brew.get_build_objects(build_nvrs, brew_session) if _is_image(build)]
def get_container_golang_versions(advisory_id: str):
all_builds = errata.get_brew_builds(advisory_id)
all_build_objs = brew.get_build_objects([b.nvr for b in all_builds])
for build in all_build_objs:
golang_version = None
name = build.get('name')
try:
parents = build['extra']['image']['parent_image_builds']
except KeyError:
print('Could not get parent image info for {}'.format(name))
continue
for p, pinfo in parents.items():
if 'builder' in p:
golang_version = pinfo.get('nvr')
if golang_version is not None:
print('{}:\t{}'.format(name, golang_version))
def _get_builds(self, ids_or_nvrs: Iterable[Union[int,
str]]) -> List[Dict]:
""" Get build dicts from Brew. This method uses an internal cache to avoid unnecessary queries.
:params ids_or_nvrs: list of build IDs or NVRs
:return: a list of Brew build dicts
"""
cache_miss = set(ids_or_nvrs) - self._build_cache.keys()
if cache_miss:
cache_miss = [
strip_epoch(item) if isinstance(item, str) else item
for item in cache_miss
]
builds = get_build_objects(cache_miss, self._koji_api)
for id_or_nvre, build in zip(cache_miss, builds):
if build:
self._cache_build(build)
else:
self._build_cache[
id_or_nvre] = None # None indicates the build ID or NVRE doesn't exist
return [self._build_cache[id] for id in ids_or_nvrs]
def verify_cvp_cli(runtime: Runtime, all_images, nvrs, optional_checks,
all_optional_checks, fix, message):
""" Verify CVP test results
Example 1: Verify CVP test results for all latest 4.4 image builds, also warn those with failed content_set_check
$ elliott --group openshift-4.4 verify-cvp --all --include-optional-check content_set_check
Example 2: Apply patches to ocp-build-data to fix the redundant content sets error:
$ elliott --group openshift-4.4 verify-cvp --all --include-optional-check content_set_check --fix
Note:
1. If `--message` is not given, `--fix` will leave changed ocp-build-data files uncommitted.
2. Make sure your ocp-build-data directory is clean before running `--fix`.
"""
if bool(all_images) + bool(nvrs) != 1:
raise click.BadParameter('You must use one of --all or --build.')
if all_optional_checks and optional_checks:
raise click.BadParameter(
'Use only one of --all-optional-checks or --include-optional-check.'
)
runtime.initialize(mode='images')
tag_pv_map = runtime.gitdata.load_data(
key='erratatool',
replace_vars=runtime.group_config.vars.primitive()
if runtime.group_config.vars else
{}).data.get('brew_tag_product_version_mapping')
brew_session = koji.ClientSession(runtime.group_config.urls.brewhub
or constants.BREW_HUB)
builds = []
if all_images:
runtime.logger.info("Getting latest image builds from Brew...")
builds = get_latest_image_builds(brew_session, tag_pv_map.keys(),
runtime.image_metas)
elif nvrs:
runtime.logger.info(f"Finding {len(builds)} builds from Brew...")
builds = brew.get_build_objects(nvrs, brew_session)
runtime.logger.info(f"Found {len(builds)} image builds.")
resultsdb_api = ResultsDBAPI()
nvrs = [b["nvr"] for b in builds]
runtime.logger.info(
f"Getting CVP test results for {len(builds)} image builds...")
latest_cvp_results = get_latest_cvp_results(runtime, resultsdb_api, nvrs)
# print a summary for all CVP results
good_results = [] # good means PASSED or INFO
bad_results = [] # bad means NEEDS_INSPECTION or FAILED
incomplete_nvrs = []
for nvr, result in zip(nvrs, latest_cvp_results):
if not result:
incomplete_nvrs.append(nvr)
continue
outcome = result.get(
"outcome"
) # only PASSED, FAILED, INFO, NEEDS_INSPECTION are now valid outcome values (https://resultsdb20.docs.apiary.io/#introduction/changes-since-1.0)
if outcome in {"PASSED", "INFO"}:
good_results.append(result)
elif outcome in {"NEEDS_INSPECTION", "FAILED"}:
bad_results.append(result)
green_prefix("good: {}".format(len(good_results)))
click.echo(", ", nl=False)
red_prefix("bad: {}".format(len(bad_results)))
click.echo(", ", nl=False)
yellow_print("incomplete: {}".format(len(incomplete_nvrs)))
if bad_results:
red_print("The following builds didn't pass CVP tests:")
for r in bad_results:
nvr = r["data"]["item"][0]
red_print(f"{nvr} {r['outcome']}: {r['ref_url']}")
if incomplete_nvrs:
yellow_print(
"We couldn't find CVP test results for the following builds:")
for nvr in incomplete_nvrs:
yellow_print(nvr)
if not optional_checks and not all_optional_checks:
return # no need to print failed optional CVP checks
# Find failed optional CVP checks in case some of the tiem *will* become required.
optional_checks = set(optional_checks)
complete_results = good_results + bad_results
runtime.logger.info(
f"Getting optional checks for {len(complete_results)} CVP tests...")
optional_check_results = get_optional_checks(runtime, complete_results)
component_distgit_keys = {
} # a dict of brew component names to distgit keys
content_set_repo_names = {
} # a map of x86_64 content set names to group.yml repo names
if fix: # Fixing redundant content sets requires those dicts
for image in runtime.image_metas():
component_distgit_keys[
image.get_component_name()] = image.distgit_key
for repo_name, repo_info in runtime.group_config.get("repos",
{}).items():
content_set_name = repo_info.get(
'content_set', {}).get('x86_64') or repo_info.get(
'content_set', {}).get('default')
if content_set_name:
content_set_repo_names[content_set_name] = repo_name
ocp_build_data_updated = False
for cvp_result, checks in zip(complete_results, optional_check_results):
# example optional checks: http://external-ci-coldstorage.datahub.redhat.com/cvp/cvp-product-test/hive-container-v4.6.0-202008010302.p0/da01e36c-8c69-4a19-be7d-ba4593a7b085/sanity-tests-optional-results.json
bad_checks = [
check for check in checks["checks"]
if check["status"] != "PASS" and (
all_optional_checks or check["name"] in optional_checks)
]
if not bad_checks:
continue
nvr = cvp_result["data"]["item"][0]
yellow_print("----------")
yellow_print(
f"Build {nvr} has {len(bad_checks)} problematic CVP optional checks:"
)
for check in bad_checks:
yellow_print(f"* {check['name']} {check['status']}")
if fix and check["name"] == "content_set_check":
if "Some content sets are redundant." in check["logs"]:
# fix redundant content sets
name = nvr.rsplit('-', 2)[0]
distgit_keys = component_distgit_keys.get(name)
if not distgit_keys:
runtime.logger.warning(
f"Will not apply the redundant content sets fix to image {name}: We don't know its distgit key."
)
continue
amd64_content_sets = list(
filter(lambda item: item.get("arch") == "amd64",
check["logs"][-1])
) # seems only x86_64 (amd64) content sets are defined in ocp-build-data.
if not amd64_content_sets:
runtime.logger.warning(
f"Will not apply the redundant content sets fix to image {name}: It doesn't have redundant x86_64 (amd64) content sets"
)
continue
amd64_redundant_cs = amd64_content_sets[0]["redundant_cs"]
redundant_repos = [
content_set_repo_names[cs] for cs in amd64_redundant_cs
if cs in content_set_repo_names
]
if len(redundant_repos) != len(amd64_redundant_cs):
runtime.logger.error(
f"Not all content sets have a repo entry in group.yml: #content_sets is {len(amd64_redundant_cs)}, #repos is {len(redundant_repos)}"
)
runtime.logger.info(
f"Applying redundant content sets fix to {distgit_keys}..."
)
fix_redundant_content_set(runtime, distgit_keys,
redundant_repos)
ocp_build_data_updated = True
runtime.logger.info(
f"Fixed redundant content sets for {distgit_keys}")
yellow_print(
f"See {cvp_result['ref_url']}sanity-tests-optional-results.json for more details."
)
if message and ocp_build_data_updated:
runtime.gitdata.commit(message)
def test_get_build_objects(self):
build_infos = {
"logging-fluentd-container-v3.11.141-2": {
"cg_id": None,
"package_name": "logging-fluentd-container",
"extra": {
"submitter": "osbs",
"image": {
"media_types": [
"application/vnd.docker.distribution.manifest.list.v2+json",
"application/vnd.docker.distribution.manifest.v1+json",
"application/vnd.docker.distribution.manifest.v2+json"
],
"help":
None,
"index": {
"pull": [
"brew-pulp-docker01.web.prod.ext.phx2.redhat.com:8888/openshift3/ose-logging-fluentd@sha256:1df5eacdd98923590afdc85330aaac0488de96e991b24a7f4cb60113b7a66e80",
"brew-pulp-docker01.web.prod.ext.phx2.redhat.com:8888/openshift3/ose-logging-fluentd:v3.11.141-2"
],
"digests": {
"application/vnd.docker.distribution.manifest.list.v2+json":
"sha256:1df5eacdd98923590afdc85330aaac0488de96e991b24a7f4cb60113b7a66e80"
},
"tags": ["v3.11.141-2"]
},
"autorebuild":
False,
"isolated":
False,
"yum_repourls": [
"http://pkgs.devel.redhat.com/cgit/containers/logging-fluentd/plain/.oit/signed.repo?h=rhaos-3.11-rhel-7"
],
"parent_build_id":
955726,
"parent_images": ["openshift/ose-base:rhel7"],
"parent_image_builds": {
"openshift/ose-base:rhel7": {
"id":
955726,
"nvr":
"openshift-enterprise-base-container-v4.0-201908250221"
}
}
},
"container_koji_task_id": 23188768
},
"creation_time": "2019-08-26 07:34:32.613833",
"completion_time": "2019-08-26 07:34:31",
"package_id": 67151,
"cg_name": None,
"id": 956245,
"build_id": 956245,
"epoch": None,
"source":
"git://pkgs.devel.redhat.com/containers/logging-fluentd#7f4bcdc798fd72414a29dc1010c448e1ed52f591",
"state": 1,
"version": "v3.11.141",
"completion_ts": 1566804871.0,
"owner_id": 4078,
"owner_name": "ocp-build/buildvm.openshift.eng.bos.redhat.com",
"nvr": "logging-fluentd-container-v3.11.141-2",
"start_time": "2019-08-26 07:03:41",
"creation_event_id": 26029088,
"start_ts": 1566803021.0,
"volume_id": 0,
"creation_ts": 1566804872.61383,
"name": "logging-fluentd-container",
"task_id": None,
"volume_name": "DEFAULT",
"release": "2"
},
"logging-fluentd-container-v4.1.14-201908291507": {
"cg_id": None,
"package_name": "logging-fluentd-container",
"extra": {
"submitter": "osbs",
"image": {
"media_types": [
"application/vnd.docker.distribution.manifest.list.v2+json",
"application/vnd.docker.distribution.manifest.v1+json",
"application/vnd.docker.distribution.manifest.v2+json"
],
"help":
None,
"index": {
"unique_tags": [
"rhaos-4.1-rhel-7-containers-candidate-94076-20190829211225"
],
"pull": [
"brew-pulp-docker01.web.prod.ext.phx2.redhat.com:8888/openshift/ose-logging-fluentd@sha256:7503f828aaf80e04b2aaab0b88626b97a20e5600ba75fef8b764e02cc1164a7c",
"brew-pulp-docker01.web.prod.ext.phx2.redhat.com:8888/openshift/ose-logging-fluentd:v4.1.14-201908291507"
],
"floating_tags": [
"latest", "v4.1.14", "v4.1.14.20190829.150756",
"v4.1"
],
"digests": {
"application/vnd.docker.distribution.manifest.list.v2+json":
"sha256:7503f828aaf80e04b2aaab0b88626b97a20e5600ba75fef8b764e02cc1164a7c"
},
"tags": ["v4.1.14-201908291507"]
},
"autorebuild":
False,
"isolated":
False,
"yum_repourls": [
"http://pkgs.devel.redhat.com/cgit/containers/logging-fluentd/plain/.oit/signed.repo?h=rhaos-4.1-rhel-7"
],
"parent_build_id":
958278,
"parent_images": [
"rhscl/ruby-25-rhel7:latest",
"openshift/ose-base:ubi7"
],
"parent_image_builds": {
"openshift/ose-base:ubi7": {
"id":
958278,
"nvr":
"openshift-enterprise-base-container-v4.0-201908290538"
},
"rhscl/ruby-25-rhel7:latest": {
"id": 957642,
"nvr": "rh-ruby25-container-2.5-50"
}
}
},
"container_koji_task_id": 23241046
},
"creation_time": "2019-08-29 21:42:46.062037",
"completion_time": "2019-08-29 21:42:44",
"package_id": 67151,
"cg_name": None,
"id": 958765,
"build_id": 958765,
"epoch": None,
"source":
"git://pkgs.devel.redhat.com/containers/logging-fluentd#ecac10b38f035ea2f9ea62b9efa63c051667ebbb",
"state": 1,
"version": "v4.1.14",
"completion_ts": 1567114964.0,
"owner_id": 4078,
"owner_name": "ocp-build/buildvm.openshift.eng.bos.redhat.com",
"nvr": "logging-fluentd-container-v4.1.14-201908291507",
"start_time": "2019-08-29 21:12:51",
"creation_event_id": 26063093,
"start_ts": 1567113171.0,
"volume_id": 0,
"creation_ts": 1567114966.06204,
"name": "logging-fluentd-container",
"task_id": None,
"volume_name": "DEFAULT",
"release": "201908291507"
},
"logging-fluentd-container-v4.1.15-201909041605": {
"cg_id": None,
"package_name": "logging-fluentd-container",
"extra": {
"submitter": "osbs",
"image": {
"media_types": [
"application/vnd.docker.distribution.manifest.list.v2+json",
"application/vnd.docker.distribution.manifest.v1+json",
"application/vnd.docker.distribution.manifest.v2+json"
],
"help":
None,
"index": {
"unique_tags": [
"rhaos-4.1-rhel-7-containers-candidate-96970-20190904214308"
],
"pull": [
"brew-pulp-docker01.web.prod.ext.phx2.redhat.com:8888/openshift/ose-logging-fluentd@sha256:1ce1555b58982a29354c293948ee6c788743a08f39a0c530be791cb9bdaf4189",
"brew-pulp-docker01.web.prod.ext.phx2.redhat.com:8888/openshift/ose-logging-fluentd:v4.1.15-201909041605"
],
"floating_tags": [
"latest", "v4.1.15", "v4.1",
"v4.1.15.20190904.160545"
],
"digests": {
"application/vnd.docker.distribution.manifest.list.v2+json":
"sha256:1ce1555b58982a29354c293948ee6c788743a08f39a0c530be791cb9bdaf4189"
},
"tags": ["v4.1.15-201909041605"]
},
"autorebuild":
False,
"isolated":
False,
"yum_repourls": [
"http://pkgs.devel.redhat.com/cgit/containers/logging-fluentd/plain/.oit/signed.repo?h=rhaos-4.1-rhel-7"
],
"parent_build_id":
961131,
"parent_images": [
"rhscl/ruby-25-rhel7:latest",
"openshift/ose-base:ubi7"
],
"parent_image_builds": {
"openshift/ose-base:ubi7": {
"id":
961131,
"nvr":
"openshift-enterprise-base-container-v4.0-201909040323"
},
"rhscl/ruby-25-rhel7:latest": {
"id": 957642,
"nvr": "rh-ruby25-container-2.5-50"
}
}
},
"container_koji_task_id": 23365465
},
"creation_time": "2019-09-04 22:17:36.432110",
"completion_time": "2019-09-04 22:17:35",
"package_id": 67151,
"cg_name": None,
"id": 962144,
"build_id": 962144,
"epoch": None,
"source":
"git://pkgs.devel.redhat.com/containers/logging-fluentd#31cf3d4264dabb8892fb4b5921e5ff4d5d0ab2de",
"state": 1,
"version": "v4.1.15",
"completion_ts": 1567635455.0,
"owner_id": 4078,
"owner_name": "ocp-build/buildvm.openshift.eng.bos.redhat.com",
"nvr": "logging-fluentd-container-v4.1.15-201909041605",
"start_time": "2019-09-04 21:43:32",
"creation_event_id": 26176078,
"start_ts": 1567633412.0,
"volume_id": 0,
"creation_ts": 1567635456.43211,
"name": "logging-fluentd-container",
"task_id": None,
"volume_name": "DEFAULT",
"release": "201909041605"
},
}
def fake_get_build(nvr):
return mock.MagicMock(result=build_infos[nvr])
fake_session = mock.MagicMock()
fake_context_manager = fake_session.multicall.return_value.__enter__.return_value
fake_context_manager.getBuild.side_effect = fake_get_build
expected = list(build_infos.values())
actual = brew.get_build_objects(build_infos.keys(), fake_session)
self.assertListEqual(actual, expected)
async def verify_cvp_cli(runtime: Runtime, all_images, nvrs, optional_checks, all_optional_checks, fix, message):
""" Verify CVP test results
Example 1: Verify CVP test results for all latest 4.4 image builds, also warn those with failed content_set_check
$ elliott --group openshift-4.4 verify-cvp --all --include-optional-check content_set_check
Example 2: Apply patches to ocp-build-data to fix the redundant content sets error:
$ elliott --group openshift-4.4 verify-cvp --all --include-optional-check content_set_check --fix
Note:
1. If `--message` is not given, `--fix` will leave changed ocp-build-data files uncommitted.
2. Make sure your ocp-build-data directory is clean before running `--fix`.
"""
if bool(all_images) + bool(nvrs) != 1:
raise click.BadParameter('You must use one of --all or --build.')
if all_optional_checks and optional_checks:
raise click.BadParameter('Use only one of --all-optional-checks or --include-optional-check.')
runtime.initialize(mode='images')
brew_session = koji.ClientSession(runtime.group_config.urls.brewhub or constants.BREW_HUB)
builds = []
if all_images:
image_metas = runtime.image_metas()
builds = await get_latest_image_builds(image_metas)
elif nvrs:
runtime.logger.info(f"Finding {len(builds)} builds from Brew...")
builds = brew.get_build_objects(nvrs, brew_session)
runtime.logger.info(f"Found {len(builds)} image builds.")
resultsdb_api = ResultsDBAPI()
nvrs = [b["nvr"] for b in builds]
runtime.logger.info(f"Getting CVP test results for {len(builds)} image builds...")
latest_cvp_results = await get_latest_cvp_results(runtime, resultsdb_api, nvrs)
# print a summary for all CVP results
good_results = [] # good means PASSED or INFO
bad_results = [] # bad means NEEDS_INSPECTION or FAILED
incomplete_nvrs = []
for nvr, result in zip(nvrs, latest_cvp_results):
if not result:
incomplete_nvrs.append(nvr)
continue
outcome = result.get("outcome") # only PASSED, FAILED, INFO, NEEDS_INSPECTION are now valid outcome values (https://resultsdb20.docs.apiary.io/#introduction/changes-since-1.0)
if outcome in {"PASSED", "INFO"}:
good_results.append(result)
elif outcome in {"NEEDS_INSPECTION", "FAILED"}:
bad_results.append(result)
green_prefix("good: {}".format(len(good_results)))
click.echo(", ", nl=False)
red_prefix("bad: {}".format(len(bad_results)))
click.echo(", ", nl=False)
yellow_print("incomplete: {}".format(len(incomplete_nvrs)))
if bad_results:
red_print("The following builds didn't pass CVP tests:")
for r in bad_results:
nvr = r["data"]["item"][0]
red_print(f"{nvr} {r['outcome']}: {r['ref_url']}")
if incomplete_nvrs:
yellow_print("We couldn't find CVP test results for the following builds:")
for nvr in incomplete_nvrs:
yellow_print(nvr)
if not optional_checks and not all_optional_checks:
return # no need to print failed optional CVP checks
# Find failed optional CVP checks in case some of the tiem *will* become required.
optional_checks = set(optional_checks)
complete_results = good_results + bad_results
runtime.logger.info(f"Getting optional checks for {len(complete_results)} CVP tests...")
optional_check_results = await get_optional_checks(runtime, complete_results)
component_distgit_keys = {} # a dict of brew component names to distgit keys
content_set_to_repo_names = {} # a map of content set names to group.yml repo names
for image in runtime.image_metas():
component_distgit_keys[image.get_component_name()] = image.distgit_key
for repo_name, repo_info in runtime.group_config.get("repos", {}).items():
for arch, cs_name in repo_info.get('content_set', {}).items():
if arch == "optional":
continue # not a real arch name
content_set_to_repo_names[cs_name] = repo_name
nvr_to_builds = {build["nvr"]: build for build in builds}
ocp_build_data_updated = False
failed_with_not_covered_rpms = set()
failed_with_redundant_repos = set()
only_failed_in_non_x86_with_not_covered_rpms = set()
only_failed_in_non_x86_with_redundant_repos = set()
for cvp_result, checks in zip(complete_results, optional_check_results):
# example optional checks: http://external-ci-coldstorage.datahub.redhat.com/cvp/cvp-product-test/hive-container-v4.6.0-202008010302.p0/da01e36c-8c69-4a19-be7d-ba4593a7b085/sanity-tests-optional-results.json
bad_checks = [check for check in checks["checks"] if check["status"] != "PASS" and (all_optional_checks or check["name"] in optional_checks)]
if not bad_checks:
continue
nvr = cvp_result["data"]["item"][0]
build = nvr_to_builds[nvr]
yellow_print("----------")
yellow_print(f"Build {nvr} (https://brewweb.engineering.redhat.com/brew/buildinfo?buildID={nvr_to_builds[nvr]['id']}) has {len(bad_checks)} problematic CVP optional checks:")
for check in bad_checks:
yellow_print(f"* {check['name']} {check['status']}")
try:
amd64_result = list(filter(lambda item: item.get("arch") == "amd64", check["logs"][-1]))
except AttributeError:
red_print("CVP result malformed.")
if len(amd64_result) != 1:
red_print("WHAT?! This build doesn't include an amd64 image? This shouldn't happen. Check Brew and CVP logs with the CVP team!")
continue
amd64_result = amd64_result[0]
image_component_name = nvr.rsplit('-', 2)[0]
distgit_key = component_distgit_keys.get(image_component_name)
amd64_redundant_cs = amd64_result.get("redundant_cs", [])
amd64_redundant_repos = {content_set_to_repo_names[cs] for cs in amd64_redundant_cs}
def _strip_arch_suffix(rpm):
# rh-nodejs10-3.2-3.el7.x86_64 -> rh-nodejs10-3.2-3.el7
rpm_split = rpm.rsplit(".", 1)
return rpm_split[0]
amd64_not_covered_rpms = {_strip_arch_suffix(rpm) for rpm in amd64_result.get("not_covered_rpms", [])}
if check["name"] == "content_set_check":
details = check["logs"][-1] # example: http://external-ci-coldstorage.datahub.redhat.com/cvp/cvp-product-test/logging-fluentd-container-v4.6.0-202008261251.p0/dd9f2024-5440-4f33-b508-472ccf258439/sanity-tests-optional-results.json
if not details:
red_print("content_set_check failed without any explanation. Report to CVP team!")
continue
if len(details) > 1: # if this build is multi-arch, check if all per-arch results are consistent
for result in details:
if result["arch"] == "amd64":
continue
redundant_repos = {content_set_to_repo_names[cs] for cs in result.get("redundant_cs", [])}
if redundant_repos != amd64_redundant_repos:
only_failed_in_non_x86_with_redundant_repos.add(nvr)
red_print(f"""content_set_check for {nvr} arch {result["arch"]} has different redundant_cs result from the one for amd64:
{result["arch"]} has redundant_cs {result.get("redundant_cs")},
but amd64 has redundant_cs {amd64_redundant_cs}.
Not sure what happened. Please see Brew and CVP logs and/or check with the CVP team.""")
not_covered_rpms = {_strip_arch_suffix(rpm) for rpm in result.get("not_covered_rpms", [])}
if not_covered_rpms != amd64_not_covered_rpms:
only_failed_in_non_x86_with_not_covered_rpms.add(nvr)
red_print(f"""content_set_check for {nvr} arch {result["arch"]} has different not_covered_rpms result from the one for amd64:
{result["arch"]} has extra not_covered_rpms {not_covered_rpms - amd64_not_covered_rpms},
and missing not_covered_rpms {amd64_not_covered_rpms - not_covered_rpms}.
Not sure what happened. Check Brew and CVP logs with the CVP team!""")
if amd64_not_covered_rpms: # This build has not_covered_rpms
failed_with_not_covered_rpms.add(nvr)
yellow_print(f"Image {distgit_key} has not_covered_rpms: {amd64_not_covered_rpms}")
brew_repos = await find_repos_for_rpms(amd64_not_covered_rpms, build)
yellow_print(f"Those repos shown in Brew logs might be a good hint: {brew_repos}")
runtime.logger.info("Looking for parent image's content_sets...")
parent = get_parent_build_ids([build])[0]
if parent:
parent_build = brew.get_build_objects([parent])[0]
parent_cs = await get_content_sets_for_build(parent_build)
parent_enabled_repos = {content_set_to_repo_names[cs] for cs in parent_cs.get("x86_64", [])}
enabled_repos = set(runtime.image_map[distgit_key].config.get("enabled_repos", []))
missing_repos = parent_enabled_repos - enabled_repos
yellow_print(f"""The following repos are defined in parent {parent_build["nvr"]} {component_distgit_keys.get(parent_build["name"], "?")}.yml but not in
{component_distgit_keys[build["name"]]}.yml: {missing_repos}""")
if fix and missing_repos:
runtime.logger.info("Trying to merge parent image's content_sets...")
fix_missing_content_set(runtime, distgit_key, missing_repos)
ocp_build_data_updated = True
runtime.logger.info(f"{distgit_key}.yml patched")
if amd64_redundant_repos: # This build has redundant_cs
failed_with_redundant_repos.add(nvr)
yellow_print(f"Image {distgit_key} has redundant repos: {amd64_redundant_repos}")
if not fix:
yellow_print(f"Please add the following repos to non_shipping_repos in {distgit_key}.yml: {amd64_redundant_repos}")
else:
runtime.logger.info(f"Applying redundant content sets fix to {distgit_key}.yml...")
fix_redundant_content_set(runtime, distgit_key, amd64_redundant_repos)
ocp_build_data_updated = True
runtime.logger.info(f"{distgit_key}.yml patched")
print(f"See {cvp_result['ref_url']}sanity-tests-optional-results.json for more details.")
if failed_with_not_covered_rpms or failed_with_redundant_repos:
yellow_print(f"{len(failed_with_not_covered_rpms | failed_with_redundant_repos)} images failed content_sets.\n Where")
if failed_with_not_covered_rpms:
yellow_print(f"\t{len(failed_with_not_covered_rpms)} images failed content_sets check because of not_covered_rpms:")
for rpm in failed_with_not_covered_rpms:
line = f"\t\t{rpm}"
if rpm in only_failed_in_non_x86_with_not_covered_rpms:
line += " - non-x86 arches are different from x86 one"
yellow_print(line)
if failed_with_redundant_repos:
yellow_print(f"\t{len(failed_with_redundant_repos)} images failed content_sets check because of redundant_repos:")
for rpm in failed_with_redundant_repos:
line = f"\t\t{rpm}"
if rpm in only_failed_in_non_x86_with_redundant_repos:
line += " - non-x86 arches are different from x86 one"
yellow_print(line)
if message and ocp_build_data_updated:
runtime.gitdata.commit(message)
def create(ctx, advisories, out_dir, out_layout, components, force):
""" Create tarball sources for advisories.
To create tarball sources for Brew component (package) logging-fluentd-container that was shipped on advisories 45606, 45527, and 46049:
$ elliott tarball-sources create --component logging-fluentd-container --out-dir=out/ 45606 45527 46049
"""
if not force and os.path.isdir(out_dir) and os.listdir(out_dir):
util.red_print("Output directory {} is not empty.\n\
Use --force to add new tarball sources to an existing directory.".format(
os.path.abspath(out_dir)))
exit(1)
mkdirs(out_dir)
working_dir = os.path.join(ctx.obj.working_dir, "tarball-sources")
LOGGER.debug("Use working directory {}.".format(
os.path.abspath(working_dir)))
mkdirs(working_dir)
# `nvr_dirs` is a dict with brew build NVRs as keys, values are
# a set of directories for the generated tarballs,
# since a build can be attached to multiple advisories.
# For example:
# nvr_dirs = {
# "logging-fluentd-container-v3.11.141-2": {
# "RHOSE/RHEL-7-OSE-3.11/45606/release/"
# },
# "logging-fluentd-container-v4.1.14-201908291507": {
# "RHOSE/RHEL-7-OSE-4.1/45527/release/"
# },
# "logging-fluentd-container-v4.1.15-201909041605": {
# "RHOSE/RHEL-7-OSE-4.1/46049/release/"
# }
# }
nvr_dirs = {} # type: Dict[str, Set[str]]
# Getting build NVRs for specified Koji/Brew components from advisories
# NOTE This is SLOW. However doing this in parallel doesn't work
# due to a race condition existing in the implementation of `errata_tool.Erratum`'s parant class ErrataConnector.
for advisory in advisories:
click.echo("Finding builds from advisory {}...".format(advisory))
builds = tarball_sources.find_builds_from_advisory(
advisory, components)
if not builds:
util.yellow_print(
"No matched builds found from advisory {}. Wrong advisory number?"
.format(advisory))
continue
util.green_print("Found {} matched build(s) from advisory {}".format(
len(builds), advisory))
for nvr, product, product_version in builds:
util.green_print("\t{}\t{}\t{}".format(nvr, product,
product_version))
for nvr, product, product_version in builds:
if nvr not in nvr_dirs:
nvr_dirs[nvr] = set()
if out_layout == "flat":
nvr_dirs[nvr].add(out_dir)
else:
nvr_dirs[nvr].add(
os.path.join(out_dir, product_version, str(advisory),
"release"))
if not nvr_dirs:
util.red_print(
"Exiting because no matched builds from all specified advisories.")
exit(1)
# Check build infos from Koji/Brew
# in order to figure out the source Git repo and commit hash for each build.
click.echo("Fetching build infos for {} from Koji/Brew...".format(
", ".join(nvr_dirs.keys())))
brew_session = koji.ClientSession(constants.BREW_HUB)
brew_builds = brew.get_build_objects(nvr_dirs.keys(), brew_session)
# Ready to generate tarballs
tarball_sources_list = []
for build_info in brew_builds:
nvr = build_info["nvr"]
tarball_filename = nvr + ".tar.gz"
click.echo("Generating tarball source {} for {}...".format(
tarball_filename, nvr))
with tempfile.NamedTemporaryFile(suffix="-" + tarball_filename,
dir=working_dir) as temp_tarball:
temp_tarball_path = temp_tarball.name
LOGGER.debug(
"Temporary tarball file is {}".format(temp_tarball_path))
tarball_sources.generate_tarball_source(
temp_tarball, nvr + "/",
os.path.join(working_dir, "repos", build_info["name"]),
build_info["source"])
for dest_dir in nvr_dirs[nvr]:
mkdirs(dest_dir)
tarball_abspath = os.path.abspath(
os.path.join(dest_dir, tarball_filename))
if os.path.exists(tarball_abspath):
util.yellow_print(
"File {} will be overwritten.".format(tarball_abspath))
LOGGER.debug("Copying {} to {}...".format(
temp_tarball_path, tarball_abspath))
shutil.copyfile(
temp_tarball_path,
tarball_abspath) # `shutil.copyfile` uses default umask
tarball_sources_list.append(tarball_abspath)
util.green_print(
"Created tarball source {}.".format(tarball_abspath))
print_success_message(tarball_sources_list, out_dir)
def tag_builds_cli(runtime: Runtime, advisories: Tuple[int],
default_advisory_type: str, product_version: str,
builds: Tuple[str], tag: str, dont_untag: bool,
dry_run: bool):
""" Tag builds into Brew tag and optionally untag unspecified builds.
Example 1: Tag RHEL7 RPMs that on ocp-build-data recorded advisory into rhaos-4.3-rhel-7-image-build
$ elliott --group=openshift-4.3 tag-builds --use-default-advisory rpm --product-version RHEL-7-OSE-4.3 --tag rhaos-4.3-rhel-7-image-build
Example 2: Tag RHEL8 RPMs that are on advisory 55016 into rhaos-4.3-rhel-8-image-build
$ elliott --group=openshift-4.3 tag-builds --advisory 55016 --product-version OSE-4.4-RHEL-8 --tag rhaos-4.3-rhel-8-image-build
Example 3: Tag specified builds into rhaos-4.3-rhel-8-image-build
$ elliott --group=openshift-4.3 tag-builds --build buildah-1.11.6-6.rhaos4.3.el8 --build openshift-4.3.23-202005230952.g1.b596217.el8 --tag rhaos-4.3-rhel-8-image-build
"""
if advisories and builds:
raise click.BadParameter('Use only one of --build or --advisory/-a.')
if advisories and default_advisory_type:
raise click.BadParameter(
'Use only one of --use-default-advisory or --advisory/-a.')
if default_advisory_type and builds:
raise click.BadParameter(
'Use only one of --build or --use-default-advisory.')
if product_version and not advisories and not default_advisory_type:
raise click.BadParameter(
'--product-version should only be used with --use-default-advisory or --advisory/-a.'
)
runtime.initialize()
logger = runtime.logger
if default_advisory_type:
advisories = (find_default_advisory(runtime, default_advisory_type), )
all_builds = set() # All Brew builds that should be in the tag
if advisories:
errata_session = requests.session()
for advisory in advisories:
logger.info(
f"Fetching attached Brew builds from advisory {advisory}...")
errata_builds = errata.get_builds(advisory, errata_session)
product_versions = list(errata_builds.keys())
logger.debug(
f"Advisory {advisory} has builds for {len(product_versions)} product versions: {product_versions}"
)
if product_version: # Only this product version should be concerned
product_versions = [product_version]
for pv in product_versions:
logger.debug(f"Extract Errata builds for product version {pv}")
nvrs = _extract_nvrs_from_errata_build_list(errata_builds, pv)
logger.info(
f"Found {len(nvrs)} builds from advisory {advisory} with product version {pv}"
)
logger.debug(
f"The following builds are found for product version {pv}:\n\t{list(nvrs)}"
)
all_builds |= set(nvrs)
brew_session = koji.ClientSession(runtime.group_config.urls.brewhub
or constants.BREW_HUB)
if builds: # NVRs are directly specified with --build
build_objs = brew.get_build_objects(list(builds), brew_session)
all_builds = {build["nvr"] for build in build_objs}
click.echo(
f"The following {len(all_builds)} build(s) should be in tag {tag}:")
for nvr in all_builds:
green_print(f"\t{nvr}")
# get NVRs that have been tagged
tagged_build_objs = brew_session.listTagged(tag,
latest=False,
inherit=False)
tagged_builds = {build["nvr"] for build in tagged_build_objs}
# get NVRs that should be tagged
missing_builds = all_builds - tagged_builds
click.echo(f"{len(missing_builds)} build(s) need to be tagged into {tag}:")
for nvr in missing_builds:
green_print(f"\t{nvr}")
# get NVRs that should be untagged
extra_builds = tagged_builds - all_builds
click.echo(f"{len(extra_builds)} build(s) need to be untagged from {tag}:")
for nvr in extra_builds:
green_print(f"\t{nvr}")
if dry_run:
yellow_print("Dry run: Do nothing.")
return
brew_session.gssapi_login()
if not dont_untag:
# untag extra builds
extra_builds = list(extra_builds)
logger.info(f"Untagging {len(extra_builds)} build(s) from {tag}...")
multicall_tasks = brew.untag_builds(tag, extra_builds, brew_session)
failed_to_untag = []
for index, task in enumerate(multicall_tasks):
try:
task.result
click.echo(f"{nvr} has been successfully untagged from {tag}")
except Exception as ex:
nvr = extra_builds[index]
failed_to_untag.append(nvr)
logger.error(f"Failed to untag {nvr}: {ex}")
# tag missing builds
missing_builds = list(missing_builds)
task_id_nvr_map = {}
logger.info(f"Tagging {len(missing_builds)} build(s) into {tag}...")
multicall_tasks = brew.tag_builds(tag, missing_builds, brew_session)
failed_to_tag = []
for index, task in enumerate(multicall_tasks):
nvr = missing_builds[index]
try:
task_id = task.result
task_id_nvr_map[task_id] = nvr
except Exception as ex:
failed_to_tag.append(nvr)
logger.error(f"Failed to tag {nvr}: {ex}")
if task_id_nvr_map:
# wait for tag task to finish
logger.info("Waiting for tag tasks to finish")
brew.wait_tasks(task_id_nvr_map.keys(), brew_session, logger=logger)
# get tagging results
stopped_tasks = list(task_id_nvr_map.keys())
with brew_session.multicall(strict=False) as m:
multicall_tasks = []
for task_id in stopped_tasks:
multicall_tasks.append(
m.getTaskResult(task_id, raise_fault=False))
for index, t in enumerate(multicall_tasks):
task_id = stopped_tasks[index]
nvr = task_id_nvr_map[task_id]
tag_res = t.result
logger.debug(
f"Tagging task {task_id} {nvr} returned result {tag_res}")
click.echo(f"{nvr} has been successfully tagged into {tag}")
if tag_res and 'faultCode' in tag_res:
if "already tagged" not in tag_res["faultString"]:
failed_to_tag.append(nvr)
logger.error(
f'Failed to tag {nvr} into {tag}: {tag_res["faultString"]}'
)
if failed_to_untag:
red_print("The following builds were failed to untag:")
for nvr in failed_to_untag:
red_print(f"\t{nvr}")
elif not dont_untag:
green_print(
f"All unspecified builds have been successfully untagged from {tag}."
)
if failed_to_tag:
red_print("The following builds were failed to tag:")
for nvr in failed_to_tag:
red_print(f"\t{nvr}")
else:
green_print(f"All builds have been successfully tagged into {tag}.")
if failed_to_untag or failed_to_tag:
raise exceptions.ElliottFatalError(
"Not all builds were successfully tagged/untagged.")
