async def verify_payload(ctx, payload, advisory):
"""Cross-check that the builds present in PAYLOAD match the builds
attached to ADVISORY. The payload is treated as the source of
truth. If something is absent or different in the advisory it is
treated as an error with the advisory.
\b
PAYLOAD - Full pullspec of the payload to verify
ADVISORY - Numerical ID of the advisory
Two checks are made:
\b
1. Missing in Advisory - No payload components are absent from the given advisory
2. Payload Advisory Mismatch - The version-release of each payload item match what is in the advisory
Results are summarily printed at the end of the run. They are also
written out to summary_results.json.
Verify builds in the given payload match the builds attached to
advisory 41567
\b
$ elliott verify-payload quay.io/openshift-release-dev/ocp-release:4.1.0-rc.6 41567
"""
all_advisory_nvrs = elliottlib.errata.get_advisory_nvrs(advisory)
click.echo("Found {} builds".format(len(all_advisory_nvrs)))
all_payload_nvrs = {}
click.echo("Fetching release info")
release_export_cmd = 'oc adm release info {} -o json'.format(payload)
rc, stdout, stderr = exectools.cmd_gather(release_export_cmd)
if rc != 0:
# Probably no point in continuing.. can't contact brew?
print("Unable to run oc release info: out={} ; err={}".format(
stdout, stderr))
exit(1)
else:
click.echo("Got release info")
payload_json = json.loads(stdout)
green_prefix("Looping over payload images: ")
click.echo("{} images to check".format(
len(payload_json['references']['spec']['tags'])))
cmds = [['oc', 'image', 'info', '-o', 'json', tag['from']['name']]
for tag in payload_json['references']['spec']['tags']]
green_prefix("Querying image infos...")
cmd_results = await asyncio.gather(
*[exectools.cmd_gather_async(cmd) for cmd in cmds])
for image, cmd, cmd_result in zip(
payload_json['references']['spec']['tags'], cmds, cmd_results):
click.echo("----")
image_name = image['name']
rc, stdout, stderr = cmd_result
if rc != 0:
# Probably no point in continuing.. can't contact brew?
red_prefix("Unable to run oc image info: ")
red_print(f"cmd={cmd!r}, out={stdout} ; err={stderr}")
exit(1)
image_info = json.loads(stdout)
labels = image_info['config']['config']['Labels']
# The machine-os-content image doesn't follow the standard
# pattern. We need to skip that image when we find it, it is
# not attached to advisories.
if 'com.coreos.ostree-commit' in labels:
yellow_prefix("Skipping machine-os-content image: ")
click.echo("Not required for checks")
continue
component = labels['com.redhat.component']
n = image_name
click.echo("Payload name: {}".format(n))
click.echo("Brew name: {}".format(component))
if labels:
v = labels['version']
r = labels['release']
all_payload_nvrs[component] = "{}-{}".format(v, r)
else:
print("For image {} Labels doesn't exist, image_info: {}".format(
image_name, image_info))
missing_in_errata = {}
payload_doesnt_match_errata = {}
in_pending_advisory = []
in_shipped_advisory = []
output = {
'missing_in_advisory': missing_in_errata,
'payload_advisory_mismatch': payload_doesnt_match_errata,
"in_pending_advisory": in_pending_advisory,
"in_shipped_advisory": in_shipped_advisory,
}
green_prefix("Analyzing data: ")
click.echo("{} images to consider from payload".format(
len(all_payload_nvrs)))
for image, vr in all_payload_nvrs.items():
yellow_prefix("Cross-checking from payload: ")
click.echo("{}-{}".format(image, vr))
if image not in all_advisory_nvrs:
missing_in_errata[image] = "{}-{}".format(image, vr)
click.echo("{} in payload not found in advisory".format(
"{}-{}".format(image, vr)))
elif image in all_advisory_nvrs and vr != all_advisory_nvrs[image]:
click.echo(
"{} from payload has version {} which does not match {} from advisory"
.format(image, vr, all_advisory_nvrs[image]))
payload_doesnt_match_errata[image] = {
'payload': vr,
'errata': all_advisory_nvrs[image]
}
if missing_in_errata: # check if missing images are already shipped or pending to ship
advisory_nvrs: Dict[int, List[str]] = {
} # a dict mapping advisory numbers to lists of NVRs
green_print(
f"Checking if {len(missing_in_errata)} missing images are shipped..."
)
for nvr in missing_in_errata.copy().values():
# get the list of advisories that this build has been attached to
build = elliottlib.errata.get_brew_build(nvr)
# filter out dropped advisories
advisories = [
ad for ad in build.all_errata
if ad["status"] != "DROPPED_NO_SHIP"
]
if not advisories:
red_print(f"Build {nvr} is not attached to any advisories.")
continue
for advisory in advisories:
if advisory["status"] == "SHIPPED_LIVE":
green_print(
f"Missing build {nvr} has been shipped with advisory {advisory}."
)
else:
yellow_print(
f"Missing build {nvr} is in another pending advisory.")
advisory_nvrs.setdefault(advisory["id"], []).append(nvr)
name = nvr.rsplit("-", 2)[0]
del missing_in_errata[name]
if advisory_nvrs:
click.echo(
f"Getting information of {len(advisory_nvrs)} advisories...")
for advisory, nvrs in advisory_nvrs.items():
advisory_obj = elliottlib.errata.get_raw_erratum(advisory)
adv_type, adv_info = next(iter(advisory_obj["errata"].items()))
item = {
"id": advisory,
"type": adv_type.upper(),
"url": elliottlib.constants.errata_url + f"/{advisory}",
"summary": adv_info["synopsis"],
"state": adv_info["status"],
"nvrs": nvrs,
}
if adv_info["status"] == "SHIPPED_LIVE":
in_shipped_advisory.append(item)
else:
in_pending_advisory.append(item)
green_print("Summary results:")
click.echo(json.dumps(output, indent=4))
with open('summary_results.json', 'w') as fp:
json.dump(output, fp, indent=4)
green_prefix("Wrote out summary results: ")
click.echo("summary_results.json")
def poll_signed(runtime, minutes, advisory, default_advisory_type, noop):
"""Poll for the signed-status of RPM builds attached to
ADVISORY. Returns rc=0 when all builds have been signed. Returns non-0
after MINUTES have passed and all builds have not been signed. This
non-0 return code is the number of unsigned builds remaining. All
builds must show 'signed' for this command to succeed.
NOTE: The two advisory options are mutually exclusive.
For testing in pipeline scripts this sub-command accepts a --noop
option. When --noop is used the value of --minutes is irrelevant. This
command will print out the signed state of all attached builds and
then exit with rc=0 if all builds are signed and non-0 if builds are
still unsigned. In the non-0 case the return code is the number of
unsigned builds.
Wait 15 minutes for the default 4.2 advisory to show all RPMS have
been signed:
$ elliott -g openshift-4.2 poll-signed --use-default-advisory rpm
Wait 5 mintes for the provided 4.2 advisory to show all RPMs have
been signed:
$ elliott -g openshift-4.2 poll-signed -m 5 --advisory 123456
Print the signed status of all attached builds, exit
immediately. Return code is the number of unsigned builds.
\b
$ elliott -g openshift-4.2 poll-signed --noop --use-default-advisory rpm
"""
if not (bool(advisory) ^ bool(default_advisory_type)):
raise click.BadParameter(
"Use only one of --use-default-advisory or --advisory")
runtime.initialize(no_group=default_advisory_type is None)
if default_advisory_type is not None:
advisory = find_default_advisory(runtime, default_advisory_type)
if not noop:
click.echo("Polling up to {} minutes for all RPMs to be signed".format(
minutes))
try:
e = elliottlib.errata.Advisory(errata_id=advisory)
all_builds = set([])
all_signed = False
# `errata_builds` is a dict with brew tags as keys, values are
# lists of builds on the advisory with that tag
for k, v in e.errata_builds.items():
all_builds = all_builds.union(set(v))
green_prefix("Fetching initial states: ")
click.echo("{} builds to check".format(len(all_builds)))
start_time = datetime.datetime.now()
while datetime.datetime.now() - start_time < datetime.timedelta(
minutes=minutes):
pbar_header("Getting build signatures: ", "Should be pretty quick",
all_builds)
pool = ThreadPool(cpu_count())
# Look up builds concurrently
click.secho("[", nl=False)
build_sigs = pool.map(
lambda build: progress_func(
lambda: elliottlib.errata.build_signed(build), '*'),
all_builds)
# Wait for results
pool.close()
pool.join()
click.echo(']')
if all(build_sigs):
all_signed = True
break
elif noop:
# Escape the time-loop
break
else:
yellow_prefix("Not all builds signed: ")
click.echo("re-checking")
continue
if not all_signed:
red_prefix("Signing incomplete: ")
if noop:
click.echo("All builds not signed. ")
else:
click.echo(
"All builds not signed in given window ({} minutes). ".
format(minutes))
exit(1)
else:
green_prefix("All builds signed: ")
click.echo("Enjoy!")
except ErrataException as ex:
raise ElliottFatalError(getattr(ex, 'message', repr(ex)))
async def verify_payload(ctx, payload, advisory):
"""Cross-check that the builds present in PAYLOAD match the builds
attached to ADVISORY. The payload is treated as the source of
truth. If something is absent or different in the advisory it is
treated as an error with the advisory.
\b
PAYLOAD - Full pullspec of the payload to verify
ADVISORY - Numerical ID of the advisory
Two checks are made:
\b
1. Missing in Advisory - No payload components are absent from the given advisory
2. Payload Advisory Mismatch - The version-release of each payload item match what is in the advisory
Results are summarily printed at the end of the run. They are also
written out to summary_results.json.
Verify builds in the given payload match the builds attached to
advisory 41567
\b
$ elliott verify-payload quay.io/openshift-release-dev/ocp-release:4.1.0-rc.6 41567
"""
try:
green_prefix("Fetching advisory builds: ")
click.echo("Advisory - {}".format(advisory))
builds = elliottlib.errata.get_builds(advisory)
except GSSError:
exit_unauthenticated()
except elliottlib.exceptions.ErrataToolError as ex:
raise ElliottFatalError(getattr(ex, 'message', repr(ex)))
all_advisory_nvrs = {}
# Results come back with top level keys which are brew tags
green_prefix("Looping over tags: ")
click.echo("{} tags to check".format(len(builds)))
for tag in builds.keys():
# Each top level has a key 'builds' which is a list of dicts
green_prefix("Looping over builds in tag: ")
click.echo("{} with {} builds".format(tag, len(builds[tag]['builds'])))
for build in builds[tag]['builds']:
# Each dict has a top level key which might be the actual
# 'nvr' but I don't have enough data to know for sure
# yet. Also I don't know when there might be more than one
# key in the build dict. We'll loop over it to be sure.
for name in build.keys():
n, v, r = name.rsplit('-', 2)
version_release = "{}-{}".format(v, r)
all_advisory_nvrs[n] = version_release
click.echo("Found {} builds".format(len(all_advisory_nvrs)))
all_payload_nvrs = {}
click.echo("Fetching release info")
release_export_cmd = 'oc adm release info {} -o json'.format(payload)
rc, stdout, stderr = exectools.cmd_gather(release_export_cmd)
if rc != 0:
# Probably no point in continuing.. can't contact brew?
print("Unable to run oc release info: out={} ; err={}".format(
stdout, stderr))
exit(1)
else:
click.echo("Got release info")
payload_json = json.loads(stdout)
green_prefix("Looping over payload images: ")
click.echo("{} images to check".format(
len(payload_json['references']['spec']['tags'])))
cmds = [['oc', 'image', 'info', '-o', 'json', tag['from']['name']]
for tag in payload_json['references']['spec']['tags']]
green_prefix("Querying image infos...")
cmd_results = await asyncio.gather(
*[exectools.cmd_gather_async(cmd) for cmd in cmds])
for image, cmd, cmd_result in zip(
payload_json['references']['spec']['tags'], cmds, cmd_results):
click.echo("----")
image_name = image['name']
rc, stdout, stderr = cmd_result
if rc != 0:
# Probably no point in continuing.. can't contact brew?
red_prefix("Unable to run oc image info: ")
red_print(f"cmd={cmd!r}, out={stdout} ; err={stderr}")
exit(1)
image_info = json.loads(stdout)
labels = image_info['config']['config']['Labels']
# The machine-os-content image doesn't follow the standard
# pattern. We need to skip that image when we find it, it is
# not attached to advisories.
if 'com.coreos.ostree-commit' in labels:
yellow_prefix("Skipping machine-os-content image: ")
click.echo("Not required for checks")
continue
component = labels['com.redhat.component']
n = image_name
click.echo("Payload name: {}".format(n))
click.echo("Brew name: {}".format(component))
if labels:
v = labels['version']
r = labels['release']
all_payload_nvrs[component] = "{}-{}".format(v, r)
else:
print("For image {} Labels doesn't exist, image_info: {}".format(
image_name, image_info))
missing_in_errata = {}
payload_doesnt_match_errata = {}
in_other_advisories = {}
output = {
'missing_in_advisory': missing_in_errata,
'payload_advisory_mismatch': payload_doesnt_match_errata,
"in_other_advisories": in_other_advisories,
}
green_prefix("Analyzing data: ")
click.echo("{} images to consider from payload".format(
len(all_payload_nvrs)))
for image, vr in all_payload_nvrs.items():
yellow_prefix("Cross-checking from payload: ")
click.echo("{}-{}".format(image, vr))
if image not in all_advisory_nvrs:
missing_in_errata[image] = "{}-{}".format(image, vr)
click.echo("{} in payload not found in advisory".format(
"{}-{}".format(image, vr)))
elif image in all_advisory_nvrs and vr != all_advisory_nvrs[image]:
click.echo(
"{} from payload has version {} which does not match {} from advisory"
.format(image, vr, all_advisory_nvrs[image]))
payload_doesnt_match_errata[image] = {
'payload': vr,
'errata': all_advisory_nvrs[image]
}
if missing_in_errata:
green_print(
f"Checking if {len(missing_in_errata)} missing images are shipped..."
)
nvrs = list(missing_in_errata.values())
tag_lists = elliottlib.brew.get_builds_tags(nvrs)
for nvr, tags in zip(nvrs, tag_lists):
name = nvr.rsplit("-", 2)[0]
if any(map(lambda tag: tag["name"].endswith('-released'), tags)):
green_print(f"Build {nvr} is shipped. Skipping...")
del missing_in_errata[name]
elif any(map(lambda tag: tag["name"].endswith('-pending'), tags)):
green_print(f"Build {nvr} is in another advisory.")
del missing_in_errata[name]
in_other_advisories[name] = nvr
green_print("Summary results:")
click.echo(json.dumps(output, indent=4))
with open('summary_results.json', 'w') as fp:
json.dump(output, fp, indent=4)
green_prefix("Wrote out summary results: ")
click.echo("summary_results.json")