def commit_and_push_bundle(self, commit_msg):
"""Try to commit and push bundle distgit repository if there were any content changes.
:param string commit_msg: Commit message
:return bool True if new changes were committed and pushed, False otherwise
"""
with pushd.Dir(self.bundle_clone_path):
exectools.cmd_assert(["git", "add", "-A"])
rc, _, _ = exectools.cmd_gather(
["git", "diff-index", "--quiet", "HEAD"])
if rc == 0:
self.runtime.logger.warning("Nothing new to commit.")
return False
exectools.cmd_assert('rhpkg {} commit -m "{}"'.format(
self.rhpkg_opts, commit_msg))
_, is_shallow, _ = exectools.cmd_gather(
["git", "rev-parse", "--is-shallow-repository"])
if is_shallow.strip() == "true":
exectools.cmd_assert(["git", "fetch", "--unshallow"],
retries=3)
cmd = f'rhpkg {self.rhpkg_opts} push'
if not self.dry_run:
exectools.cmd_assert(cmd)
else:
self.runtime.logger.warning("[DRY RUN] Would have run %s", cmd)
return True
def fetch_image_sha(self, image, arch):
"""Use skopeo to obtain the SHA of a given image
We want the image manifest shasum because internal registry/cri-o can't handle manifest lists yet.
More info: http://post-office.corp.redhat.com/archives/aos-team-art/2019-October/msg02010.html
:param string image: Image name + version (format: openshift/my-image:v4.1.16-201901010000)
:param string arch: Same image has different SHAs per architecture
:return string Digest (format: sha256:a1b2c3d4...)
"""
registry = self.runtime.group_config.urls.brew_image_host.rstrip("/")
ns = self.runtime.group_config.urls.brew_image_namespace
if ns:
image = "{}/{}".format(ns, image.replace('/', '-'))
if arch == 'manifest-list':
cmd = 'skopeo inspect docker://{}/{}'.format(registry, image)
rc, out, err = exectools.retry(
retries=3, task_f=lambda *_: exectools.cmd_gather(cmd))
return json.loads(out)['Digest']
cmd = 'skopeo inspect --raw docker://{}/{}'.format(registry, image)
rc, out, err = exectools.retry(
retries=3, task_f=lambda *_: exectools.cmd_gather(cmd))
arch = 'amd64' if arch == 'x86_64' else arch # x86_64 is called amd64 in skopeo
def select_arch(manifests):
return manifests['platform']['architecture'] == arch
return filter(select_arch, json.loads(out)['manifests'])[0]['digest']
def get_latest_bundle_build(self):
"""Get NVR of latest bundle build tagged on given target
:return string: NVR of latest bundle build, or "" if there is none.
"""
_rc, out, _err = exectools.cmd_gather(
'brew latest-build --quiet {} {}'.format(
self.target, self.bundle_brew_component))
return out.split(' ')[0]
def get_all_builds(self):
"""Ask brew for all releases of a package"""
cmd = 'brew list-tagged --quiet {} {}'.format(self.brew_tag, self.metadata_component)
_rc, stdout, _stderr = exectools.cmd_gather(cmd)
for line in stdout.splitlines():
yield line.split(' ')[0]
def images_streams_mirror(runtime, streams, only_if_missing, live_test_mode,
dry_run):
runtime.initialize(clone_distgits=False, clone_source=False)
runtime.assert_mutation_is_permitted()
if streams:
user_specified = True
else:
user_specified = False
streams = runtime.get_stream_names()
streams_config = runtime.streams
for stream in streams:
if streams_config[stream] is Missing:
raise IOError(
f'Did not find stream {stream} in streams.yml for this group')
config = streams_config[stream]
if config.mirror is True or user_specified:
upstream_dest = config.upstream_image
if upstream_dest is Missing:
raise IOError(
f'Unable to mirror stream {stream} as upstream_image is not defined'
)
# If the configuration specifies a upstream_image_base, then ART is responsible for mirroring
# that location and NOT the upstream_image. DPTP will take the upstream_image_base and
# formulate the upstream_image.
if config.upstream_image_base is not Missing:
upstream_dest = config.upstream_image_base
brew_image = config.image
brew_pullspec = runtime.resolve_brew_image_url(brew_image)
if only_if_missing:
check_cmd = f'oc image info {upstream_dest}'
rc, check_out, check_err = exectools.cmd_gather(check_cmd)
if 'does not exist' not in check_err: # should be 'error: image does not exist or you don't have permission to access the repository'
print(
f'Image {upstream_dest} seems to exist already; skipping because of --only-if-missing'
)
continue
if live_test_mode:
upstream_dest += '.test'
cmd = f'oc image mirror {brew_pullspec} {upstream_dest}'
if runtime.registry_config_dir is not None:
cmd += f" --registry-config={get_docker_config_json(runtime.registry_config_dir)}"
if dry_run:
print(f'For {stream}, would have run: {cmd}')
else:
exectools.cmd_assert(cmd, retries=3, realtime=True)
def images_streams_check_upstream(runtime, live_test_mode):
runtime.initialize(clone_distgits=False, clone_source=False)
streams = runtime.get_stream_names()
streams_config = runtime.streams
istags_status = []
for stream in streams:
config = streams_config[stream]
if not (config.transform or config.mirror):
continue
upstream_dest = config.upstream_image
_, dest_ns, dest_istag = upstream_dest.rsplit('/', maxsplit=2)
if live_test_mode:
dest_istag += '.test'
rc, stdout, stderr = exectools.cmd_gather(
f'oc get -n {dest_ns} istag {dest_istag} --no-headers')
if rc:
istags_status.append(
f'ERROR: {stream}\nIs not yet represented upstream in {dest_ns} istag/{dest_istag}'
)
else:
istags_status.append(
f'OK: {stream} exists, but check whether it is recently updated\n{stdout}'
)
group_label = runtime.group_config.name
if live_test_mode:
group_label += '.test'
bc_stdout, bc_stderr = exectools.cmd_assert(
f'oc -n ci get -o=wide buildconfigs -l art-builder-group={group_label}'
)
builds_stdout, builds_stderr = exectools.cmd_assert(
f'oc -n ci get -o=wide builds -l art-builder-group={group_label}')
ds_stdout, ds_stderr = exectools.cmd_assert(
f'oc -n ci get ds -l art-builder-group={group_label}')
print(
'Daemonset status (pins image to prevent gc on node; verify that READY=CURRENT):'
)
print(ds_stdout or ds_stderr)
print('Build configs:')
print(bc_stdout or bc_stderr)
print('Recent builds:')
print(builds_stdout or builds_stderr)
print('Upstream imagestream tag status')
for istag_status in istags_status:
print(istag_status)
print()
def commit_and_push_bundle(self, commit_msg):
"""Try to commit and push bundle distgit repository if there were any content changes.
:param string commit_msg: Commit message
:return bool True if new changes were committed and pushed, False otherwise
"""
with pushd.Dir(self.bundle_clone_path):
try:
exectools.cmd_assert('git add .')
exectools.cmd_assert('rhpkg{}commit -m "{}"'.format(
self.rhpkg_opts, commit_msg))
rc, out, err = exectools.cmd_gather('rhpkg{}push'.format(
self.rhpkg_opts))
return True
except Exception:
return False # Bundle repository might be already up-to-date, nothing new to commit
def is_commit_in_public_upstream(revision: str, public_upstream_branch: str, source_dir: str):
"""
Determine if the public upstream branch includes the specified commit.
:param revision: Git commit hash or reference
:param public_upstream_branch: Git branch of the public upstream source
:param source_dir: Path to the local Git repository
"""
cmd = ["git", "merge-base", "--is-ancestor", "--", revision, "public_upstream/" + public_upstream_branch]
# The command exits with status 0 if true, or with status 1 if not. Errors are signaled by a non-zero status that is not 1.
# https://git-scm.com/docs/git-merge-base#Documentation/git-merge-base.txt---is-ancestor
rc, out, err = exectools.cmd_gather(cmd)
if rc == 0:
return True
if rc == 1:
return False
raise IOError(f"Couldn't determine if the commit {revision} is in the public upstream source repo. `git merge-base` exited with {rc}, stdout={out}, stderr={err}")
def build_metadata_container(self):
"""Build the metadata container using rhpkg
:return: bool True if build succeeded, False otherwise
:raise: Exception if command failed (rc != 0)
"""
with pushd.Dir('{}/{}'.format(self.working_dir, self.metadata_repo)):
cmd = 'timeout 600 rhpkg {} {}container-build --nowait --target {}'.format(
self.runtime.rhpkg_config, ('--user {} '.format(
self.rhpkg_user) if self.rhpkg_user else ''), self.target)
rc, stdout, stderr = exectools.cmd_gather(cmd)
if rc != 0:
raise Exception('{} failed! rc={} stdout={} stderr={}'.format(
cmd, rc, stdout.strip(), stderr.strip()))
return self.watch_brew_task(
self.extract_brew_task_id(stdout.strip())) is None
def trigger_bundle_container_build(self):
"""Ask brew for a container-build of operator's bundle
:return bool True if brew task was successfully created, False otherwise
"""
with pushd.Dir(self.bundle_clone_path):
rc, out, err = exectools.cmd_gather(
'rhpkg{}container-build --nowait --target {}'.format(
self.rhpkg_opts, self.target))
if rc != 0:
msg = 'Unable to create brew task: rc={} out={} err={}'.format(
rc, out, err)
self.runtime.logger.info(msg)
return False
self.task_url = re.search(r'Task info:\s(.+)', out).group(1)
self.task_id = re.search(r'Created task:\s(\d+)', out).group(1)
return True
def run_doozer(self, *args):
user_arg = []
if self.dz_user:
user_arg = ['--user', self.dz_user]
cmd = [
*DOOZER_CMD, *user_arg, '--cache-dir', self.dz_cache_dir,
'--working-dir', self.dz_working_dir, *args
]
print(f'Running doozer with: {cmd}', file=sys.stderr)
rc, out, err = exectools.cmd_gather(cmd, strip=True)
if rc:
self.logger.error('Doozer executed with non-zero exit status.')
self.logger.error(f'Stderror: {err}')
raise IOError('Doozer exited with error; see tests_functional.log')
return out.strip(), err.strip()
def test_gather_fail(self):
"""
"""
(status, stdout,
stderr) = exectools.cmd_gather(["/usr/bin/sed", "-e", "f"])
status_expected = 1
stdout_expected = ""
stderr_expected = "/usr/bin/sed: -e expression #1, char 1: unknown command: `f'\n"
self.assertEqual(status_expected, status)
self.assertEqual(stdout, stdout_expected)
self.assertEqual(stderr, stderr_expected)
# check that the log file has all of the tests.
log_file = open(self.test_file, 'r')
lines = log_file.readlines()
self.assertEqual(len(lines), 6)
def test_gather_success(self):
"""
"""
(status, stdout,
stderr) = exectools.cmd_gather("/usr/bin/echo hello there")
status_expected = 0
stdout_expected = "hello there\n"
stderr_expected = ""
self.assertEqual(status_expected, status)
self.assertEqual(stdout, stdout_expected)
self.assertEqual(stderr, stderr_expected)
# check that the log file has all of the tests.
log_file = open(self.test_file, 'r')
lines = log_file.readlines()
self.assertEqual(len(lines), 6)
def trigger_bundle_container_build(self) -> Tuple[Optional[int], Optional[str]]:
"""Ask brew for a container-build of operator's bundle
:return: (task_id, task_url) if brew task was successfully created, (None, None) otherwise
"""
if self.dry_run:
self.runtime.logger.warning("[DRY RUN] Would have triggered bundle build.")
return 12345, "https://brewweb.example.com/brew/taskinfo?taskID=12345"
with pushd.Dir(self.bundle_clone_path):
rc, out, err = exectools.cmd_gather(
'rhpkg {} container-build --nowait --target {}'.format(self.rhpkg_opts, self.target)
)
if rc != 0:
msg = 'Unable to create brew task: rc={} out={} err={}'.format(rc, out, err)
self.runtime.logger.warning(msg)
return False
task_url = re.search(r'Task info:\s(.+)', out).group(1)
task_id = int(re.search(r'Created task:\s(\d+)', out).group(1))
return task_id, task_url
def covscan(self, result_archive, repo_type='unsigned', local_repo=[]):
self.logger.info('Setting up for coverity scan')
all_js = 'all_results.js'
diff_js = 'diff_results.js'
all_html = 'all_results.html'
diff_html = 'diff_results.html'
waived_flag = 'waived.flag'
archive_path = pathlib.Path(result_archive)
dg_archive_path = archive_path.joinpath(self.distgit_key)
dg_archive_path.mkdir(parents=True,
exist_ok=True) # /<archive-dir>/<dg-key>
builders = self.config['from'].builder
if builders is Missing:
self.logger.info(
'No builder images -- does not appear to be container first. Skipping.'
)
return
dgr = self.distgit_repo()
with Dir(dgr.distgit_dir):
dg_commit_hash, _ = exectools.cmd_assert('git rev-parse HEAD',
strip=True)
archive_commit_results_path = dg_archive_path.joinpath(
dg_commit_hash) # /<archive-dir>/<dg-key>/<hash>
archive_all_results_js_path = archive_commit_results_path.joinpath(
all_js) # /<archive-dir>/<dg-key>/<hash>/all_results.js
archive_all_results_html_path = archive_commit_results_path.joinpath(
all_html) # /<archive-dir>/<dg-key>/<hash>/all_results.html
archive_diff_results_js_path = archive_commit_results_path.joinpath(
diff_js)
archive_diff_results_html_path = archive_commit_results_path.joinpath(
diff_html)
archive_waived_flag_path = archive_commit_results_path.joinpath(
waived_flag)
def write_record():
diff = json.loads(
archive_diff_results_js_path.read_text(encoding='utf-8'))
diff_count = len(diff['issues'])
if diff_count == 0:
self.logger.info('No new issues found during scan')
archive_waived_flag_path.write_text(
'') # No new differences, mark as waived
owners = ",".join(self.config.owners or [])
self.runtime.add_record(
'covscan',
distgit=self.qualified_name,
distgit_key=self.distgit_key,
commit_results_path=str(archive_commit_results_path),
all_results_js_path=str(archive_all_results_js_path),
all_results_html_path=str(archive_all_results_html_path),
diff_results_js_path=str(archive_diff_results_js_path),
diff_results_html_path=str(archive_diff_results_html_path),
diff_count=str(diff_count),
waive_path=str(archive_waived_flag_path),
waived=str(archive_waived_flag_path.exists()).lower(),
owners=owners,
image=self.config.name,
commit_hash=dg_commit_hash)
if archive_diff_results_html_path.exists():
self.logger.info(
f'This commit already has scan results ({str(archive_all_results_js_path)}; skipping scan'
)
write_record()
return
archive_commit_results_path.mkdir(parents=True, exist_ok=True)
dg_path = pathlib.Path(Dir.getcwd())
cov_path = dg_path.joinpath('cov')
dockerfile_path = dg_path.joinpath('Dockerfile')
if not dockerfile_path.exists():
self.logger.error(
'Dockerfile does not exist in distgit; not rebased yet?')
return
dfp = DockerfileParser(str(dockerfile_path))
covscan_builder_df_path = dg_path.joinpath(
'Dockerfile.covscan.builder')
with covscan_builder_df_path.open(mode='w+') as df_out:
first_parent = dfp.parent_images[0]
ns, repo_tag = first_parent.split(
'/'
) # e.g. openshift/golang-builder:latest => [openshift, golang-builder:latest]
if '@' in repo_tag:
repo, tag = repo_tag.split(
'@'
) # e.g. golang-builder@sha256:12345 => golang-builder & tag=sha256:12345
tag = '@' + tag
else:
if ':' in repo_tag:
repo, tag = repo_tag.split(
':'
) # e.g. golang-builder:latest => golang-builder & tag=latest
else:
repo = repo_tag
tag = 'latest'
tag = ':' + tag
first_parent_url = f'registry-proxy.engineering.redhat.com/rh-osbs/{ns}-{repo}{tag}'
# build a local image name we can use.
# We will build a local scanner image based on the target distgit's first parent.
# It will have coverity tools installed.
m = hashlib.md5()
m.update(first_parent.encode('utf-8'))
local_builder_tag = f'{repo}-{m.hexdigest()}'
vol_mount_arg = ''
make_image_repo_files = ''
if local_repo:
for idx, lr in enumerate(local_repo):
make_image_repo_files += f"""
# Create a repo able to pull from the local filesystem and prioritize it for speed.
RUN echo '[covscan_local_{idx}]' > /etc/yum.repos.d/covscan_local_{idx}.repo
RUN echo 'baseurl=file:///covscan_local_{idx}' >> /etc/yum.repos.d/covscan_local_{idx}.repo
RUN echo skip_if_unavailable=True >> /etc/yum.repos.d/covscan_local_{idx}.repo
RUN echo gpgcheck=0 >> /etc/yum.repos.d/covscan_local_{idx}.repo
RUN echo enabled=1 >> /etc/yum.repos.d/covscan_local_{idx}.repo
RUN echo enabled_metadata=1 >> /etc/yum.repos.d/covscan_local_{idx}.repo
RUN echo priority=1 >> /etc/yum.repos.d/covscan_local_{idx}.repo
"""
vol_mount_arg += f' -mount {lr}:/covscan_local_{idx}'
else:
make_image_repo_files = 'RUN wget https://cov01.lab.eng.brq.redhat.com/coverity/install/covscan/covscan-rhel-7.repo -O /etc/yum.repos.d/covscan.repo\n'
df_out.write(f'''FROM {first_parent_url}
LABEL DOOZER_COVSCAN_PARENT_IMAGE=true
LABEL DOOZER_COVSCAN_FIRST_PARENT={local_builder_tag}
LABEL DOOZER_COVSCAN_GROUP={self.runtime.group_config.name}
{make_image_repo_files}
RUN yum install -y wget
RUN wget {self.cgit_url(".oit/" + repo_type + ".repo")} -O /etc/yum.repos.d/oit.repo
RUN yum install -y python36
# Enable epel for csmock
RUN wget https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
RUN yum -y install epel-release-latest-7.noarch.rpm
# Certs necessary to install from covscan repos
RUN wget https://password.corp.redhat.com/RH-IT-Root-CA.crt -O /etc/pki/ca-trust/source/anchors/RH-IT-Root-CA.crt --no-check-certificate
RUN wget https://password.corp.redhat.com/legacy.crt -O /etc/pki/ca-trust/source/anchors/legacy.crt --no-check-certificate
RUN wget https://engineering.redhat.com/Eng-CA.crt -O /etc/pki/ca-trust/source/anchors/Eng-CA.crt --no-check-certificate
RUN update-ca-trust
RUN update-ca-trust enable
RUN yum install -y cov-sa csmock csmock-plugin-coverity csdiff
''')
rc, out, err = exectools.cmd_gather(
f'docker image inspect {local_builder_tag}')
if rc != 0:
rc, _, _ = exectools.cmd_gather(
f'imagebuilder {vol_mount_arg} -t {local_builder_tag} -f {str(covscan_builder_df_path)} {str(dg_path)}'
)
if rc != 0:
self.logger.error(
f'Unable to create scanner image based on builder image: {first_parent_url}'
)
# TODO: log this as a record and make sure the pipeline warns artist.
# until covscan can be installed on rhel-8, this is expected for some images.
return
# We should now have an image tagged local_builder_tag based on the original image's builder image
# but also with covscan tools installed.
covscan_df_path = dg_path.joinpath('Dockerfile.covscan')
runs = ['#!/bin/bash', 'set -o xtrace']
setup = []
first_from = False
for entry in dfp.structure:
content = entry['content']
instruction = entry['instruction'].upper()
if instruction.upper() == 'FROM':
if first_from:
break
first_from = True
if instruction == 'COPY':
setup.append(content)
if instruction == 'ENV':
setup.append(content)
if instruction == 'RUN':
runs.append(content.strip()[4:])
if instruction == 'WORKDIR':
setup.append(
content
) # Pass into setup so things like ADD work as expected
path = content.strip()[7:]
runs.append(
'mkdir -p ' + path
) # Also pass into RUN so they have the correct working dir
runs.append('cd ' + path)
run_file = '\n'.join(runs)
self.logger.info(f'Constructed run file:\n{run_file}')
build_script_name = 'doozer_covscan_runit.sh'
with dg_path.joinpath(build_script_name).open(mode='w+') as runit:
runit.write(run_file)
with covscan_df_path.open(mode='w+') as df_out:
df_out.write(f'FROM {local_builder_tag}\n')
df_out.write(
f'LABEL DOOZER_COVSCAN_GROUP={self.runtime.group_config.name}\n'
)
df_out.write(f'ADD {build_script_name} /\n')
df_out.write(f'RUN chmod +x /{build_script_name}\n')
df_out.write('\n'.join(setup) + '\n')
df_out.write('ENV PATH=/opt/coverity/bin:${PATH}\n')
run_tag = f'{local_builder_tag}.{self.image_name_short}'
rc, out, err = exectools.cmd_gather(
f'docker image inspect {run_tag}')
if rc != 0:
exectools.cmd_assert(
f'imagebuilder -t {run_tag} -f {str(covscan_df_path)} {str(dg_path)}'
)
cov_path.mkdir(exist_ok=True)
emit_path = cov_path.joinpath('emit')
if not emit_path.exists():
rc, out, err = exectools.cmd_gather(
f'docker run --hostname=covscan --rm -v {str(cov_path)}:/cov:z {run_tag} cov-build --dir=/cov /{build_script_name}'
)
if rc != 0:
self.logger.error('Did not achieve full compilation')
builg_log_path = cov_path.joinpath('build-log.txt')
build_log = builg_log_path.read_text(encoding='utf-8')
if '[WARNING] No files were emitted' in build_log:
self.logger.error(
f'Build did not emit anything. Check out the build-log.txt: {builg_log_path}'
)
# TODO: log this as a record and make sure the pipeline warns artist
return
else:
self.logger.info(
'covscan emit already exists -- skipping this step')
def run_docker_cov(cmd):
return exectools.cmd_assert(
f'docker run --hostname=covscan --rm -v {str(cov_path)}:/cov:z {run_tag} {cmd}'
)
summary_path = cov_path.joinpath('output', 'summary.txt')
if not summary_path.exists(
) or 'Time taken by analysis' not in summary_path.read_text(
encoding='utf-8'):
# This can take an extremely long time and use virtually all CPU
run_docker_cov(
'cov-analyze --dir=/cov "--wait-for-license" "-co" "ASSERT_SIDE_EFFECT:macro_name_lacks:^assert_(return|se)\\$" "-co" "BAD_FREE:allow_first_field:true" "--include-java" "--fb-max-mem=4096" "--security" "--concurrency" --allow-unmerged-emits'
)
else:
self.logger.info(
'covscan analysis already exists -- skipping this step')
all_results_js, _ = run_docker_cov(
'cov-format-errors --json-output-v2 /dev/stdout --dir=/cov')
all_results_js_path = cov_path.joinpath(all_js)
all_results_js_path.write_text(all_results_js, encoding='utf-8')
all_results_html = "<html>Error generating HTML report.</html>"
try:
# Rarely, cshtml just outputs empty html and rc==1; just ignore it.
all_results_html, _ = run_docker_cov(f'cshtml /cov/{all_js}')
except:
self.logger.warning(
f'Error generating HTML report for {str(archive_all_results_js_path)}'
)
pass
all_results_html_path = cov_path.joinpath(all_html)
all_results_html_path.write_text(all_results_html,
encoding='utf-8')
run_docker_cov(f'chown -R {os.getuid()}:{os.getgid()} /cov'
) # Otherwise, root will own these files
# Write out the files to the archive directory as well
archive_all_results_js_path.write_text(all_results_js,
encoding='utf-8')
archive_all_results_html_path.write_text(all_results_html,
encoding='utf-8')
# Now on to computing diffs
# Search backwards through commit history; try to find a hash for this distgit that has been scanned before
diff_results_js = all_results_js # Unless we find an old has, diff results matches all results
commit_log, _ = exectools.cmd_assert(
"git --no-pager log --pretty='%H' -1000")
for old_commit in commit_log.split()[1:]:
old_all_results_js_path = dg_archive_path.joinpath(
old_commit, all_js)
old_are_results_waived_path = dg_archive_path.joinpath(
old_commit, waived_flag)
# Only compute diff from commit if results were actually waived
# This file should be created by the Jenkins / scanning pipeline.
if old_are_results_waived_path.exists():
diff_results_js, _ = exectools.cmd_assert(
f'csdiff {str(archive_all_results_js_path)} {str(old_all_results_js_path)}'
)
break
archive_diff_results_js_path.write_text(diff_results_js,
encoding='utf-8')
diff_results_html = "<html>Error generating HTML report.</html>"
try:
# Rarely, cshtml just outputs empty html and rc==1; just ignore it.
diff_results_html, _ = exectools.cmd_assert(
f'cshtml {str(archive_diff_results_js_path)}')
except:
self.logger.warning(
f'Error generating HTML report for {str(archive_diff_results_js_path)}'
)
pass
archive_diff_results_html_path.write_text(diff_results_html,
encoding='utf-8')
write_record()
def get_latest_build(self):
cmd = 'brew latest-build {} {} --quiet'.format(
self.target, self.metadata_component_name)
_rc, stdout, _stderr = exectools.retry(
retries=3, task_f=lambda *_: exectools.cmd_gather(cmd))
return stdout.split(' ')[0]
def get_brew_buildinfo(self):
"""Output of this command is used to extract the operator name and its commit hash
"""
cmd = 'brew buildinfo {}'.format(self.nvr)
return exectools.retry(retries=3,
task_f=lambda *_: exectools.cmd_gather(cmd))
def _check_nightly_consistency(assembly_inspector: AssemblyInspector,
nightly: str,
arch: str) -> List[AssemblyIssue]:
runtime = assembly_inspector.runtime
def terminal_issue(msg: str) -> List[AssemblyIssue]:
return [AssemblyIssue(msg, component='reference-releases')]
issues: List[str]
runtime.logger.info(f'Processing nightly: {nightly}')
major_minor, brew_cpu_arch, priv = isolate_nightly_name_components(
nightly)
if major_minor != runtime.get_minor_version():
return terminal_issue(
f'Specified nightly {nightly} does not match group major.minor'
)
rc_suffix = go_suffix_for_arch(brew_cpu_arch, priv)
retries: int = 3
release_json_str = ''
rc = -1
pullspec = f'registry.ci.openshift.org/ocp{rc_suffix}/release{rc_suffix}:{nightly}'
while retries > 0:
rc, release_json_str, err = exectools.cmd_gather(
f'oc adm release info {pullspec} -o=json')
if rc == 0:
break
runtime.logger.warn(
f'Error accessing nightly release info for {pullspec}: {err}')
retries -= 1
if rc != 0:
return terminal_issue(
f'Unable to gather nightly release info details: {pullspec}; garbage collected?'
)
release_info = Model(dict_to_model=json.loads(release_json_str))
if not release_info.references.spec.tags:
return terminal_issue(f'Could not find tags in nightly {nightly}')
issues: List[AssemblyIssue] = list()
payload_entries: Dict[
str, PayloadGenerator.
PayloadEntry] = PayloadGenerator.find_payload_entries(
assembly_inspector, arch, '')
for component_tag in release_info.references.spec.tags: # For each tag in the imagestream
payload_tag_name: str = component_tag.name # e.g. "aws-ebs-csi-driver"
payload_tag_pullspec: str = component_tag[
'from'].name # quay pullspec
if '@' not in payload_tag_pullspec:
# This speaks to an invalid nightly, so raise and exception
raise IOError(
f'Expected pullspec in {nightly}:{payload_tag_name} to be sha digest but found invalid: {payload_tag_pullspec}'
)
pullspec_sha = payload_tag_pullspec.rsplit('@', 1)[-1]
entry = payload_entries.get(payload_tag_name, None)
if not entry:
raise IOError(
f'Did not find {nightly} payload tag {payload_tag_name} in computed assembly payload'
)
if entry.archive_inspector:
if entry.archive_inspector.get_archive_digest(
) != pullspec_sha:
# Impermissible because the artist should remove the reference nightlies from the assembly definition
issues.append(
AssemblyIssue(
f'{nightly} contains {payload_tag_name} sha {pullspec_sha} but assembly computed archive: {entry.archive_inspector.get_archive_id()} and {entry.archive_inspector.get_archive_pullspec()}',
component='reference-releases'))
elif entry.rhcos_build:
if entry.rhcos_build.get_machine_os_content_digest(
) != pullspec_sha:
# Impermissible because the artist should remove the reference nightlies from the assembly definition
issues.append(
AssemblyIssue(
f'{nightly} contains {payload_tag_name} sha {pullspec_sha} but assembly computed rhcos: {entry.rhcos_build} and {entry.rhcos_build.get_machine_os_content_digest()}',
component='reference-releases'))
else:
raise IOError(f'Unsupported payload entry {entry}')
return issues
def _covscan_prepare_parent(cc: CoverityContext, parent_image_name,
parent_tag) -> bool:
"""
Builds an image for the specified parent image and layers coverity tools on top of it.
This image is called the parent image derivative and will be used during the actual
coverity scan.
:param cc: The coverity scan context
:param parent_image_name: The name of the image as found in the distgit Dockerfile
:param parent_tag: A string with which to tag the image after building.
:return: Returns True if the image is built successfully
"""
dg_path = cc.dg_path
rc, _, _ = exectools.cmd_gather(f'{cc.podman_cmd} inspect {parent_tag}',
set_env=cc.podman_env)
if rc != 0:
cc.logger.info(
f'Creating parent image derivative with covscan tools installed as {parent_tag} for parent {parent_image_name}'
)
df_parent_path = dg_path.joinpath(f'Dockerfile.{parent_tag}')
repo_injection_lines, mount_args = cc.parent_repo_injection_info()
rhel_repo_gen_sh = '_rhel_repo_gen.sh'
with dg_path.joinpath(rhel_repo_gen_sh).open(mode='w') as f:
f.write('''
#!/bin/sh
set -o xtrace
if cat /etc/redhat-release | grep "release 8"; then
cp /tmp/oit.repo /etc/yum.repos.d/oit.repo
# For an el8 layer, make sure baseos & appstream are
# available for tools like python to install.
cat <<EOF > /etc/yum.repos.d/el8.repo
[rhel-8-appstream-rpms-x86_64]
baseurl = http://rhsm-pulp.corp.redhat.com/content/dist/rhel8/8/x86_64/appstream/os/
enabled = 1
name = rhel-8-appstream-rpms-x86_64
gpgcheck = 0
gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release
[rhel-8-baseos-rpms-x86_64]
baseurl = http://rhsm-pulp.corp.redhat.com/content/dist/rhel8/8/x86_64/baseos/os/
enabled = 1
name = rhel-8-baseos-rpms-x86_64
gpgcheck = 0
gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release
EOF
# Enable epel for csmock
curl https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm --output epel8.rpm
yum -y install epel8.rpm
else
# For rhel-7, just enable the basic rhel repos so that we
# can install python and other dependencies.
cat <<EOF > /etc/yum.repos.d/el7.repo
[rhel-server-rpms-x86_64]
baseurl = http://rhsm-pulp.corp.redhat.com/content/dist/rhel/server/7/7Server/x86_64/os/
enabled = 1
name = rhel-server-rpms-x86_64
gpgcheck = 0
gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release
[rhel-server-optional-rpms-x86_64]
baseurl = http://rhsm-pulp.corp.redhat.com/content/dist/rhel/server/7/7Server/x86_64/optional/os/
enabled = 0
name = rhel-server-optional-rpms-x86_64
gpgcheck = 0
gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release
[rhel-server-extras-rpms-x86_64]
baseurl = http://rhsm-pulp.corp.redhat.com/content/dist/rhel/server/7/7Server/x86_64/extras/os/
enabled = 0
name = rhel-server-extras-rpms-x86_64
gpgcheck = 0
gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release
EOF
# Enable epel for csmock
curl https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm --output epel7.rpm
yum -y install epel7.rpm
fi
''')
with df_parent_path.open(mode='w+', encoding='utf-8') as df_parent_out:
parent_image_url = parent_image_name
if 'redhat.registry' not in parent_image_name:
parent_image_url = cc.runtime.resolve_brew_image_url(
parent_image_name)
df_parent_out.write(f'''
FROM {parent_image_url}
LABEL DOOZER_COVSCAN_GROUP_PARENT={cc.runtime.group_config.name}
USER 0
# Add typical build repos to the image, but don't add to /etc/yum.repos.d
# until we know whether we are on el7 or el8. As of 4.8, repos are only
# appropriate for el8, so this repo file should only be installed in el8.
ADD .oit/{cc.repo_type}.repo /tmp/oit.repo
# Install covscan repos
{repo_injection_lines}
# Act on oit.repo and enable rhel repos
ADD {rhel_repo_gen_sh} .
RUN chmod +x {rhel_repo_gen_sh} && ./{rhel_repo_gen_sh}
RUN yum install -y python36
# Certs necessary to install from covscan repos
RUN curl -k https://password.corp.redhat.com/RH-IT-Root-CA.crt --output /etc/pki/ca-trust/source/anchors/RH-IT-Root-CA.crt
RUN curl -k https://engineering.redhat.com/Eng-CA.crt --output /etc/pki/ca-trust/source/anchors/Eng-CA.crt
RUN update-ca-trust
RUN update-ca-trust enable
RUN yum install -y cov-sa csmock csmock-plugin-coverity csdiff
''')
df_parent_out.write('ENV PATH=/opt/coverity/bin:${PATH}\n'
) # Ensure coverity is in the path
# This will have prepared a parent image we can use during the actual covscan Dockerfile build
rc, stdout, stderr = exectools.cmd_gather(
f'{cc.podman_cmd} build {mount_args} -t {parent_tag} -f {str(df_parent_path)} {str(dg_path)}',
set_env=cc.podman_env)
cc.logger.info(f'''Output from covscan build for {cc.image.distgit_key}
stdout: {stdout}
stderr: {stderr}
''')
if rc != 0:
cc.logger.error(
f'Error preparing builder image derivative {parent_tag} from {parent_image_name} with {str(df_parent_path)}'
)
# TODO: log this as a record and make sure the pipeline warns artist
return False
else:
cc.logger.info(
f'Parent image already exists with covscan tools {parent_tag} for {parent_image_name}'
)
return True
def run_covscan(cc: CoverityContext) -> bool:
dg_path = cc.dg_path
with Dir(dg_path):
dockerfile_path = dg_path.joinpath('Dockerfile')
if not dockerfile_path.exists():
cc.logger.error(
'Dockerfile does not exist in distgit; not rebased yet?')
return False
dfp = DockerfileParser(str(dockerfile_path))
if cc.are_results_done():
cc.logger.info(
f'Scan results already exist for {cc.dg_commit_hash}; skipping scan'
)
# Even if it is complete, write a record for Jenkins so that results can be sent to prodsec.
for i in range(len(dfp.parent_images)):
records_results(cc,
stage_number=i + 1,
waived_cov_path_root=None,
write_only=True)
return True
def compute_parent_tag(parent_image_name):
parent_sig = hashlib.md5(
parent_image_name.encode("utf-8")).hexdigest()
return f'parent-{parent_sig}'
covscan_df = dg_path.joinpath('Dockerfile.covscan')
with covscan_df.open(mode='w+') as df_out:
df_line = 0
stage_number = 0
for entry in dfp.structure:
def append_analysis(stage_number):
# We will have monitored compilation processes, but we need to scan for non-compiled code
# like python / nodejs.
# cov-capture will search for files like .js, typescript, python in the source directory;
# cov-analyze will then search for issues within those non-compiled files.
# https://community.synopsys.com/s/question/0D52H000054zcvZSAQ/i-would-like-to-know-the-coverity-static-analysis-process-for-node-js-could-you-please-provide-me-the-sample-steps-to-run-coverity-for-node-js
# Why cov-manage-emit?
# coverity requires a consistent hostname for each tool run. podman does not allow the hostname to be
# set and it varies over the course of the build. That is why we reset the hostname in emit before each
# tool run.
container_stage_cov_dir = str(
cc.container_stage_cov_path(stage_number))
analysis_script_name = f'_gen_{cc.image.image_name_short}_stage_{stage_number}_analysis.sh'
with open(dg_path.joinpath(analysis_script_name),
mode='w+',
encoding='utf-8') as sh:
sh.write(f'''
#!/bin/sh
set -o xtrace
set -eo pipefail
if [[ -f "{container_stage_cov_dir}/all_results.js" ]]; then
echo "Results have already been analyzed for this this stage -- found all_results.js; skipping analysis"
exit 0
fi
if [[ "{stage_number}" == "1" ]]; then
# hostname changes between steps in the Dockerfile; reset to current before running coverity tools.
# use || true because it is possible nothing has been emitted before this step
cov-manage-emit --dir={container_stage_cov_dir} reset-host-name || true
echo "Running un-compiled source search as hostname: $(hostname)"
timeout 3h cov-capture --dir {container_stage_cov_dir} --source-dir /covscan-src || echo "Error running source detection"
fi
if ls {container_stage_cov_dir}/emit/*/config; then
echo "Running analysis phase as hostname: $(hostname)"
# hostname changes between steps in the Dockerfile; reset to current before running coverity tools.
cov-manage-emit --dir={container_stage_cov_dir} reset-host-name || true
if timeout 3h cov-analyze --dir={container_stage_cov_dir} "--wait-for-license" "-co" "ASSERT_SIDE_EFFECT:macro_name_lacks:^assert_(return|se)\\$" "-co" "BAD_FREE:allow_first_field:true" "--include-java" "--fb-max-mem=4096" "--all" "--security" "--concurrency" --allow-unmerged-emits > /tmp/analysis.txt 2>&1 ; then
echo "Analysis completed successfully"
cat /tmp/analysis.txt
else
# In some cases, no translation units were emitted and analyze will exit with an error; ignore that error
# if it is because nothing was emitted.
cat /tmp/analysis.txt
if cat /tmp/analysis.txt | grep "contains no translation units"; then
echo "Nothing was emitted; ignoring analyze failure."
exit 0
else
echo "Analysis failed for unknown reason!"
exit 1
fi
fi
cov-format-errors --json-output-v2 /dev/stdout --dir={container_stage_cov_dir} > {container_stage_cov_dir}/{COVSCAN_ALL_JS_FILENAME}
else
echo "No units have been emitted for analysis by this stage; skipping analysis"
fi
''')
df_out.write(f'''
ADD {analysis_script_name} /
RUN chmod +x /{analysis_script_name}
# Finally, run the analysis step script.
# Route stderr to stdout so everything is in one stream; otherwise, it is hard to correlate a command with its stderr.
RUN /{analysis_script_name} 2>&1
''')
# Before running cov-analyze, make sure that all_js doesn't exist (i.e. we haven't already run it
# in this workspace AND summary.txt exist (i.e. at least one item in this stage emitted results).
df_out.write(f'''
# Dockerfile steps run as root; chang permissions back to doozer user before leaving stage
RUN chown -R {os.getuid()}:{os.getgid()} {container_stage_cov_dir}
''')
df_line += 1
content = entry['content']
instruction = entry['instruction'].upper()
if instruction == 'USER':
# Stay as root
continue
if instruction == 'FROM':
stage_number += 1
if stage_number > 1:
# We are about to transition stages, do the analysis first.
append_analysis(stage_number - 1)
image_name_components = content.split(
) # [ 'FROM', image-name, (possible 'AS', ...) ]
image_name = image_name_components[1]
parent_tag = compute_parent_tag(image_name)
if not _covscan_prepare_parent(cc, image_name, parent_tag):
return False
image_name_components[1] = parent_tag
df_out.write(' '.join(image_name_components) + '\n')
# Label these images so we can find a delete them later
df_out.write(
f'LABEL DOOZER_COVSCAN_RUNNER={cc.runtime.group_config.name}\n'
)
df_out.write(
f'LABEL DOOZER_COVSCAN_COMPONENT={cc.image.distgit_key}\n'
)
df_out.write(
'ENTRYPOINT []\n'
) # Ensure all invocations use /bin/sh -c, the default docker entrypoint
df_out.write(
'USER 0\n') # Just make sure all images are consistent
# Each stage will have its own cov output directory
df_out.write(f'''
RUN mkdir -p {cc.container_stage_cov_path(stage_number)}
# If we are reusing a workspace, coverity cannot pick up where it left off; clear anything already emitted
RUN rm -rf {cc.container_stage_cov_path(stage_number)}/emit
''')
# For each new stage, we also need to make sure we have the appropriate repos enabled for this image
df_out.write(f'''
# Ensure that the build process can access the same RPMs that the build can during a brew build
RUN curl {cc.image.cgit_file_url(".oit/" + cc.repo_type + ".repo")} --output /etc/yum.repos.d/oit.repo 2>&1
''')
continue
if instruction in ('ENTRYPOINT', 'CMD'):
df_out.write(f'# Disabling: {content}')
continue
if instruction == 'RUN':
container_stage_cov_dir = str(
cc.container_stage_cov_path(stage_number))
# For RUN commands, we need to execute the command under the watchful eye of coverity
# tools. Create a batch file that will wrap the command
command_to_run = content.strip()[4:] # Remove 'RUN '
temp_script_name = f'_gen_{cc.image.image_name_short}_stage_{stage_number}_line_{df_line}.sh'
with open(dg_path.joinpath(temp_script_name),
mode='w+',
encoding='utf-8') as sh:
sh.write(f'''
#!/bin/sh
set -o xtrace
set -eo pipefail
echo "Running build as hostname: $(hostname)"
{command_to_run}
''')
df_out.write(f'''
ADD {temp_script_name} .
RUN chmod +x {temp_script_name}
# Finally, run the script while coverity is watching. If there is already a summary file, assume we have already run
# the build in this working directory.
# The hostname changes with each run, so reset-host-name before cov-build.
# Route stderr to stdout so everything is in one stream; otherwise, it is hard to tell which command failed.
RUN cov-manage-emit --dir={container_stage_cov_dir} reset-host-name; timeout 3h cov-build --dir={container_stage_cov_dir} ./{temp_script_name} 2>&1
''')
else: # e.g. COPY, ENV, WORKDIR...
# Just pass it straight through to the covscan Dockerfile
df_out.write(f'{content}\n')
append_analysis(stage_number)
# The dockerfile which will run the coverity builds and analysis for each stage has been created.
# Now, run the build (and execute those steps). The output will be to <cov_path>/<stage_number>
run_tag = f'{cc.image.image_name_short}_{cc.runtime.group_config.name}'
rc, stdout, stderr = exectools.cmd_gather(
f'{cc.podman_cmd} build -v {str(cc.cov_root_path)}:/cov:z -v {str(dg_path)}:/covscan-src:z -t {run_tag} -f {str(covscan_df)} {str(dg_path)}',
set_env=cc.podman_env)
cc.logger.info(f'''Output from covscan build for {cc.image.distgit_key}
stdout: {stdout}
stderr: {stderr}
''')
_, cleanup_out, cleanup_err = exectools.cmd_gather(
f'{cc.podman_cmd} rmi -f {run_tag}', set_env=cc.podman_env)
cc.logger.info(f'''Output from image clean up {cc.image.distgit_key}
stdout: {cleanup_out}
stderr: {cleanup_err}
''')
if rc != 0:
cc.logger.error(
f'Error running covscan build for {cc.image.distgit_key} ({str(covscan_df)})'
)
# TODO: log this as a record and make sure the pipeline warns artist
return False
# For each stage, let's now compute diffs & store results
waived_cov_path_root = cc.find_nearest_waived_cov_root_path()
for i in range(len(dfp.parent_images)):
records_results(cc,
stage_number=i + 1,
waived_cov_path_root=waived_cov_path_root)
return True
def records_results(cc: CoverityContext,
stage_number,
waived_cov_path_root=None,
write_only=False):
dest_result_path = cc.get_stage_results_path(stage_number)
dest_all_js_path = dest_result_path.joinpath(COVSCAN_ALL_JS_FILENAME)
dest_diff_js_path = dest_result_path.joinpath(COVSCAN_DIFF_JS_FILENAME)
dest_all_results_html_path = dest_result_path.joinpath(
COVSCAN_ALL_HTML_FILENAME)
dest_diff_results_html_path = dest_result_path.joinpath(
COVSCAN_DIFF_HTML_FILENAME)
def write_record():
owners = ",".join(cc.image.config.owners or [])
all_json = json.loads(dest_all_js_path.read_text(encoding='utf-8'))
diff_json = json.loads(dest_diff_js_path.read_text(encoding='utf-8'))
diff_count = len(diff_json.get('issues', []))
all_count = len(all_json.get('issues', []))
host_stage_waived_flag_path = cc.get_stage_results_waive_path(
stage_number)
cc.image.runtime.add_record(
'covscan',
distgit=cc.image.qualified_name,
distgit_key=cc.image.distgit_key,
commit_results_path=str(dest_result_path),
all_results_js_path=str(dest_all_js_path),
all_results_html_path=str(dest_all_results_html_path),
diff_results_js_path=str(dest_diff_js_path),
diff_results_html_path=str(dest_diff_results_html_path),
diff_count=str(diff_count),
all_count=str(all_count),
stage_number=str(stage_number),
waive_path=str(host_stage_waived_flag_path),
waived=str(host_stage_waived_flag_path.exists()).lower(),
owners=owners,
image=cc.image.config.name,
commit_hash=cc.dg_commit_hash)
if write_only:
if dest_all_js_path.exists():
# Results are already computed and in results; just write the record
write_record()
else:
# This stage was analyzed previously, but had no results (i.e. it did not build code).
pass
return
host_stage_cov_path = cc.host_stage_cov_path(stage_number)
source_all_js_path: pathlib.Path = host_stage_cov_path.joinpath(
COVSCAN_ALL_JS_FILENAME)
if not source_all_js_path.exists():
# No results for this stage; nothing to report
return
source_all_js = source_all_js_path.read_text(encoding='utf-8')
if not source_all_js.strip():
return
dest_all_js_path.write_text(source_all_js, encoding='utf-8')
host_stage_output_path = host_stage_cov_path.joinpath('output')
source_summary_path = host_stage_output_path.joinpath('summary.txt')
if source_summary_path.exists():
dest_summary_path = dest_result_path.joinpath('summary.txt')
# The only reason there would not be a summary.txt is if we are testing with fake data.
# If we find it, copy it into the results directory.
dest_summary_path.write_text(
source_summary_path.read_text(encoding='utf-8'), encoding='utf-8')
source_buildlog_path = host_stage_cov_path.joinpath('build-log.txt')
if source_buildlog_path.exists():
dest_buildlog_path = dest_result_path.joinpath('build-log.txt')
# The only reason there would not be a build-log.txt is if we are testing with fake data.
# If we find it, copy it into the results directory.
dest_buildlog_path.write_text(
source_buildlog_path.read_text(encoding='utf-8'), encoding='utf-8')
dest_diff_js_path = dest_result_path.joinpath(COVSCAN_DIFF_JS_FILENAME)
waived_stage_cov_path = cc.get_nearest_waived_cov_path(
waived_cov_path_root, stage_number)
if not waived_stage_cov_path or not waived_stage_cov_path.joinpath(
COVSCAN_ALL_JS_FILENAME).exists():
# No previous commit results to compare against; diff will be same as all
dest_diff_js_path.write_text(source_all_js, encoding='utf-8')
else:
waived_all_results_js_path = waived_stage_cov_path.joinpath(
COVSCAN_ALL_JS_FILENAME)
diff_results_js, _ = exectools.cmd_assert(
f'csdiff {str(source_all_js_path)} {str(waived_all_results_js_path)}'
)
dest_diff_js_path.write_text(diff_results_js, encoding='utf-8')
for entry in ((source_all_js_path, dest_all_results_html_path),
(dest_diff_js_path, dest_diff_results_html_path)):
js_path, html_out_path = entry
rc, html, stderr = exectools.cmd_gather(f'cshtml {str(js_path)}')
if rc != 0:
# Rarely, cshtml just outputs empty html and rc==1; just ignore it.
html = "<html>Error generating HTML report.</html>"
cc.logger.warning(
f'Error generating HTML report for {str(js_path)}: {stderr}')
pass
html_out_path.write_text(html, encoding='utf-8')
write_record()
# The output directory may be multiple gigabytes for each phase. Remove it
# to reduce workspace size during runtime.
shutil.rmtree(str(host_stage_output_path), ignore_errors=True)
