def copy_operator_package_yaml_to_metadata(self): exectools.cmd_assert('cp {} {}/{}/{}'.format( self.operator_package_yaml_filename, self.working_dir, self.metadata_repo, self.metadata_manifests_dir ))
def remove_metadata_channel_dir(self): exectools.cmd_assert('rm -rf {}/{}/{}/{}'.format( self.working_dir, self.metadata_repo, self.metadata_manifests_dir, self.channel ))
def fetch_image_sha(self, image, arch): """Use skopeo to obtain the SHA of a given image We want the image manifest shasum because internal registry/cri-o can't handle manifest lists yet. More info: http://post-office.corp.redhat.com/archives/aos-team-art/2019-October/msg02010.html :param string image: Image name + version (format: openshift/my-image:v4.1.16-201901010000) :param string arch: Same image has different SHAs per architecture :return string Digest (format: sha256:a1b2c3d4...) """ registry = self.runtime.group_config.urls.brew_image_host.rstrip("/") ns = self.runtime.group_config.urls.brew_image_namespace if ns: image = "{}/{}".format(ns, image.replace('/', '-')) if arch == 'manifest-list': cmd = 'skopeo inspect docker://{}/{}'.format(registry, image) out, err = exectools.cmd_assert(cmd, retries=3) return json.loads(out)['Digest'] cmd = 'skopeo inspect --raw docker://{}/{}'.format(registry, image) out, err = exectools.cmd_assert(cmd, retries=3) arch = 'amd64' if arch == 'x86_64' else arch # x86_64 is called amd64 in skopeo def select_arch(manifests): return manifests['platform']['architecture'] == arch return list(filter(select_arch, json.loads(out)['manifests']))[0]['digest']
def act(self, *args, **kwargs): """ Run the command :param context: A context dict. `context.set_env` is a `dict` of env vars to set for command (overriding existing). """ context = kwargs["context"] set_env = context["set_env"] cmd_assert(self.command, set_env=set_env)
def checkout_repo(self, repo, commit_hash): """Checkout a repository to a particular commit hash :param string repo: The repository in which the checkout operation will be performed :param string commit_hash: The desired point to checkout the repository """ with pushd.Dir('{}/{}'.format(self.working_dir, repo)): exectools.cmd_assert('git checkout {}'.format(commit_hash))
def clean_bundle_contents(self): """Delete all files currently present in the bundle repository Generating bundle files is an idempotent operation, so it is much easier to clean up everything and re-create them instead of parsing and figuring out what changed At the end, only relevant diff, if any, will be committed. """ exectools.cmd_assert(["git", "-C", self.bundle_clone_path, "rm", "--ignore-unmatch", "-rf", "."])
def generate_bundle_annotations(self): """Create an annotations YAML file for the bundle, using info extracted from operator's package YAML """ annotations_file = '{}/metadata/annotations.yaml'.format(self.bundle_clone_path) exectools.cmd_assert('mkdir -p {}'.format(os.path.dirname(annotations_file))) with io.open(annotations_file, 'w', encoding='utf-8') as writer: writer.write(yaml.dump({'annotations': self.operator_framework_tags}))
def copy_channel_manifests_from_operator_to_metadata(self): exectools.cmd_assert('cp -r {}/{}/{}/{} {}/{}/{}'.format( self.working_dir, self.operator_name, self.operator_manifests_dir, self.channel, self.working_dir, self.metadata_repo, self.metadata_manifests_dir ))
def delete_metadata_arch_manifests_dir(self, arch): """Delete previous arch-specific manifests, should they exist. """ exectools.cmd_assert('rm -rf {}/{}/{}/{}-{}'.format( self.working_dir, self.metadata_repo, self.metadata_manifests_dir, self.channel, arch ))
def images_streams_mirror(runtime, streams, only_if_missing, live_test_mode, dry_run): runtime.initialize(clone_distgits=False, clone_source=False) runtime.assert_mutation_is_permitted() if streams: user_specified = True else: user_specified = False streams = runtime.get_stream_names() streams_config = runtime.streams for stream in streams: if streams_config[stream] is Missing: raise IOError( f'Did not find stream {stream} in streams.yml for this group') config = streams_config[stream] if config.mirror is True or user_specified: upstream_dest = config.upstream_image if upstream_dest is Missing: raise IOError( f'Unable to mirror stream {stream} as upstream_image is not defined' ) # If the configuration specifies a upstream_image_base, then ART is responsible for mirroring # that location and NOT the upstream_image. DPTP will take the upstream_image_base and # formulate the upstream_image. if config.upstream_image_base is not Missing: upstream_dest = config.upstream_image_base brew_image = config.image brew_pullspec = runtime.resolve_brew_image_url(brew_image) if only_if_missing: check_cmd = f'oc image info {upstream_dest}' rc, check_out, check_err = exectools.cmd_gather(check_cmd) if 'does not exist' not in check_err: # should be 'error: image does not exist or you don't have permission to access the repository' print( f'Image {upstream_dest} seems to exist already; skipping because of --only-if-missing' ) continue if live_test_mode: upstream_dest += '.test' cmd = f'oc image mirror {brew_pullspec} {upstream_dest}' if runtime.registry_config_dir is not None: cmd += f" --registry-config={get_docker_config_json(runtime.registry_config_dir)}" if dry_run: print(f'For {stream}, would have run: {cmd}') else: exectools.cmd_assert(cmd, retries=3, realtime=True)
def images_streams_check_upstream(runtime, live_test_mode): runtime.initialize(clone_distgits=False, clone_source=False) streams = runtime.get_stream_names() streams_config = runtime.streams istags_status = [] for stream in streams: config = streams_config[stream] if not (config.transform or config.mirror): continue upstream_dest = config.upstream_image _, dest_ns, dest_istag = upstream_dest.rsplit('/', maxsplit=2) if live_test_mode: dest_istag += '.test' rc, stdout, stderr = exectools.cmd_gather( f'oc get -n {dest_ns} istag {dest_istag} --no-headers') if rc: istags_status.append( f'ERROR: {stream}\nIs not yet represented upstream in {dest_ns} istag/{dest_istag}' ) else: istags_status.append( f'OK: {stream} exists, but check whether it is recently updated\n{stdout}' ) group_label = runtime.group_config.name if live_test_mode: group_label += '.test' bc_stdout, bc_stderr = exectools.cmd_assert( f'oc -n ci get -o=wide buildconfigs -l art-builder-group={group_label}' ) builds_stdout, builds_stderr = exectools.cmd_assert( f'oc -n ci get -o=wide builds -l art-builder-group={group_label}') ds_stdout, ds_stderr = exectools.cmd_assert( f'oc -n ci get ds -l art-builder-group={group_label}') print( 'Daemonset status (pins image to prevent gc on node; verify that READY=CURRENT):' ) print(ds_stdout or ds_stderr) print('Build configs:') print(bc_stdout or bc_stderr) print('Recent builds:') print(builds_stdout or builds_stderr) print('Upstream imagestream tag status') for istag_status in istags_status: print(istag_status) print()
def create_manifests_copy_for_arch(self, arch): """Copy current channel manifests to <current channel>-<arch> Example: cp /path/to/manifests/4.2 /path/to/manifests/4.2-s390x """ exectools.cmd_assert('cp -r {}/{}/{}/{} {}/{}/{}/{}-{}'.format( self.working_dir, self.operator_name, self.operator_manifests_dir, self.channel, self.working_dir, self.metadata_repo, self.metadata_manifests_dir, self.channel, arch)) filename = glob.glob( '{}/{}/{}/{}-{}/*.clusterserviceversion.yaml'.format( self.working_dir, self.metadata_repo, self.metadata_manifests_dir, self.channel, arch))[0] self.change_arch_csv_metadata_name(filename, arch)
def clone_bundle(self): """Clone corresponding bundle distgit repository of given operator NVR """ dg_dir = Path(self.bundle_clone_path) if dg_dir.exists(): self.runtime.logger.info( "Distgit directory already exists; skipping clone: %s", dg_dir) if self.runtime.upcycle: self.runtime.logger.warning( "Refreshing source for '%s' due to --upcycle", dg_dir) exectools.cmd_assert( ["git", "-C", str(dg_dir), "clean", "-fdx"]) exectools.cmd_assert([ "git", "-C", str(dg_dir), "fetch", "--depth", "1", "origin", self.branch ], retries=3) exectools.cmd_assert([ "git", "-C", str(dg_dir), "checkout", "-B", self.branch, "--track", f"origin/{self.branch}", "--force" ]) return dg_dir.parent.mkdir(parents=True, exist_ok=True) exectools.cmd_assert( 'rhpkg {} clone --depth 1 --branch {} {} {}'.format( self.rhpkg_opts, self.branch, self.bundle_repo_name, self.bundle_clone_path), retries=3)
def commit_and_push_bundle(self, commit_msg): """Try to commit and push bundle distgit repository if there were any content changes. :param string commit_msg: Commit message :return bool True if new changes were committed and pushed, False otherwise """ with pushd.Dir(self.bundle_clone_path): exectools.cmd_assert(["git", "add", "-A"]) rc, _, _ = exectools.cmd_gather( ["git", "diff-index", "--quiet", "HEAD"]) if rc == 0: self.runtime.logger.warning("Nothing new to commit.") return False exectools.cmd_assert('rhpkg {} commit -m "{}"'.format( self.rhpkg_opts, commit_msg)) _, is_shallow, _ = exectools.cmd_gather( ["git", "rev-parse", "--is-shallow-repository"]) if is_shallow.strip() == "true": exectools.cmd_assert(["git", "fetch", "--unshallow"], retries=3) cmd = f'rhpkg {self.rhpkg_opts} push' if not self.dry_run: exectools.cmd_assert(cmd) else: self.runtime.logger.warning("[DRY RUN] Would have run %s", cmd) return True
def clone_operator(self): """Clone operator distgit repository to doozer working dir """ dg_dir = Path(self.operator_clone_path) tag = f'{self.operator_dict["version"]}-{self.operator_dict["release"]}' if dg_dir.exists(): self.runtime.logger.info( "Distgit directory already exists; skipping clone: %s", dg_dir) if self.runtime.upcycle: self.runtime.logger.warning( "Refreshing source for '%s' due to --upcycle", dg_dir) exectools.cmd_assert( ["git", "-C", str(dg_dir), "clean", "-fdx"]) exectools.cmd_assert([ "git", "-C", str(dg_dir), "fetch", "--depth", "1", "origin", "tag", tag ], retries=3) exectools.cmd_assert([ "git", "-C", str(dg_dir), "reset", "--hard", "FETCH_HEAD" ]) return dg_dir.parent.mkdir(parents=True, exist_ok=True) exectools.cmd_assert( 'rhpkg {} clone --depth 1 --branch {} {} {}'.format( self.rhpkg_opts, tag, self.operator_repo_name, self.operator_clone_path), retries=3)
def test_cmd_assert_fail(self): """ """ # Try a failing command 3 times, at 1 sec intervals with self.assertRaises(IOError): exectools.cmd_assert("/usr/bin/false", 3, 1) # check that the log file has all of the tests. log_file = open(self.test_file, 'r') lines = log_file.readlines() log_file.close() self.assertEqual(len(lines), 12)
def test_cmd_assert_success(self): """ """ try: exectools.cmd_assert("/bin/true") except IOError as error: self.Fail("/bin/truereturned failure: {}".format(error)) # check that the log file has all of the tests. log_file = open(self.test_file, 'r') lines = log_file.readlines() log_file.close() self.assertEqual(len(lines), 4)
def clone_repo(self, repo, branch): """Clone a repository using rhpkg :param string repo: Name of the repository to be cloned :param string branch: Which branch of the repository should be cloned """ cmd = 'timeout 600 rhpkg ' cmd += self.runtime.rhpkg_config cmd += '--user {} '.format(self.rhpkg_user) if self.rhpkg_user else '' cmd += 'clone containers/{} --branch {}'.format(repo, branch) delete_repo = 'rm -rf {}/{}'.format(self.working_dir, repo) with pushd.Dir(self.working_dir): exectools.cmd_assert(cmd, retries=3, on_retry=delete_repo)
def __init__(self, runtime, pullspec_for_tag: Dict[str, str], brew_arch: str): self.runtime = runtime self.brew_arch = brew_arch self.pullspec_for_tag = pullspec_for_tag self.build_id = None # Remember the pullspec(s) provided in case it does not match what is in the releases.yaml. # Because of an incident where we needed to repush RHCOS and get a new SHA for 4.10 GA, # trust the exact pullspec in releases.yml instead of what we find in the RHCOS release # browser. for tag, pullspec in pullspec_for_tag.items(): image_info_str, _ = exectools.cmd_assert(f'oc image info -o json {pullspec}', retries=3) image_info = Model(json.loads(image_info_str)) build_id = image_info.config.config.Labels.version if not build_id: raise Exception(f'Unable to determine RHCOS build_id from tag {tag} pullspec {pullspec}. Retrieved image info: {image_info_str}') if self.build_id and self.build_id != build_id: raise Exception(f'Found divergent RHCOS build_id for {pullspec_for_tag}. {build_id} versus {self.build_id}') self.build_id = build_id # The first digits of the RHCOS build are the major.minor of the rhcos stream name. # Which, near branch cut, might not match the actual release stream. # Sadly we don't have any other labels or anything to look at to determine the stream. version = self.build_id.split('.')[0] self.stream_version = version[0] + '.' + version[1:] # e.g. 43.82.202102081639.0 -> "4.3" try: finder = RHCOSBuildFinder(runtime, self.stream_version, self.brew_arch) self._build_meta = finder.rhcos_build_meta(self.build_id, meta_type='meta') self._os_commitmeta = finder.rhcos_build_meta(self.build_id, meta_type='commitmeta') except Exception: # Fall back to trying to find a custom build finder = RHCOSBuildFinder(runtime, self.stream_version, self.brew_arch, custom=True) self._build_meta = finder.rhcos_build_meta(self.build_id, meta_type='meta') self._os_commitmeta = finder.rhcos_build_meta(self.build_id, meta_type='commitmeta')
def get_build_from_payload(payload_pullspec): rhcos_tag = 'machine-os-content' out, err = exectools.cmd_assert([ "oc", "adm", "release", "info", "--image-for", rhcos_tag, "--", payload_pullspec ]) if err: raise Exception(f"Error running oc adm: {err}") rhcos_pullspec = out.split('\n')[0] out, err = exectools.cmd_assert( ["oc", "image", "info", "-o", "json", rhcos_pullspec]) if err: raise Exception(f"Error running oc adm: {err}") image_info = json.loads(out) build_id = image_info["config"]["config"]["Labels"]["version"] return build_id
def commit_and_push_bundle(self, commit_msg): """Try to commit and push bundle distgit repository if there were any content changes. :param string commit_msg: Commit message :return bool True if new changes were committed and pushed, False otherwise """ with pushd.Dir(self.bundle_clone_path): try: exectools.cmd_assert('git add .') exectools.cmd_assert('rhpkg{}commit -m "{}"'.format( self.rhpkg_opts, commit_msg)) rc, out, err = exectools.cmd_gather('rhpkg{}push'.format( self.rhpkg_opts)) return True except Exception: return False # Bundle repository might be already up-to-date, nothing new to commit
def _get_upstream_source(runtime, image_meta): """ Analyzes an image metadata to find the upstream URL and branch associated with its content. :param runtime: The runtime object :param image_meta: The metadata to inspect :return: A tuple containing (url, branch) for the upstream source OR (None, None) if there is no upstream source. """ if "git" in image_meta.config.content.source: source_repo_url = image_meta.config.content.source.git.url source_repo_branch = image_meta.config.content.source.git.branch.target branch_check, err = exectools.cmd_assert( f'git ls-remote --heads {source_repo_url} {source_repo_branch}', strip=True) if not branch_check: # Output is empty if branch does not exist source_repo_branch = image_meta.config.content.source.git.branch.fallback if source_repo_branch is Missing: raise IOError( f'Unable to detect source repository branch for {image_meta.distgit_key}' ) elif "alias" in image_meta.config.content.source: alias = image_meta.config.content.source.alias if alias not in runtime.group_config.sources: raise IOError( f'Unable to find source alias {alias} for {image_meta.distgit_key}' ) source_repo_url = runtime.group_config.sources[alias].url source_repo_branch = runtime.group_config.sources[alias].branch.target else: # No upstream source, no PR to open return None, None return source_repo_url, source_repo_branch
def find_manifest_list_sha(pull_spec): cmd = 'oc image info --filter-by-os=linux/amd64 -o json {}'.format(pull_spec) out, err = exectools.cmd_assert(cmd, retries=3) image_data = json.loads(out) if 'listDigest' not in image_data: raise ValueError('Specified image is not a manifest-list.') return image_data['listDigest']
def retrieve_image_info(self, pullspec: str) -> Model: """pull/cache/return json info for a container pullspec""" if pullspec not in image_info_cache: image_json_str, _ = exectools.cmd_assert( f"oc image info {pullspec} -o=json --filter-by-os=amd64", retries=3) image_info_cache[pullspec] = Model(json.loads(image_json_str)) return image_info_cache[pullspec]
def update_metadata_repo(self, metadata_branch): """Update the corresponding metadata repository of an operator :param string metadata_branch: Which branch of the metadata repository should be updated :return: bool True if metadata repo was updated, False if there was nothing to update """ exectools.cmd_assert('mkdir -p {}'.format(self.working_dir)) self.clone_repo(self.operator_name, self.operator_branch) self.clone_repo(self.metadata_repo, metadata_branch) self.checkout_repo(self.operator_name, self.commit_hash) self.update_metadata_manifests_dir() self.update_current_csv_shasums() self.merge_streams_on_top_level_package_yaml() self.create_metadata_dockerfile() return self.commit_and_push_metadata_repo()
def delete_and_clone(): self.delete_repo(repo) cmd = 'timeout 600 rhpkg ' cmd += '--user {} '.format( self.rhpkg_user) if self.rhpkg_user else '' cmd += 'clone containers/{} --branch {}'.format(repo, branch) return exectools.cmd_assert(cmd)
def setup_and_fetch_public_upstream_source(public_source_url: str, public_upstream_branch: str, source_dir: str): """ Fetch public upstream source for specified Git repository. Set up public_upstream remote if needed. :param public_source_url: HTTPS Git URL of the public upstream source :param public_upstream_branch: Git branch of the public upstream source :param source_dir: Path to the local Git repository """ out, err = exectools.cmd_assert(["git", "-C", source_dir, "remote"]) if 'public_upstream' not in out.strip().split(): exectools.cmd_assert([ "git", "-C", source_dir, "remote", "add", "--", "public_upstream", public_source_url ]) else: exectools.cmd_assert([ "git", "-C", source_dir, "remote", "set-url", "--", "public_upstream", public_source_url ]) exectools.cmd_assert([ "git", "-C", source_dir, "fetch", "--", "public_upstream", public_upstream_branch ], retries=3, set_env=constants.GIT_NO_PROMPTS)
def commit_and_push_metadata_repo(self): """Commit and push changes made on the metadata repository, using rhpkg """ with pushd.Dir('{}/{}'.format(self.working_dir, self.metadata_repo)): try: exectools.cmd_assert('git add .') user_option = '--user {} '.format( self.rhpkg_user) if self.rhpkg_user else '' exectools.cmd_assert( 'rhpkg {} {}commit -m "Update operator metadata"'.format( self.runtime.rhpkg_config, user_option)) exectools.retry( retries=3, task_f=lambda: exectools.cmd_assert( 'timeout 600 rhpkg {}push'.format(user_option))) return True except Exception: # The metadata repo might be already up to date, so we don't have anything new to commit return False
def images_streams_start_buildconfigs(runtime, live_test_mode): runtime.initialize(clone_distgits=False, clone_source=False) group_label = runtime.group_config.name if live_test_mode: group_label += '.test' bc_stdout, bc_stderr = exectools.cmd_assert( f'oc -n ci get -o=name buildconfigs -l art-builder-group={group_label}' ) bc_stdout = bc_stdout.strip() if bc_stdout: for name in bc_stdout.splitlines(): print(f'Triggering: {name}') stdout, stderr = exectools.cmd_assert( f'oc -n ci start-build {name}') print(' ' + stdout or stderr) else: print(f'No buildconfigs associated with this group: {group_label}')
def test_create_metadata_dockerfile(self): # using the real filesystem, because DockerfileParser library keeps # opening and closing files at every operation, really hard to mock exectools.cmd_assert('mkdir -p /tmp/my-operator') exectools.cmd_assert('mkdir -p /tmp/my-dev-operator-metadata') with open('/tmp/my-operator/Dockerfile', 'w') as f: f.write("""FROM openshift/foo-bar-operator:v0.1.2.20190826.143750 ENV SOURCE_GIT_COMMIT=... SOURCE_DATE_EPOCH=00000 BUILD_VERSION=vX.Y.Z ADD deploy/olm-catalog/path/to/manifests /manifests LABEL \ com.redhat.component="my-operator-container" \ name="openshift/ose-my-operator" \ com.redhat.delivery.appregistry="true" \ version="vX.Y.Z" \ release="201908261419" """) nvr = '...irrelevant...' stream = 'dev' runtime = '...irrelevant...' cached_attrs = {'working_dir': '/tmp', 'operator_name': 'my-operator'} operator_metadata.OperatorMetadataBuilder( nvr, stream, runtime, **cached_attrs).create_metadata_dockerfile() with open('/tmp/my-dev-operator-metadata/Dockerfile', 'r') as f: self.assertItemsEqual([l.strip() for l in f.readlines()], [ 'FROM scratch', 'COPY ./manifests /manifests', 'LABEL version=vX.Y.Z', 'LABEL com.redhat.delivery.appregistry=true', 'LABEL name=openshift/ose-my-operator-metadata', 'LABEL com.redhat.component=my-operator-metadata-container', ]) # Cleaning up shutil.rmtree('/tmp/my-operator') shutil.rmtree('/tmp/my-dev-operator-metadata')