def test_conversion_from_ssh_source(self):
source = "[email protected]:myorg/myproject.git"
actual = util.convert_remote_git_to_https(source)
expected = "https://github.com/myorg/myproject"
self.assertEqual(actual, expected)
def test_conversion_from_https_org_source(self):
source = "https://github.com/myorg/"
actual = util.convert_remote_git_to_https(source)
expected = "https://github.com/myorg"
self.assertEqual(actual, expected)
def test_conversion_from_https_source_with_dotgit_elsewhere(self):
source = "https://foo.gitlab.com/myorg/myproject"
actual = util.convert_remote_git_to_https(source)
expected = "https://foo.gitlab.com/myorg/myproject"
self.assertEqual(actual, expected)
def test_conversion_from_https_source_with_dotgit_suffix(self):
source = "https://github.com/myorg/myproject.git"
actual = util.convert_remote_git_to_https(source)
expected = "https://github.com/myorg/myproject"
self.assertEqual(actual, expected)
def find_mismatched_siblings(
build_image_inspectors: Iterable[Optional[BrewBuildImageInspector]]
) -> List[Tuple[BrewBuildImageInspector, BrewBuildImageInspector]]:
"""
Sibling images are those built from the same repository. We need to throw an error
if there are sibling built from different commits.
:return: Returns a list of (BrewBuildImageInspector,BrewBuildImageInspector) where the first item is a mismatched sibling of the second
"""
class RepoBuildRecord(NamedTuple):
build_image_inspector: BrewBuildImageInspector
source_git_commit: str
# Maps SOURCE_GIT_URL -> RepoBuildRecord(SOURCE_GIT_COMMIT, DISTGIT_KEY, NVR). Where the Tuple is the first build
# encountered claiming it is sourced from the SOURCE_GIT_URL
repo_builds: Dict[str, RepoBuildRecord] = dict()
mismatched_siblings: List[Tuple[BrewBuildImageInspector,
BrewBuildImageInspector]] = []
for build_image_inspector in build_image_inspectors:
if not build_image_inspector:
# No build for this component at present.
continue
# Here we check the raw config - before it is affected by assembly overrides. Why?
# If an artist overrides one sibling's git url, but not another, the following
# scan would not be able to detect that they were siblings. Instead, we rely on the
# original image metadata to determine sibling-ness.
source_url = build_image_inspector.get_image_meta(
).raw_config.content.source.git.url
source_git_commit = build_image_inspector.get_source_git_commit()
if not source_url or not source_git_commit:
# This is true for distgit only components.
continue
# Make sure URLs are comparable regardless of git: or https:
source_url = convert_remote_git_to_https(source_url)
potential_conflict: RepoBuildRecord = repo_builds.get(
source_url, None)
if potential_conflict:
# Another component has build from this repo before. Make
# sure it built from the same commit.
if potential_conflict.source_git_commit != source_git_commit:
mismatched_siblings.append(
(build_image_inspector,
potential_conflict.build_image_inspector))
red_print(
f"The following NVRs are siblings but built from different commits: {potential_conflict.build_image_inspector.get_nvr()} and {build_image_inspector.get_nvr()}",
file=sys.stderr)
else:
# No conflict, so this is our first encounter for this repo; add it to our tracking dict.
repo_builds[source_url] = RepoBuildRecord(
build_image_inspector=build_image_inspector,
source_git_commit=source_git_commit)
return mismatched_siblings
def test_conversion_from_ssh_org_source(self):
expected = "https://github.com/myorg"
for source in [
"[email protected]:myorg/", "[email protected]/myorg/",
'ssh://[email protected]/myorg',
'ssh://[email protected]:myorg'
]:
actual = util.convert_remote_git_to_https(source)
self.assertEqual(actual, expected)
def test_convert_remote_git_to_https(self):
# git@ to https
self.assertEqual(
util.convert_remote_git_to_https(
'[email protected]:openshift/aos-cd-jobs.git'),
'https://github.com/openshift/aos-cd-jobs')
# https to https (no-op)
self.assertEqual(
util.convert_remote_git_to_https(
'https://github.com/openshift/aos-cd-jobs'),
'https://github.com/openshift/aos-cd-jobs')
# https to https, remove suffix
self.assertEqual(
util.convert_remote_git_to_https(
'https://github.com/openshift/aos-cd-jobs.git'),
'https://github.com/openshift/aos-cd-jobs')
# ssh to https
self.assertEqual(
util.convert_remote_git_to_https(
'ssh://[email protected]/openshift/aos-cd-jobs.git'),
'https://github.com/openshift/aos-cd-jobs')
def check_group_image_consistency(
self,
build_inspector: BrewBuildImageInspector) -> List[AssemblyIssue]:
"""
Evaluate the current assembly build and an image in the group and check whether they are consistent with
:param build_inspector: The brew build to check
:return: Returns a (potentially empty) list of reasons the image should be rebuilt.
"""
image_meta = build_inspector.get_image_meta()
self.runtime.logger.info(
f'Checking group image for consistency: {image_meta.distgit_key}...'
)
issues: List[AssemblyIssue] = []
installed_packages = build_inspector.get_all_installed_package_build_dicts(
)
dgk = build_inspector.get_image_meta().distgit_key
"""
If the assembly defined any RPM package dependencies at the group or image
member level, we want to check to make sure that installed RPMs in the
build image match the override package.
If reading this, keep in mind that a single package/build may create several
RPMs. Both assemblies and this method deal with the package level - not
individual RPMs.
"""
member_package_overrides, all_package_overrides = image_meta.get_assembly_rpm_package_dependencies(
el_ver=image_meta.branch_el_target())
if member_package_overrides or all_package_overrides:
for package_name, required_nvr in all_package_overrides.items():
if package_name in member_package_overrides and package_name not in installed_packages:
# A dependency was defined explicitly in an assembly member, but it is not installed.
# i.e. the artists expected something to be installed, but it wasn't found in the final image.
# Raise an issue. In rare circumstances the RPM may be used by early stage of the Dockerfile
# and not in the final. In this case, it should be permitted in the assembly definition.
issues.append(
AssemblyIssue(
f'Expected image to contain assembly member override dependencies NVR {required_nvr} but it was not installed',
component=dgk,
code=AssemblyIssueCode.MISSING_INHERITED_DEPENDENCY
))
if package_name in installed_packages:
installed_build_dict: Dict = installed_packages[
package_name]
installed_nvr = installed_build_dict['nvr']
if required_nvr != installed_nvr:
issues.append(
AssemblyIssue(
f'Expected image to contain assembly override dependencies NVR {required_nvr} but found {installed_nvr} installed',
component=dgk,
code=AssemblyIssueCode.
CONFLICTING_INHERITED_DEPENDENCY))
"""
If an image contains an RPM from the doozer group, make sure it is the current
RPM for the assembly.
"""
el_ver = build_inspector.get_rhel_base_version()
if el_ver: # We might not find an el_ver for an image (e.g. FROM scratch)
for dgk, assembly_rpm_build in self.get_group_rpm_build_dicts(
el_ver).items():
if not assembly_rpm_build:
# The RPM doesn't claim to build for this image's RHEL base, so ignore it.
continue
package_name = assembly_rpm_build['package_name']
assembly_nvr = assembly_rpm_build['nvr']
if package_name in installed_packages:
installed_nvr = installed_packages[package_name]['nvr']
if installed_nvr != assembly_nvr:
issues.append(
AssemblyIssue(
f'Expected image to contain assembly RPM build {assembly_nvr} but found {installed_nvr} installed',
component=dgk,
code=AssemblyIssueCode.
CONFLICTING_GROUP_RPM_INSTALLED))
"""
Assess whether the image build has the upstream
source git repo and git commit that may have been declared/
overridden in an assembly definition.
"""
content_git_url = image_meta.config.content.source.git.url
if content_git_url:
# Make sure things are in https form so we can compare
content_git_url, _ = self.runtime.get_public_upstream(
util.convert_remote_git_to_https(content_git_url))
build_git_url = util.convert_remote_git_to_https(
build_inspector.get_source_git_url())
if content_git_url != build_git_url:
# Impermissible as artist can just fix upstream git source in assembly definition
issues.append(
AssemblyIssue(
f'Expected image git source from metadata {content_git_url} but found {build_git_url} as the upstream source of the brew build',
component=dgk))
try:
target_branch = image_meta.config.content.source.git.branch.target
if target_branch:
_ = int(target_branch,
16) # parse the name as a git commit
# if we reach here, a git commit hash was declared as the
# upstream source of the image's content. We should verify
# it perfectly matches what we find in the assembly build.
build_commit = build_inspector.get_source_git_commit()
if target_branch != build_commit:
# Impermissible as artist can just fix the assembly definition.
issues.append(
AssemblyIssue(
f'Expected image build git commit {target_branch} but {build_commit} was found in the build',
component=dgk))
except ValueError:
# The meta's target branch a normal branch name
# and not a git commit. When this is the case,
# we don't try to assert anything about the build's
# git commit.
pass
return issues
