def change_from_in_df(self, build_dir: BuildDir) -> None:
self.log.info(
"Updating FROM instructions in %s Dockerfile to pin parent images",
build_dir.platform)
dfp = build_dir.dockerfile
df_images = self.workflow.data.dockerfile_images
build_base = df_images.base_image
if not df_images.base_from_scratch:
# do some sanity checks to defend against bugs and rogue plugins
self._sanity_check(dfp.baseimage, build_base)
# check for lost parent images
missing_set = set()
for df_img in dfp.parent_images:
if base_image_is_scratch(df_img):
continue
try:
df_images[df_img]
except KeyError:
missing_set.add(df_img)
if missing_set:
# this would indicate another plugin modified parent_images out of sync
# with the Dockerfile or some other code bug
raise ParentImageMissing(
f"Lost parent image(s) from Dockerfile images and their "
f"parents:"
f" {missing_set}")
# replace image tags with manifest digests generated by check_base_image
new_parents = []
for df_img in dfp.parent_images:
if base_image_is_scratch(df_img):
new_parents.append(df_img)
else:
new_parents.append(df_images[df_img].to_str())
# update parent_images in Dockerfile
dfp.parent_images = new_parents
if df_images.base_from_scratch:
self.log.debug("base image '%s' left unchanged", dfp.baseimage)
else:
self.log.debug(
"for base image '%s' using image with pullspec '%s'",
dfp.baseimage, build_base)
def set_base_image(self, base_image, parents_pulled=True, insecure=False, dockercfg_path=None):
self.base_from_scratch = base_image_is_scratch(base_image)
if not self.custom_base_image:
self.custom_base_image = base_image_is_custom(base_image)
self.base_image = ImageName.parse(base_image)
self.original_base_image = self.original_base_image or self.base_image
self.recreate_parent_images()
if not self.base_from_scratch:
self.parent_images[self.original_base_image] = self.base_image
def layer_index(self):
if self._layer_index is None:
# Default layer index is 1, because base and 'FROM scratch' images
# *always* have 2 layers
self._layer_index = 1
if not base_image_is_scratch(self.dfp.baseimage):
inspect = self.workflow.builder.base_image_inspect
self._layer_index = len(inspect[INSPECT_ROOTFS][INSPECT_ROOTFS_LAYERS])
return self._layer_index
def set_base_image(self, base_image, parents_pulled=True, insecure=False):
self.base_from_scratch = base_image_is_scratch(base_image)
if not self.custom_base_image:
self.custom_base_image = base_image_is_custom(base_image)
self.base_image = ImageName.parse(base_image)
self.original_base_image = self.original_base_image or self.base_image
self.recreate_parent_images()
if not self.base_from_scratch:
self.parent_images[self.original_base_image] = self.base_image
def set_df_path(self, path):
self._df_path = path
dfp = df_parser(path)
base = dfp.baseimage
if base is None:
raise RuntimeError("no base image specified in Dockerfile")
self.set_base_image(base)
logger.debug("base image specified in dockerfile = '%s'",
self.base_image)
self.parent_images.clear()
custom_base_images = set()
for image in dfp.parent_images:
image_name = ImageName.parse(image)
if base_image_is_scratch(image_name.get_repo()):
image_name.tag = None
self.parents_ordered.append(image_name.to_str())
continue
image_str = image_name.to_str()
if base_image_is_custom(image_str):
custom_base_images.add(image_str)
self.custom_parent_image = True
self.parents_ordered.append(image_str)
self.parent_images[image_name] = None
if len(custom_base_images) > 1:
raise NotImplementedError("multiple different custom base images"
" aren't allowed in Dockerfile")
# validate user has not specified COPY --from=image
builders = []
for stmt in dfp.structure:
if stmt['instruction'] == 'FROM':
# extract "bar" from "foo as bar" and record as build stage
match = re.search(r'\S+ \s+ as \s+ (\S+)', stmt['value'],
re.I | re.X)
builders.append(match.group(1) if match else None)
elif stmt['instruction'] == 'COPY':
match = re.search(r'--from=(\S+)', stmt['value'], re.I)
if not match:
continue
stage = match.group(1)
# error unless the --from is the index or name of a stage we've seen
if any(stage in [str(idx), builder]
for idx, builder in enumerate(builders)):
continue
raise RuntimeError(
dedent("""\
OSBS does not support COPY --from unless it matches a build stage.
Dockerfile instruction was:
{}
To use an image with COPY --from, specify it in a stage with FROM, e.g.
FROM {} AS source
FROM ...
COPY --from=source <src> <dest>
""").format(stmt['content'], stage))
def set_base_image(self, base_image, parents_pulled=True, insecure=False):
self.base_from_scratch = base_image_is_scratch(base_image)
self.base_image = ImageName.parse(base_image)
self.original_base_image = self.original_base_image or self.base_image
self.recreate_parent_images()
if not self.base_from_scratch:
self.parent_images[self.original_base_image] = self.base_image
self.parents_pulled = parents_pulled
self.base_image_insecure = insecure
logger.info("set base image to '%s' with original base '%s'",
self.base_image, self.original_base_image)
def set_df_path(self, path):
self._df_path = path
dfp = df_parser(path)
base = dfp.baseimage
if base is None:
raise RuntimeError("no base image specified in Dockerfile")
self.set_base_image(base)
logger.debug("base image specified in dockerfile = '%s'", self.base_image)
self.parent_images.clear()
custom_base_images = set()
for image in dfp.parent_images:
image_name = ImageName.parse(image)
if base_image_is_scratch(image_name.get_repo()):
image_name.tag = None
self.parents_ordered.append(image_name.to_str())
continue
image_str = image_name.to_str()
if base_image_is_custom(image_str):
custom_base_images.add(image_str)
self.custom_parent_image = True
self.parents_ordered.append(image_str)
self.parent_images[image_name] = None
if len(custom_base_images) > 1:
raise NotImplementedError("multiple different custom base images"
" aren't allowed in Dockerfile")
# validate user has not specified COPY --from=image
builders = []
for stmt in dfp.structure:
if stmt['instruction'] == 'FROM':
# extract "bar" from "foo as bar" and record as build stage
match = re.search(r'\S+ \s+ as \s+ (\S+)', stmt['value'], re.I | re.X)
builders.append(match.group(1) if match else None)
elif stmt['instruction'] == 'COPY':
match = re.search(r'--from=(\S+)', stmt['value'], re.I)
if not match:
continue
stage = match.group(1)
# error unless the --from is the index or name of a stage we've seen
if any(stage in [str(idx), builder] for idx, builder in enumerate(builders)):
continue
raise RuntimeError(dedent("""\
OSBS does not support COPY --from unless it matches a build stage.
Dockerfile instruction was:
{}
To use an image with COPY --from, specify it in a stage with FROM, e.g.
FROM {} AS source
FROM ...
COPY --from=source <src> <dest>
""").format(stmt['content'], stage))
def set_base_image(self, base_image, parents_pulled=True, insecure=False, dockercfg_path=None):
self.base_from_scratch = base_image_is_scratch(base_image)
if not self.custom_base_image:
self.custom_base_image = base_image_is_custom(base_image)
self.base_image = ImageName.parse(base_image)
self.original_base_image = self.original_base_image or self.base_image
self.recreate_parent_images()
if not self.base_from_scratch:
self.parent_images[self.original_base_image] = self.base_image
self.parents_pulled = parents_pulled
self.base_image_insecure = insecure
self.base_image_dockercfg_path = dockercfg_path
logger.info("set base image to '%s' with original base '%s'", self.base_image,
self.original_base_image)
def run(self):
"""
Check parent images to ensure they only come from allowed registries.
"""
self.manifest_list_cache.clear()
digest_fetching_exceptions = []
for parent in self.workflow.data.dockerfile_images.keys():
if base_image_is_custom(parent.to_str()) or base_image_is_scratch(
parent.to_str()):
continue
image = parent
# base_image_key is an ImageName, so compare parent as an ImageName also
if image == self.workflow.data.dockerfile_images.base_image_key:
image = self._resolve_base_image()
self._ensure_image_registry(image)
self._validate_platforms_in_image(image)
try:
digest = self._fetch_manifest_digest(image)
except RuntimeError as exc:
digest_fetching_exceptions.append(exc)
continue
image_with_digest = self._pin_to_digest(image, digest)
self.log.info("Replacing image '%s' with '%s'", image,
image_with_digest)
self.workflow.data.dockerfile_images[parent] = image_with_digest
self.workflow.data.parent_images_digests[str(
image_with_digest)] = digest
if digest_fetching_exceptions:
raise RuntimeError(
'Error when extracting parent images manifest digests: {}'.
format(digest_fetching_exceptions))
def run(self):
builder = self.workflow.builder
dfp = df_parser(builder.df_path)
organization = get_registries_organization(self.workflow)
df_base = ImageName.parse(dfp.baseimage)
if organization and not base_image_is_custom(dfp.baseimage):
df_base.enclose(organization)
build_base = builder.base_image
if not self.workflow.builder.base_from_scratch:
# do some sanity checks to defend against bugs and rogue plugins
self._sanity_check(df_base, build_base, builder)
self.log.info("parent_images '%s'", builder.parent_images)
unresolved = [
key for key, val in builder.parent_images.items() if not val
]
if unresolved:
# this would generally mean pull_base_image didn't run and/or
# custom plugins modified parent_images; treat it as an error.
raise ParentImageUnresolved(
"Parent image(s) unresolved: {}".format(unresolved))
# enclose images from dfp
enclosed_parent_images = []
for df_img in dfp.parent_images:
if base_image_is_scratch(df_img):
enclosed_parent_images.append(df_img)
continue
parent = ImageName.parse(df_img)
if organization and not base_image_is_custom(df_img):
parent.enclose(organization)
enclosed_parent_images.append(parent)
missing = [
df_img for df_img in enclosed_parent_images
if df_img not in builder.parent_images
]
missing_set = set(missing)
if SCRATCH_FROM in missing_set:
missing_set.remove(SCRATCH_FROM)
if missing_set:
# this would indicate another plugin modified parent_images out of sync
# with the Dockerfile or some other code bug
raise ParentImageMissing(
"Lost parent image(s) from Dockerfile: {}".format(missing_set))
# docker inspect all parent images so we can address them by Id
parent_image_ids = {}
for img, new_img in builder.parent_images.items():
inspection = builder.parent_image_inspect(new_img)
try:
parent_image_ids[img] = inspection['Id']
except KeyError: # unexpected code bugs or maybe docker weirdness
self.log.error(
"Id for image %s is missing in inspection: '%s'", new_img,
inspection)
raise NoIdInspection("Could not inspect Id for image " +
str(new_img))
# update the parents in Dockerfile
new_parents = []
for parent in enclosed_parent_images:
if base_image_is_scratch(parent):
new_parents.append(parent)
continue
pid = parent_image_ids[parent]
self.log.info("changed FROM: '%s' -> '%s'", parent, pid)
new_parents.append(pid)
dfp.parent_images = new_parents
# update builder's representation of what will be built
builder.parent_images = parent_image_ids
if self.workflow.builder.base_from_scratch:
return
builder.set_base_image(parent_image_ids[df_base])
self.log.debug("for base image '%s' using local image '%s', id '%s'",
df_base, build_base, parent_image_ids[df_base])
def image_is_inspectable(image: Union[str, ImageName]) -> bool:
"""Check if we should expect the image to be inspectable."""
im = str(image)
return not (util.base_image_is_scratch(im)
or util.base_image_is_custom(im))
def run(self):
builder = self.workflow.builder
dfp = df_parser(builder.df_path)
builder.original_df = dfp.content
df_base = dfp.baseimage
build_base = builder.dockerfile_images.base_image
if not self.workflow.builder.dockerfile_images.base_from_scratch:
# do some sanity checks to defend against bugs and rogue plugins
self._sanity_check(dfp.baseimage, build_base, builder)
self.log.info("parent_images '%s'", builder.dockerfile_images.keys())
unresolved = [
key for key, val in builder.dockerfile_images.items() if not val
]
if unresolved:
# this would generally mean pull_base_image didn't run and/or
# custom plugins modified parent_images; treat it as an error.
raise ParentImageUnresolved(
"Parent image(s) unresolved: {}".format(unresolved))
# check for lost parent images
missing_set = set()
for df_img in dfp.parent_images:
if base_image_is_scratch(df_img):
continue
try:
builder.dockerfile_images[df_img]
except KeyError:
missing_set.add(df_img)
if missing_set:
# this would indicate another plugin modified parent_images out of sync
# with the Dockerfile or some other code bug
raise ParentImageMissing(
"Lost parent image(s) from Dockerfile: {}".format(missing_set))
# docker inspect all parent images so we can address them by Id
parent_image_ids = {}
new_parents = []
for df_img in dfp.parent_images:
if base_image_is_scratch(df_img):
new_parents.append(df_img)
continue
local_image = builder.dockerfile_images[df_img]
inspection = builder.parent_image_inspect(local_image)
try:
parent_image_ids[df_img] = inspection['Id']
new_parents.append(inspection['Id'])
except KeyError as exc: # unexpected code bugs or maybe docker weirdness
self.log.error(
"Id for image %s is missing in inspection: '%s'", df_img,
inspection)
raise NoIdInspection("Could not inspect Id for image " +
df_img) from exc
# update builder's representation of what will be built
for df_img in dfp.parent_images:
if base_image_is_scratch(df_img):
continue
builder.dockerfile_images[df_img] = parent_image_ids[df_img]
# update parent_images in Dockerfile
dfp.parent_images = new_parents
if builder.dockerfile_images.base_from_scratch:
return
self.log.debug("for base image '%s' using local image '%s', id '%s'",
df_base, build_base, parent_image_ids[df_base])
def run(self):
builder = self.workflow.builder
dfp = df_parser(builder.df_path)
organization = get_registries_organization(self.workflow)
df_base = ImageName.parse(dfp.baseimage)
if organization and not base_image_is_custom(dfp.baseimage):
df_base.enclose(organization)
build_base = builder.base_image
if not self.workflow.builder.base_from_scratch:
# do some sanity checks to defend against bugs and rogue plugins
self._sanity_check(df_base, build_base, builder)
self.log.info("parent_images '%s'", builder.parent_images)
unresolved = [key for key, val in builder.parent_images.items() if not val]
if unresolved:
# this would generally mean pull_base_image didn't run and/or
# custom plugins modified parent_images; treat it as an error.
raise ParentImageUnresolved("Parent image(s) unresolved: {}".format(unresolved))
# enclose images from dfp
enclosed_parent_images = []
for df_img in dfp.parent_images:
if base_image_is_scratch(df_img):
enclosed_parent_images.append(df_img)
continue
parent = ImageName.parse(df_img)
if organization and not base_image_is_custom(df_img):
parent.enclose(organization)
enclosed_parent_images.append(parent)
missing = [df_img for df_img in enclosed_parent_images
if df_img not in builder.parent_images]
missing_set = set(missing)
if SCRATCH_FROM in missing_set:
missing_set.remove(SCRATCH_FROM)
if missing_set:
# this would indicate another plugin modified parent_images out of sync
# with the Dockerfile or some other code bug
raise ParentImageMissing("Lost parent image(s) from Dockerfile: {}".format(missing_set))
# docker inspect all parent images so we can address them by Id
parent_image_ids = {}
for img, new_img in builder.parent_images.items():
inspection = builder.parent_image_inspect(new_img)
try:
parent_image_ids[img] = inspection['Id']
except KeyError: # unexpected code bugs or maybe docker weirdness
self.log.error(
"Id for image %s is missing in inspection: '%s'",
new_img, inspection)
raise NoIdInspection("Could not inspect Id for image " + str(new_img))
# update the parents in Dockerfile
new_parents = []
for parent in enclosed_parent_images:
if base_image_is_scratch(parent):
new_parents.append(parent)
continue
pid = parent_image_ids[parent]
self.log.info("changed FROM: '%s' -> '%s'", parent, pid)
new_parents.append(pid)
dfp.parent_images = new_parents
# update builder's representation of what will be built
builder.parent_images = parent_image_ids
if self.workflow.builder.base_from_scratch:
return
builder.set_base_image(parent_image_ids[df_base])
self.log.debug(
"for base image '%s' using local image '%s', id '%s'",
df_base, build_base, parent_image_ids[df_base]
)