def detect(image, force, no_force_detect):
f = None
if (no_force_detect):
ch.VERBOSE("not detecting --force config, per --no-force-detect")
else:
# Try to find a real fakeroot config.
for (tag, cfg) in DEFAULT_CONFIGS.items():
try:
f = Fakeroot(image, tag, cfg, force)
break
except Config_Aint_Matched:
pass
# Report findings.
if (f is None):
msg = "--force not available (no suitable config found)"
if (force):
ch.WARNING(msg)
else:
ch.VERBOSE(msg)
else:
if (force):
adj = "will use"
else:
adj = "available"
ch.INFO("%s --force: %s: %s" % (adj, f.tag, f.name))
# Wrap up
if (f is None):
f = Fakeroot_Noop()
return f
def init_maybe(self, img_path, args, env):
if (not self.needs_inject(args)):
ch.VERBOSE("workarounds: init: instruction doesn't need injection")
return
if (self.init_done):
ch.VERBOSE("workarounds: init: already initialized")
return
for (i, (test_cmd, init_cmd)) in enumerate(self.init, 1):
ch.INFO("workarounds: init step %s: checking: $ %s" %
(i, test_cmd))
args = ["/bin/sh", "-c", test_cmd]
exit_code = ch.ch_run_modify(img_path, args, env, fail_ok=True)
if (exit_code == 0):
ch.INFO(
"workarounds: init step %d: exit code %d, step not needed"
% (i, exit_code))
else:
if (not self.inject_p):
ch.INFO("workarounds: init step %d: no --force, skipping" %
i)
else:
ch.INFO("workarounds: init step %d: $ %s" % (i, init_cmd))
args = ["/bin/sh", "-c", init_cmd]
ch.ch_run_modify(img_path, args, env)
self.init_done = True
def download(self):
"Download image metadata and layers and put them in the download cache."
# Spec: https://docs.docker.com/registry/spec/manifest-v2-2/
ch.VERBOSE("downloading image: %s" % self.image)
try:
# fat manifest
if (ch.arch != "yolo"):
try:
self.fatman_load()
if (ch.arch not in self.architectures):
ch.FATAL("requested arch unavailable: %s" % ch.arch,
("available: %s" %
" ".join(sorted(self.architectures.keys()))))
except ch.No_Fatman_Error:
if (ch.arch == "amd64"):
# We're guessing that enough arch-unaware images are amd64 to
# barge ahead if requested architecture is amd64.
ch.arch = "yolo"
ch.WARNING("image is architecture-unaware")
ch.WARNING(
"requested arch is amd64; using --arch=yolo")
else:
ch.FATAL("image is architecture-unaware",
"consider --arch=yolo")
# manifest
self.manifest_load()
except ch.Not_In_Registry_Error:
ch.FATAL("not in registry: %s" % self.registry.ref)
# config
ch.VERBOSE("config path: %s" % self.config_path)
if (self.config_path is not None):
if (os.path.exists(self.config_path) and self.use_cache):
ch.INFO("config: using existing file")
else:
self.registry.blob_to_file(self.config_hash, self.config_path,
"config: downloading")
# layers
for (i, lh) in enumerate(self.layer_hashes, start=1):
path = self.layer_path(lh)
ch.VERBOSE("layer path: %s" % path)
msg = "layer %d/%d: %s" % (i, len(self.layer_hashes), lh[:7])
if (os.path.exists(path) and self.use_cache):
ch.INFO("%s: using existing file" % msg)
else:
self.registry.blob_to_file(lh, path, "%s: downloading" % msg)
# done
self.registry.close()
def execute_(self):
# Complain about unsupported stuff.
if (self.options.pop("platform", False)):
self.unsupported_yet_fatal("--platform", 778)
# Any remaining options are invalid.
self.options_assert_empty()
# Update image globals.
global image_i
image_i += 1
global image_alias
image_alias = self.alias
if (image_i == image_ct - 1):
# Last image; use tag unchanged.
tag = cli.tag
elif (image_i > image_ct - 1):
# Too many images!
ch.FATAL("expected %d stages but found at least %d" %
(image_ct, image_i + 1))
else:
# Not last image; append stage index to tag.
tag = "%s/_stage%d" % (cli.tag, image_i)
image = ch.Image(ch.Image_Ref(tag))
images[image_i] = image
if (self.alias is not None):
images[self.alias] = image
ch.VERBOSE("image path: %s" % image.unpack_path)
# Other error checking.
if (str(image.ref) == str(self.base_ref)):
ch.FATAL("output image ref same as FROM: %s" % self.base_ref)
# Initialize image.
self.base_image = ch.Image(self.base_ref)
if (os.path.isdir(self.base_image.unpack_path)):
ch.VERBOSE("base image found: %s" % self.base_image.unpack_path)
else:
ch.VERBOSE("base image not found, pulling")
# a young hen, especially one less than one year old.
pullet = pull.Image_Puller(self.base_image, not cli.no_cache)
pullet.pull_to_unpacked()
pullet.done()
image.copy_unpacked(self.base_image)
image.metadata_load()
env.reset()
# Find fakeroot configuration, if any.
global fakeroot_config
fakeroot_config = fakeroot.detect(image.unpack_path, cli.force,
cli.no_force_detect)
def download(self):
"Download image metadata and layers and put them in the download cache."
# Spec: https://docs.docker.com/registry/spec/manifest-v2-2/
ch.VERBOSE("downloading image: %s" % self.image)
ch.mkdirs(ch.storage.download_cache)
# fat manifest
if (ch.arch != "yolo"):
self.fatman_load()
if (self.architectures is not None):
if (ch.arch not in self.architectures):
ch.FATAL(
"requested arch unavailable: %s not one of: %s" %
(ch.arch, " ".join(sorted(self.architectures.keys()))))
elif (ch.arch == "amd64"):
# We're guessing that enough arch-unaware images are amd64 to
# barge ahead if requested architecture is amd64.
ch.arch = "yolo"
ch.WARNING("image is architecture-unaware")
ch.WARNING("requested arch is amd64; switching to --arch=yolo")
else:
ch.FATAL("image is architecture-unaware; try --arch=yolo (?)")
# manifest
self.manifest_load()
# config
ch.VERBOSE("config path: %s" % self.config_path)
if (self.config_path is not None):
if (os.path.exists(self.config_path) and self.use_cache):
ch.INFO("config: using existing file")
else:
ch.INFO("config: downloading")
self.registry.blob_to_file(self.config_hash, self.config_path)
# layers
for (i, lh) in enumerate(self.layer_hashes, start=1):
path = self.layer_path(lh)
ch.VERBOSE("layer path: %s" % path)
ch.INFO("layer %d/%d: %s: " % (i, len(self.layer_hashes), lh[:7]),
end="")
if (os.path.exists(path) and self.use_cache):
ch.INFO("using existing file")
else:
ch.INFO("downloading")
self.registry.blob_to_file(lh, path)
def main(cli):
ch.dependencies_check()
# Set things up.
ref = ch.Image_Ref(cli.image_ref)
if (cli.parse_only):
ch.INFO(ref.as_verbose_str)
sys.exit(0)
image = ch.Image(ref, cli.image_dir)
ch.INFO("pulling image: %s" % ref)
if (cli.image_dir is not None):
ch.INFO( "destination: %s" % image.unpack_path)
else:
ch.VERBOSE("destination: %s" % image.unpack_path)
ch.VERBOSE("use cache: %s" % (not cli.no_cache))
ch.VERBOSE("download cache: %s" % ch.storage.download_cache)
pullet = Image_Puller(image)
ch.VERBOSE("manifest: %s" % pullet.manifest_path)
pullet.pull_to_unpacked(use_cache=(not cli.no_cache),
last_layer=cli.last_layer)
ch.done_notify()
def inject_run(self, args):
if (not self.needs_inject(args)):
ch.VERBOSE("workarounds: RUN: instruction doesn't need injection")
return args
assert (self.init_done)
if (not self.inject_p):
ch.INFO("workarounds: RUN: available here with --force")
return args
args = self.each + args
self.inject_ct += 1
ch.INFO("workarounds: RUN: new command: %s" % args)
return args
def __init__(self, image_path, tag, cfg, inject_p):
ch.VERBOSE("workarounds: testing config: %s" % tag)
file_path = "%s/%s" % (image_path, cfg["match"][0])
if (not (os.path.isfile(file_path)
and ch.grep_p(file_path, cfg["match"][1]))):
raise Config_Aint_Matched(tag)
self.tag = tag
self.inject_ct = 0
self.inject_p = inject_p
for i in ("name", "init", "cmds", "each"):
setattr(self, i, cfg[i])
self.init_done = False
def download(self, use_cache):
"""Download image metadata and layers and put them in the download
cache. If use_cache is True (the default), anything already in the
cache is skipped, otherwise download it anyway, overwriting what's in
the cache."""
# Spec: https://docs.docker.com/registry/spec/manifest-v2-2/
dl = ch.Registry_HTTP(self.image.ref)
ch.VERBOSE("downloading image: %s" % dl.ref)
ch.mkdirs(ch.storage.download_cache)
# manifest
if (os.path.exists(self.manifest_path) and use_cache):
ch.INFO("manifest: using existing file")
else:
ch.INFO("manifest: downloading")
dl.manifest_to_file(self.manifest_path)
self.manifest_load()
# config
ch.VERBOSE("config path: %s" % self.config_path)
if (self.config_path is not None):
if (os.path.exists(self.config_path) and use_cache):
ch.INFO("config: using existing file")
else:
ch.INFO("config: downloading")
dl.blob_to_file(self.config_hash, self.config_path)
# layers
for (i, lh) in enumerate(self.layer_hashes, start=1):
path = self.layer_path(lh)
ch.VERBOSE("layer path: %s" % path)
ch.INFO("layer %d/%d: %s: "% (i, len(self.layer_hashes), lh[:7]),
end="")
if (os.path.exists(path) and use_cache):
ch.INFO("using existing file")
else:
ch.INFO("downloading")
dl.blob_to_file(lh, path)
dl.close()
def main(cli):
# Set things up.
ref = ch.Image_Ref(cli.image_ref)
if (cli.parse_only):
print(ref.as_verbose_str)
sys.exit(0)
image = ch.Image(ref, cli.image_dir)
ch.INFO("pulling image: %s" % ref)
ch.INFO("requesting arch: %s" % ch.arch)
if (cli.image_dir is not None):
ch.INFO("destination: %s" % image.unpack_path)
else:
ch.VERBOSE("destination: %s" % image.unpack_path)
pullet = Image_Puller(image, not cli.no_cache)
pullet.pull_to_unpacked(cli.last_layer)
pullet.done()
ch.done_notify()
def main(cli):
src_ref = ch.Image_Ref(cli.source_ref)
ch.INFO("pushing image: %s" % src_ref)
image = ch.Image(src_ref, cli.image)
# FIXME: validate it's an image using Megan's new function (PR #908)
if (not os.path.isdir(image.unpack_path)):
if (cli.image is not None):
ch.FATAL("can't push: %s does not appear to be an image" %
cli.image)
else:
ch.FATAL("can't push: no image %s" % src_ref)
if (cli.image is not None):
ch.INFO("image path: %s" % image.unpack_path)
else:
ch.VERBOSE("image path: %s" % image.unpack_path)
if (cli.dest_ref is not None):
dst_ref = ch.Image_Ref(cli.dest_ref)
ch.INFO("destination: %s" % dst_ref)
else:
dst_ref = ch.Image_Ref(cli.source_ref)
up = Image_Pusher(image, dst_ref)
up.push()
ch.done_notify()
def manifest_load(self, continue_404=False):
"""Download the manifest file if needed, parse it, and set
self.config_hash and self.layer_hashes. By default, if the image does
not exist, exit with error; if continue_404, then log the condition
but do not exit. In this case, self.config_hash and self.layer_hashes
will both be None."""
def bad_key(key):
ch.FATAL("manifest: %s: no key: %s" % (self.manifest_path, key))
self.config_hash = None
self.layer_hashes = None
# obtain the manifest
try:
# internal manifest library, e.g. for "FROM scratch"
manifest = manifests_internal[str(self.image.ref)]
ch.INFO("manifest: using internal library")
except KeyError:
# download the file if needed, then parse it
if (ch.arch == "yolo" or self.architectures is None):
digest = None
else:
digest = self.architectures[ch.arch]
ch.DEBUG("manifest digest: %s" % digest)
if (os.path.exists(self.manifest_path) and self.use_cache):
ch.INFO("manifest: using existing file")
else:
ch.INFO("manifest: downloading")
self.registry.manifest_to_file(self.manifest_path,
digest=digest,
continue_404=continue_404)
if (not os.path.exists(self.manifest_path)):
# response was 404 (or equivalent)
ch.INFO("manifest: none found")
return
manifest = ch.json_from_file(self.manifest_path, "manifest")
# validate schema version
try:
version = manifest['schemaVersion']
except KeyError:
bad_key("schemaVersion")
if (version not in {1, 2}):
ch.FATAL("unsupported manifest schema version: %s" % repr(version))
# load config hash
#
# FIXME: Manifest version 1 does not list a config blob. It does have
# things (plural) that look like a config at history/v1Compatibility as
# an embedded JSON string :P but I haven't dug into it.
if (version == 1):
ch.VERBOSE("no config; manifest schema version 1")
self.config_hash = None
else: # version == 2
try:
self.config_hash = manifest["config"]["digest"]
if (self.config_hash is not None):
self.config_hash = ch.digest_trim(self.config_hash)
except KeyError:
bad_key("config/digest")
# load layer hashes
if (version == 1):
key1 = "fsLayers"
key2 = "blobSum"
else: # version == 2
key1 = "layers"
key2 = "digest"
if (key1 not in manifest):
bad_key(key1)
self.layer_hashes = list()
for i in manifest[key1]:
if (key2 not in i):
bad_key("%s/%s" % (key1, key2))
self.layer_hashes.append(ch.digest_trim(i[key2]))
if (version == 1):
self.layer_hashes.reverse()
def cleanup(self):
ch.INFO("cleaning up")
# Delete the tarballs since we can't yet cache them.
for (_, tar_c) in self.layers:
ch.VERBOSE("deleting tarball: %s" % tar_c)
ch.unlink(tar_c)
def main(cli_):
# CLI namespace. :P
global cli
cli = cli_
# Check argument validity.
if (cli.force and cli.no_force_detect):
ch.FATAL("--force and --no-force-detect are incompatible")
# Infer input file if needed.
if (cli.file is None):
cli.file = cli.context + "/Dockerfile"
# Infer image name if needed.
if (cli.tag is None):
path = os.path.basename(cli.file)
if ("." in path):
(base, ext_all) = str(path).split(".", maxsplit=1)
(base_all, ext_last) = str(path).rsplit(".", maxsplit=1)
else:
base = None
ext_last = None
if (base == "Dockerfile"):
cli.tag = ext_all
ch.VERBOSE("inferring name from Dockerfile extension: %s" %
cli.tag)
elif (ext_last == "dockerfile"):
cli.tag = base_all
ch.VERBOSE("inferring name from Dockerfile basename: %s" % cli.tag)
elif (os.path.abspath(cli.context) != "/"):
cli.tag = os.path.basename(os.path.abspath(cli.context))
ch.VERBOSE("inferring name from context directory: %s" % cli.tag)
else:
assert (os.path.abspath(cli.context) == "/")
cli.tag = "root"
ch.VERBOSE("inferring name with root context directory: %s" %
cli.tag)
cli.tag = re.sub(r"[^a-z0-9_.-]", "", cli.tag.lower())
ch.INFO("inferred image name: %s" % cli.tag)
# Deal with build arguments.
def build_arg_get(arg):
kv = arg.split("=")
if (len(kv) == 2):
return kv
else:
v = os.getenv(kv[0])
if (v is None):
ch.FATAL("--build-arg: %s: no value and not in environment" %
kv[0])
return (kv[0], v)
cli.build_arg = dict(build_arg_get(i) for i in cli.build_arg)
ch.DEBUG(cli)
# Guess whether the context is a URL, and error out if so. This can be a
# typical looking URL e.g. "https://..." or also something like
# "[email protected]:...". The line noise in the second line of the regex is
# to match this second form. Username and host characters from
# https://tools.ietf.org/html/rfc3986.
if (re.search(
r""" ^((git|git+ssh|http|https|ssh)://
| ^[\w.~%!$&'\(\)\*\+,;=-]+@[\w.~%!$&'\(\)\*\+,;=-]+:)""",
cli.context, re.VERBOSE) is not None):
ch.FATAL("not yet supported: issue #773: URL context: %s" %
cli.context)
if (os.path.exists(cli.context + "/.dockerignore")):
ch.WARNING(
"not yet supported, ignored: issue #777: .dockerignore file")
# Set up build environment.
global env
env = Environment()
# Read input file.
if (cli.file == "-" or cli.context == "-"):
text = ch.ossafe(sys.stdin.read, "can't read stdin")
else:
fp = ch.open_(cli.file, "rt")
text = ch.ossafe(fp.read, "can't read: %s" % cli.file)
fp.close()
# Parse it.
parser = lark.Lark("?start: dockerfile\n" + ch.GRAMMAR,
parser="earley",
propagate_positions=True)
# Avoid Lark issue #237: lark.exceptions.UnexpectedEOF if the file does not
# end in newline.
text += "\n"
try:
tree = parser.parse(text)
except lark.exceptions.UnexpectedInput as x:
ch.VERBOSE(x) # noise about what was expected in the grammar
ch.FATAL("can't parse: %s:%d,%d\n\n%s" %
(cli.file, x.line, x.column, x.get_context(text, 39)))
ch.VERBOSE(tree.pretty())
# Sometimes we exit after parsing.
if (cli.parse_only):
sys.exit(0)
# Count the number of stages (i.e., FROM instructions)
global image_ct
image_ct = sum(1 for i in ch.tree_children(tree, "from_"))
# Traverse the tree and do what it says.
#
# We don't actually care whether the tree is traversed breadth-first or
# depth-first, but we *do* care that instruction nodes are visited in
# order. Neither visit() nor visit_topdown() are documented as of
# 2020-06-11 [1], but examining source code [2] shows that visit_topdown()
# uses Tree.iter_trees_topdown(), which *is* documented to be in-order [3].
#
# This change seems to have been made in 0.8.6 (see PR #761); before then,
# visit() was in order. Therefore, we call that instead, if visit_topdown()
# is not present, to improve compatibility (see issue #792).
#
# [1]: https://lark-parser.readthedocs.io/en/latest/visitors/#visitors
# [2]: https://github.com/lark-parser/lark/blob/445c8d4/lark/visitors.py#L211
# [3]: https://lark-parser.readthedocs.io/en/latest/classes/#tree
ml = Main_Loop()
if (hasattr(ml, 'visit_topdown')):
ml.visit_topdown(tree)
else:
ml.visit(tree)
# Check that all build arguments were consumed.
if (len(cli.build_arg) != 0):
ch.FATAL("--build-arg: not consumed: " +
" ".join(cli.build_arg.keys()))
# Print summary & we're done.
if (ml.instruction_ct == 0):
ch.FATAL("no instructions found: %s" % cli.file)
assert (image_i + 1 == image_ct) # should have errored already if not
if (cli.force):
if (fakeroot_config.inject_ct == 0):
assert (not fakeroot_config.init_done)
ch.WARNING("--force specified, but nothing to do")
else:
ch.INFO("--force: init OK & modified %d RUN instructions" %
fakeroot_config.inject_ct)
ch.INFO("grown in %d instructions: %s" %
(ml.instruction_ct, images[image_i]))
def execute_(self):
if (cli.context == "-"):
ch.FATAL("can't COPY: no context because \"-\" given")
if (len(self.srcs) < 1):
ch.FATAL("can't COPY: must specify at least one source")
# Complain about unsupported stuff.
if (self.options.pop("chown", False)):
self.unsupported_forever_warn("--chown")
# Any remaining options are invalid.
self.options_assert_empty()
# Find the context directory.
if (self.from_ is None):
context = cli.context
else:
if (self.from_ == image_i or self.from_ == image_alias):
ch.FATAL("COPY --from: stage %s is the current stage" %
self.from_)
if (not self.from_ in images):
# FIXME: Would be nice to also report if a named stage is below.
if (isinstance(self.from_, int) and self.from_ < image_ct):
if (self.from_ < 0):
ch.FATAL(
"COPY --from: invalid negative stage index %d" %
self.from_)
else:
ch.FATAL("COPY --from: stage %d does not exist yet" %
self.from_)
else:
ch.FATAL("COPY --from: stage %s does not exist" %
self.from_)
context = images[self.from_].unpack_path
context_canon = os.path.realpath(context)
ch.VERBOSE("context: %s" % context)
# Expand source wildcards.
srcs = list()
for src in self.srcs:
matches = glob.glob("%s/%s" %
(context, src)) # glob can't take Path
if (len(matches) == 0):
ch.FATAL("can't copy: not found: %s" % src)
for i in matches:
srcs.append(i)
ch.VERBOSE("source: %s" % i)
# Validate sources are within context directory. (Can't convert to
# canonical paths yet because we need the source path as given.)
for src in srcs:
src_canon = os.path.realpath(src)
if (not os.path.commonpath([src_canon, context_canon
]).startswith(context_canon)):
ch.FATAL("can't COPY from outside context: %s" % src)
# Locate the destination.
unpack_canon = os.path.realpath(images[image_i].unpack_path)
if (self.dst.startswith("/")):
dst = ch.Path(self.dst)
else:
dst = env.workdir // self.dst
ch.VERBOSE("destination, as given: %s" % dst)
dst_canon = self.dest_realpath(unpack_canon,
dst) # strips trailing slash
ch.VERBOSE("destination, canonical: %s" % dst_canon)
if (not os.path.commonpath([dst_canon, unpack_canon
]).startswith(unpack_canon)):
ch.FATAL("can't COPY: destination not in image: %s" % dst_canon)
# Create the destination directory if needed.
if (self.dst.endswith("/") or len(srcs) > 1 or os.path.isdir(srcs[0])):
if (not os.path.exists(dst_canon)):
ch.mkdirs(dst_canon)
elif (not os.path.isdir(dst_canon)): # not symlink b/c realpath()
ch.FATAL("can't COPY: not a directory: %s" % dst_canon)
# Copy each source.
for src in srcs:
if (os.path.isfile(src)):
self.copy_src_file(src, dst_canon)
elif (os.path.isdir(src)):
self.copy_src_dir(src, dst_canon)
else:
ch.FATAL("can't COPY: unknown file type: %s" % src)
