def execute_(self):
rootfs = images[image_i].unpack_path
fakeroot_config.init_maybe(rootfs, self.cmd, env.env_build)
cmd = fakeroot_config.inject_run(self.cmd)
exit_code = ch.ch_run_modify(rootfs,
cmd,
env.env_build,
env.workdir,
cli.bind,
fail_ok=True)
if (exit_code != 0):
msg = "build failed: RUN command exited with %d" % exit_code
if (cli.force):
if (isinstance(fakeroot_config, fakeroot.Fakeroot_Noop)):
ch.FATAL(
msg, "--force specified, but no suitable config found")
else:
ch.FATAL(
msg) # --force inited OK but the build still failed
elif (not cli.no_force_detect):
if (fakeroot_config.init_done):
ch.FATAL(msg, "--force may fix it")
else:
ch.FATAL(msg, "current version of --force wouldn't help")
assert False, "unreachable code reached"
def fatman_load(self):
"""Load the fat manifest JSON file, downloading it first if needed. If
the image has a fat manifest, populate self.architectures; this may
be an empty dictionary if no valid architectures were found.
Raises:
* Not_In_Registry_Error if the image does not exist.
* No_Fatman_Error if the image exists but has no fat manifest,
i.e., is architecture-unaware. In this case self.architectures is
set to None."""
self.architectures = None
if (str(self.image.ref) in manifests_internal):
# cheat; internal manifest library matches every architecture
self.architectures = {ch.arch_host: None}
return
if (os.path.exists(self.fatman_path) and self.use_cache):
ch.INFO("manifest list: using existing file")
else:
# raises Not_In_Registry_Error if needed
self.registry.fatman_to_file(self.fatman_path,
"manifest list: downloading")
fm = ch.json_from_file(self.fatman_path, "fat manifest")
if ("layers" in fm or "fsLayers" in fm):
# FIXME (issue #1101): If it's a v2 manifest we could use it instead
# of re-requesting later. Maybe we could here move/copy it over to
# the skinny manifest path.
raise ch.No_Fatman_Error()
if ("errors" in fm):
# fm is an error blob.
(code, msg) = self.error_decode(fm)
if (code == "MANIFEST_UNKNOWN"):
ch.INFO("manifest list: no such image")
return
else:
ch.FATAL("manifest list: error: %s" % msg)
self.architectures = dict()
if ("manifests" not in fm):
ch.FATAL("manifest list has no key 'manifests'")
for m in fm["manifests"]:
try:
if (m["platform"]["os"] != "linux"):
continue
arch = m["platform"]["architecture"]
if ("variant" in m["platform"]):
arch = "%s/%s" % (arch, m["platform"]["variant"])
digest = m["digest"]
except KeyError:
ch.FATAL("manifest lists missing a required key")
if (arch in self.architectures):
ch.FATAL("manifest list: duplicate architecture: %s" % arch)
self.architectures[arch] = ch.digest_trim(digest)
if (len(self.architectures) == 0):
ch.WARNING("no valid architectures found")
def fatman_load(self):
"""Load the fat manifest JSON file, downloading it first if needed. If
the image has a fat manifest, populate self.architectures; this may
be an empty dictionary if no valid architectures were found.
It is not an error if the image has no fat manifest or the registry
reports no such image. In this architecture-unaware condition, set
self.architectures to None."""
self.architectures = None
if (str(self.image.ref) in manifests_internal):
return # no fat manifests for internal library
if (os.path.exists(self.fatman_path) and self.use_cache):
ch.INFO("manifest list: using existing file")
else:
ch.INFO("manifest list: downloading")
self.registry.fatman_to_file(self.fatman_path, True)
if (not os.path.exists(self.fatman_path)):
# Response was 404 (or equivalent).
ch.INFO("manifest list: no list found")
return
fm = ch.json_from_file(self.fatman_path, "fat manifest")
if ("layers" in fm or "fsLayers" in fm):
# If there is no fat manifest but the image exists, we get a skinny
# manifest instead. We can't use it, however, because it might be a
# v1 manifest when a v2 is available. ¯\_(ツ)_/¯
ch.INFO("manifest list: no valid list found")
return
if ("errors" in fm):
# fm is an error blob.
(code, msg) = self.error_decode(fm)
if (code == "MANIFEST_UNKNOWN"):
ch.INFO("manifest list: no such image")
return
else:
ch.FATAL("manifest list: error: %s" % msg)
self.architectures = dict()
if ("manifests" not in fm):
ch.FATAL("manifest list has no key 'manifests'")
for m in fm["manifests"]:
try:
if (m["platform"]["os"] != "linux"):
continue
arch = m["platform"]["architecture"]
if ("variant" in m["platform"]):
arch = "%s/%s" % (arch, m["platform"]["variant"])
digest = m["digest"]
except KeyError:
ch.FATAL("manifest lists missing a required key")
if (arch in self.architectures):
ch.FATAL("manifest list: duplicate architecture: %s" % arch)
self.architectures[arch] = ch.digest_trim(digest)
if (len(self.architectures) == 0):
ch.WARNING("no valid architectures found")
def manifest_load(self):
"""Parse the manifest file and set self.config_hash and
self.layer_hashes."""
def bad_key(key):
ch.FATAL("manifest: %s: no key: %s" % (self.manifest_path, key))
# read and parse the JSON
fp = ch.open_(self.manifest_path, "rt", encoding="UTF-8")
text = ch.ossafe(fp.read, "can't read: %s" % self.manifest_path)
ch.ossafe(fp.close, "can't close: %s" % self.manifest_path)
ch.DEBUG("manifest:\n%s" % text)
try:
manifest = json.loads(text)
except json.JSONDecodeError as x:
ch.FATAL("can't parse manifest file: %s:%d: %s"
% (self.manifest_path, x.lineno, x.msg))
# validate schema version
try:
version = manifest['schemaVersion']
except KeyError:
bad_key("schemaVersion")
if (version not in {1,2}):
ch.FATAL("unsupported manifest schema version: %s" % repr(version))
# load config hash
#
# FIXME: Manifest version 1 does not list a config blob. It does have
# things (plural) that look like a config at history/v1Compatibility as
# an embedded JSON string :P but I haven't dug into it.
if (version == 1):
ch.WARNING("no config; manifest schema version 1")
self.config_hash = None
else: # version == 2
try:
self.config_hash = ch.digest_trim(manifest["config"]["digest"])
except KeyError:
bad_key("config/digest")
# load layer hashes
if (version == 1):
key1 = "fsLayers"
key2 = "blobSum"
else: # version == 2
key1 = "layers"
key2 = "digest"
if (key1 not in manifest):
bad_key(key1)
self.layer_hashes = list()
for i in manifest[key1]:
if (key2 not in i):
bad_key("%s/%s" % (key1, key2))
self.layer_hashes.append(ch.digest_trim(i[key2]))
if (version == 1):
self.layer_hashes.reverse()
def download(self):
"Download image metadata and layers and put them in the download cache."
# Spec: https://docs.docker.com/registry/spec/manifest-v2-2/
ch.VERBOSE("downloading image: %s" % self.image)
try:
# fat manifest
if (ch.arch != "yolo"):
try:
self.fatman_load()
if (ch.arch not in self.architectures):
ch.FATAL("requested arch unavailable: %s" % ch.arch,
("available: %s" %
" ".join(sorted(self.architectures.keys()))))
except ch.No_Fatman_Error:
if (ch.arch == "amd64"):
# We're guessing that enough arch-unaware images are amd64 to
# barge ahead if requested architecture is amd64.
ch.arch = "yolo"
ch.WARNING("image is architecture-unaware")
ch.WARNING(
"requested arch is amd64; using --arch=yolo")
else:
ch.FATAL("image is architecture-unaware",
"consider --arch=yolo")
# manifest
self.manifest_load()
except ch.Not_In_Registry_Error:
ch.FATAL("not in registry: %s" % self.registry.ref)
# config
ch.VERBOSE("config path: %s" % self.config_path)
if (self.config_path is not None):
if (os.path.exists(self.config_path) and self.use_cache):
ch.INFO("config: using existing file")
else:
self.registry.blob_to_file(self.config_hash, self.config_path,
"config: downloading")
# layers
for (i, lh) in enumerate(self.layer_hashes, start=1):
path = self.layer_path(lh)
ch.VERBOSE("layer path: %s" % path)
msg = "layer %d/%d: %s" % (i, len(self.layer_hashes), lh[:7])
if (os.path.exists(path) and self.use_cache):
ch.INFO("%s: using existing file" % msg)
else:
self.registry.blob_to_file(lh, path, "%s: downloading" % msg)
# done
self.registry.close()
def execute_(self):
# Complain about unsupported stuff.
if (self.options.pop("platform", False)):
self.unsupported_yet_fatal("--platform", 778)
# Any remaining options are invalid.
self.options_assert_empty()
# Update image globals.
global image_i
image_i += 1
global image_alias
image_alias = self.alias
if (image_i == image_ct - 1):
# Last image; use tag unchanged.
tag = cli.tag
elif (image_i > image_ct - 1):
# Too many images!
ch.FATAL("expected %d stages but found at least %d" %
(image_ct, image_i + 1))
else:
# Not last image; append stage index to tag.
tag = "%s/_stage%d" % (cli.tag, image_i)
image = ch.Image(ch.Image_Ref(tag))
images[image_i] = image
if (self.alias is not None):
images[self.alias] = image
ch.VERBOSE("image path: %s" % image.unpack_path)
# Other error checking.
if (str(image.ref) == str(self.base_ref)):
ch.FATAL("output image ref same as FROM: %s" % self.base_ref)
# Initialize image.
self.base_image = ch.Image(self.base_ref)
if (os.path.isdir(self.base_image.unpack_path)):
ch.VERBOSE("base image found: %s" % self.base_image.unpack_path)
else:
ch.VERBOSE("base image not found, pulling")
# a young hen, especially one less than one year old.
pullet = pull.Image_Puller(self.base_image, not cli.no_cache)
pullet.pull_to_unpacked()
pullet.done()
image.copy_unpacked(self.base_image)
image.metadata_load()
env.reset()
# Find fakeroot configuration, if any.
global fakeroot_config
fakeroot_config = fakeroot.detect(image.unpack_path, cli.force,
cli.no_force_detect)
def build_arg_get(arg):
kv = arg.split("=")
if (len(kv) == 2):
return kv
else:
v = os.getenv(kv[0])
if (v is None):
ch.FATAL("--build-arg: %s: no value and not in environment" % kv[0])
return (kv[0], v)
def error_decode(self, data):
"""Decode first error message in registry error blob and return a tuple
(code, message)."""
try:
code = data["errors"][0]["code"]
msg = data["errors"][0]["message"]
except (IndexError, KeyError):
ch.FATAL("malformed error data (yes this is ironic)")
return (code, msg)
def list_(cli):
imgdir = ch.storage.unpack_base
if (cli.image_ref is None):
# list all images
if (not os.path.isdir(ch.storage.root)):
ch.FATAL("does not exist: %s" % ch.storage.root)
if (not ch.storage.valid_p):
ch.FATAL("not a storage directory: %s" % ch.storage.root)
imgs = ch.ossafe(os.listdir,
"can't list directory: %s" % ch.storage.root, imgdir)
for img in sorted(imgs):
print(ch.Image_Ref(img))
else:
# list specified image
img = ch.Image(ch.Image_Ref(cli.image_ref))
print("details of image: %s" % img.ref)
# present locally?
if (not img.unpack_exist_p):
stored = "no"
else:
img.metadata_load()
stored = "yes (%s)" % img.metadata["arch"]
print("in local storage: %s" % stored)
# present remotely?
print("full remote ref: %s" % img.ref.canonical)
pullet = pull.Image_Puller(img, not cli.no_cache)
try:
pullet.fatman_load()
remote = "yes"
arch_aware = "yes"
arch_avail = " ".join(sorted(pullet.architectures.keys()))
except ch.Not_In_Registry_Error:
remote = "no"
arch_aware = "n/a"
arch_avail = "n/a"
except ch.No_Fatman_Error:
remote = "yes"
arch_aware = "no"
arch_avail = "unknown"
pullet.done()
print("available remotely: %s" % remote)
print("remote arch-aware: %s" % arch_aware)
print("host architecture: %s" % ch.arch_host)
print("archs available: %s" % arch_avail)
def unescape(sl):
# FIXME: This is also ugly and should go in the grammar.
#
# The Dockerfile spec does not precisely define string escaping, but I'm
# guessing it's the Go rules. You will note that we are using Python rules.
# This is wrong but close enough for now (see also gripe in previous
# paragraph).
if (not (sl.startswith('"') and sl.endswith('"'))):
ch.FATAL("string literal not quoted")
return ast.literal_eval(sl)
def download(self):
"Download image metadata and layers and put them in the download cache."
# Spec: https://docs.docker.com/registry/spec/manifest-v2-2/
ch.VERBOSE("downloading image: %s" % self.image)
ch.mkdirs(ch.storage.download_cache)
# fat manifest
if (ch.arch != "yolo"):
self.fatman_load()
if (self.architectures is not None):
if (ch.arch not in self.architectures):
ch.FATAL(
"requested arch unavailable: %s not one of: %s" %
(ch.arch, " ".join(sorted(self.architectures.keys()))))
elif (ch.arch == "amd64"):
# We're guessing that enough arch-unaware images are amd64 to
# barge ahead if requested architecture is amd64.
ch.arch = "yolo"
ch.WARNING("image is architecture-unaware")
ch.WARNING("requested arch is amd64; switching to --arch=yolo")
else:
ch.FATAL("image is architecture-unaware; try --arch=yolo (?)")
# manifest
self.manifest_load()
# config
ch.VERBOSE("config path: %s" % self.config_path)
if (self.config_path is not None):
if (os.path.exists(self.config_path) and self.use_cache):
ch.INFO("config: using existing file")
else:
ch.INFO("config: downloading")
self.registry.blob_to_file(self.config_hash, self.config_path)
# layers
for (i, lh) in enumerate(self.layer_hashes, start=1):
path = self.layer_path(lh)
ch.VERBOSE("layer path: %s" % path)
ch.INFO("layer %d/%d: %s: " % (i, len(self.layer_hashes), lh[:7]),
end="")
if (os.path.exists(path) and self.use_cache):
ch.INFO("using existing file")
else:
ch.INFO("downloading")
self.registry.blob_to_file(lh, path)
def execute_(self):
# Complain about unsupported stuff.
if (self.options.pop("platform", False)):
self.unsupported_yet_fatal("--platform", 778)
# Any remaining options are invalid.
self.options_assert_empty()
# Update image globals.
global image_i
image_i += 1
global image_alias
image_alias = self.alias
if (image_i == image_ct - 1):
# Last image; use tag unchanged.
tag = cli.tag
elif (image_i > image_ct - 1):
# Too many images!
ch.FATAL("expected %d stages but found at least %d" %
(image_ct, image_i + 1))
else:
# Not last image; append stage index to tag.
tag = "%s/_stage%d" % (cli.tag, image_i)
image = ch.Image(ch.Image_Ref(tag), cli.storage + "/dlcache",
cli.storage + "/img")
images[image_i] = image
if (self.alias is not None):
images[self.alias] = image
ch.DEBUG("image path: %s" % image.unpack_path)
# Other error checking.
if (str(image.ref) == str(self.base_ref)):
ch.FATAL("output image ref same as FROM: %s" % self.base_ref)
# Initialize image.
self.base_image = ch.Image(self.base_ref, image.download_cache,
image.unpack_dir)
if (not os.path.isdir(self.base_image.unpack_path)):
ch.DEBUG("image not found, pulling: %s" %
self.base_image.unpack_path)
self.base_image.pull_to_unpacked(fixup=True)
image.copy_unpacked(self.base_image)
env.reset()
# Inject fakeroot preparatory stuff if needed.
if (not cli.no_fakeroot):
fakeroot.inject_first(image.unpack_path, env.env_build)
def variables_sub(s, variables):
# FIXME: This should go in the grammar rather than being a regex kludge.
#
# Dockerfile spec does not say what to do if substituting a value that's
# not set. We ignore those subsitutions. This is probably wrong (the shell
# substitutes the empty string).
for (k, v) in variables.items():
# FIXME: remove when issue #774 is fixed
m = re.search(r"(?<!\\)\${.+?:[+-].+?}", s)
if (m is not None):
ch.FATAL("modifiers ${foo:+bar} and ${foo:-bar} not yet supported (issue #774)")
s = re.sub(r"(?<!\\)\${?%s}?" % k, v, s)
return s
def manifest_digest_by_arch(self):
"Return skinny manifest digest for target architecture."
fatman = ch.json_from_file(self.fat_manifest_path)
arch = None
digest = None
variant = None
try:
arch, variant = ch.arch.split("/", maxsplit=1)
except ValueError:
arch = ch.arch
if ("manifests" not in fatman):
ch.FATAL("manifest list has no manifests")
for k in fatman["manifests"]:
if (k.get('platform').get('os') != 'linux'):
continue
elif (k.get('platform').get('architecture') == arch
and (variant is None
or k.get('platform').get('variant') == variant)):
digest = k.get('digest')
if (digest is None):
ch.FATAL('arch not found for image: %s; try "ch-image list"?' %
arch)
return digest
def main(cli):
src_ref = ch.Image_Ref(cli.source_ref)
ch.INFO("pushing image: %s" % src_ref)
image = ch.Image(src_ref, cli.image)
# FIXME: validate it's an image using Megan's new function (PR #908)
if (not os.path.isdir(image.unpack_path)):
if (cli.image is not None):
ch.FATAL("can't push: %s does not appear to be an image" %
cli.image)
else:
ch.FATAL("can't push: no image %s" % src_ref)
if (cli.image is not None):
ch.INFO("image path: %s" % image.unpack_path)
else:
ch.VERBOSE("image path: %s" % image.unpack_path)
if (cli.dest_ref is not None):
dst_ref = ch.Image_Ref(cli.dest_ref)
ch.INFO("destination: %s" % dst_ref)
else:
dst_ref = ch.Image_Ref(cli.source_ref)
up = Image_Pusher(image, dst_ref)
up.push()
ch.done_notify()
def import_(cli):
if (not os.path.exists(cli.path)):
ch.FATAL("can't copy: not found: %s" % cli.path)
dst = ch.Image(ch.Image_Ref(cli.image_ref))
ch.INFO("importing: %s" % cli.path)
ch.INFO("destination: %s" % dst)
if (os.path.isdir(cli.path)):
dst.copy_unpacked(cli.path)
else: # tarball, hopefully
dst.unpack([cli.path])
# initialize metadata if needed
dst.metadata_load()
dst.metadata_save()
ch.done_notify()
def __default__(self, tree):
class_ = "I_" + tree.data
if (class_ in globals()):
inst = globals()[class_](tree)
inst.announce()
if (self.instruction_ct == 0):
if ( isinstance(inst, I_directive)
or isinstance(inst, I_from_)):
pass
elif (isinstance(inst, Arg)):
ch.WARNING("ARG before FROM not yet supported; see issue #779")
else:
ch.FATAL("first instruction must be ARG or FROM")
inst.execute()
self.instruction_ct += inst.execute_increment
def __init__(self, tree):
self.lineno = tree.meta.line
self.options = {}
for st in ch.tree_children(tree, "option"):
k = ch.tree_terminal(st, "OPTION_KEY")
v = ch.tree_terminal(st, "OPTION_VALUE")
if (k in self.options):
ch.FATAL("%3d %s: repeated option --%s"
% (self.lineno, self.str_name(), k))
self.options[k] = v
# Save original options string because instructions pop() from the dict
# to process them.
self.options_str = " ".join("--%s=%s" % (k,v)
for (k,v) in self.options.items())
self.tree = tree
def dest_realpath(self, unpack_path, dst):
"""Return the canonicalized version of path dst within (canonical) image
path unpack_path. We can't use os.path.realpath() because if dst is
an absolute symlink, we need to use the *image's* root directory, not
the host. Thus, we have to resolve symlinks manually."""
unpack_path = ch.Path(unpack_path)
dst_canon = ch.Path(unpack_path)
dst = ch.Path(dst)
dst_parts = list(reversed(
dst.parts)) # easier to operate on end of list
iter_ct = 0
while (len(dst_parts) > 0):
iter_ct += 1
if (iter_ct > 100): # arbitrary
ch.FATAL("can't COPY: too many path components")
ch.TRACE("current destination: %d %s" % (iter_ct, dst_canon))
#ch.TRACE("parts remaining: %s" % dst_parts)
part = dst_parts.pop()
if (part == "/" or part == "//"): # 3 or more slashes yields "/"
ch.TRACE("skipping root")
continue
cand = dst_canon // part
ch.TRACE("checking: %s" % cand)
if (not cand.is_symlink()):
ch.TRACE("not symlink")
dst_canon = cand
else:
target = ch.Path(os.readlink(cand))
ch.TRACE("symlink to: %s" % target)
assert (len(target.parts) > 0) # POSIX says no empty symlinks
if (target.is_absolute()):
ch.TRACE("absolute")
dst_canon = ch.Path(unpack_path)
else:
ch.TRACE("relative")
dst_parts.extend(reversed(target.parts))
return dst_canon
def bad_key(key):
ch.FATAL("manifest: %s: no key: %s" % (self.manifest_path, key))
def copy_src_dir(self, src, dst):
"""Copy the contents of directory src, named by COPY, either explicitly
or with wildcards, to dst. src might be a symlink, but dst is a
canonical path. Both must be at the top level of the COPY
instruction; i.e., this function must not be called recursively. dst
must exist already and be a directory. Unlike subdirectories, the
metadata of dst will not be altered to match src."""
def onerror(x):
ch.FATAL("can't scan directory: %s: %s" % (x.filename, x.strerror))
# Use Path objects in this method because the path arithmetic was
# getting too hard with strings.
src = ch.Path(os.path.realpath(src))
dst = ch.Path(dst)
assert (os.path.isdir(src) and not os.path.islink(src))
assert (os.path.isdir(dst) and not os.path.islink(dst))
ch.DEBUG("copying named directory: %s -> %s" % (src, dst))
for (dirpath, dirnames, filenames) in os.walk(src, onerror=onerror):
dirpath = ch.Path(dirpath)
subdir = dirpath.relative_to(src)
dst_dir = dst // subdir
# dirnames can contain symlinks, which we handle as files, so we'll
# rebuild it; the walk will not descend into those "directories".
dirnames2 = dirnames.copy() # shallow copy
dirnames[:] = list() # clear in place
for d in dirnames2:
d = ch.Path(d)
src_path = dirpath // d
dst_path = dst_dir // d
ch.TRACE("dir: %s -> %s" % (src_path, dst_path))
if (os.path.islink(src_path)):
filenames.append(d) # symlink, handle as file
ch.TRACE("symlink to dir, will handle as file")
continue
else:
dirnames.append(d) # directory, descend into later
# If destination exists, but isn't a directory, remove it.
if (os.path.exists(dst_path)):
if (os.path.isdir(dst_path)
and not os.path.islink(dst_path)):
ch.TRACE("dst_path exists and is a directory")
else:
ch.TRACE("dst_path exists, not a directory, removing")
ch.unlink(dst_path)
# If destination directory doesn't exist, create it.
if (not os.path.exists(dst_path)):
ch.TRACE("mkdir dst_path")
ch.ossafe(os.mkdir, "can't mkdir: %s" % dst_path, dst_path)
# Copy metadata, now that we know the destination exists and is a
# directory.
ch.ossafe(shutil.copystat,
"can't copy metadata: %s -> %s" %
(src_path, dst_path),
src_path,
dst_path,
follow_symlinks=False)
for f in filenames:
f = ch.Path(f)
src_path = dirpath // f
dst_path = dst_dir // f
ch.TRACE("file or symlink via copy2: %s -> %s" %
(src_path, dst_path))
if (not (os.path.isfile(src_path)
or os.path.islink(src_path))):
ch.FATAL("can't COPY: unknown file type: %s" % src_path)
if (os.path.exists(dst_path)):
ch.TRACE("destination exists, removing")
if (os.path.isdir(dst_path)
and not os.path.islink(dst_path)):
ch.rmtree(dst_path)
else:
ch.unlink(dst_path)
ch.copy2(src_path, dst_path, follow_symlinks=False)
def onerror(x):
ch.FATAL("can't scan directory: %s: %s" % (x.filename, x.strerror))
def execute_(self):
# Complain about unsupported stuff.
if (self.options.pop("chown", False)):
self.unsupported_forever_warn("--chown")
# Any remaining options are invalid.
self.options_assert_empty()
# Find the source directory.
if (self.from_ is None):
context = cli.context
else:
if (self.from_ == image_i or self.from_ == image_alias):
ch.FATAL("COPY --from: stage %s is the current stage" % self.from_)
if (not self.from_ in images):
# FIXME: Would be nice to also report if a named stage is below.
if (isinstance(self.from_, int) and self.from_ < image_ct):
if (self.from_ < 0):
ch.FATAL("COPY --from: invalid negative stage index %d"
% self.from_)
else:
ch.FATAL("COPY --from: stage %d does not exist yet"
% self.from_)
else:
ch.FATAL("COPY --from: stage %s does not exist" % self.from_)
context = images[self.from_].unpack_path
ch.DEBUG("context: " + context)
# Do the copy.
srcs = list()
for src in self.srcs:
if (os.path.normpath(src).startswith("..")):
ch.FATAL("can't COPY: %s climbs outside context" % src)
for i in glob.glob(context + "/" + src):
srcs.append(i)
if (len(srcs) == 0):
ch.FATAL("can't COPY: no sources exist")
dst = images[image_i].unpack_path + "/"
if (not self.dst.startswith("/")):
dst += env.workdir + "/"
dst += self.dst
if (dst.endswith("/") or len(srcs) > 1 or os.path.isdir(srcs[0])):
# Create destination directory.
if (dst.endswith("/")):
dst = dst[:-1]
if (os.path.exists(dst) and not os.path.isdir(dst)):
ch.FATAL("can't COPY: %s exists but is not a directory" % dst)
ch.mkdirs(dst)
for src in srcs:
# Check for symlinks to outside context.
src_real = os.path.realpath(src)
context_real = os.path.realpath(context)
if (not os.path.commonpath([src_real, context_real]) \
.startswith(context_real)):
ch.FATAL("can't COPY: %s climbs outside context via symlink" % src)
# Do the copy.
if (os.path.isfile(src)): # or symlink to file
ch.DEBUG("COPY via copy2 file %s to %s" % (src, dst))
ch.copy2(src, dst, follow_symlinks=True)
elif (os.path.isdir(src)): # or symlink to directory
# Copy *contents* of src, not src itself. Note: shutil.copytree()
# has a parameter dirs_exist_ok that I think will make this easier
# in Python 3.8.
ch.DEBUG("COPY dir %s to %s" % (src, dst))
if (not os.path.isdir(dst)):
ch.FATAL("can't COPY: destination not a directory: %s to %s"
% (src, dst))
for src2_basename in ch.ossafe(
os.listdir, "can't list directory: %s" % src, src):
src2 = src + "/" + src2_basename
if (os.path.islink(src2)):
# Symlinks within directories do not get dereferenced.
ch.DEBUG("symlink via copy2: %s to %s" % (src2, dst))
ch.copy2(src2, dst, follow_symlinks=False)
elif (os.path.isfile(src2)): # not symlink to file
ch.DEBUG("file via copy2: %s to %s" % (src2, dst))
ch.copy2(src2, dst)
elif (os.path.isdir(src2)): # not symlink to directory
dst2 = dst + "/" + src2_basename
ch.DEBUG("directory via copytree: %s to %s" % (src2, dst2))
ch.copytree(src2, dst2, symlinks=True,
ignore_dangling_symlinks=True)
else:
ch.FATAL("can't COPY unknown file type: %s" % src2)
else:
ch.FATAL("can't COPY unknown file type: %s" % src)
def main(cli_):
# CLI namespace. :P
global cli
cli = cli_
# Infer input file if needed.
if (cli.file is None):
cli.file = cli.context + "/Dockerfile"
# Infer image name if needed.
if (cli.tag is None):
m = re.search(r"(([^/]+)/)?Dockerfile(\.(.+))?$",
os.path.abspath(cli.file))
if (m is not None):
if m.group(4): # extension
cli.tag = m.group(4)
elif m.group(2): # containing directory
cli.tag = m.group(2)
# Deal with build arguments.
def build_arg_get(arg):
kv = arg.split("=")
if (len(kv) == 2):
return kv
else:
v = os.getenv(kv[0])
if (v is None):
ch.FATAL("--build-arg: %s: no value and not in environment" % kv[0])
return (kv[0], v)
if (cli.build_arg is None):
cli.build_arg = list()
cli.build_arg = dict( build_arg_get(i) for i in cli.build_arg )
# Finish CLI initialization.
ch.DEBUG(cli)
ch.dependencies_check()
# Guess whether the context is a URL, and error out if so. This can be a
# typical looking URL e.g. "https://..." or also something like
# "[email protected]:...". The line noise in the second line of the regex is
# to match this second form. Username and host characters from
# https://tools.ietf.org/html/rfc3986.
if (re.search(r""" ^((git|git+ssh|http|https|ssh)://
| ^[\w.~%!$&'\(\)\*\+,;=-]+@[\w.~%!$&'\(\)\*\+,;=-]+:)""",
cli.context, re.VERBOSE) is not None):
ch.FATAL("not yet supported: issue #773: URL context: %s" % cli.context)
if (os.path.exists(cli.context + "/.dockerignore")):
ch.WARNING("not yet supported, ignored: issue #777: .dockerignore file")
# Set up build environment.
global env
env = Environment()
# Read input file.
if (cli.file == "-"):
text = ch.ossafe(sys.stdin.read, "can't read stdin")
else:
fp = ch.open_(cli.file, "rt")
text = ch.ossafe(fp.read, "can't read: %s" % cli.file)
fp.close()
# Parse it.
parser = lark.Lark("?start: dockerfile\n" + ch.GRAMMAR,
parser="earley", propagate_positions=True)
# Avoid Lark issue #237: lark.exceptions.UnexpectedEOF if the file does not
# end in newline.
text += "\n"
try:
tree = parser.parse(text)
except lark.exceptions.UnexpectedInput as x:
ch.DEBUG(x) # noise about what was expected in the grammar
ch.FATAL("can't parse: %s:%d,%d\n\n%s" % (cli.file, x.line, x.column, x.get_context(text, 39)))
ch.DEBUG(tree.pretty())
# Sometimes we exit after parsing.
if (cli.parse_only):
sys.exit(0)
# Count the number of stages (i.e., FROM instructions)
global image_ct
image_ct = sum(1 for i in ch.tree_children(tree, "from_"))
# Traverse the tree and do what it says.
#
# We don't actually care whether the tree is traversed breadth-first or
# depth-first, but we *do* care that instruction nodes are visited in
# order. Neither visit() nor visit_topdown() are documented as of
# 2020-06-11 [1], but examining source code [2] shows that visit_topdown()
# uses Tree.iter_trees_topdown(), which *is* documented to be in-order [3].
#
# This change seems to have been made in 0.8.6 (see PR #761); before then,
# visit() was in order. Therefore, we call that instead, if visit_topdown()
# is not present, to improve compatibility (see issue #792).
#
# [1]: https://lark-parser.readthedocs.io/en/latest/visitors/#visitors
# [2]: https://github.com/lark-parser/lark/blob/445c8d4/lark/visitors.py#L211
# [3]: https://lark-parser.readthedocs.io/en/latest/classes/#tree
ml = Main_Loop()
if (hasattr(ml, 'visit_topdown')):
ml.visit_topdown(tree)
else:
ml.visit(tree)
# Check that all build arguments were consumed.
if (len(cli.build_arg) != 0):
ch.FATAL("--build-arg: not consumed: " + " ".join(cli.build_arg.keys()))
# Print summary & we're done.
if (ml.instruction_ct == 0):
ch.FATAL("no instructions found: %s" % cli.file)
assert (image_i + 1 == image_ct) # should have errored already if not
ch.INFO("grown in %d instructions: %s"
% (ml.instruction_ct, images[image_i]))
def unsupported_yet_fatal(self, msg, issue_no):
ch.FATAL("not yet supported: issue #%d: %s %s"
% (issue_no, self.str_name(), msg))
def options_assert_empty(self):
try:
k = next(iter(self.options.keys()))
ch.FATAL("%s: invalid option --%s" % (self.str_name(), k))
except StopIteration:
pass
def onerror(x):
ch.FATAL("error scanning directory: %s: %s" %
(x.filename, x.strerror))
def manifest_load(self, continue_404=False):
"""Download the manifest file if needed, parse it, and set
self.config_hash and self.layer_hashes. By default, if the image does
not exist, exit with error; if continue_404, then log the condition
but do not exit. In this case, self.config_hash and self.layer_hashes
will both be None."""
def bad_key(key):
ch.FATAL("manifest: %s: no key: %s" % (self.manifest_path, key))
self.config_hash = None
self.layer_hashes = None
# obtain the manifest
try:
# internal manifest library, e.g. for "FROM scratch"
manifest = manifests_internal[str(self.image.ref)]
ch.INFO("manifest: using internal library")
except KeyError:
# download the file if needed, then parse it
if (ch.arch == "yolo" or self.architectures is None):
digest = None
else:
digest = self.architectures[ch.arch]
ch.DEBUG("manifest digest: %s" % digest)
if (os.path.exists(self.manifest_path) and self.use_cache):
ch.INFO("manifest: using existing file")
else:
ch.INFO("manifest: downloading")
self.registry.manifest_to_file(self.manifest_path,
digest=digest,
continue_404=continue_404)
if (not os.path.exists(self.manifest_path)):
# response was 404 (or equivalent)
ch.INFO("manifest: none found")
return
manifest = ch.json_from_file(self.manifest_path, "manifest")
# validate schema version
try:
version = manifest['schemaVersion']
except KeyError:
bad_key("schemaVersion")
if (version not in {1, 2}):
ch.FATAL("unsupported manifest schema version: %s" % repr(version))
# load config hash
#
# FIXME: Manifest version 1 does not list a config blob. It does have
# things (plural) that look like a config at history/v1Compatibility as
# an embedded JSON string :P but I haven't dug into it.
if (version == 1):
ch.VERBOSE("no config; manifest schema version 1")
self.config_hash = None
else: # version == 2
try:
self.config_hash = manifest["config"]["digest"]
if (self.config_hash is not None):
self.config_hash = ch.digest_trim(self.config_hash)
except KeyError:
bad_key("config/digest")
# load layer hashes
if (version == 1):
key1 = "fsLayers"
key2 = "blobSum"
else: # version == 2
key1 = "layers"
key2 = "digest"
if (key1 not in manifest):
bad_key(key1)
self.layer_hashes = list()
for i in manifest[key1]:
if (key2 not in i):
bad_key("%s/%s" % (key1, key2))
self.layer_hashes.append(ch.digest_trim(i[key2]))
if (version == 1):
self.layer_hashes.reverse()
def execute_(self):
if (cli.context == "-"):
ch.FATAL("can't COPY: no context because \"-\" given")
if (len(self.srcs) < 1):
ch.FATAL("can't COPY: must specify at least one source")
# Complain about unsupported stuff.
if (self.options.pop("chown", False)):
self.unsupported_forever_warn("--chown")
# Any remaining options are invalid.
self.options_assert_empty()
# Find the context directory.
if (self.from_ is None):
context = cli.context
else:
if (self.from_ == image_i or self.from_ == image_alias):
ch.FATAL("COPY --from: stage %s is the current stage" %
self.from_)
if (not self.from_ in images):
# FIXME: Would be nice to also report if a named stage is below.
if (isinstance(self.from_, int) and self.from_ < image_ct):
if (self.from_ < 0):
ch.FATAL(
"COPY --from: invalid negative stage index %d" %
self.from_)
else:
ch.FATAL("COPY --from: stage %d does not exist yet" %
self.from_)
else:
ch.FATAL("COPY --from: stage %s does not exist" %
self.from_)
context = images[self.from_].unpack_path
context_canon = os.path.realpath(context)
ch.VERBOSE("context: %s" % context)
# Expand source wildcards.
srcs = list()
for src in self.srcs:
matches = glob.glob("%s/%s" %
(context, src)) # glob can't take Path
if (len(matches) == 0):
ch.FATAL("can't copy: not found: %s" % src)
for i in matches:
srcs.append(i)
ch.VERBOSE("source: %s" % i)
# Validate sources are within context directory. (Can't convert to
# canonical paths yet because we need the source path as given.)
for src in srcs:
src_canon = os.path.realpath(src)
if (not os.path.commonpath([src_canon, context_canon
]).startswith(context_canon)):
ch.FATAL("can't COPY from outside context: %s" % src)
# Locate the destination.
unpack_canon = os.path.realpath(images[image_i].unpack_path)
if (self.dst.startswith("/")):
dst = ch.Path(self.dst)
else:
dst = env.workdir // self.dst
ch.VERBOSE("destination, as given: %s" % dst)
dst_canon = self.dest_realpath(unpack_canon,
dst) # strips trailing slash
ch.VERBOSE("destination, canonical: %s" % dst_canon)
if (not os.path.commonpath([dst_canon, unpack_canon
]).startswith(unpack_canon)):
ch.FATAL("can't COPY: destination not in image: %s" % dst_canon)
# Create the destination directory if needed.
if (self.dst.endswith("/") or len(srcs) > 1 or os.path.isdir(srcs[0])):
if (not os.path.exists(dst_canon)):
ch.mkdirs(dst_canon)
elif (not os.path.isdir(dst_canon)): # not symlink b/c realpath()
ch.FATAL("can't COPY: not a directory: %s" % dst_canon)
# Copy each source.
for src in srcs:
if (os.path.isfile(src)):
self.copy_src_file(src, dst_canon)
elif (os.path.isdir(src)):
self.copy_src_dir(src, dst_canon)
else:
ch.FATAL("can't COPY: unknown file type: %s" % src)
