def prepare(self):
"""Prepare self.image for pushing to self.dst_ref. Return tuple: (list
of gzipped layer tarball paths, config as a sequence of bytes,
manifest as a sequence of bytes).
There is not currently any support for re-using any previously
prepared files already in the upload cache, because we don't yet have
a way to know if these have changed until they are already build."""
ch.mkdirs(ch.storage.upload_cache)
tars_uc = self.image.tarballs_write(ch.storage.upload_cache)
tars_c = list()
config = self.config_new()
manifest = self.manifest_new()
# Prepare layers.
for (i, tar_uc) in enumerate(tars_uc, start=1):
ch.INFO("layer %d/%d: preparing" % (i, len(tars_uc)))
path_uc = ch.storage.upload_cache // tar_uc
hash_uc = ch.file_hash(path_uc)
config["rootfs"]["diff_ids"].append("sha256:" + hash_uc)
#size_uc = ch.file_size(path_uc)
path_c = ch.file_gzip(path_uc, ["-9", "--no-name"])
tar_c = path_c.name
hash_c = ch.file_hash(path_c)
size_c = ch.file_size(path_c)
tars_c.append((hash_c, path_c))
manifest["layers"].append({
"mediaType": ch.TYPE_LAYER,
"size": size_c,
"digest": "sha256:" + hash_c
})
# Prepare metadata.
ch.INFO("preparing metadata")
config_bytes = json.dumps(config, indent=2).encode("UTF-8")
config_hash = ch.bytes_hash(config_bytes)
manifest["config"]["size"] = len(config_bytes)
manifest["config"]["digest"] = "sha256:" + config_hash
ch.DEBUG("config: %s\n%s" %
(config_hash, config_bytes.decode("UTF-8")))
manifest_bytes = json.dumps(manifest, indent=2).encode("UTF-8")
ch.DEBUG("manifest:\n%s" % manifest_bytes.decode("UTF-8"))
# Store for the next steps.
self.layers = tars_c
self.config = config_bytes
self.manifest = manifest_bytes
def download(self):
"Download image metadata and layers and put them in the download cache."
# Spec: https://docs.docker.com/registry/spec/manifest-v2-2/
ch.VERBOSE("downloading image: %s" % self.image)
ch.mkdirs(ch.storage.download_cache)
# fat manifest
if (ch.arch != "yolo"):
self.fatman_load()
if (self.architectures is not None):
if (ch.arch not in self.architectures):
ch.FATAL(
"requested arch unavailable: %s not one of: %s" %
(ch.arch, " ".join(sorted(self.architectures.keys()))))
elif (ch.arch == "amd64"):
# We're guessing that enough arch-unaware images are amd64 to
# barge ahead if requested architecture is amd64.
ch.arch = "yolo"
ch.WARNING("image is architecture-unaware")
ch.WARNING("requested arch is amd64; switching to --arch=yolo")
else:
ch.FATAL("image is architecture-unaware; try --arch=yolo (?)")
# manifest
self.manifest_load()
# config
ch.VERBOSE("config path: %s" % self.config_path)
if (self.config_path is not None):
if (os.path.exists(self.config_path) and self.use_cache):
ch.INFO("config: using existing file")
else:
ch.INFO("config: downloading")
self.registry.blob_to_file(self.config_hash, self.config_path)
# layers
for (i, lh) in enumerate(self.layer_hashes, start=1):
path = self.layer_path(lh)
ch.VERBOSE("layer path: %s" % path)
ch.INFO("layer %d/%d: %s: " % (i, len(self.layer_hashes), lh[:7]),
end="")
if (os.path.exists(path) and self.use_cache):
ch.INFO("using existing file")
else:
ch.INFO("downloading")
self.registry.blob_to_file(lh, path)
def download(self, use_cache):
"""Download image metadata and layers and put them in the download
cache. If use_cache is True (the default), anything already in the
cache is skipped, otherwise download it anyway, overwriting what's in
the cache."""
# Spec: https://docs.docker.com/registry/spec/manifest-v2-2/
dl = ch.Registry_HTTP(self.image.ref)
ch.VERBOSE("downloading image: %s" % dl.ref)
ch.mkdirs(ch.storage.download_cache)
# manifest
if (os.path.exists(self.manifest_path) and use_cache):
ch.INFO("manifest: using existing file")
else:
ch.INFO("manifest: downloading")
dl.manifest_to_file(self.manifest_path)
self.manifest_load()
# config
ch.VERBOSE("config path: %s" % self.config_path)
if (self.config_path is not None):
if (os.path.exists(self.config_path) and use_cache):
ch.INFO("config: using existing file")
else:
ch.INFO("config: downloading")
dl.blob_to_file(self.config_hash, self.config_path)
# layers
for (i, lh) in enumerate(self.layer_hashes, start=1):
path = self.layer_path(lh)
ch.VERBOSE("layer path: %s" % path)
ch.INFO("layer %d/%d: %s: "% (i, len(self.layer_hashes), lh[:7]),
end="")
if (os.path.exists(path) and use_cache):
ch.INFO("using existing file")
else:
ch.INFO("downloading")
dl.blob_to_file(lh, path)
dl.close()
def execute_(self):
env.chdir(self.path)
ch.mkdirs(images[image_i].unpack_path + env.workdir)
def execute_(self):
# Complain about unsupported stuff.
if (self.options.pop("chown", False)):
self.unsupported_forever_warn("--chown")
# Any remaining options are invalid.
self.options_assert_empty()
# Find the source directory.
if (self.from_ is None):
context = cli.context
else:
if (self.from_ == image_i or self.from_ == image_alias):
ch.FATAL("COPY --from: stage %s is the current stage" % self.from_)
if (not self.from_ in images):
# FIXME: Would be nice to also report if a named stage is below.
if (isinstance(self.from_, int) and self.from_ < image_ct):
if (self.from_ < 0):
ch.FATAL("COPY --from: invalid negative stage index %d"
% self.from_)
else:
ch.FATAL("COPY --from: stage %d does not exist yet"
% self.from_)
else:
ch.FATAL("COPY --from: stage %s does not exist" % self.from_)
context = images[self.from_].unpack_path
ch.DEBUG("context: " + context)
# Do the copy.
srcs = list()
for src in self.srcs:
if (os.path.normpath(src).startswith("..")):
ch.FATAL("can't COPY: %s climbs outside context" % src)
for i in glob.glob(context + "/" + src):
srcs.append(i)
if (len(srcs) == 0):
ch.FATAL("can't COPY: no sources exist")
dst = images[image_i].unpack_path + "/"
if (not self.dst.startswith("/")):
dst += env.workdir + "/"
dst += self.dst
if (dst.endswith("/") or len(srcs) > 1 or os.path.isdir(srcs[0])):
# Create destination directory.
if (dst.endswith("/")):
dst = dst[:-1]
if (os.path.exists(dst) and not os.path.isdir(dst)):
ch.FATAL("can't COPY: %s exists but is not a directory" % dst)
ch.mkdirs(dst)
for src in srcs:
# Check for symlinks to outside context.
src_real = os.path.realpath(src)
context_real = os.path.realpath(context)
if (not os.path.commonpath([src_real, context_real]) \
.startswith(context_real)):
ch.FATAL("can't COPY: %s climbs outside context via symlink" % src)
# Do the copy.
if (os.path.isfile(src)): # or symlink to file
ch.DEBUG("COPY via copy2 file %s to %s" % (src, dst))
ch.copy2(src, dst, follow_symlinks=True)
elif (os.path.isdir(src)): # or symlink to directory
# Copy *contents* of src, not src itself. Note: shutil.copytree()
# has a parameter dirs_exist_ok that I think will make this easier
# in Python 3.8.
ch.DEBUG("COPY dir %s to %s" % (src, dst))
if (not os.path.isdir(dst)):
ch.FATAL("can't COPY: destination not a directory: %s to %s"
% (src, dst))
for src2_basename in ch.ossafe(
os.listdir, "can't list directory: %s" % src, src):
src2 = src + "/" + src2_basename
if (os.path.islink(src2)):
# Symlinks within directories do not get dereferenced.
ch.DEBUG("symlink via copy2: %s to %s" % (src2, dst))
ch.copy2(src2, dst, follow_symlinks=False)
elif (os.path.isfile(src2)): # not symlink to file
ch.DEBUG("file via copy2: %s to %s" % (src2, dst))
ch.copy2(src2, dst)
elif (os.path.isdir(src2)): # not symlink to directory
dst2 = dst + "/" + src2_basename
ch.DEBUG("directory via copytree: %s to %s" % (src2, dst2))
ch.copytree(src2, dst2, symlinks=True,
ignore_dangling_symlinks=True)
else:
ch.FATAL("can't COPY unknown file type: %s" % src2)
else:
ch.FATAL("can't COPY unknown file type: %s" % src)
def execute_(self):
if (cli.context == "-"):
ch.FATAL("can't COPY: no context because \"-\" given")
if (len(self.srcs) < 1):
ch.FATAL("can't COPY: must specify at least one source")
# Complain about unsupported stuff.
if (self.options.pop("chown", False)):
self.unsupported_forever_warn("--chown")
# Any remaining options are invalid.
self.options_assert_empty()
# Find the context directory.
if (self.from_ is None):
context = cli.context
else:
if (self.from_ == image_i or self.from_ == image_alias):
ch.FATAL("COPY --from: stage %s is the current stage" %
self.from_)
if (not self.from_ in images):
# FIXME: Would be nice to also report if a named stage is below.
if (isinstance(self.from_, int) and self.from_ < image_ct):
if (self.from_ < 0):
ch.FATAL(
"COPY --from: invalid negative stage index %d" %
self.from_)
else:
ch.FATAL("COPY --from: stage %d does not exist yet" %
self.from_)
else:
ch.FATAL("COPY --from: stage %s does not exist" %
self.from_)
context = images[self.from_].unpack_path
context_canon = os.path.realpath(context)
ch.VERBOSE("context: %s" % context)
# Expand source wildcards.
srcs = list()
for src in self.srcs:
matches = glob.glob("%s/%s" %
(context, src)) # glob can't take Path
if (len(matches) == 0):
ch.FATAL("can't copy: not found: %s" % src)
for i in matches:
srcs.append(i)
ch.VERBOSE("source: %s" % i)
# Validate sources are within context directory. (Can't convert to
# canonical paths yet because we need the source path as given.)
for src in srcs:
src_canon = os.path.realpath(src)
if (not os.path.commonpath([src_canon, context_canon
]).startswith(context_canon)):
ch.FATAL("can't COPY from outside context: %s" % src)
# Locate the destination.
unpack_canon = os.path.realpath(images[image_i].unpack_path)
if (self.dst.startswith("/")):
dst = ch.Path(self.dst)
else:
dst = env.workdir // self.dst
ch.VERBOSE("destination, as given: %s" % dst)
dst_canon = self.dest_realpath(unpack_canon,
dst) # strips trailing slash
ch.VERBOSE("destination, canonical: %s" % dst_canon)
if (not os.path.commonpath([dst_canon, unpack_canon
]).startswith(unpack_canon)):
ch.FATAL("can't COPY: destination not in image: %s" % dst_canon)
# Create the destination directory if needed.
if (self.dst.endswith("/") or len(srcs) > 1 or os.path.isdir(srcs[0])):
if (not os.path.exists(dst_canon)):
ch.mkdirs(dst_canon)
elif (not os.path.isdir(dst_canon)): # not symlink b/c realpath()
ch.FATAL("can't COPY: not a directory: %s" % dst_canon)
# Copy each source.
for src in srcs:
if (os.path.isfile(src)):
self.copy_src_file(src, dst_canon)
elif (os.path.isdir(src)):
self.copy_src_dir(src, dst_canon)
else:
ch.FATAL("can't COPY: unknown file type: %s" % src)