def manifest_load(self): """Parse the manifest file and set self.config_hash and self.layer_hashes.""" def bad_key(key): ch.FATAL("manifest: %s: no key: %s" % (self.manifest_path, key)) # read and parse the JSON fp = ch.open_(self.manifest_path, "rt", encoding="UTF-8") text = ch.ossafe(fp.read, "can't read: %s" % self.manifest_path) ch.ossafe(fp.close, "can't close: %s" % self.manifest_path) ch.DEBUG("manifest:\n%s" % text) try: manifest = json.loads(text) except json.JSONDecodeError as x: ch.FATAL("can't parse manifest file: %s:%d: %s" % (self.manifest_path, x.lineno, x.msg)) # validate schema version try: version = manifest['schemaVersion'] except KeyError: bad_key("schemaVersion") if (version not in {1,2}): ch.FATAL("unsupported manifest schema version: %s" % repr(version)) # load config hash # # FIXME: Manifest version 1 does not list a config blob. It does have # things (plural) that look like a config at history/v1Compatibility as # an embedded JSON string :P but I haven't dug into it. if (version == 1): ch.WARNING("no config; manifest schema version 1") self.config_hash = None else: # version == 2 try: self.config_hash = ch.digest_trim(manifest["config"]["digest"]) except KeyError: bad_key("config/digest") # load layer hashes if (version == 1): key1 = "fsLayers" key2 = "blobSum" else: # version == 2 key1 = "layers" key2 = "digest" if (key1 not in manifest): bad_key(key1) self.layer_hashes = list() for i in manifest[key1]: if (key2 not in i): bad_key("%s/%s" % (key1, key2)) self.layer_hashes.append(ch.digest_trim(i[key2])) if (version == 1): self.layer_hashes.reverse()
def list_(cli): ch.dependencies_check() imgdir = ch.storage.unpack_base if (cli.image_ref is None): # list all images if (not os.path.isdir(ch.storage.root)): ch.INFO("does not exist: %s" % ch.storage.root) return if (not ch.storage.valid_p()): ch.INFO("not a storage directory: %s" % ch.storage.root) return imgs = ch.ossafe(os.listdir, "can't list directory: %s" % ch.storage.root, imgdir) for img in sorted(imgs): print(ch.Image_Ref(img)) else: # list specified image img = ch.Image(ch.Image_Ref(cli.image_ref)) print("details of image: %s" % img.ref) # present locally? if (not img.unpack_exist_p): stored = "no" else: img.metadata_load() stored = "yes (%s)" % img.metadata["arch"] print("in local storage: %s" % stored) # present remotely? print("full remote ref: %s" % img.ref.canonical) pullet = pull.Image_Puller(img, not cli.no_cache) pullet.fatman_load() if (pullet.architectures is not None): remote = "yes" arch_aware = "yes" arch_avail = " ".join(sorted(pullet.architectures.keys())) else: pullet.manifest_load(True) if (pullet.layer_hashes is not None): remote = "yes" arch_aware = "no" arch_avail = "unknown" else: remote = "no" arch_aware = "n/a" arch_avail = "n/a" pullet.done() print("available remotely: %s" % remote) print("remote arch-aware: %s" % arch_aware) print("host architecture: %s" % ch.arch_host) print("archs available: %s" % arch_avail)
def main(cli_): # CLI namespace. :P global cli cli = cli_ # Infer input file if needed. if (cli.file is None): cli.file = cli.context + "/Dockerfile" # Infer image name if needed. if (cli.tag is None): m = re.search(r"(([^/]+)/)?Dockerfile(\.(.+))?$", os.path.abspath(cli.file)) if (m is not None): if m.group(4): # extension cli.tag = m.group(4) elif m.group(2): # containing directory cli.tag = m.group(2) # Deal with build arguments. def build_arg_get(arg): kv = arg.split("=") if (len(kv) == 2): return kv else: v = os.getenv(kv[0]) if (v is None): ch.FATAL("--build-arg: %s: no value and not in environment" % kv[0]) return (kv[0], v) if (cli.build_arg is None): cli.build_arg = list() cli.build_arg = dict( build_arg_get(i) for i in cli.build_arg ) # Finish CLI initialization. ch.DEBUG(cli) ch.dependencies_check() # Guess whether the context is a URL, and error out if so. This can be a # typical looking URL e.g. "https://..." or also something like # "[email protected]:...". The line noise in the second line of the regex is # to match this second form. Username and host characters from # https://tools.ietf.org/html/rfc3986. if (re.search(r""" ^((git|git+ssh|http|https|ssh):// | ^[\w.~%!$&'\(\)\*\+,;=-]+@[\w.~%!$&'\(\)\*\+,;=-]+:)""", cli.context, re.VERBOSE) is not None): ch.FATAL("not yet supported: issue #773: URL context: %s" % cli.context) if (os.path.exists(cli.context + "/.dockerignore")): ch.WARNING("not yet supported, ignored: issue #777: .dockerignore file") # Set up build environment. global env env = Environment() # Read input file. if (cli.file == "-"): text = ch.ossafe(sys.stdin.read, "can't read stdin") else: fp = ch.open_(cli.file, "rt") text = ch.ossafe(fp.read, "can't read: %s" % cli.file) fp.close() # Parse it. parser = lark.Lark("?start: dockerfile\n" + ch.GRAMMAR, parser="earley", propagate_positions=True) # Avoid Lark issue #237: lark.exceptions.UnexpectedEOF if the file does not # end in newline. text += "\n" try: tree = parser.parse(text) except lark.exceptions.UnexpectedInput as x: ch.DEBUG(x) # noise about what was expected in the grammar ch.FATAL("can't parse: %s:%d,%d\n\n%s" % (cli.file, x.line, x.column, x.get_context(text, 39))) ch.DEBUG(tree.pretty()) # Sometimes we exit after parsing. if (cli.parse_only): sys.exit(0) # Count the number of stages (i.e., FROM instructions) global image_ct image_ct = sum(1 for i in ch.tree_children(tree, "from_")) # Traverse the tree and do what it says. # # We don't actually care whether the tree is traversed breadth-first or # depth-first, but we *do* care that instruction nodes are visited in # order. Neither visit() nor visit_topdown() are documented as of # 2020-06-11 [1], but examining source code [2] shows that visit_topdown() # uses Tree.iter_trees_topdown(), which *is* documented to be in-order [3]. # # This change seems to have been made in 0.8.6 (see PR #761); before then, # visit() was in order. Therefore, we call that instead, if visit_topdown() # is not present, to improve compatibility (see issue #792). # # [1]: https://lark-parser.readthedocs.io/en/latest/visitors/#visitors # [2]: https://github.com/lark-parser/lark/blob/445c8d4/lark/visitors.py#L211 # [3]: https://lark-parser.readthedocs.io/en/latest/classes/#tree ml = Main_Loop() if (hasattr(ml, 'visit_topdown')): ml.visit_topdown(tree) else: ml.visit(tree) # Check that all build arguments were consumed. if (len(cli.build_arg) != 0): ch.FATAL("--build-arg: not consumed: " + " ".join(cli.build_arg.keys())) # Print summary & we're done. if (ml.instruction_ct == 0): ch.FATAL("no instructions found: %s" % cli.file) assert (image_i + 1 == image_ct) # should have errored already if not ch.INFO("grown in %d instructions: %s" % (ml.instruction_ct, images[image_i]))
def execute_(self): # Complain about unsupported stuff. if (self.options.pop("chown", False)): self.unsupported_forever_warn("--chown") # Any remaining options are invalid. self.options_assert_empty() # Find the source directory. if (self.from_ is None): context = cli.context else: if (self.from_ == image_i or self.from_ == image_alias): ch.FATAL("COPY --from: stage %s is the current stage" % self.from_) if (not self.from_ in images): # FIXME: Would be nice to also report if a named stage is below. if (isinstance(self.from_, int) and self.from_ < image_ct): if (self.from_ < 0): ch.FATAL("COPY --from: invalid negative stage index %d" % self.from_) else: ch.FATAL("COPY --from: stage %d does not exist yet" % self.from_) else: ch.FATAL("COPY --from: stage %s does not exist" % self.from_) context = images[self.from_].unpack_path ch.DEBUG("context: " + context) # Do the copy. srcs = list() for src in self.srcs: if (os.path.normpath(src).startswith("..")): ch.FATAL("can't COPY: %s climbs outside context" % src) for i in glob.glob(context + "/" + src): srcs.append(i) if (len(srcs) == 0): ch.FATAL("can't COPY: no sources exist") dst = images[image_i].unpack_path + "/" if (not self.dst.startswith("/")): dst += env.workdir + "/" dst += self.dst if (dst.endswith("/") or len(srcs) > 1 or os.path.isdir(srcs[0])): # Create destination directory. if (dst.endswith("/")): dst = dst[:-1] if (os.path.exists(dst) and not os.path.isdir(dst)): ch.FATAL("can't COPY: %s exists but is not a directory" % dst) ch.mkdirs(dst) for src in srcs: # Check for symlinks to outside context. src_real = os.path.realpath(src) context_real = os.path.realpath(context) if (not os.path.commonpath([src_real, context_real]) \ .startswith(context_real)): ch.FATAL("can't COPY: %s climbs outside context via symlink" % src) # Do the copy. if (os.path.isfile(src)): # or symlink to file ch.DEBUG("COPY via copy2 file %s to %s" % (src, dst)) ch.copy2(src, dst, follow_symlinks=True) elif (os.path.isdir(src)): # or symlink to directory # Copy *contents* of src, not src itself. Note: shutil.copytree() # has a parameter dirs_exist_ok that I think will make this easier # in Python 3.8. ch.DEBUG("COPY dir %s to %s" % (src, dst)) if (not os.path.isdir(dst)): ch.FATAL("can't COPY: destination not a directory: %s to %s" % (src, dst)) for src2_basename in ch.ossafe( os.listdir, "can't list directory: %s" % src, src): src2 = src + "/" + src2_basename if (os.path.islink(src2)): # Symlinks within directories do not get dereferenced. ch.DEBUG("symlink via copy2: %s to %s" % (src2, dst)) ch.copy2(src2, dst, follow_symlinks=False) elif (os.path.isfile(src2)): # not symlink to file ch.DEBUG("file via copy2: %s to %s" % (src2, dst)) ch.copy2(src2, dst) elif (os.path.isdir(src2)): # not symlink to directory dst2 = dst + "/" + src2_basename ch.DEBUG("directory via copytree: %s to %s" % (src2, dst2)) ch.copytree(src2, dst2, symlinks=True, ignore_dangling_symlinks=True) else: ch.FATAL("can't COPY unknown file type: %s" % src2) else: ch.FATAL("can't COPY unknown file type: %s" % src)
def list_(cli): ch.dependencies_check() imgdir = ch.storage.unpack_base imgs = ch.ossafe(os.listdir, "can't list directory: %s" % imgdir, imgdir) for img in sorted(imgs): print(ch.Image_Ref(img))
def copy_src_dir(self, src, dst): """Copy the contents of directory src, named by COPY, either explicitly or with wildcards, to dst. src might be a symlink, but dst is a canonical path. Both must be at the top level of the COPY instruction; i.e., this function must not be called recursively. dst must exist already and be a directory. Unlike subdirectories, the metadata of dst will not be altered to match src.""" def onerror(x): ch.FATAL("can't scan directory: %s: %s" % (x.filename, x.strerror)) # Use Path objects in this method because the path arithmetic was # getting too hard with strings. src = ch.Path(os.path.realpath(src)) dst = ch.Path(dst) assert (os.path.isdir(src) and not os.path.islink(src)) assert (os.path.isdir(dst) and not os.path.islink(dst)) ch.DEBUG("copying named directory: %s -> %s" % (src, dst)) for (dirpath, dirnames, filenames) in os.walk(src, onerror=onerror): dirpath = ch.Path(dirpath) subdir = dirpath.relative_to(src) dst_dir = dst // subdir # dirnames can contain symlinks, which we handle as files, so we'll # rebuild it; the walk will not descend into those "directories". dirnames2 = dirnames.copy() # shallow copy dirnames[:] = list() # clear in place for d in dirnames2: d = ch.Path(d) src_path = dirpath // d dst_path = dst_dir // d ch.TRACE("dir: %s -> %s" % (src_path, dst_path)) if (os.path.islink(src_path)): filenames.append(d) # symlink, handle as file ch.TRACE("symlink to dir, will handle as file") continue else: dirnames.append(d) # directory, descend into later # If destination exists, but isn't a directory, remove it. if (os.path.exists(dst_path)): if (os.path.isdir(dst_path) and not os.path.islink(dst_path)): ch.TRACE("dst_path exists and is a directory") else: ch.TRACE("dst_path exists, not a directory, removing") ch.unlink(dst_path) # If destination directory doesn't exist, create it. if (not os.path.exists(dst_path)): ch.TRACE("mkdir dst_path") ch.ossafe(os.mkdir, "can't mkdir: %s" % dst_path, dst_path) # Copy metadata, now that we know the destination exists and is a # directory. ch.ossafe(shutil.copystat, "can't copy metadata: %s -> %s" % (src_path, dst_path), src_path, dst_path, follow_symlinks=False) for f in filenames: f = ch.Path(f) src_path = dirpath // f dst_path = dst_dir // f ch.TRACE("file or symlink via copy2: %s -> %s" % (src_path, dst_path)) if (not (os.path.isfile(src_path) or os.path.islink(src_path))): ch.FATAL("can't COPY: unknown file type: %s" % src_path) if (os.path.exists(dst_path)): ch.TRACE("destination exists, removing") if (os.path.isdir(dst_path) and not os.path.islink(dst_path)): ch.rmtree(dst_path) else: ch.unlink(dst_path) ch.copy2(src_path, dst_path, follow_symlinks=False)