def detect(image, force, no_force_detect): f = None if (no_force_detect): ch.VERBOSE("not detecting --force config, per --no-force-detect") else: # Try to find a real fakeroot config. for (tag, cfg) in DEFAULT_CONFIGS.items(): try: f = Fakeroot(image, tag, cfg, force) break except Config_Aint_Matched: pass # Report findings. if (f is None): msg = "--force not available (no suitable config found)" if (force): ch.WARNING(msg) else: ch.VERBOSE(msg) else: if (force): adj = "will use" else: adj = "available" ch.INFO("%s --force: %s: %s" % (adj, f.tag, f.name)) # Wrap up if (f is None): f = Fakeroot_Noop() return f
def download(self): "Download image metadata and layers and put them in the download cache." # Spec: https://docs.docker.com/registry/spec/manifest-v2-2/ ch.VERBOSE("downloading image: %s" % self.image) try: # fat manifest if (ch.arch != "yolo"): try: self.fatman_load() if (ch.arch not in self.architectures): ch.FATAL("requested arch unavailable: %s" % ch.arch, ("available: %s" % " ".join(sorted(self.architectures.keys())))) except ch.No_Fatman_Error: if (ch.arch == "amd64"): # We're guessing that enough arch-unaware images are amd64 to # barge ahead if requested architecture is amd64. ch.arch = "yolo" ch.WARNING("image is architecture-unaware") ch.WARNING( "requested arch is amd64; using --arch=yolo") else: ch.FATAL("image is architecture-unaware", "consider --arch=yolo") # manifest self.manifest_load() except ch.Not_In_Registry_Error: ch.FATAL("not in registry: %s" % self.registry.ref) # config ch.VERBOSE("config path: %s" % self.config_path) if (self.config_path is not None): if (os.path.exists(self.config_path) and self.use_cache): ch.INFO("config: using existing file") else: self.registry.blob_to_file(self.config_hash, self.config_path, "config: downloading") # layers for (i, lh) in enumerate(self.layer_hashes, start=1): path = self.layer_path(lh) ch.VERBOSE("layer path: %s" % path) msg = "layer %d/%d: %s" % (i, len(self.layer_hashes), lh[:7]) if (os.path.exists(path) and self.use_cache): ch.INFO("%s: using existing file" % msg) else: self.registry.blob_to_file(lh, path, "%s: downloading" % msg) # done self.registry.close()
def fatman_load(self): """Load the fat manifest JSON file, downloading it first if needed. If the image has a fat manifest, populate self.architectures; this may be an empty dictionary if no valid architectures were found. Raises: * Not_In_Registry_Error if the image does not exist. * No_Fatman_Error if the image exists but has no fat manifest, i.e., is architecture-unaware. In this case self.architectures is set to None.""" self.architectures = None if (str(self.image.ref) in manifests_internal): # cheat; internal manifest library matches every architecture self.architectures = {ch.arch_host: None} return if (os.path.exists(self.fatman_path) and self.use_cache): ch.INFO("manifest list: using existing file") else: # raises Not_In_Registry_Error if needed self.registry.fatman_to_file(self.fatman_path, "manifest list: downloading") fm = ch.json_from_file(self.fatman_path, "fat manifest") if ("layers" in fm or "fsLayers" in fm): # FIXME (issue #1101): If it's a v2 manifest we could use it instead # of re-requesting later. Maybe we could here move/copy it over to # the skinny manifest path. raise ch.No_Fatman_Error() if ("errors" in fm): # fm is an error blob. (code, msg) = self.error_decode(fm) if (code == "MANIFEST_UNKNOWN"): ch.INFO("manifest list: no such image") return else: ch.FATAL("manifest list: error: %s" % msg) self.architectures = dict() if ("manifests" not in fm): ch.FATAL("manifest list has no key 'manifests'") for m in fm["manifests"]: try: if (m["platform"]["os"] != "linux"): continue arch = m["platform"]["architecture"] if ("variant" in m["platform"]): arch = "%s/%s" % (arch, m["platform"]["variant"]) digest = m["digest"] except KeyError: ch.FATAL("manifest lists missing a required key") if (arch in self.architectures): ch.FATAL("manifest list: duplicate architecture: %s" % arch) self.architectures[arch] = ch.digest_trim(digest) if (len(self.architectures) == 0): ch.WARNING("no valid architectures found")
def download(self): "Download image metadata and layers and put them in the download cache." # Spec: https://docs.docker.com/registry/spec/manifest-v2-2/ ch.VERBOSE("downloading image: %s" % self.image) ch.mkdirs(ch.storage.download_cache) # fat manifest if (ch.arch != "yolo"): self.fatman_load() if (self.architectures is not None): if (ch.arch not in self.architectures): ch.FATAL( "requested arch unavailable: %s not one of: %s" % (ch.arch, " ".join(sorted(self.architectures.keys())))) elif (ch.arch == "amd64"): # We're guessing that enough arch-unaware images are amd64 to # barge ahead if requested architecture is amd64. ch.arch = "yolo" ch.WARNING("image is architecture-unaware") ch.WARNING("requested arch is amd64; switching to --arch=yolo") else: ch.FATAL("image is architecture-unaware; try --arch=yolo (?)") # manifest self.manifest_load() # config ch.VERBOSE("config path: %s" % self.config_path) if (self.config_path is not None): if (os.path.exists(self.config_path) and self.use_cache): ch.INFO("config: using existing file") else: ch.INFO("config: downloading") self.registry.blob_to_file(self.config_hash, self.config_path) # layers for (i, lh) in enumerate(self.layer_hashes, start=1): path = self.layer_path(lh) ch.VERBOSE("layer path: %s" % path) ch.INFO("layer %d/%d: %s: " % (i, len(self.layer_hashes), lh[:7]), end="") if (os.path.exists(path) and self.use_cache): ch.INFO("using existing file") else: ch.INFO("downloading") self.registry.blob_to_file(lh, path)
def fatman_load(self): """Load the fat manifest JSON file, downloading it first if needed. If the image has a fat manifest, populate self.architectures; this may be an empty dictionary if no valid architectures were found. It is not an error if the image has no fat manifest or the registry reports no such image. In this architecture-unaware condition, set self.architectures to None.""" self.architectures = None if (str(self.image.ref) in manifests_internal): return # no fat manifests for internal library if (os.path.exists(self.fatman_path) and self.use_cache): ch.INFO("manifest list: using existing file") else: ch.INFO("manifest list: downloading") self.registry.fatman_to_file(self.fatman_path, True) if (not os.path.exists(self.fatman_path)): # Response was 404 (or equivalent). ch.INFO("manifest list: no list found") return fm = ch.json_from_file(self.fatman_path, "fat manifest") if ("layers" in fm or "fsLayers" in fm): # If there is no fat manifest but the image exists, we get a skinny # manifest instead. We can't use it, however, because it might be a # v1 manifest when a v2 is available. ¯\_(ツ)_/¯ ch.INFO("manifest list: no valid list found") return if ("errors" in fm): # fm is an error blob. (code, msg) = self.error_decode(fm) if (code == "MANIFEST_UNKNOWN"): ch.INFO("manifest list: no such image") return else: ch.FATAL("manifest list: error: %s" % msg) self.architectures = dict() if ("manifests" not in fm): ch.FATAL("manifest list has no key 'manifests'") for m in fm["manifests"]: try: if (m["platform"]["os"] != "linux"): continue arch = m["platform"]["architecture"] if ("variant" in m["platform"]): arch = "%s/%s" % (arch, m["platform"]["variant"]) digest = m["digest"] except KeyError: ch.FATAL("manifest lists missing a required key") if (arch in self.architectures): ch.FATAL("manifest list: duplicate architecture: %s" % arch) self.architectures[arch] = ch.digest_trim(digest) if (len(self.architectures) == 0): ch.WARNING("no valid architectures found")
def manifest_load(self): """Parse the manifest file and set self.config_hash and self.layer_hashes.""" def bad_key(key): ch.FATAL("manifest: %s: no key: %s" % (self.manifest_path, key)) # read and parse the JSON fp = ch.open_(self.manifest_path, "rt", encoding="UTF-8") text = ch.ossafe(fp.read, "can't read: %s" % self.manifest_path) ch.ossafe(fp.close, "can't close: %s" % self.manifest_path) ch.DEBUG("manifest:\n%s" % text) try: manifest = json.loads(text) except json.JSONDecodeError as x: ch.FATAL("can't parse manifest file: %s:%d: %s" % (self.manifest_path, x.lineno, x.msg)) # validate schema version try: version = manifest['schemaVersion'] except KeyError: bad_key("schemaVersion") if (version not in {1,2}): ch.FATAL("unsupported manifest schema version: %s" % repr(version)) # load config hash # # FIXME: Manifest version 1 does not list a config blob. It does have # things (plural) that look like a config at history/v1Compatibility as # an embedded JSON string :P but I haven't dug into it. if (version == 1): ch.WARNING("no config; manifest schema version 1") self.config_hash = None else: # version == 2 try: self.config_hash = ch.digest_trim(manifest["config"]["digest"]) except KeyError: bad_key("config/digest") # load layer hashes if (version == 1): key1 = "fsLayers" key2 = "blobSum" else: # version == 2 key1 = "layers" key2 = "digest" if (key1 not in manifest): bad_key(key1) self.layer_hashes = list() for i in manifest[key1]: if (key2 not in i): bad_key("%s/%s" % (key1, key2)) self.layer_hashes.append(ch.digest_trim(i[key2])) if (version == 1): self.layer_hashes.reverse()
def __default__(self, tree): class_ = "I_" + tree.data if (class_ in globals()): inst = globals()[class_](tree) inst.announce() if (self.instruction_ct == 0): if ( isinstance(inst, I_directive) or isinstance(inst, I_from_)): pass elif (isinstance(inst, Arg)): ch.WARNING("ARG before FROM not yet supported; see issue #779") else: ch.FATAL("first instruction must be ARG or FROM") inst.execute() self.instruction_ct += inst.execute_increment
def pull_to_unpacked(self, last_layer=None): "Pull and flatten image." self.download() layer_paths = [self.layer_path(h) for h in self.layer_hashes] self.image.unpack(layer_paths, last_layer) self.image.metadata_replace(self.config_path) # Check architecture we got. This is limited because image metadata does # not store the variant. Move fast and break things, I guess. arch_image = self.image.metadata["arch"] or "unknown" arch_short = ch.arch.split("/")[0] arch_host_short = ch.arch_host.split("/")[0] if (arch_image != "unknown" and arch_image != arch_host_short): host_mismatch = " (may not match host %s)" % ch.arch_host else: host_mismatch = "" ch.INFO("image arch: %s%s" % (arch_image, host_mismatch)) if (ch.arch != "yolo" and arch_short != arch_image): ch.WARNING("image architecture does not match requested: %s ≠ %s" % (ch.arch, image_arch))
def main(cli_): # CLI namespace. :P global cli cli = cli_ # Infer input file if needed. if (cli.file is None): cli.file = cli.context + "/Dockerfile" # Infer image name if needed. if (cli.tag is None): m = re.search(r"(([^/]+)/)?Dockerfile(\.(.+))?$", os.path.abspath(cli.file)) if (m is not None): if m.group(4): # extension cli.tag = m.group(4) elif m.group(2): # containing directory cli.tag = m.group(2) # Deal with build arguments. def build_arg_get(arg): kv = arg.split("=") if (len(kv) == 2): return kv else: v = os.getenv(kv[0]) if (v is None): ch.FATAL("--build-arg: %s: no value and not in environment" % kv[0]) return (kv[0], v) if (cli.build_arg is None): cli.build_arg = list() cli.build_arg = dict( build_arg_get(i) for i in cli.build_arg ) # Finish CLI initialization. ch.DEBUG(cli) ch.dependencies_check() # Guess whether the context is a URL, and error out if so. This can be a # typical looking URL e.g. "https://..." or also something like # "[email protected]:...". The line noise in the second line of the regex is # to match this second form. Username and host characters from # https://tools.ietf.org/html/rfc3986. if (re.search(r""" ^((git|git+ssh|http|https|ssh):// | ^[\w.~%!$&'\(\)\*\+,;=-]+@[\w.~%!$&'\(\)\*\+,;=-]+:)""", cli.context, re.VERBOSE) is not None): ch.FATAL("not yet supported: issue #773: URL context: %s" % cli.context) if (os.path.exists(cli.context + "/.dockerignore")): ch.WARNING("not yet supported, ignored: issue #777: .dockerignore file") # Set up build environment. global env env = Environment() # Read input file. if (cli.file == "-"): text = ch.ossafe(sys.stdin.read, "can't read stdin") else: fp = ch.open_(cli.file, "rt") text = ch.ossafe(fp.read, "can't read: %s" % cli.file) fp.close() # Parse it. parser = lark.Lark("?start: dockerfile\n" + ch.GRAMMAR, parser="earley", propagate_positions=True) # Avoid Lark issue #237: lark.exceptions.UnexpectedEOF if the file does not # end in newline. text += "\n" try: tree = parser.parse(text) except lark.exceptions.UnexpectedInput as x: ch.DEBUG(x) # noise about what was expected in the grammar ch.FATAL("can't parse: %s:%d,%d\n\n%s" % (cli.file, x.line, x.column, x.get_context(text, 39))) ch.DEBUG(tree.pretty()) # Sometimes we exit after parsing. if (cli.parse_only): sys.exit(0) # Count the number of stages (i.e., FROM instructions) global image_ct image_ct = sum(1 for i in ch.tree_children(tree, "from_")) # Traverse the tree and do what it says. # # We don't actually care whether the tree is traversed breadth-first or # depth-first, but we *do* care that instruction nodes are visited in # order. Neither visit() nor visit_topdown() are documented as of # 2020-06-11 [1], but examining source code [2] shows that visit_topdown() # uses Tree.iter_trees_topdown(), which *is* documented to be in-order [3]. # # This change seems to have been made in 0.8.6 (see PR #761); before then, # visit() was in order. Therefore, we call that instead, if visit_topdown() # is not present, to improve compatibility (see issue #792). # # [1]: https://lark-parser.readthedocs.io/en/latest/visitors/#visitors # [2]: https://github.com/lark-parser/lark/blob/445c8d4/lark/visitors.py#L211 # [3]: https://lark-parser.readthedocs.io/en/latest/classes/#tree ml = Main_Loop() if (hasattr(ml, 'visit_topdown')): ml.visit_topdown(tree) else: ml.visit(tree) # Check that all build arguments were consumed. if (len(cli.build_arg) != 0): ch.FATAL("--build-arg: not consumed: " + " ".join(cli.build_arg.keys())) # Print summary & we're done. if (ml.instruction_ct == 0): ch.FATAL("no instructions found: %s" % cli.file) assert (image_i + 1 == image_ct) # should have errored already if not ch.INFO("grown in %d instructions: %s" % (ml.instruction_ct, images[image_i]))
def announce(self): ch.WARNING("not supported, ignored: parser directives")
def unsupported_yet_warn(self, msg, issue_no): ch.WARNING("not yet supported, ignored: issue #%d: %s %s" % (issue_no, self.str_name(), msg))
def unsupported_forever_warn(self, msg): ch.WARNING("not supported, ignored: %s %s" % (self.str_name(), msg))