def detect(image, force, no_force_detect): f = None if (no_force_detect): ch.VERBOSE("not detecting --force config, per --no-force-detect") else: # Try to find a real fakeroot config. for (tag, cfg) in DEFAULT_CONFIGS.items(): try: f = Fakeroot(image, tag, cfg, force) break except Config_Aint_Matched: pass # Report findings. if (f is None): msg = "--force not available (no suitable config found)" if (force): ch.WARNING(msg) else: ch.VERBOSE(msg) else: if (force): adj = "will use" else: adj = "available" ch.INFO("%s --force: %s: %s" % (adj, f.tag, f.name)) # Wrap up if (f is None): f = Fakeroot_Noop() return f
def init_maybe(self, img_path, args, env): if (not self.needs_inject(args)): ch.VERBOSE("workarounds: init: instruction doesn't need injection") return if (self.init_done): ch.VERBOSE("workarounds: init: already initialized") return for (i, (test_cmd, init_cmd)) in enumerate(self.init, 1): ch.INFO("workarounds: init step %s: checking: $ %s" % (i, test_cmd)) args = ["/bin/sh", "-c", test_cmd] exit_code = ch.ch_run_modify(img_path, args, env, fail_ok=True) if (exit_code == 0): ch.INFO( "workarounds: init step %d: exit code %d, step not needed" % (i, exit_code)) else: if (not self.inject_p): ch.INFO("workarounds: init step %d: no --force, skipping" % i) else: ch.INFO("workarounds: init step %d: $ %s" % (i, init_cmd)) args = ["/bin/sh", "-c", init_cmd] ch.ch_run_modify(img_path, args, env) self.init_done = True
def download(self): "Download image metadata and layers and put them in the download cache." # Spec: https://docs.docker.com/registry/spec/manifest-v2-2/ ch.VERBOSE("downloading image: %s" % self.image) try: # fat manifest if (ch.arch != "yolo"): try: self.fatman_load() if (ch.arch not in self.architectures): ch.FATAL("requested arch unavailable: %s" % ch.arch, ("available: %s" % " ".join(sorted(self.architectures.keys())))) except ch.No_Fatman_Error: if (ch.arch == "amd64"): # We're guessing that enough arch-unaware images are amd64 to # barge ahead if requested architecture is amd64. ch.arch = "yolo" ch.WARNING("image is architecture-unaware") ch.WARNING( "requested arch is amd64; using --arch=yolo") else: ch.FATAL("image is architecture-unaware", "consider --arch=yolo") # manifest self.manifest_load() except ch.Not_In_Registry_Error: ch.FATAL("not in registry: %s" % self.registry.ref) # config ch.VERBOSE("config path: %s" % self.config_path) if (self.config_path is not None): if (os.path.exists(self.config_path) and self.use_cache): ch.INFO("config: using existing file") else: self.registry.blob_to_file(self.config_hash, self.config_path, "config: downloading") # layers for (i, lh) in enumerate(self.layer_hashes, start=1): path = self.layer_path(lh) ch.VERBOSE("layer path: %s" % path) msg = "layer %d/%d: %s" % (i, len(self.layer_hashes), lh[:7]) if (os.path.exists(path) and self.use_cache): ch.INFO("%s: using existing file" % msg) else: self.registry.blob_to_file(lh, path, "%s: downloading" % msg) # done self.registry.close()
def execute_(self): # Complain about unsupported stuff. if (self.options.pop("platform", False)): self.unsupported_yet_fatal("--platform", 778) # Any remaining options are invalid. self.options_assert_empty() # Update image globals. global image_i image_i += 1 global image_alias image_alias = self.alias if (image_i == image_ct - 1): # Last image; use tag unchanged. tag = cli.tag elif (image_i > image_ct - 1): # Too many images! ch.FATAL("expected %d stages but found at least %d" % (image_ct, image_i + 1)) else: # Not last image; append stage index to tag. tag = "%s/_stage%d" % (cli.tag, image_i) image = ch.Image(ch.Image_Ref(tag)) images[image_i] = image if (self.alias is not None): images[self.alias] = image ch.VERBOSE("image path: %s" % image.unpack_path) # Other error checking. if (str(image.ref) == str(self.base_ref)): ch.FATAL("output image ref same as FROM: %s" % self.base_ref) # Initialize image. self.base_image = ch.Image(self.base_ref) if (os.path.isdir(self.base_image.unpack_path)): ch.VERBOSE("base image found: %s" % self.base_image.unpack_path) else: ch.VERBOSE("base image not found, pulling") # a young hen, especially one less than one year old. pullet = pull.Image_Puller(self.base_image, not cli.no_cache) pullet.pull_to_unpacked() pullet.done() image.copy_unpacked(self.base_image) image.metadata_load() env.reset() # Find fakeroot configuration, if any. global fakeroot_config fakeroot_config = fakeroot.detect(image.unpack_path, cli.force, cli.no_force_detect)
def download(self): "Download image metadata and layers and put them in the download cache." # Spec: https://docs.docker.com/registry/spec/manifest-v2-2/ ch.VERBOSE("downloading image: %s" % self.image) ch.mkdirs(ch.storage.download_cache) # fat manifest if (ch.arch != "yolo"): self.fatman_load() if (self.architectures is not None): if (ch.arch not in self.architectures): ch.FATAL( "requested arch unavailable: %s not one of: %s" % (ch.arch, " ".join(sorted(self.architectures.keys())))) elif (ch.arch == "amd64"): # We're guessing that enough arch-unaware images are amd64 to # barge ahead if requested architecture is amd64. ch.arch = "yolo" ch.WARNING("image is architecture-unaware") ch.WARNING("requested arch is amd64; switching to --arch=yolo") else: ch.FATAL("image is architecture-unaware; try --arch=yolo (?)") # manifest self.manifest_load() # config ch.VERBOSE("config path: %s" % self.config_path) if (self.config_path is not None): if (os.path.exists(self.config_path) and self.use_cache): ch.INFO("config: using existing file") else: ch.INFO("config: downloading") self.registry.blob_to_file(self.config_hash, self.config_path) # layers for (i, lh) in enumerate(self.layer_hashes, start=1): path = self.layer_path(lh) ch.VERBOSE("layer path: %s" % path) ch.INFO("layer %d/%d: %s: " % (i, len(self.layer_hashes), lh[:7]), end="") if (os.path.exists(path) and self.use_cache): ch.INFO("using existing file") else: ch.INFO("downloading") self.registry.blob_to_file(lh, path)
def main(cli): ch.dependencies_check() # Set things up. ref = ch.Image_Ref(cli.image_ref) if (cli.parse_only): ch.INFO(ref.as_verbose_str) sys.exit(0) image = ch.Image(ref, cli.image_dir) ch.INFO("pulling image: %s" % ref) if (cli.image_dir is not None): ch.INFO( "destination: %s" % image.unpack_path) else: ch.VERBOSE("destination: %s" % image.unpack_path) ch.VERBOSE("use cache: %s" % (not cli.no_cache)) ch.VERBOSE("download cache: %s" % ch.storage.download_cache) pullet = Image_Puller(image) ch.VERBOSE("manifest: %s" % pullet.manifest_path) pullet.pull_to_unpacked(use_cache=(not cli.no_cache), last_layer=cli.last_layer) ch.done_notify()
def inject_run(self, args): if (not self.needs_inject(args)): ch.VERBOSE("workarounds: RUN: instruction doesn't need injection") return args assert (self.init_done) if (not self.inject_p): ch.INFO("workarounds: RUN: available here with --force") return args args = self.each + args self.inject_ct += 1 ch.INFO("workarounds: RUN: new command: %s" % args) return args
def __init__(self, image_path, tag, cfg, inject_p): ch.VERBOSE("workarounds: testing config: %s" % tag) file_path = "%s/%s" % (image_path, cfg["match"][0]) if (not (os.path.isfile(file_path) and ch.grep_p(file_path, cfg["match"][1]))): raise Config_Aint_Matched(tag) self.tag = tag self.inject_ct = 0 self.inject_p = inject_p for i in ("name", "init", "cmds", "each"): setattr(self, i, cfg[i]) self.init_done = False
def download(self, use_cache): """Download image metadata and layers and put them in the download cache. If use_cache is True (the default), anything already in the cache is skipped, otherwise download it anyway, overwriting what's in the cache.""" # Spec: https://docs.docker.com/registry/spec/manifest-v2-2/ dl = ch.Registry_HTTP(self.image.ref) ch.VERBOSE("downloading image: %s" % dl.ref) ch.mkdirs(ch.storage.download_cache) # manifest if (os.path.exists(self.manifest_path) and use_cache): ch.INFO("manifest: using existing file") else: ch.INFO("manifest: downloading") dl.manifest_to_file(self.manifest_path) self.manifest_load() # config ch.VERBOSE("config path: %s" % self.config_path) if (self.config_path is not None): if (os.path.exists(self.config_path) and use_cache): ch.INFO("config: using existing file") else: ch.INFO("config: downloading") dl.blob_to_file(self.config_hash, self.config_path) # layers for (i, lh) in enumerate(self.layer_hashes, start=1): path = self.layer_path(lh) ch.VERBOSE("layer path: %s" % path) ch.INFO("layer %d/%d: %s: "% (i, len(self.layer_hashes), lh[:7]), end="") if (os.path.exists(path) and use_cache): ch.INFO("using existing file") else: ch.INFO("downloading") dl.blob_to_file(lh, path) dl.close()
def main(cli): # Set things up. ref = ch.Image_Ref(cli.image_ref) if (cli.parse_only): print(ref.as_verbose_str) sys.exit(0) image = ch.Image(ref, cli.image_dir) ch.INFO("pulling image: %s" % ref) ch.INFO("requesting arch: %s" % ch.arch) if (cli.image_dir is not None): ch.INFO("destination: %s" % image.unpack_path) else: ch.VERBOSE("destination: %s" % image.unpack_path) pullet = Image_Puller(image, not cli.no_cache) pullet.pull_to_unpacked(cli.last_layer) pullet.done() ch.done_notify()
def main(cli): src_ref = ch.Image_Ref(cli.source_ref) ch.INFO("pushing image: %s" % src_ref) image = ch.Image(src_ref, cli.image) # FIXME: validate it's an image using Megan's new function (PR #908) if (not os.path.isdir(image.unpack_path)): if (cli.image is not None): ch.FATAL("can't push: %s does not appear to be an image" % cli.image) else: ch.FATAL("can't push: no image %s" % src_ref) if (cli.image is not None): ch.INFO("image path: %s" % image.unpack_path) else: ch.VERBOSE("image path: %s" % image.unpack_path) if (cli.dest_ref is not None): dst_ref = ch.Image_Ref(cli.dest_ref) ch.INFO("destination: %s" % dst_ref) else: dst_ref = ch.Image_Ref(cli.source_ref) up = Image_Pusher(image, dst_ref) up.push() ch.done_notify()
def manifest_load(self, continue_404=False): """Download the manifest file if needed, parse it, and set self.config_hash and self.layer_hashes. By default, if the image does not exist, exit with error; if continue_404, then log the condition but do not exit. In this case, self.config_hash and self.layer_hashes will both be None.""" def bad_key(key): ch.FATAL("manifest: %s: no key: %s" % (self.manifest_path, key)) self.config_hash = None self.layer_hashes = None # obtain the manifest try: # internal manifest library, e.g. for "FROM scratch" manifest = manifests_internal[str(self.image.ref)] ch.INFO("manifest: using internal library") except KeyError: # download the file if needed, then parse it if (ch.arch == "yolo" or self.architectures is None): digest = None else: digest = self.architectures[ch.arch] ch.DEBUG("manifest digest: %s" % digest) if (os.path.exists(self.manifest_path) and self.use_cache): ch.INFO("manifest: using existing file") else: ch.INFO("manifest: downloading") self.registry.manifest_to_file(self.manifest_path, digest=digest, continue_404=continue_404) if (not os.path.exists(self.manifest_path)): # response was 404 (or equivalent) ch.INFO("manifest: none found") return manifest = ch.json_from_file(self.manifest_path, "manifest") # validate schema version try: version = manifest['schemaVersion'] except KeyError: bad_key("schemaVersion") if (version not in {1, 2}): ch.FATAL("unsupported manifest schema version: %s" % repr(version)) # load config hash # # FIXME: Manifest version 1 does not list a config blob. It does have # things (plural) that look like a config at history/v1Compatibility as # an embedded JSON string :P but I haven't dug into it. if (version == 1): ch.VERBOSE("no config; manifest schema version 1") self.config_hash = None else: # version == 2 try: self.config_hash = manifest["config"]["digest"] if (self.config_hash is not None): self.config_hash = ch.digest_trim(self.config_hash) except KeyError: bad_key("config/digest") # load layer hashes if (version == 1): key1 = "fsLayers" key2 = "blobSum" else: # version == 2 key1 = "layers" key2 = "digest" if (key1 not in manifest): bad_key(key1) self.layer_hashes = list() for i in manifest[key1]: if (key2 not in i): bad_key("%s/%s" % (key1, key2)) self.layer_hashes.append(ch.digest_trim(i[key2])) if (version == 1): self.layer_hashes.reverse()
def cleanup(self): ch.INFO("cleaning up") # Delete the tarballs since we can't yet cache them. for (_, tar_c) in self.layers: ch.VERBOSE("deleting tarball: %s" % tar_c) ch.unlink(tar_c)
def main(cli_): # CLI namespace. :P global cli cli = cli_ # Check argument validity. if (cli.force and cli.no_force_detect): ch.FATAL("--force and --no-force-detect are incompatible") # Infer input file if needed. if (cli.file is None): cli.file = cli.context + "/Dockerfile" # Infer image name if needed. if (cli.tag is None): path = os.path.basename(cli.file) if ("." in path): (base, ext_all) = str(path).split(".", maxsplit=1) (base_all, ext_last) = str(path).rsplit(".", maxsplit=1) else: base = None ext_last = None if (base == "Dockerfile"): cli.tag = ext_all ch.VERBOSE("inferring name from Dockerfile extension: %s" % cli.tag) elif (ext_last == "dockerfile"): cli.tag = base_all ch.VERBOSE("inferring name from Dockerfile basename: %s" % cli.tag) elif (os.path.abspath(cli.context) != "/"): cli.tag = os.path.basename(os.path.abspath(cli.context)) ch.VERBOSE("inferring name from context directory: %s" % cli.tag) else: assert (os.path.abspath(cli.context) == "/") cli.tag = "root" ch.VERBOSE("inferring name with root context directory: %s" % cli.tag) cli.tag = re.sub(r"[^a-z0-9_.-]", "", cli.tag.lower()) ch.INFO("inferred image name: %s" % cli.tag) # Deal with build arguments. def build_arg_get(arg): kv = arg.split("=") if (len(kv) == 2): return kv else: v = os.getenv(kv[0]) if (v is None): ch.FATAL("--build-arg: %s: no value and not in environment" % kv[0]) return (kv[0], v) cli.build_arg = dict(build_arg_get(i) for i in cli.build_arg) ch.DEBUG(cli) # Guess whether the context is a URL, and error out if so. This can be a # typical looking URL e.g. "https://..." or also something like # "[email protected]:...". The line noise in the second line of the regex is # to match this second form. Username and host characters from # https://tools.ietf.org/html/rfc3986. if (re.search( r""" ^((git|git+ssh|http|https|ssh):// | ^[\w.~%!$&'\(\)\*\+,;=-]+@[\w.~%!$&'\(\)\*\+,;=-]+:)""", cli.context, re.VERBOSE) is not None): ch.FATAL("not yet supported: issue #773: URL context: %s" % cli.context) if (os.path.exists(cli.context + "/.dockerignore")): ch.WARNING( "not yet supported, ignored: issue #777: .dockerignore file") # Set up build environment. global env env = Environment() # Read input file. if (cli.file == "-" or cli.context == "-"): text = ch.ossafe(sys.stdin.read, "can't read stdin") else: fp = ch.open_(cli.file, "rt") text = ch.ossafe(fp.read, "can't read: %s" % cli.file) fp.close() # Parse it. parser = lark.Lark("?start: dockerfile\n" + ch.GRAMMAR, parser="earley", propagate_positions=True) # Avoid Lark issue #237: lark.exceptions.UnexpectedEOF if the file does not # end in newline. text += "\n" try: tree = parser.parse(text) except lark.exceptions.UnexpectedInput as x: ch.VERBOSE(x) # noise about what was expected in the grammar ch.FATAL("can't parse: %s:%d,%d\n\n%s" % (cli.file, x.line, x.column, x.get_context(text, 39))) ch.VERBOSE(tree.pretty()) # Sometimes we exit after parsing. if (cli.parse_only): sys.exit(0) # Count the number of stages (i.e., FROM instructions) global image_ct image_ct = sum(1 for i in ch.tree_children(tree, "from_")) # Traverse the tree and do what it says. # # We don't actually care whether the tree is traversed breadth-first or # depth-first, but we *do* care that instruction nodes are visited in # order. Neither visit() nor visit_topdown() are documented as of # 2020-06-11 [1], but examining source code [2] shows that visit_topdown() # uses Tree.iter_trees_topdown(), which *is* documented to be in-order [3]. # # This change seems to have been made in 0.8.6 (see PR #761); before then, # visit() was in order. Therefore, we call that instead, if visit_topdown() # is not present, to improve compatibility (see issue #792). # # [1]: https://lark-parser.readthedocs.io/en/latest/visitors/#visitors # [2]: https://github.com/lark-parser/lark/blob/445c8d4/lark/visitors.py#L211 # [3]: https://lark-parser.readthedocs.io/en/latest/classes/#tree ml = Main_Loop() if (hasattr(ml, 'visit_topdown')): ml.visit_topdown(tree) else: ml.visit(tree) # Check that all build arguments were consumed. if (len(cli.build_arg) != 0): ch.FATAL("--build-arg: not consumed: " + " ".join(cli.build_arg.keys())) # Print summary & we're done. if (ml.instruction_ct == 0): ch.FATAL("no instructions found: %s" % cli.file) assert (image_i + 1 == image_ct) # should have errored already if not if (cli.force): if (fakeroot_config.inject_ct == 0): assert (not fakeroot_config.init_done) ch.WARNING("--force specified, but nothing to do") else: ch.INFO("--force: init OK & modified %d RUN instructions" % fakeroot_config.inject_ct) ch.INFO("grown in %d instructions: %s" % (ml.instruction_ct, images[image_i]))
def execute_(self): if (cli.context == "-"): ch.FATAL("can't COPY: no context because \"-\" given") if (len(self.srcs) < 1): ch.FATAL("can't COPY: must specify at least one source") # Complain about unsupported stuff. if (self.options.pop("chown", False)): self.unsupported_forever_warn("--chown") # Any remaining options are invalid. self.options_assert_empty() # Find the context directory. if (self.from_ is None): context = cli.context else: if (self.from_ == image_i or self.from_ == image_alias): ch.FATAL("COPY --from: stage %s is the current stage" % self.from_) if (not self.from_ in images): # FIXME: Would be nice to also report if a named stage is below. if (isinstance(self.from_, int) and self.from_ < image_ct): if (self.from_ < 0): ch.FATAL( "COPY --from: invalid negative stage index %d" % self.from_) else: ch.FATAL("COPY --from: stage %d does not exist yet" % self.from_) else: ch.FATAL("COPY --from: stage %s does not exist" % self.from_) context = images[self.from_].unpack_path context_canon = os.path.realpath(context) ch.VERBOSE("context: %s" % context) # Expand source wildcards. srcs = list() for src in self.srcs: matches = glob.glob("%s/%s" % (context, src)) # glob can't take Path if (len(matches) == 0): ch.FATAL("can't copy: not found: %s" % src) for i in matches: srcs.append(i) ch.VERBOSE("source: %s" % i) # Validate sources are within context directory. (Can't convert to # canonical paths yet because we need the source path as given.) for src in srcs: src_canon = os.path.realpath(src) if (not os.path.commonpath([src_canon, context_canon ]).startswith(context_canon)): ch.FATAL("can't COPY from outside context: %s" % src) # Locate the destination. unpack_canon = os.path.realpath(images[image_i].unpack_path) if (self.dst.startswith("/")): dst = ch.Path(self.dst) else: dst = env.workdir // self.dst ch.VERBOSE("destination, as given: %s" % dst) dst_canon = self.dest_realpath(unpack_canon, dst) # strips trailing slash ch.VERBOSE("destination, canonical: %s" % dst_canon) if (not os.path.commonpath([dst_canon, unpack_canon ]).startswith(unpack_canon)): ch.FATAL("can't COPY: destination not in image: %s" % dst_canon) # Create the destination directory if needed. if (self.dst.endswith("/") or len(srcs) > 1 or os.path.isdir(srcs[0])): if (not os.path.exists(dst_canon)): ch.mkdirs(dst_canon) elif (not os.path.isdir(dst_canon)): # not symlink b/c realpath() ch.FATAL("can't COPY: not a directory: %s" % dst_canon) # Copy each source. for src in srcs: if (os.path.isfile(src)): self.copy_src_file(src, dst_canon) elif (os.path.isdir(src)): self.copy_src_dir(src, dst_canon) else: ch.FATAL("can't COPY: unknown file type: %s" % src)