def append(self): if notebook_helper.is_in_notebook(): notebook_helper.export_notebook_to_tar_gz( self.notebook_file, TEMP_TAR_GZ_FILENAME, converted_filename=self.get_python_entrypoint()) else: utils.generate_context_tarball(".", TEMP_TAR_GZ_FILENAME) transport = transport_pool.Http(httplib2.Http, size=_THREADS) src = docker_name.Tag(self.base_image) creds = docker_creds.DefaultKeychain.Resolve(src) with v2_2_image.FromRegistry(src, creds, transport) as src_image: with open(TEMP_TAR_GZ_FILENAME, 'rb') as f: new_img = append.Layer(src_image, f.read()) if self.image_tag is None: self.image_tag = new_img.digest().split(":")[1] dst = docker_name.Tag(self.full_image_name()) creds = docker_creds.DefaultKeychain.Resolve(dst) with docker_session.Push(dst, creds, transport, threads=_THREADS, mount=[src.as_repository()]) as session: logger.warn("Uploading {}".format(self.full_image_name())) session.upload(new_img) os.remove(TEMP_TAR_GZ_FILENAME) logger.warn("Pushed image {}".format(self.full_image_name()))
def main(): args = parser.parse_args() if not args.src_image or not args.tarball or not args.dst_image: raise Exception('--src-image, --dst-image and --tarball are required ' 'arguments.') transport = transport_pool.Http(httplib2.Http, size=_THREADS) # This library can support push-by-digest, but the likelihood of a user # correctly providing us with the digest without using this library # directly is essentially nil. src = docker_name.Tag(args.src_image) dst = docker_name.Tag(args.dst_image) # Resolve the appropriate credential to use based on the standard Docker # client logic. creds = docker_creds.DefaultKeychain.Resolve(src) with v2_2_image.FromRegistry(src, creds, transport) as src_image: with open(args.tarball, 'rb') as f: new_img = append.Layer(src_image, f.read()) creds = docker_creds.DefaultKeychain.Resolve(dst) with docker_session.Push(dst, creds, transport, threads=_THREADS) as session: session.upload(new_img)
def main(): logging_setup.DefineCommandLineArgs(parser) args = parser.parse_args() logging_setup.Init(args=args) transport = transport_pool.Http(httplib2.Http, size=_THREADS) # This library can support push-by-digest, but the likelihood of a user # correctly providing us with the digest without using this library # directly is essentially nil. src = docker_name.Tag(args.src_image) dst = docker_name.Tag(args.dst_image) # Resolve the appropriate credential to use based on the standard Docker # client logic. creds = docker_creds.DefaultKeychain.Resolve(src) logging.info('Pulling v2.2 image from %r ...', src) with v2_2_image.FromRegistry(src, creds, transport) as src_image: with open(args.tarball, 'rb') as f: new_img = append.Layer(src_image, f.read()) creds = docker_creds.DefaultKeychain.Resolve(dst) with docker_session.Push(dst, creds, transport, threads=_THREADS, mount=[src.as_repository()]) as session: logging.info('Starting upload ...') session.upload(new_img) digest = new_img.digest() print(('{name} was published with digest: {digest}'.format( name=dst, digest=digest)))
def Publish(transport, image_chroot, name=None, tarball=None, config=None, digest=None, layer=None): if not name: raise Exception('Expected "name" kwarg') if not config and (layer or digest): raise Exception( name + ': Using "layer" or "digest" requires "config" to be specified.') if config: with open(config, 'r') as reader: config = reader.read() elif tarball: with v2_2_image.FromTarball(tarball) as base: config = base.config_file() else: raise Exception(name + ': Either "config" or "tarball" must be specified.') if digest or layer: digest = digest.split(',') layer = layer.split(',') if len(digest) != len(layer): raise Exception( name + ': "digest" and "layer" must have matching lengths.') else: digest = [] layer = [] name_to_replace = name if image_chroot: name_to_publish = docker_name.Tag(os.path.join(image_chroot, name), strict=False) else: # Without a chroot, the left-hand-side must be a valid tag. name_to_publish = docker_name.Tag(name, strict=False) # Resolve the appropriate credential to use based on the standard Docker # client logic. creds = docker_creds.DefaultKeychain.Resolve(name_to_publish) with v2_2_session.Push(name_to_publish, creds, transport, threads=_THREADS) as session: with v2_2_image.FromDisk(config, zip(digest or [], layer or []), legacy_base=tarball) as v2_2_img: session.upload(v2_2_img) return (name_to_replace, docker_name.Digest('{repository}@{digest}'.format( repository=name_to_publish.as_repository(), digest=v2_2_img.digest())))
def main(): args = parser.parse_args() creds = docker_creds.Anonymous() transport = httplib2.Http() repo = docker_name.Repository(args.repository) latest = docker_name.Tag(str(repo) + ":latest") with docker_image.FromRegistry(latest, creds, transport) as img: latest_digest = img.digest() debug = docker_name.Tag(str(repo) + ":debug") with docker_image.FromRegistry(debug, creds, transport) as img: if img.exists(): debug_digest = img.digest() else: debug_digest = latest_digest with open(args.output, 'w') as f: f.write("""\ # Copyright 2017 The Bazel Authors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. \"\"\" Generated file with dependencies for language rule.\"\"\" # !!!! THIS IS A GENERATED FILE TO NOT EDIT IT BY HAND !!!! # # To regenerate this file, run ./update_deps.sh from the root of the # git repository. DIGESTS = {{ # "{debug_tag}" circa {date} "debug": "{debug}", # "{latest_tag}" circa {date} "latest": "{latest}", }} """.format(debug_tag=debug, debug=debug_digest, latest_tag=latest, latest=latest_digest, date=time.strftime("%Y-%m-%d %H:%M %z")))
def Run(self, args): # pylint: disable=missing-docstring def Push(image, dest_name, creds, http_obj, src_name, session_push_type): try: with session_push_type(dest_name, creds, http_obj) as push: push.upload(image) log.CreatedResource(dest_name) log.UpdatedResource(src_name) except docker_http.V2DiagnosticException as err: raise util.GcloudifyRecoverableV2Errors( err, { 403: 'Tagging failed, access denied: {0}'.format(dest_name) }) http_obj = http.Http() src_name = util.GetDockerImageFromTagOrDigest(args.src_image) dest_name = docker_name.Tag(args.dest_image) console_io.PromptContinue('This will tag {0} with {1}'.format( src_name, dest_name), default=True, cancel_on_no=True) creds = util.CredentialProvider() with v2_2_image.FromRegistry(src_name, creds, http_obj) as v2_2_img: if v2_2_img.exists(): Push(v2_2_img, dest_name, creds, http_obj, src_name, v2_2_session.Push) return with v2_image.FromRegistry(src_name, creds, http_obj) as v2_img: Push(v2_img, dest_name, creds, http_obj, src_name, v2_session.Push)
def walk_string(s): try: as_tag = docker_name.Tag(s) if as_tag in overrides: return str(overrides[as_tag]) # Resolve the tag to digest using the standard # Docker keychain logic. creds = docker_creds.DefaultKeychain.Resolve(as_tag) with v2_2_image.FromRegistry(as_tag, creds, transport) as img: if img.exists(): digest = str( docker_name.Digest('{repository}@{digest}'.format( repository=as_tag.as_repository(), digest=util.Digest(img.manifest())))) else: # If the tag doesn't exists as v2.2, then try as v2. with v2_image.FromRegistry(as_tag, creds, transport) as img: digest = str( docker_name.Digest('{repository}@{digest}'.format( repository=as_tag.as_repository(), digest=v2_util.Digest(img.manifest())))) # Make sure we consistently resolve all instances of a tag, # since it is technically possible to have a race here. overrides[as_tag] = digest return digest except: return s
def test_tag_to_digest_cached(self): tag = docker_name.Tag('gcr.io/foo/bar:baz') expected_digest = 'gcr.io/foo/bar@sha256:deadbeef' actual_digest = resolver.TagToDigest(tag, { tag: expected_digest, }, _BAD_TRANSPORT) self.assertEqual(actual_digest, expected_digest)
def main(): args = parser.parse_args() if not args.name or not args.tarball: raise Exception('--name and --tarball are required arguments.') transport = transport_pool.Http(httplib2.Http, size=8) if '@' in args.name: name = docker_name.Digest(args.name) else: name = docker_name.Tag(args.name) # Resolve the appropriate credential to use based on the standard Docker # client logic. creds = docker_creds.DefaultKeychain.Resolve(name) with tarfile.open(name=args.tarball, mode='w') as tar: with v2_2_image.FromRegistry(name, creds, transport) as v2_2_img: if v2_2_img.exists(): save.tarball(_make_tag_if_digest(name), v2_2_img, tar) return with v2_image.FromRegistry(name, creds, transport) as v2_img: with v2_compat.V22FromV2(v2_img) as v2_2_img: save.tarball(_make_tag_if_digest(name), v2_2_img, tar) return
def Run(self, args): def Push(image, dest_name, creds, http_obj, src_name, session_push_type): with session_push_type(dest_name, creds, http_obj) as push: push.upload(image) log.CreatedResource(dest_name) log.UpdatedResource(src_name) http_obj = http.Http() src_name = util.GetDockerImageFromTagOrDigest(args.src_image) dest_name = docker_name.Tag(args.dest_image) console_io.PromptContinue('This will tag {0} with {1}'.format( src_name, dest_name), default=True, cancel_on_no=True) creds = util.CredentialProvider() with v2_2_image.FromRegistry(src_name, creds, http_obj) as v2_2_img: if v2_2_img.exists(): Push(v2_2_img, dest_name, creds, http_obj, src_name, v2_2_session.Push) return with v2_image.FromRegistry(src_name, creds, http_obj) as v2_img: Push(v2_img, dest_name, creds, http_obj, src_name, v2_session.Push)
def ImageDigestForContainerImage(name, tag): """Given a container image and tag, returns the digest for that image version. Args: name: the gcr.io registry name plus the image name tag: the image tag Returns: The digest of the image, or None if there is no such image. Raises: googlecloudsdk.core.UnsupportedRegistryError: If the path is valid, but belongs to an unsupported registry. i_util.InvalidImageNameError: If the image name is invalid. i_util.TokenRefreshError: If there is an error refreshing credentials needed to access the GCR repo. i_util.UserRecoverableV2Error: If a user-recoverable error occurs accessing the GCR repo. """ def _TaggedImage(): """Display the fully-qualified name.""" return '{}:{}'.format(name, tag) name = i_util.ValidateRepositoryPath(name) with i_util.WrapExpectedDockerlessErrors(name): with docker_image.FromRegistry( basic_creds=i_util.CredentialProvider(), name=docker_name.Tag(_TaggedImage()), transport=http.Http()) as r: return r.digest()
def create_tag_to_file_content_map(stamp_info, tag_file_pairs): """ Creates a Docker image tag to file content map. Args: stamp_info - Tag substitutions to make in the input tags, e.g. {BUILD_USER} tag_file_pairs - List of input tags and file names (e.g. ...:image=@bazel-out/...image.0.config) """ tag_to_file_content = {} if tag_file_pairs: for entry in tag_file_pairs: elts = entry.split('=') if len(elts) != 2: raise Exception('Expected associative list key=value, got: %s' % entry) (fq_tag, filename) = elts formatted_tag = fq_tag.format(**stamp_info) tag = docker_name.Tag(formatted_tag, strict=False) file_contents = utils.ExtractValue(filename) # Add the mapping in one direction. tag_to_file_content[tag] = file_contents return tag_to_file_content
def walk_string(s): try: as_tag = docker_name.Tag(s) digest = tag_to_digest(as_tag) return digest except: return s
def main(): args = parser.parse_args() if not args.name or not args.directory: raise Exception('--name and --directory are required arguments.') transport = transport_pool.Http(httplib2.Http, size=_THREADS) if '@' in args.name: name = docker_name.Digest(args.name) else: name = docker_name.Tag(args.name) # Resolve the appropriate credential to use based on the standard Docker # client logic. creds = docker_creds.DefaultKeychain.Resolve(name) with v2_2_image.FromRegistry(name, creds, transport) as v2_2_img: if v2_2_img.exists(): save.fast(v2_2_img, args.directory, threads=_THREADS) return with v2_image.FromRegistry(name, creds, transport) as v2_img: with v2_compat.V22FromV2(v2_img) as v2_2_img: save.fast(v2_2_img, args.directory, threads=_THREADS) return
def StringToDigest(string, overrides, transport): """Turn a string into a stringified digest.""" if string in overrides: return str(overrides[string]) # Attempt to turn the string into a tag, this may throw. tag = docker_name.Tag(string) def fully_qualify_digest(digest): return docker_name.Digest('{registry}/{repo}@{digest}'.format( registry=tag.registry, repo=tag.repository, digest=digest)) # Resolve the tag to digest using the standard # Docker keychain logic. creds = docker_creds.DefaultKeychain.Resolve(tag) with v2_2_image.FromRegistry(tag, creds, transport) as img: if img.exists(): digest = fully_qualify_digest(img.digest()) overrides[string] = digest return str(digest) # If the tag doesn't exists as v2.2, then try as v2. with v2_image.FromRegistry(tag, creds, transport) as img: digest = fully_qualify_digest(img.digest()) overrides[string] = digest return str(digest)
def _make_tag_if_digest( name ): if isinstance(name, docker_name.Tag): return name return docker_name.Tag('{repo}:{tag}'.format( repo=str(name.as_repository()), tag=_DEFAULT_TAG))
def main(): args = parser.parse_args() if not args.name or not args.directory: raise Exception('--name and --directory are required arguments.') transport = transport_pool.Http(httplib2.Http, size=_THREADS) if '@' in args.name: name = docker_name.Digest(args.name) else: name = docker_name.Tag(args.name) # OCI Image Manifest is compatible with Docker Image Manifest Version 2, # Schema 2. We indicate support for both formats by passing both media types # as 'Accept' headers. # # For reference: # OCI: https://github.com/opencontainers/image-spec # Docker: https://docs.docker.com/registry/spec/manifest-v2-2/ accept = docker_http.SUPPORTED_MANIFEST_MIMES # Resolve the appropriate credential to use based on the standard Docker # client logic. creds = docker_creds.DefaultKeychain.Resolve(name) with v2_2_image.FromRegistry(name, creds, transport, accept) as v2_2_img: if v2_2_img.exists(): save.fast(v2_2_img, args.directory, threads=_THREADS) return with v2_image.FromRegistry(name, creds, transport) as v2_img: with v2_compat.V22FromV2(v2_img) as v2_2_img: save.fast(v2_2_img, args.directory, threads=_THREADS) return
def _tag(self, base_image, namespace, checksum): # TODO(mattmoor): pick up the latest containerregistry then use: # fingerprint = '%s %s' % (base_image.digest(), checksum) fingerprint = checksum return docker_name.Tag('{base}/{namespace}:{tag}'.format( base=str(self._repo), namespace=namespace, tag=hashlib.sha256(fingerprint).hexdigest()))
def _tag(self, cache_key, repo=None): fingerprint = '%s %s' % (self._base_image.digest(), cache_key) if self._cache_version: fingerprint += ' ' + self._cache_version return docker_name.Tag('{repo}/{namespace}:{tag}'.format( repo=repo or str(self._repo), namespace=self._namespace, tag=hashlib.sha256(fingerprint).hexdigest()))
def main(args): args = parser.parse_args(args) logging.getLogger().setLevel(_LEVEL_MAP[args.verbosity]) logging.basicConfig( format='%(asctime)s.%(msecs)03d %(levelname)-8s %(message)s', datefmt='%Y-%m-%d,%H:%M:%S') transport = transport_pool.Http(httplib2.Http, size=_THREADS) # TODO(mattmoor): Support digest base images. base_name = docker_name.Tag(args.base) base_creds = docker_creds.DefaultKeychain.Resolve(base_name) target_image = docker_name.Tag(args.name) target_creds = docker_creds.DefaultKeychain.Resolve(target_image) ctx = context.Workspace(args.directory) cash = cache.Registry(target_image.as_repository(), target_creds, transport, threads=_THREADS, mount=[base_name]) bldr = builder.From(ctx) with docker_image.FromRegistry(base_name, base_creds, transport) as base_image: # Create (or pull from cache) the base image with the # package descriptor installation overlaid. logging.info('Generating dependency layer...') with bldr.CreatePackageBase(base_image, cash, args.cache) as deps: # Construct the application layer from the context. logging.info('Generating app layer...') app_layer, diff_id = bldr.BuildAppLayer() with append.Layer(deps, app_layer, diff_id=diff_id) as app_image: if args.output_path: with tarfile.open(name=args.output_path, mode='w') as tar: save.tarball(target_image, app_image, tar) logging.info("{0} tarball located at {1}".format( str(target_image), args.output_path)) return with docker_session.Push(target_image, target_creds, transport, threads=_THREADS, mount=[base_name]) as session: logging.info('Pushing final image...') session.upload(app_image)
def _make_tag_if_digest( name ): if isinstance(name, docker_name.Tag): return name return docker_name.Tag('{registry}/{repository}:{tag}'.format( registry=name.registry, repository=name.repository, tag=_DEFAULT_TAG))
def __init__(self, ctx, cache_namespace, args, descriptor_files): super(RuntimeBase, self).__init__(ctx) self._cache_namespace = cache_namespace if args.entrypoint: args.entrypoint = args.entrypoint.split(" ") if args.sh_c_prefix: args.entrypoint = ['bash', '-c', " ".join(args.entrypoint)] if args.exposed_ports: args.exposed_ports = args.exposed_ports.split(",") args.cache_key_version = "%s %s" % (args.cache_key_version, args.cache_salt) self._args = args self._base_name = docker_name.Tag(self._args.base, strict=False) self._base_creds = docker_creds.DefaultKeychain.Resolve( self._base_name) self._target_image = docker_name.Tag(self._args.name, strict=False) self._target_creds = docker_creds.DefaultKeychain.Resolve( self._target_image) self._transport = transport_pool.Http( httplib2.Http, size=constants.THREADS) if args.tar_base_image_path: self._base_image = docker_image.FromTarball( args.tar_base_image_path) else: self._base_image = docker_image.FromRegistry( self._base_name, self._base_creds, self._transport) self._base_image.__enter__() cache_repo = args.cache_repository if not cache_repo: cache_repo = self._target_image.as_repository() if args.ttl: ttl = args.ttl else: ttl = ftl_util.get_ttl(descriptor_files, ctx) self._cache = cache.Registry( repo=cache_repo, namespace=self._cache_namespace, creds=self._target_creds, transport=self._transport, ttl=ttl, threads=constants.THREADS, mount=[self._base_name], use_global=args.global_cache, should_cache=args.cache, should_upload=args.upload) self._descriptor_files = descriptor_files
def main(root_dir, dst_image): transport = transport_pool.Http(httplib2.Http, size=32) cache_dir = os.path.join(root_dir, '.cache') if not os.path.exists(cache_dir): os.makedirs(cache_dir) base = docker_name.Tag('gcr.io/google-appengine/nodejs:latest') creds = docker_creds.DefaultKeychain.Resolve(base) with docker_image.FromRegistry(base, creds, transport) as base_img: npm = NPM(root_dir, base_img) for i in range(10): start = time.time() dst = docker_name.Tag(dst_image) creds = docker_creds.DefaultKeychain.Resolve(dst) with docker_session.Push(dst, creds, transport, threads=32) as session: img = handle_app(npm, cache_dir) session.upload(img) print time.time() - start
def build(self): """Will be called when the build needs to start""" transport = transport_pool.Http(httplib2.Http) src = docker_name.Tag(self.base_image, strict=False) logger.warning("Building image using Append builder...") start = timer() new_img = self._build(transport, src) end = timer() logger.warning("Image successfully built in {}s.".format(end - start)) dst = docker_name.Tag(self.full_image_name(self.context_hash), strict=False) if self.push: self.timed_push(transport, src, new_img, dst) else: # TODO(r2d4): # Load image into local daemon. This wouldn't provide any speedup # over using the docker daemon directly. pass
def push(self, transport, src, img): dst = docker_name.Tag( self.full_image_name(self.context_hash), strict=False) creds = docker_creds.DefaultKeychain.Resolve(dst) with docker_session.Push( dst, creds, transport, mount=[src.as_repository()]) as session: logger.warn("Uploading {}".format(self.image_tag)) session.upload(img) os.remove(self.context_file)
def _parse_image_reference(image_reference): util.check_type(image_reference, str) if '@' in image_reference: name = docker_name.Digest(image_reference) else: name = docker_name.Tag(image_reference) return name
def _fetch_lyr_shas(self, img_name): name = docker_name.Tag(img_name) creds = docker_creds.DefaultKeychain.Resolve(name) transport = transport_pool.Http(httplib2.Http, size=_THREADS) with docker_image.FromRegistry(name, creds, transport) as img: lyrs = json.loads(img.manifest())['layers'] lyr_shas = [] for lyr in lyrs: lyr_shas.append(lyr['digest']) return set(lyr_shas)
def test_complex_walk(self): present = 'gcr.io/foo/bar:baz' not_present = 'gcr.io/foo/bar:blah' expected = 'foo@sha256:deadbeef' unexpected = 'bar@sha256:baadf00d' values = { docker_name.Tag(present): expected, docker_name.Tag(not_present): unexpected, } input = """ key1: key2: - value1 - {present} """.format(present=present) output = resolver.Resolve(input, lambda x: values[x]) self.assertTrue(expected in output) self.assertFalse(unexpected in output)
def build(self): """Will be called when the build needs to start""" transport = transport_pool.Http(httplib2.Http) src = docker_name.Tag(self.base_image, strict=False) logger.warn("Building image...") start = timer() new_img = self._build(transport, src) end = timer() logger.warn("Image successfully built in {}s.".format(end-start)) self.timed_push(transport, src, new_img)
def Run(self, args): # pylint: disable=missing-docstring def Push(image, dest_names, creds, http_obj, src_name, session_push_type): for dest_name in dest_names: with session_push_type(dest_name, creds, http_obj) as push: push.upload(image) log.CreatedResource(dest_name) log.UpdatedResource(src_name) http_obj = http.Http() src_name = util.GetDockerImageFromTagOrDigest(args.src_image) dest_names = [] for dest_image in args.dest_image: try: dest_name = docker_name.Tag(dest_image) except docker_name.BadNameException as e: raise util.InvalidImageNameError(six.text_type(e)) if '/' not in dest_name.repository: raise exceptions.Error( 'Pushing to project root-level images is disabled. ' 'Please designate an image within a project, ' 'e.g. gcr.io/project-id/my-image:tag') dest_names.append(dest_name) console_io.PromptContinue('This will tag {} with:\n{}'.format( src_name, '\n'.join(six.text_type(dest_name) for dest_name in dest_names)), default=True, cancel_on_no=True) creds = util.CredentialProvider() with util.WrapExpectedDockerlessErrors(): with docker_image_list.FromRegistry(src_name, creds, http_obj) as manifest_list: if manifest_list.exists(): Push(manifest_list, dest_names, creds, http_obj, src_name, v2_2_session.Push) return with v2_2_image.FromRegistry(src_name, creds, http_obj, accepted_mimes=docker_http. SUPPORTED_MANIFEST_MIMES) as v2_2_img: if v2_2_img.exists(): Push(v2_2_img, dest_names, creds, http_obj, src_name, v2_2_session.Push) return with v2_image.FromRegistry(src_name, creds, http_obj) as v2_img: Push(v2_img, dest_names, creds, http_obj, src_name, v2_session.Push)