def test_img_exists(self): with docker_image.FromTarball('testing/lib/test.tar') as img: self.registry.setImage('gcr.io/foobar/baz', img) with docker_image.FromRegistry('gcr.io/foobar/baz') as img: self.assertTrue(img.exists()) with docker_image.FromRegistry('does_not_exist') as img: self.assertFalse(img.exists())
def test_from_registry_and_push(self): with docker_image.FromTarball('testing/lib/test.tar') as img: self.registry.setImage('gcr.io/foobar/baz', img) with docker_image.FromRegistry('gcr.io/foobar/baz') as img: with docker_session.Push('gcr.io/foo/bar:testing', "", None) as push: push.upload(img) with docker_image.FromRegistry('gcr.io/foo/bar:testing') as img: self.assertTrue(img.exists())
def main(): args = parser.parse_args() creds = docker_creds.Anonymous() transport = httplib2.Http() repo = docker_name.Repository(args.repository) latest = docker_name.Tag(str(repo) + ":latest") with docker_image.FromRegistry(latest, creds, transport) as img: latest_digest = img.digest() debug = docker_name.Tag(str(repo) + ":debug") with docker_image.FromRegistry(debug, creds, transport) as img: if img.exists(): debug_digest = img.digest() else: debug_digest = latest_digest with open(args.output, 'w') as f: f.write("""\ # Copyright 2017 The Bazel Authors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. \"\"\" Generated file with dependencies for language rule.\"\"\" # !!!! THIS IS A GENERATED FILE TO NOT EDIT IT BY HAND !!!! # # To regenerate this file, run ./update_deps.sh from the root of the # git repository. DIGESTS = {{ # "{debug_tag}" circa {date} "debug": "{debug}", # "{latest_tag}" circa {date} "latest": "{latest}", }} """.format(debug_tag=debug, debug=debug_digest, latest_tag=latest, latest=latest_digest, date=time.strftime("%Y-%m-%d %H:%M %z")))
def walk_string(s): try: as_tag = docker_name.Tag(s) if as_tag in overrides: return str(overrides[as_tag]) # Resolve the tag to digest using the standard # Docker keychain logic. creds = docker_creds.DefaultKeychain.Resolve(as_tag) with v2_2_image.FromRegistry(as_tag, creds, transport) as img: if img.exists(): digest = str( docker_name.Digest('{repository}@{digest}'.format( repository=as_tag.as_repository(), digest=util.Digest(img.manifest())))) else: # If the tag doesn't exists as v2.2, then try as v2. with v2_image.FromRegistry(as_tag, creds, transport) as img: digest = str( docker_name.Digest('{repository}@{digest}'.format( repository=as_tag.as_repository(), digest=v2_util.Digest(img.manifest())))) # Make sure we consistently resolve all instances of a tag, # since it is technically possible to have a race here. overrides[as_tag] = digest return digest except: return s
def main(): logging_setup.DefineCommandLineArgs(parser) args = parser.parse_args() logging_setup.Init(args=args) transport = transport_pool.Http(httplib2.Http, size=_THREADS) # This library can support push-by-digest, but the likelihood of a user # correctly providing us with the digest without using this library # directly is essentially nil. src = docker_name.Tag(args.src_image) dst = docker_name.Tag(args.dst_image) # Resolve the appropriate credential to use based on the standard Docker # client logic. creds = docker_creds.DefaultKeychain.Resolve(src) logging.info('Pulling v2.2 image from %r ...', src) with v2_2_image.FromRegistry(src, creds, transport) as src_image: with open(args.tarball, 'rb') as f: new_img = append.Layer(src_image, f.read()) creds = docker_creds.DefaultKeychain.Resolve(dst) with docker_session.Push(dst, creds, transport, threads=_THREADS, mount=[src.as_repository()]) as session: logging.info('Starting upload ...') session.upload(new_img) digest = new_img.digest() print(('{name} was published with digest: {digest}'.format( name=dst, digest=digest)))
def Run(self, args): # pylint: disable=missing-docstring def Push(image, dest_name, creds, http_obj, src_name, session_push_type): try: with session_push_type(dest_name, creds, http_obj) as push: push.upload(image) log.CreatedResource(dest_name) log.UpdatedResource(src_name) except docker_http.V2DiagnosticException as err: raise util.GcloudifyRecoverableV2Errors( err, { 403: 'Tagging failed, access denied: {0}'.format(dest_name) }) http_obj = http.Http() src_name = util.GetDockerImageFromTagOrDigest(args.src_image) dest_name = docker_name.Tag(args.dest_image) console_io.PromptContinue('This will tag {0} with {1}'.format( src_name, dest_name), default=True, cancel_on_no=True) creds = util.CredentialProvider() with v2_2_image.FromRegistry(src_name, creds, http_obj) as v2_2_img: if v2_2_img.exists(): Push(v2_2_img, dest_name, creds, http_obj, src_name, v2_2_session.Push) return with v2_image.FromRegistry(src_name, creds, http_obj) as v2_img: Push(v2_img, dest_name, creds, http_obj, src_name, v2_session.Push)
def Run(self, args): def Push(image, dest_name, creds, http_obj, src_name, session_push_type): with session_push_type(dest_name, creds, http_obj) as push: push.upload(image) log.CreatedResource(dest_name) log.UpdatedResource(src_name) http_obj = http.Http() src_name = util.GetDockerImageFromTagOrDigest(args.src_image) dest_name = docker_name.Tag(args.dest_image) console_io.PromptContinue('This will tag {0} with {1}'.format( src_name, dest_name), default=True, cancel_on_no=True) creds = util.CredentialProvider() with v2_2_image.FromRegistry(src_name, creds, http_obj) as v2_2_img: if v2_2_img.exists(): Push(v2_2_img, dest_name, creds, http_obj, src_name, v2_2_session.Push) return with v2_image.FromRegistry(src_name, creds, http_obj) as v2_img: Push(v2_img, dest_name, creds, http_obj, src_name, v2_session.Push)
def ImageDigestForContainerImage(name, tag): """Given a container image and tag, returns the digest for that image version. Args: name: the gcr.io registry name plus the image name tag: the image tag Returns: The digest of the image, or None if there is no such image. Raises: googlecloudsdk.core.UnsupportedRegistryError: If the path is valid, but belongs to an unsupported registry. i_util.InvalidImageNameError: If the image name is invalid. i_util.TokenRefreshError: If there is an error refreshing credentials needed to access the GCR repo. i_util.UserRecoverableV2Error: If a user-recoverable error occurs accessing the GCR repo. """ def _TaggedImage(): """Display the fully-qualified name.""" return '{}:{}'.format(name, tag) name = i_util.ValidateRepositoryPath(name) with i_util.WrapExpectedDockerlessErrors(name): with docker_image.FromRegistry( basic_creds=i_util.CredentialProvider(), name=docker_name.Tag(_TaggedImage()), transport=http.Http()) as r: return r.digest()
def images(self): """Returns a list of tuples whose elements are (name, platform, image). Raises: InvalidMediaTypeError: a child with an unexpected media type was found. """ manifests = json.loads(self.manifest())['manifests'] results = [] for entry in manifests: digest = entry['digest'] base = self._name.as_repository() # pytype: disable=attribute-error name = docker_name.Digest('{base}@{digest}'.format(base=base, digest=digest)) media_type = entry['mediaType'] if media_type in docker_http.MANIFEST_LIST_MIMES: image = FromRegistry(name, self._creds, self._original_transport) elif media_type in docker_http.SUPPORTED_MANIFEST_MIMES: image = v2_2_image.FromRegistry(name, self._creds, self._original_transport, [media_type]) else: raise InvalidMediaTypeError('Invalid media type: ' + media_type) platform = Platform( entry['platform']) if 'platform' in entry else None results.append((name, platform, image)) return results
def StringToDigest(string, overrides, transport): """Turn a string into a stringified digest.""" if string in overrides: return str(overrides[string]) # Attempt to turn the string into a tag, this may throw. tag = docker_name.Tag(string) def fully_qualify_digest(digest): return docker_name.Digest('{registry}/{repo}@{digest}'.format( registry=tag.registry, repo=tag.repository, digest=digest)) # Resolve the tag to digest using the standard # Docker keychain logic. creds = docker_creds.DefaultKeychain.Resolve(tag) with v2_2_image.FromRegistry(tag, creds, transport) as img: if img.exists(): digest = fully_qualify_digest(img.digest()) overrides[string] = digest return str(digest) # If the tag doesn't exists as v2.2, then try as v2. with v2_image.FromRegistry(tag, creds, transport) as img: digest = fully_qualify_digest(img.digest()) overrides[string] = digest return str(digest)
def main(): args = parser.parse_args() if not args.name or not args.directory: raise Exception('--name and --directory are required arguments.') transport = transport_pool.Http(httplib2.Http, size=_THREADS) if '@' in args.name: name = docker_name.Digest(args.name) else: name = docker_name.Tag(args.name) # OCI Image Manifest is compatible with Docker Image Manifest Version 2, # Schema 2. We indicate support for both formats by passing both media types # as 'Accept' headers. # # For reference: # OCI: https://github.com/opencontainers/image-spec # Docker: https://docs.docker.com/registry/spec/manifest-v2-2/ accept = docker_http.SUPPORTED_MANIFEST_MIMES # Resolve the appropriate credential to use based on the standard Docker # client logic. creds = docker_creds.DefaultKeychain.Resolve(name) with v2_2_image.FromRegistry(name, creds, transport, accept) as v2_2_img: if v2_2_img.exists(): save.fast(v2_2_img, args.directory, threads=_THREADS) return with v2_image.FromRegistry(name, creds, transport) as v2_img: with v2_compat.V22FromV2(v2_img) as v2_2_img: save.fast(v2_2_img, args.directory, threads=_THREADS) return
def ResolveV22Tag(tag): with v2_2_image.FromRegistry( basic_creds=CredentialProvider(), name=tag, transport=http.Http()) as v2_2_img: if v2_2_img.exists(): return v2_2_util.Digest(v2_2_img.manifest()) return None
def Run(self, args): """This is what gets called when the user runs this command. Args: args: an argparse namespace. All the arguments that were provided to this command invocation. Returns: Some value that we want to have printed later. """ repository_arg = args.repository if not repository_arg: repository_arg = 'gcr.io/{0}'.format( properties.VALUES.core.project.Get(required=True)) # Throws if invalid. repository = util.ValidateRepositoryPath(repository_arg) def _DisplayName(c): """Display the fully-qualified name.""" return '{0}/{1}'.format(repository, c) http_obj = http.Http() with docker_image.FromRegistry(basic_creds=util.CredentialProvider(), name=repository, transport=http_obj) as r: images = [{'name': _DisplayName(c)} for c in r.children()] return images
def main(): args = parser.parse_args() if not args.src_image or not args.tarball or not args.dst_image: raise Exception('--src-image, --dst-image and --tarball are required ' 'arguments.') transport = transport_pool.Http(httplib2.Http, size=_THREADS) # This library can support push-by-digest, but the likelihood of a user # correctly providing us with the digest without using this library # directly is essentially nil. src = docker_name.Tag(args.src_image) dst = docker_name.Tag(args.dst_image) # Resolve the appropriate credential to use based on the standard Docker # client logic. creds = docker_creds.DefaultKeychain.Resolve(src) with v2_2_image.FromRegistry(src, creds, transport) as src_image: with open(args.tarball, 'rb') as f: new_img = append.Layer(src_image, f.read()) creds = docker_creds.DefaultKeychain.Resolve(dst) with docker_session.Push(dst, creds, transport, threads=_THREADS) as session: session.upload(new_img)
def pulled_image( image_reference: str, credentials_lookup: typing.Callable[[image_reference, oa.Privileges, bool], oa.OciConfig], ): _tag_or_digest_reference(image_reference) transport = _mk_transport_pool() image_reference = ou.normalise_image_reference(image_reference) image_reference = docker_name.from_string(image_reference) creds = _mk_credentials( image_reference=image_reference, credentials_lookup=credentials_lookup, ) # OCI Image Manifest is compatible with Docker Image Manifest Version 2, # Schema 2. We indicate support for both formats by passing both media types # as 'Accept' headers. # # For reference: # OCI: https://github.com/opencontainers/image-spec # Docker: https://docs.docker.com/registry/spec/manifest-v2-2/ accept = docker_http.SUPPORTED_MANIFEST_MIMES try: logger.info(f'Pulling v2.2 image from {image_reference}..') with v2_2_image.FromRegistry(image_reference, creds, transport, accept) as v2_2_img: if v2_2_img.exists(): yield v2_2_img return # XXX TODO: use streaming rather than writing to local FS # if outfile is given, we must use it instead of an ano logger.debug(f'Pulling manifest list from {image_reference}..') with image_list.FromRegistry(image_reference, creds, transport) as img_list: if img_list.exists(): platform = image_list.Platform({ 'architecture': _PROCESSOR_ARCHITECTURE, 'os': _OPERATING_SYSTEM, }) # pytype: disable=wrong-arg-types with img_list.resolve(platform) as default_child: yield default_child return logger.info(f'Pulling v2 image from {image_reference}..') with v2_image.FromRegistry(image_reference, creds, transport) as v2_img: if v2_img.exists(): with v2_compat.V22FromV2(v2_img) as v2_2_img: yield v2_2_img return raise om.OciImageNotFoundException( f'failed to retrieve {image_reference=} - does it exist?') except Exception as e: raise e
def main(): args = parser.parse_args() if not args.name or not args.directory: raise Exception('--name and --directory are required arguments.') transport = transport_pool.Http(httplib2.Http, size=_THREADS) if '@' in args.name: name = docker_name.Digest(args.name) else: name = docker_name.Tag(args.name) # Resolve the appropriate credential to use based on the standard Docker # client logic. creds = docker_creds.DefaultKeychain.Resolve(name) with v2_2_image.FromRegistry(name, creds, transport) as v2_2_img: if v2_2_img.exists(): save.fast(v2_2_img, args.directory, threads=_THREADS) return with v2_image.FromRegistry(name, creds, transport) as v2_img: with v2_compat.V22FromV2(v2_img) as v2_2_img: save.fast(v2_2_img, args.directory, threads=_THREADS) return
def images(self): """Returns a list of tuples whose elements are (name, platform, image). Raises: InvalidMediaTypeError: a child with an unexpected media type was found. """ manifests = json.loads(self.manifest())['manifests'] results = [] for entry in manifests: digest = entry['digest'] name = docker_name.Digest('{base}@{digest}'.format( base=self._name.as_repository(), digest=digest)) # TODO(user): Support Image Index. if entry['mediaType'] == docker_http.MANIFEST_LIST_MIME: image = FromRegistry(name, self._creds, self._original_transport) elif entry['mediaType'] == docker_http.MANIFEST_SCHEMA2_MIME: image = v2_2_image.FromRegistry(name, self._creds, self._original_transport) else: raise InvalidMediaTypeError('Invalid media type: ' + entry['mediaType']) platform = Platform(entry['platform']) if 'platform' in entry else None results.append((name, platform, image)) return results
def append(self): if notebook_helper.is_in_notebook(): notebook_helper.export_notebook_to_tar_gz( self.notebook_file, TEMP_TAR_GZ_FILENAME, converted_filename=self.get_python_entrypoint()) else: utils.generate_context_tarball(".", TEMP_TAR_GZ_FILENAME) transport = transport_pool.Http(httplib2.Http, size=_THREADS) src = docker_name.Tag(self.base_image) creds = docker_creds.DefaultKeychain.Resolve(src) with v2_2_image.FromRegistry(src, creds, transport) as src_image: with open(TEMP_TAR_GZ_FILENAME, 'rb') as f: new_img = append.Layer(src_image, f.read()) if self.image_tag is None: self.image_tag = new_img.digest().split(":")[1] dst = docker_name.Tag(self.full_image_name()) creds = docker_creds.DefaultKeychain.Resolve(dst) with docker_session.Push(dst, creds, transport, threads=_THREADS, mount=[src.as_repository()]) as session: logger.warn("Uploading {}".format(self.full_image_name())) session.upload(new_img) os.remove(TEMP_TAR_GZ_FILENAME) logger.warn("Pushed image {}".format(self.full_image_name()))
def ResolveV22Tag(tag): with v2_2_image.FromRegistry( basic_creds=CredentialProvider(), name=tag, transport=http.Http(), accepted_mimes=v2_2_docker_http.SUPPORTED_MANIFEST_MIMES) as v2_2_img: if v2_2_img.exists(): return v2_2_img.digest() return None
def main(): args = parser.parse_args() if not args.name or not args.tarball: raise Exception('--name and --tarball are required arguments.') transport = transport_pool.Http(httplib2.Http, size=8) if '@' in args.name: name = docker_name.Digest(args.name) else: name = docker_name.Tag(args.name) # Resolve the appropriate credential to use based on the standard Docker # client logic. creds = docker_creds.DefaultKeychain.Resolve(name) with tarfile.open(name=args.tarball, mode='w') as tar: with v2_2_image.FromRegistry(name, creds, transport) as v2_2_img: if v2_2_img.exists(): save.tarball(_make_tag_if_digest(name), v2_2_img, tar) return with v2_image.FromRegistry(name, creds, transport) as v2_img: with v2_compat.V22FromV2(v2_img) as v2_2_img: save.tarball(_make_tag_if_digest(name), v2_2_img, tar) return
def GetDockerDigestFromPrefix(digest): """Gets a full digest string given a potential prefix. Args: digest: The digest prefix Returns: The full digest, or the same prefix if no full digest is found. Raises: InvalidImageNameError: if the prefix supplied isn't unique. """ repository_path, prefix = digest.split('@', 1) repository = ValidateRepositoryPath(repository_path) with v2_2_image.FromRegistry(basic_creds=CredentialProvider(), name=repository, transport=http.Http()) as image: matches = [d for d in image.manifests() if d.startswith(prefix)] if len(matches) == 1: return repository_path + '@' + matches.pop() elif len(matches) > 1: raise InvalidImageNameError( '{0} is not a unique digest prefix. Options are {1}.]'.format( prefix, ', '.join(map(str, matches)))) return digest
def Run(self, args): """This is what gets called when the user runs this command. Args: args: an argparse namespace. All the arguments that were provided to this command invocation. Raises: InvalidImageNameError: If the user specified an invalid image name. Returns: Some value that we want to have printed later. """ repository = util.ValidateRepositoryPath(args.image) http_obj = http.Http() with docker_image.FromRegistry( basic_creds=util.CredentialProvider(), name=repository, transport=http_obj) as image: try: return util.TransformManifests( image.manifests(), repository, show_occurrences=args.show_occurrences, occurrence_filter=args.occurrence_filter) except docker_http.V2DiagnosticException as err: raise util.GcloudifyRecoverableV2Errors(err, { 403: 'Access denied: {0}'.format(repository), 404: 'Not found: {0}'.format(repository) })
def to_hash_reference(image_name: str): transport = _mk_transport(size=1) image_name = normalise_image_reference(image_name) image_reference = _parse_image_reference(image_name) creds = _mk_credentials(image_reference=image_reference) accept = docker_http.SUPPORTED_MANIFEST_MIMES digest = None with image_list.FromRegistry(image_reference, creds, transport) as img_list: if img_list.exists(): digest = img_list.digest() else: logger.debug('no manifest found') # look for image with v2_2_image.FromRegistry(image_reference, creds, transport, accept) as v2_2_img: if v2_2_img.exists(): digest = v2_2_img.digest() else: logger.debug('no img v2.2 found') if not digest: # fallback to v2 with v2_image.FromRegistry(image_reference, creds, transport) as v2_img: if v2_img.exists(): digest = v2_img.digest() else: logger.debug('no img v2 found') raise RuntimeError(f'could not access img-metadata for {image_name}') name = image_name.rsplit(':', 1)[0] return f'{name}@{digest}'
def Get(self, base_image, namespace, checksum): entry = self._tag(base_image, namespace, checksum) with docker_image.FromRegistry(entry, self._creds, self._transport) as img: if img.exists(): print('Found cached base image: %s.' % entry) return img return None
def getEntryFromCreds(entry, creds, transport): """Given a cache entry and a set of credentials authenticated to a cache registry, check if the entry exists in the cache.""" with docker_image.FromRegistry(entry, creds, transport) as img: if img.exists(): logging.info('Found cached base image: %s.' % entry) return img logging.info('No cached base image found for entry: %s.' % entry)
def test_add_tag(self, mock_from_registry, mock_push): mock_from_registry.return_value = docker_image.FromRegistry() mock_push.return_value = docker_session.Push() self.r.add_tags(_FULL_REPO + '@sha256:' + _DIGEST2, _FULL_REPO + ':' + _TAG2, False) assert mock_from_registry.called assert mock_push.called
def _pull_image(image_reference: str, outfileobj=None): import ci.util ci.util.not_none(image_reference) transport = _mk_transport() image_reference = normalise_image_reference(image_reference) image_reference = _parse_image_reference(image_reference) creds = _mk_credentials(image_reference=image_reference) # OCI Image Manifest is compatible with Docker Image Manifest Version 2, # Schema 2. We indicate support for both formats by passing both media types # as 'Accept' headers. # # For reference: # OCI: https://github.com/opencontainers/image-spec # Docker: https://docs.docker.com/registry/spec/manifest-v2-2/ accept = docker_http.SUPPORTED_MANIFEST_MIMES try: # XXX TODO: use streaming rather than writing to local FS # if outfile is given, we must use it instead of an ano outfileobj = outfileobj if outfileobj else tempfile.TemporaryFile() with tarfile.open(fileobj=outfileobj, mode='w:') as tar: ci.util.verbose(f'Pulling manifest list from {image_reference}..') with image_list.FromRegistry(image_reference, creds, transport) as img_list: if img_list.exists(): platform = image_list.Platform({ 'architecture': _PROCESSOR_ARCHITECTURE, 'os': _OPERATING_SYSTEM, }) # pytype: disable=wrong-arg-types with img_list.resolve(platform) as default_child: save.tarball(_make_tag_if_digest(image_reference), default_child, tar) return outfileobj # pytype: enable=wrong-arg-types ci.util.info(f'Pulling v2.2 image from {image_reference}..') with v2_2_image.FromRegistry(image_reference, creds, transport, accept) as v2_2_img: if v2_2_img.exists(): save.tarball(_make_tag_if_digest(image_reference), v2_2_img, tar) return outfileobj ci.util.info(f'Pulling v2 image from {image_reference}..') with v2_image.FromRegistry(image_reference, creds, transport) as v2_img: with v2_compat.V22FromV2(v2_img) as v2_2_img: save.tarball(_make_tag_if_digest(image_reference), v2_2_img, tar) return outfileobj except Exception as e: outfileobj.close() ci.util.fail(f'Error pulling and saving image {image_reference}: {e}')
def test_dry_run(self, mock_from_registry, mock_push, mock_get_digest): mock_from_registry.return_value = docker_image.FromRegistry() mock_push.return_value = docker_session.Push() mock_get_digest.return_value = _DIGEST1 self.r.reconcile_tags(self.data, True) assert mock_from_registry.called assert mock_push.called
def _build(self, transport, src): file, hash = self.preprocessor.context_tar_gz() self.context_file, self.context_hash = file, hash self.image_tag = self.full_image_name(self.context_hash) creds = docker_creds.DefaultKeychain.Resolve(src) with v2_2_image.FromRegistry(src, creds, transport) as src_image: with open(self.context_file, 'rb') as f: new_img = append.Layer(src_image, f.read()) return new_img
def _get_digests(self, repo): name = docker_name.Repository(repo) creds = docker_creds.DefaultKeychain.Resolve(name) transport = transport_pool.Http(httplib2.Http) with docker_image.FromRegistry(name, creds, transport) as img: digests = [d[len('sha256:'):] for d in img.manifests()] return digests raise AssertionError('Unable to get digests from {0}'.format(repo))