def test_normalise_image_reference(): # do not change fully qualified reference reference = 'foo.io/my/image:1.2.3' assert ou.normalise_image_reference(reference) == reference # prepend default registry (docker.io) if no host given reference = 'my/image:1.2.3' assert ou.normalise_image_reference(reference) == 'registry-1.docker.io/' + reference # insert 'library' if no "owner" is given reference = 'alpine:1.2.3' assert ou.normalise_image_reference(reference) == 'registry-1.docker.io/library/' + reference
def pulled_image( image_reference: str, credentials_lookup: typing.Callable[[image_reference, oa.Privileges, bool], oa.OciConfig], ): _tag_or_digest_reference(image_reference) transport = _mk_transport_pool() image_reference = ou.normalise_image_reference(image_reference) image_reference = docker_name.from_string(image_reference) creds = _mk_credentials( image_reference=image_reference, credentials_lookup=credentials_lookup, ) # OCI Image Manifest is compatible with Docker Image Manifest Version 2, # Schema 2. We indicate support for both formats by passing both media types # as 'Accept' headers. # # For reference: # OCI: https://github.com/opencontainers/image-spec # Docker: https://docs.docker.com/registry/spec/manifest-v2-2/ accept = docker_http.SUPPORTED_MANIFEST_MIMES try: logger.info(f'Pulling v2.2 image from {image_reference}..') with v2_2_image.FromRegistry(image_reference, creds, transport, accept) as v2_2_img: if v2_2_img.exists(): yield v2_2_img return # XXX TODO: use streaming rather than writing to local FS # if outfile is given, we must use it instead of an ano logger.debug(f'Pulling manifest list from {image_reference}..') with image_list.FromRegistry(image_reference, creds, transport) as img_list: if img_list.exists(): platform = image_list.Platform({ 'architecture': _PROCESSOR_ARCHITECTURE, 'os': _OPERATING_SYSTEM, }) # pytype: disable=wrong-arg-types with img_list.resolve(platform) as default_child: yield default_child return logger.info(f'Pulling v2 image from {image_reference}..') with v2_image.FromRegistry(image_reference, creds, transport) as v2_img: if v2_img.exists(): with v2_compat.V22FromV2(v2_img) as v2_2_img: yield v2_2_img return raise om.OciImageNotFoundException( f'failed to retrieve {image_reference=} - does it exist?') except Exception as e: raise e
def _mk_transport( image_name: str, credentials_lookup: typing.Callable[[image_reference, oa.Privileges, bool], oa.OciConfig], action: str = docker_http.PULL, privileges: oa.Privileges = oa.Privileges.READONLY, ): if isinstance(image_name, str): image_name = ou.normalise_image_reference(image_name) credentials = _mk_credentials( image_reference=str(image_name), credentials_lookup=credentials_lookup, privileges=privileges, ) if isinstance(image_name, str): image_name = docker_name.from_string(name=image_name) transport_pool = _mk_transport_pool(size=1) transport = docker_http.Transport( name=image_name, creds=credentials, transport=transport_pool, action=docker_http.PULL, ) return transport
def tags( image_name: str, credentials_lookup: typing.Callable[[image_reference, oa.Privileges, bool], oa.OciConfig], ) -> typing.Sequence[str]: ''' returns a sequence of all `tags` for the given image_name ''' if isinstance(image_name, str): image_name = ou.normalise_image_reference(image_name) from containerregistry.client.v2_2 import docker_http transport = _ou._mk_transport( image_name=image_name, credentials_lookup=credentials_lookup, action=docker_http.PULL, ) if isinstance(image_name, str): from containerregistry.client import docker_name image_name = docker_name.from_string(image_name) url = f'https://{image_name.registry}/v2/{image_name.repository}/tags/list' res, body_bytes = transport.Request(url, (200, )) parsed = json.loads(body_bytes) # XXX parsed['manifest'] might be used to e.g. determine stale images, and purge them tags = parsed['tags'] return tags
def rm_tag(image_reference: str): transport = _mk_transport_pool() image_reference = ou.normalise_image_reference(image_reference) image_reference = docker_name.from_string(image_reference) creds = _mk_credentials(image_reference=image_reference) docker_session.Delete( name=image_reference, creds=creds, transport=transport, ) logger.info(f'untagged {image_reference=} - note: did not purge blobs!')
def publish_container_image_from_tarfile( tar_file: typing.Union[str, typing.IO], oci_client, image_reference: str, architecture: Architecture, os: OperatingSystem = OperatingSystem.LINUX, additional_tags: typing.List[str] = [], ): image_reference = ou.normalise_image_reference( image_reference=image_reference) image_name = image_reference.rsplit(':', 1)[0] image_references = (image_reference, ) + tuple( [f'{image_name}:{tag}' for tag in additional_tags]) uncompressed_hash = hashlib.sha256() with tempfile.TemporaryFile() as gzip_file: compressed_hash = hashlib.sha256() length = 0 src_length = 0 crc = 0 with lzma.open(tar_file) as f: gzip_file.write(gzip_header := gziputil.gzip_header( fname=b'layer.tar')) compressed_hash.update(gzip_header) length += len(gzip_header) compressor = gziputil.zlib_compressobj() while chunk := f.read(1024): uncompressed_hash.update(chunk) crc = zlib.crc32(chunk, crc) src_length += len(chunk) chunk = compressor.compress(chunk) compressed_hash.update(chunk) length += len(chunk) gzip_file.write(chunk) gzip_file.write((remainder := compressor.flush()))
def _pull_image( image_reference: str, credentials_lookup: typing.Callable[[image_reference, oa.Privileges, bool], oa.OciConfig], outfileobj=None, ): if not image_reference: raise ValueError(image_reference) image_reference = ou.normalise_image_reference( image_reference=image_reference) outfileobj = outfileobj if outfileobj else tempfile.TemporaryFile() with tarfile.open(fileobj=outfileobj, mode='w:') as tar: with pulled_image( image_reference=image_reference, credentials_lookup=credentials_lookup, ) as image: image_reference = docker_name.from_string(image_reference) save.tarball(_make_tag_if_digest(image_reference), image, tar) return outfileobj
def _push_image(image_reference: str, image_file: str, credentials_lookup: typing.Callable[ [image_reference, oa.Privileges, bool], oa.OciConfig], threads=8): if not image_reference: raise ValueError(image_reference) if not os.path.isfile(image_file): raise ValueError(f'not an exiting file: {image_file=}') transport = _mk_transport_pool() image_reference = ou.normalise_image_reference(image_reference) image_reference = docker_name.from_string(image_reference) creds = _mk_credentials( image_reference=image_reference, credentials_lookup=credentials_lookup, privileges=oa.Privileges.READWRITE, ) # XXX fail if no creds were found with v2_2_image.FromTarball(image_file) as v2_2_img: try: with docker_session.Push( image_reference, creds, transport, threads=threads, ) as session: session.upload(v2_2_img) digest = v2_2_img.digest() logger.info( f'{image_reference} was uploaded - digest: {digest}') except Exception as e: import traceback traceback.print_exc() raise e
def publish_container_image_from_kaniko_tarfile( image_tarfile_path: str, oci_client: oc.Client, image_reference: str, additional_tags: typing.List[str] = (), ): image_reference = ou.normalise_image_reference( image_reference=image_reference) image_name = image_reference.rsplit(':', 1)[0] image_references = (image_reference, ) + tuple( [f'{image_name}:{tag}' for tag in additional_tags]) with ok.read_kaniko_image_tar(tar_path=image_tarfile_path) as image: chunk_size = 1024 * 1024 for kaniko_blob in image.blobs(): oci_client.put_blob( image_reference=image_reference, digest=kaniko_blob.digest_str(), octets_count=kaniko_blob.size, data=kaniko_blob, max_chunk=chunk_size, ) oci_client.blob( image_reference=image_reference, digest=kaniko_blob.digest_str(), absent_ok=True, ) manifest_bytes = json.dumps(dataclasses.asdict( image.oci_manifest())).encode('utf-8') for tgt_ref in image_references: logger.info(f'publishing manifest {tgt_ref=}') oci_client.put_manifest( image_reference=tgt_ref, manifest=manifest_bytes, )
def to_hash_reference(image_name: str): transport = _mk_transport_pool(size=1) image_name = ou.normalise_image_reference(image_name) image_reference = docker_name.from_string(image_name) creds = _mk_credentials(image_reference=image_reference) accept = docker_http.SUPPORTED_MANIFEST_MIMES digest = None with image_list.FromRegistry(image_reference, creds, transport) as img_list: if img_list.exists(): digest = img_list.digest() else: logger.debug('no manifest found') # look for image with v2_2_image.FromRegistry(image_reference, creds, transport, accept) as v2_2_img: if v2_2_img.exists(): digest = v2_2_img.digest() else: logger.debug('no img v2.2 found') if not digest: # fallback to v2 with v2_image.FromRegistry(image_reference, creds, transport) as v2_img: if v2_img.exists(): digest = v2_img.digest() else: logger.debug('no img v2 found') raise RuntimeError( f'could not access img-metadata for {image_name}') name = image_name.rsplit(':', 1)[0] return f'{name}@{digest}'
def image_exists( image_reference: str, credentials_lookup: typing.Callable[[image_reference, oa.Privileges, bool], oa.OciConfig], ) -> bool: ''' returns a boolean value indicating whether or not the given OCI Artifact exists ''' transport = _ou._mk_transport_pool(size=1) image_reference = ou.normalise_image_reference( image_reference=image_reference) image_reference = _ou.docker_name.from_string(image_reference) creds = _ou._mk_credentials( image_reference=image_reference, credentials_lookup=credentials_lookup, ) # keep import local to avoid exposure to module's users from containerregistry.client.v2_2 import docker_image_list as image_list with image_list.FromRegistry(image_reference, creds, transport) as img_list: if img_list.exists(): return True # keep import local to avoid exposure to module's users from containerregistry.client.v2_2 import docker_image as v2_2_image accept = _ou.docker_http.SUPPORTED_MANIFEST_MIMES with v2_2_image.FromRegistry(image_reference, creds, transport, accept) as v2_2_img: if v2_2_img.exists(): return True return False
def publish_container_image_from_kaniko_tarfile( image_tarfile_path: str, oci_client: oc.Client, image_reference: str, additional_tags: typing.List[str] = (), manifest_mimetype: str = om.OCI_MANIFEST_SCHEMA_V2_MIME, ): image_reference = ou.normalise_image_reference( image_reference=image_reference) image_name = image_reference.rsplit(':', 1)[0] image_references = (image_reference, ) + tuple( [f'{image_name}:{tag}' for tag in additional_tags]) with ok.read_kaniko_image_tar(tar_path=image_tarfile_path) as image: chunk_size = 1024 * 1024 for kaniko_blob in image.blobs(): oci_client.put_blob( image_reference=image_reference, digest=kaniko_blob.digest_str(), octets_count=kaniko_blob.size, data=kaniko_blob, max_chunk=chunk_size, ) # optionally patch manifest's mimetype (e.g. required for docker-hub) manifest_dict = dataclasses.asdict(image.oci_manifest()) manifest_dict['mediaType'] = manifest_mimetype manifest_bytes = json.dumps(manifest_dict, ).encode('utf-8') for tgt_ref in image_references: logger.info(f'publishing manifest {tgt_ref=}') oci_client.put_manifest( image_reference=tgt_ref, manifest=manifest_bytes, )
def replicate_artifact( src_image_reference: str, tgt_image_reference: str, credentials_lookup: typing.Callable[[image_reference, oa.Privileges, bool], oa.OciConfig], ): ''' verbatimly replicate the OCI Artifact from src -> tgt without taking any assumptions about the transported contents. This in particular allows contents to be replicated that are not e.g. "docker-compliant" OCI Images. ''' src_image_reference = ou.normalise_image_reference(src_image_reference) tgt_image_reference = ou.normalise_image_reference(tgt_image_reference) client = oc.Client(credentials_lookup=credentials_lookup) # we need the unaltered - manifest for verbatim replication raw_manifest = client.manifest_raw( image_reference=src_image_reference, ).text manifest = json.loads(raw_manifest) schema_version = int(manifest['schemaVersion']) if schema_version == 1: manifest = dacite.from_dict(data_class=om.OciImageManifestV1, data=json.loads(raw_manifest)) manifest = client.manifest(src_image_reference) elif schema_version == 2: manifest = dacite.from_dict(data_class=om.OciImageManifest, data=json.loads(raw_manifest)) for idx, layer in enumerate(manifest.blobs()): # need to specially handle manifest (may be absent for v2 / legacy images) is_manifest = idx == 0 blob_res = client.blob( image_reference=src_image_reference, digest=layer.digest, absent_ok=is_manifest, ) if not blob_res: # fallback to non-verbatim replication # XXX we definitely should _not_ read entire blobs into memory # this is done by the used containerregistry lib, so we do not make things worse # here - however this must not remain so! logger.warning('falling back to non-verbatim replication ' '{src_image_reference=} {tgt_image_reference=}') with tempfile.NamedTemporaryFile() as tmp_fh: retrieve_container_image( image_reference=src_image_reference, credentials_lookup=credentials_lookup, outfileobj=tmp_fh, ) publish_container_image( image_reference=tgt_image_reference, image_file_obj=tmp_fh, credentials_lookup=credentials_lookup, ) return client.put_blob( image_reference=tgt_image_reference, digest=layer.digest, octets_count=layer.size, data=blob_res, ) client.put_manifest( image_reference=tgt_image_reference, manifest=raw_manifest, )
def replicate_artifact( src_image_reference: str, tgt_image_reference: str, credentials_lookup: oa.credentials_lookup = None, routes: oc.OciRoutes = oc.OciRoutes(), oci_client: oc.Client = None, ): ''' verbatimly replicate the OCI Artifact from src -> tgt without taking any assumptions about the transported contents. This in particular allows contents to be replicated that are not e.g. "docker-compliant" OCI Images. ''' if not (bool(credentials_lookup) ^ bool(oci_client)): raise ValueError( 'either credentials-lookup + routes, xor client must be passed') src_image_reference = ou.normalise_image_reference(src_image_reference) tgt_image_reference = ou.normalise_image_reference(tgt_image_reference) if not oci_client: client = oc.Client( credentials_lookup=credentials_lookup, routes=routes, ) else: client = oci_client # we need the unaltered - manifest for verbatim replication raw_manifest = client.manifest_raw( image_reference=src_image_reference, ).text manifest = json.loads(raw_manifest) schema_version = int(manifest['schemaVersion']) if schema_version == 1: manifest = dacite.from_dict(data_class=om.OciImageManifestV1, data=json.loads(raw_manifest)) manifest = client.manifest(src_image_reference) elif schema_version == 2: manifest = dacite.from_dict(data_class=om.OciImageManifest, data=json.loads(raw_manifest)) for idx, layer in enumerate(manifest.blobs()): # need to specially handle manifest (may be absent for v2 / legacy images) is_manifest = idx == 0 head_res = client.head_blob( image_reference=tgt_image_reference, digest=layer.digest, ) if head_res.ok: logger.info( f'skipping blob download {layer.digest=} - already exists in tgt' ) continue # no need to download if blob already exists in tgt blob_res = client.blob( image_reference=src_image_reference, digest=layer.digest, absent_ok=is_manifest, ) if not blob_res and is_manifest: # fallback to non-verbatim replication; synthesise cfg logger.warning('falling back to non-verbatim replication ' '{src_image_reference=} {tgt_image_reference=}') fake_cfg = od.docker_cfg( ) # TODO: check whether we need to pass-in cfg fake_cfg_dict = dataclasses.asdict(fake_cfg) fake_cfg_raw = json.dumps(fake_cfg_dict).encode('utf-8') client.put_blob( image_reference=tgt_image_reference, digest=f'sha256:{hashlib.sha256(fake_cfg_raw).hexdigest()}', octets_count=len(fake_cfg_raw), data=fake_cfg_raw, ) continue client.put_blob( image_reference=tgt_image_reference, digest=layer.digest, octets_count=layer.size, data=blob_res, ) client.put_manifest( image_reference=tgt_image_reference, manifest=raw_manifest, )