def test_normalise_image_reference():
# do not change fully qualified reference
reference = 'foo.io/my/image:1.2.3'
assert ou.normalise_image_reference(reference) == reference
# prepend default registry (docker.io) if no host given
reference = 'my/image:1.2.3'
assert ou.normalise_image_reference(reference) == 'registry-1.docker.io/' + reference
# insert 'library' if no "owner" is given
reference = 'alpine:1.2.3'
assert ou.normalise_image_reference(reference) == 'registry-1.docker.io/library/' + reference
def pulled_image(
image_reference: str,
credentials_lookup: typing.Callable[[image_reference, oa.Privileges, bool],
oa.OciConfig],
):
_tag_or_digest_reference(image_reference)
transport = _mk_transport_pool()
image_reference = ou.normalise_image_reference(image_reference)
image_reference = docker_name.from_string(image_reference)
creds = _mk_credentials(
image_reference=image_reference,
credentials_lookup=credentials_lookup,
)
# OCI Image Manifest is compatible with Docker Image Manifest Version 2,
# Schema 2. We indicate support for both formats by passing both media types
# as 'Accept' headers.
#
# For reference:
# OCI: https://github.com/opencontainers/image-spec
# Docker: https://docs.docker.com/registry/spec/manifest-v2-2/
accept = docker_http.SUPPORTED_MANIFEST_MIMES
try:
logger.info(f'Pulling v2.2 image from {image_reference}..')
with v2_2_image.FromRegistry(image_reference, creds, transport,
accept) as v2_2_img:
if v2_2_img.exists():
yield v2_2_img
return
# XXX TODO: use streaming rather than writing to local FS
# if outfile is given, we must use it instead of an ano
logger.debug(f'Pulling manifest list from {image_reference}..')
with image_list.FromRegistry(image_reference, creds,
transport) as img_list:
if img_list.exists():
platform = image_list.Platform({
'architecture': _PROCESSOR_ARCHITECTURE,
'os': _OPERATING_SYSTEM,
})
# pytype: disable=wrong-arg-types
with img_list.resolve(platform) as default_child:
yield default_child
return
logger.info(f'Pulling v2 image from {image_reference}..')
with v2_image.FromRegistry(image_reference, creds,
transport) as v2_img:
if v2_img.exists():
with v2_compat.V22FromV2(v2_img) as v2_2_img:
yield v2_2_img
return
raise om.OciImageNotFoundException(
f'failed to retrieve {image_reference=} - does it exist?')
except Exception as e:
raise e
def _mk_transport(
image_name: str,
credentials_lookup: typing.Callable[[image_reference, oa.Privileges, bool],
oa.OciConfig],
action: str = docker_http.PULL,
privileges: oa.Privileges = oa.Privileges.READONLY,
):
if isinstance(image_name, str):
image_name = ou.normalise_image_reference(image_name)
credentials = _mk_credentials(
image_reference=str(image_name),
credentials_lookup=credentials_lookup,
privileges=privileges,
)
if isinstance(image_name, str):
image_name = docker_name.from_string(name=image_name)
transport_pool = _mk_transport_pool(size=1)
transport = docker_http.Transport(
name=image_name,
creds=credentials,
transport=transport_pool,
action=docker_http.PULL,
)
return transport
def tags(
image_name: str,
credentials_lookup: typing.Callable[[image_reference, oa.Privileges, bool],
oa.OciConfig],
) -> typing.Sequence[str]:
'''
returns a sequence of all `tags` for the given image_name
'''
if isinstance(image_name, str):
image_name = ou.normalise_image_reference(image_name)
from containerregistry.client.v2_2 import docker_http
transport = _ou._mk_transport(
image_name=image_name,
credentials_lookup=credentials_lookup,
action=docker_http.PULL,
)
if isinstance(image_name, str):
from containerregistry.client import docker_name
image_name = docker_name.from_string(image_name)
url = f'https://{image_name.registry}/v2/{image_name.repository}/tags/list'
res, body_bytes = transport.Request(url, (200, ))
parsed = json.loads(body_bytes)
# XXX parsed['manifest'] might be used to e.g. determine stale images, and purge them
tags = parsed['tags']
return tags
def rm_tag(image_reference: str):
transport = _mk_transport_pool()
image_reference = ou.normalise_image_reference(image_reference)
image_reference = docker_name.from_string(image_reference)
creds = _mk_credentials(image_reference=image_reference)
docker_session.Delete(
name=image_reference,
creds=creds,
transport=transport,
)
logger.info(f'untagged {image_reference=} - note: did not purge blobs!')
def publish_container_image_from_tarfile(
tar_file: typing.Union[str, typing.IO],
oci_client,
image_reference: str,
architecture: Architecture,
os: OperatingSystem = OperatingSystem.LINUX,
additional_tags: typing.List[str] = [],
):
image_reference = ou.normalise_image_reference(
image_reference=image_reference)
image_name = image_reference.rsplit(':', 1)[0]
image_references = (image_reference, ) + tuple(
[f'{image_name}:{tag}' for tag in additional_tags])
uncompressed_hash = hashlib.sha256()
with tempfile.TemporaryFile() as gzip_file:
compressed_hash = hashlib.sha256()
length = 0
src_length = 0
crc = 0
with lzma.open(tar_file) as f:
gzip_file.write(gzip_header := gziputil.gzip_header(
fname=b'layer.tar'))
compressed_hash.update(gzip_header)
length += len(gzip_header)
compressor = gziputil.zlib_compressobj()
while chunk := f.read(1024):
uncompressed_hash.update(chunk)
crc = zlib.crc32(chunk, crc)
src_length += len(chunk)
chunk = compressor.compress(chunk)
compressed_hash.update(chunk)
length += len(chunk)
gzip_file.write(chunk)
gzip_file.write((remainder := compressor.flush()))
def _pull_image(
image_reference: str,
credentials_lookup: typing.Callable[[image_reference, oa.Privileges, bool],
oa.OciConfig],
outfileobj=None,
):
if not image_reference:
raise ValueError(image_reference)
image_reference = ou.normalise_image_reference(
image_reference=image_reference)
outfileobj = outfileobj if outfileobj else tempfile.TemporaryFile()
with tarfile.open(fileobj=outfileobj, mode='w:') as tar:
with pulled_image(
image_reference=image_reference,
credentials_lookup=credentials_lookup,
) as image:
image_reference = docker_name.from_string(image_reference)
save.tarball(_make_tag_if_digest(image_reference), image, tar)
return outfileobj
def _push_image(image_reference: str,
image_file: str,
credentials_lookup: typing.Callable[
[image_reference, oa.Privileges, bool], oa.OciConfig],
threads=8):
if not image_reference:
raise ValueError(image_reference)
if not os.path.isfile(image_file):
raise ValueError(f'not an exiting file: {image_file=}')
transport = _mk_transport_pool()
image_reference = ou.normalise_image_reference(image_reference)
image_reference = docker_name.from_string(image_reference)
creds = _mk_credentials(
image_reference=image_reference,
credentials_lookup=credentials_lookup,
privileges=oa.Privileges.READWRITE,
)
# XXX fail if no creds were found
with v2_2_image.FromTarball(image_file) as v2_2_img:
try:
with docker_session.Push(
image_reference,
creds,
transport,
threads=threads,
) as session:
session.upload(v2_2_img)
digest = v2_2_img.digest()
logger.info(
f'{image_reference} was uploaded - digest: {digest}')
except Exception as e:
import traceback
traceback.print_exc()
raise e
def publish_container_image_from_kaniko_tarfile(
image_tarfile_path: str,
oci_client: oc.Client,
image_reference: str,
additional_tags: typing.List[str] = (),
):
image_reference = ou.normalise_image_reference(
image_reference=image_reference)
image_name = image_reference.rsplit(':', 1)[0]
image_references = (image_reference, ) + tuple(
[f'{image_name}:{tag}' for tag in additional_tags])
with ok.read_kaniko_image_tar(tar_path=image_tarfile_path) as image:
chunk_size = 1024 * 1024
for kaniko_blob in image.blobs():
oci_client.put_blob(
image_reference=image_reference,
digest=kaniko_blob.digest_str(),
octets_count=kaniko_blob.size,
data=kaniko_blob,
max_chunk=chunk_size,
)
oci_client.blob(
image_reference=image_reference,
digest=kaniko_blob.digest_str(),
absent_ok=True,
)
manifest_bytes = json.dumps(dataclasses.asdict(
image.oci_manifest())).encode('utf-8')
for tgt_ref in image_references:
logger.info(f'publishing manifest {tgt_ref=}')
oci_client.put_manifest(
image_reference=tgt_ref,
manifest=manifest_bytes,
)
def to_hash_reference(image_name: str):
transport = _mk_transport_pool(size=1)
image_name = ou.normalise_image_reference(image_name)
image_reference = docker_name.from_string(image_name)
creds = _mk_credentials(image_reference=image_reference)
accept = docker_http.SUPPORTED_MANIFEST_MIMES
digest = None
with image_list.FromRegistry(image_reference, creds,
transport) as img_list:
if img_list.exists():
digest = img_list.digest()
else:
logger.debug('no manifest found')
# look for image
with v2_2_image.FromRegistry(image_reference, creds, transport,
accept) as v2_2_img:
if v2_2_img.exists():
digest = v2_2_img.digest()
else:
logger.debug('no img v2.2 found')
if not digest:
# fallback to v2
with v2_image.FromRegistry(image_reference, creds,
transport) as v2_img:
if v2_img.exists():
digest = v2_img.digest()
else:
logger.debug('no img v2 found')
raise RuntimeError(
f'could not access img-metadata for {image_name}')
name = image_name.rsplit(':', 1)[0]
return f'{name}@{digest}'
def image_exists(
image_reference: str,
credentials_lookup: typing.Callable[[image_reference, oa.Privileges, bool],
oa.OciConfig],
) -> bool:
'''
returns a boolean value indicating whether or not the given OCI Artifact exists
'''
transport = _ou._mk_transport_pool(size=1)
image_reference = ou.normalise_image_reference(
image_reference=image_reference)
image_reference = _ou.docker_name.from_string(image_reference)
creds = _ou._mk_credentials(
image_reference=image_reference,
credentials_lookup=credentials_lookup,
)
# keep import local to avoid exposure to module's users
from containerregistry.client.v2_2 import docker_image_list as image_list
with image_list.FromRegistry(image_reference, creds,
transport) as img_list:
if img_list.exists():
return True
# keep import local to avoid exposure to module's users
from containerregistry.client.v2_2 import docker_image as v2_2_image
accept = _ou.docker_http.SUPPORTED_MANIFEST_MIMES
with v2_2_image.FromRegistry(image_reference, creds, transport,
accept) as v2_2_img:
if v2_2_img.exists():
return True
return False
def publish_container_image_from_kaniko_tarfile(
image_tarfile_path: str,
oci_client: oc.Client,
image_reference: str,
additional_tags: typing.List[str] = (),
manifest_mimetype: str = om.OCI_MANIFEST_SCHEMA_V2_MIME,
):
image_reference = ou.normalise_image_reference(
image_reference=image_reference)
image_name = image_reference.rsplit(':', 1)[0]
image_references = (image_reference, ) + tuple(
[f'{image_name}:{tag}' for tag in additional_tags])
with ok.read_kaniko_image_tar(tar_path=image_tarfile_path) as image:
chunk_size = 1024 * 1024
for kaniko_blob in image.blobs():
oci_client.put_blob(
image_reference=image_reference,
digest=kaniko_blob.digest_str(),
octets_count=kaniko_blob.size,
data=kaniko_blob,
max_chunk=chunk_size,
)
# optionally patch manifest's mimetype (e.g. required for docker-hub)
manifest_dict = dataclasses.asdict(image.oci_manifest())
manifest_dict['mediaType'] = manifest_mimetype
manifest_bytes = json.dumps(manifest_dict, ).encode('utf-8')
for tgt_ref in image_references:
logger.info(f'publishing manifest {tgt_ref=}')
oci_client.put_manifest(
image_reference=tgt_ref,
manifest=manifest_bytes,
)
def replicate_artifact(
src_image_reference: str,
tgt_image_reference: str,
credentials_lookup: typing.Callable[[image_reference, oa.Privileges, bool],
oa.OciConfig],
):
'''
verbatimly replicate the OCI Artifact from src -> tgt without taking any assumptions
about the transported contents. This in particular allows contents to be replicated
that are not e.g. "docker-compliant" OCI Images.
'''
src_image_reference = ou.normalise_image_reference(src_image_reference)
tgt_image_reference = ou.normalise_image_reference(tgt_image_reference)
client = oc.Client(credentials_lookup=credentials_lookup)
# we need the unaltered - manifest for verbatim replication
raw_manifest = client.manifest_raw(
image_reference=src_image_reference, ).text
manifest = json.loads(raw_manifest)
schema_version = int(manifest['schemaVersion'])
if schema_version == 1:
manifest = dacite.from_dict(data_class=om.OciImageManifestV1,
data=json.loads(raw_manifest))
manifest = client.manifest(src_image_reference)
elif schema_version == 2:
manifest = dacite.from_dict(data_class=om.OciImageManifest,
data=json.loads(raw_manifest))
for idx, layer in enumerate(manifest.blobs()):
# need to specially handle manifest (may be absent for v2 / legacy images)
is_manifest = idx == 0
blob_res = client.blob(
image_reference=src_image_reference,
digest=layer.digest,
absent_ok=is_manifest,
)
if not blob_res:
# fallback to non-verbatim replication
# XXX we definitely should _not_ read entire blobs into memory
# this is done by the used containerregistry lib, so we do not make things worse
# here - however this must not remain so!
logger.warning('falling back to non-verbatim replication '
'{src_image_reference=} {tgt_image_reference=}')
with tempfile.NamedTemporaryFile() as tmp_fh:
retrieve_container_image(
image_reference=src_image_reference,
credentials_lookup=credentials_lookup,
outfileobj=tmp_fh,
)
publish_container_image(
image_reference=tgt_image_reference,
image_file_obj=tmp_fh,
credentials_lookup=credentials_lookup,
)
return
client.put_blob(
image_reference=tgt_image_reference,
digest=layer.digest,
octets_count=layer.size,
data=blob_res,
)
client.put_manifest(
image_reference=tgt_image_reference,
manifest=raw_manifest,
)
def replicate_artifact(
src_image_reference: str,
tgt_image_reference: str,
credentials_lookup: oa.credentials_lookup = None,
routes: oc.OciRoutes = oc.OciRoutes(),
oci_client: oc.Client = None,
):
'''
verbatimly replicate the OCI Artifact from src -> tgt without taking any assumptions
about the transported contents. This in particular allows contents to be replicated
that are not e.g. "docker-compliant" OCI Images.
'''
if not (bool(credentials_lookup) ^ bool(oci_client)):
raise ValueError(
'either credentials-lookup + routes, xor client must be passed')
src_image_reference = ou.normalise_image_reference(src_image_reference)
tgt_image_reference = ou.normalise_image_reference(tgt_image_reference)
if not oci_client:
client = oc.Client(
credentials_lookup=credentials_lookup,
routes=routes,
)
else:
client = oci_client
# we need the unaltered - manifest for verbatim replication
raw_manifest = client.manifest_raw(
image_reference=src_image_reference, ).text
manifest = json.loads(raw_manifest)
schema_version = int(manifest['schemaVersion'])
if schema_version == 1:
manifest = dacite.from_dict(data_class=om.OciImageManifestV1,
data=json.loads(raw_manifest))
manifest = client.manifest(src_image_reference)
elif schema_version == 2:
manifest = dacite.from_dict(data_class=om.OciImageManifest,
data=json.loads(raw_manifest))
for idx, layer in enumerate(manifest.blobs()):
# need to specially handle manifest (may be absent for v2 / legacy images)
is_manifest = idx == 0
head_res = client.head_blob(
image_reference=tgt_image_reference,
digest=layer.digest,
)
if head_res.ok:
logger.info(
f'skipping blob download {layer.digest=} - already exists in tgt'
)
continue # no need to download if blob already exists in tgt
blob_res = client.blob(
image_reference=src_image_reference,
digest=layer.digest,
absent_ok=is_manifest,
)
if not blob_res and is_manifest:
# fallback to non-verbatim replication; synthesise cfg
logger.warning('falling back to non-verbatim replication '
'{src_image_reference=} {tgt_image_reference=}')
fake_cfg = od.docker_cfg(
) # TODO: check whether we need to pass-in cfg
fake_cfg_dict = dataclasses.asdict(fake_cfg)
fake_cfg_raw = json.dumps(fake_cfg_dict).encode('utf-8')
client.put_blob(
image_reference=tgt_image_reference,
digest=f'sha256:{hashlib.sha256(fake_cfg_raw).hexdigest()}',
octets_count=len(fake_cfg_raw),
data=fake_cfg_raw,
)
continue
client.put_blob(
image_reference=tgt_image_reference,
digest=layer.digest,
octets_count=layer.size,
data=blob_res,
)
client.put_manifest(
image_reference=tgt_image_reference,
manifest=raw_manifest,
)
