def resolve_tag(image_loc: str):
logging.info(f"Resolving {image_loc}")
domain, org, repo, tag = parse_image_loc(image_loc)
dxf = DXF(domain, f"{org}/{repo}", tag, None)
hash = sha256(dxf.get_manifest(tag_to_pin).encode("utf-8")).hexdigest()
resolved = f"{domain}/{org}/{repo}@sha256:{hash}"
logging.info(f"{resolved}")
return resolved
def get_image_digest(self, repository: str, tag: str) -> str:
from dxf import DXF
from dxf.exceptions import DXFUnauthorizedError
try:
repository = f'library/{repository}' if '/' not in repository else repository
auth = 'Basic ' + self.auth if self.auth is not None else None
dxf = DXF(self.registry_host, repo=repository)
dxf.authenticate(authorization=auth, actions=['pull'])
except DXFUnauthorizedError:
raise RegistryAuthorizationException(f'Authentication with Docker registry {self.registry_host} failed. Run `docker login` first?')
try:
manifest = json.loads(dxf.get_manifest(tag))
return manifest['config']['digest']
except DXFUnauthorizedError:
raise RegistryImageNotFoundException(f'Image {repository}:{tag} not found in registry {self.registry_host}')
class DockerInjector:
"""
The main class of the Docker injector which injects new versions of a layer into
OCI images retrieved from an OCI compliant distribution API
"""
def __init__(self, host, repo, alias, user, pw):
"""
Initializes the injector by downloading both the slim and the fat image manifest
"""
def auth(dxf, response):
dxf.authenticate(user, pw, response=response)
self.dxfObject = DXF(host, repo, tlsverify=True, auth=auth)
self.image_manifest = self._get_manifest(alias)
self.fat_manifest = self._get_fat_manifest(self.image_manifest)
def setup(self, push_alias):
"""
Sets an image up for layer injection
"""
tar_digest, gz_digest = self._build_init_tar()
layer_size = self.dxfObject.blob_size(gz_digest)
self.fat_manifest.init_cvmfs_layer(tar_digest, gz_digest)
fat_man_json = self.fat_manifest.as_JSON()
manifest_digest = hash_bytes(bytes(fat_man_json, 'utf-8'))
self.dxfObject.push_blob(data=fat_man_json, digest=manifest_digest)
manifest_size = self.dxfObject.blob_size(manifest_digest)
self.image_manifest.init_cvmfs_layer(gz_digest, layer_size,
manifest_digest, manifest_size)
image_man_json = self.image_manifest.as_JSON()
self.dxfObject.set_manifest(push_alias, image_man_json)
def unpack(self, dest_dir):
"""
Unpacks the current version of a layer into the dest_dir directory in order to update it
"""
if not self.fat_manifest.is_cvmfs_prepared():
os.makedirs(dest_dir + "/cvmfs", exist_ok=True)
return
gz_digest = self.fat_manifest.get_gz_digest()
# Write out tar file
decompress_object = zlib.decompressobj(16 + zlib.MAX_WBITS)
try:
chunk_it = self.dxfObject.pull_blob(gz_digest)
except HTTPError as e:
if e.response.status_code == 404:
print("ERROR: The hash of the CVMFS layer must have changed.")
print(
"This is a known issue. Please do not reupload images to other repositories after CVMFS injection!"
)
else:
raise e
with tempfile.TemporaryFile() as tmp_file:
for chunk in chunk_it:
tmp_file.write(decompress_object.decompress(chunk))
tmp_file.write(decompress_object.flush())
tmp_file.seek(0)
tar = tarfile.TarFile(fileobj=tmp_file)
tar.extractall(dest_dir)
tar.close()
def update(self, src_dir, push_alias):
"""
Packs and uploads the contents of src_dir as a layer and injects the layer into the image.
The new layer version is stored under the tag push_alias
"""
if not self.fat_manifest.is_cvmfs_prepared():
print("Preparing image for CVMFS injection...")
self.setup(push_alias)
with tempfile.NamedTemporaryFile(delete=False) as tmp_file:
print("Bundling file into tar...")
_, error = exec_bash("tar --xattrs -C " + src_dir + " -cvf " +
tmp_file.name + " .")
if error:
raise RuntimeError("Failed to tar with error " + str(error))
tar_digest = hash_file(tmp_file.name)
print("Bundling tar into gz...")
gz_dest = tmp_file.name + ".gz"
_, error = exec_bash("gzip " + tmp_file.name)
if error:
raise RuntimeError("Failed to tar with error " + str(error))
print("Uploading...")
gz_digest = self.dxfObject.push_blob(gz_dest)
os.unlink(gz_dest)
print("Refreshing manifests...")
old_gz_digest = self.fat_manifest.get_gz_digest()
layer_size = self.dxfObject.blob_size(gz_digest)
self.fat_manifest.inject(tar_digest, gz_digest)
fat_man_json = self.fat_manifest.as_JSON()
manifest_digest = hash_bytes(bytes(fat_man_json, 'utf-8'))
self.dxfObject.push_blob(data=fat_man_json, digest=manifest_digest)
manifest_size = self.dxfObject.blob_size(manifest_digest)
self.image_manifest.inject(old_gz_digest, gz_digest, layer_size,
manifest_digest, manifest_size)
image_man_json = self.image_manifest.as_JSON()
self.dxfObject.set_manifest(push_alias, image_man_json)
def _get_manifest(self, alias):
return ImageManifest(self.dxfObject.get_manifest(alias))
def _get_fat_manifest(self, image_manifest):
fat_manifest = ""
(readIter, _) = self.dxfObject.pull_blob(
self.image_manifest.get_fat_manif_digest(),
size=True,
chunk_size=4096)
for chunk in readIter:
fat_manifest += str(chunk)[2:-1]
fat_manifest = fat_manifest.replace("\\\\", "\\")
return FatManifest(fat_manifest)
def _build_init_tar(self):
"""
Builds an empty /cvmfs tar and uploads it to the registry
:rtype: tuple
:returns: Tuple containing the tar digest and gz digest
"""
ident = self.image_manifest.get_fat_manif_digest()[5:15]
tmp_name = "/tmp/injector-" + ident
os.makedirs(tmp_name + "/cvmfs", exist_ok=True)
tar_dest = "/tmp/" + ident + ".tar"
_, error = exec_bash("tar --xattrs -C " + tmp_name + " -cvf " +
tar_dest + " .")
if error:
print("Failed to tar with error " + str(error))
return
tar_digest = hash_file(tar_dest)
_, error = exec_bash("gzip -n " + tar_dest)
if error:
print("Failed to tar with error " + str(error))
return
gz_dest = tar_dest + ".gz"
gzip_digest = self.dxfObject.push_blob(tar_dest + ".gz")
# Cleanup
os.rmdir(tmp_name + "/cvmfs")
os.rmdir(tmp_name)
os.unlink(gz_dest)
return (tar_digest, gzip_digest)
dxf.authenticate(os.environ.get("DOCKER_USERNAME"),
os.environ.get("DOCKER_TOKEN"),
response=response)
if __name__ == '__main__':
try:
logging.basicConfig(level=logging.INFO)
parsedArgs = parseArguments()
logging.info('Call arguments given: %s' % sys.argv[1:])
dxf = DXF('registry-1.docker.io',
'schulcloud/{}'.format(parsedArgs.repo), auth)
if hasattr(parsedArgs, 'add_tag') and parsedArgs.add_tag == True:
logging.info("Adding tag '{}' to '{}' in repository '{}'".format(
parsedArgs.new_tag, parsedArgs.exist_tag, parsedArgs.repo))
manifest = dxf.get_manifest('{}'.format(parsedArgs.exist_tag))
dxf.set_manifest('{}'.format(parsedArgs.new_tag), manifest)
else:
logging.info("Deleting tag '{}' in repository '{}'".format(
parsedArgs.exist_tag, parsedArgs.repo))
suretodel = query_yes_no(
"Are you sure to delete tag '{}' from '{}'".format(
parsedArgs.exist_tag, parsedArgs.repo))
if suretodel:
dxf.del_alias(parsedArgs.exist_tag)
exit(0)
except Exception as ex:
logging.exception(ex)
exit(1)
class DockerInjector:
"""
The main class of the Docker injector which injects new versions of a layer into
OCI images retrieved from an OCI compliant distribution API
"""
def __init__(self, host, repo, alias, user, pw):
"""
Initializes the injector by downloading both the slim and the fat image manifest
"""
def auth(dxf, response):
dxf.authenticate(user, pw, response=response)
self.dxfObject = DXF(host, repo, tlsverify=True, auth=auth)
self.image_manifest = self._get_manifest(alias)
self.fat_manifest = self._get_fat_manifest(self.image_manifest)
def setup(self, push_alias):
"""
Sets an image up for layer injection
"""
tar_digest, gz_digest = self._build_init_tar()
layer_size = self.dxfObject.blob_size(gz_digest)
self.fat_manifest.init_cvmfs_layer(tar_digest, gz_digest)
fat_man_json = self.fat_manifest.as_JSON()
manifest_digest = hash_bytes(bytes(fat_man_json, 'utf-8'))
self.dxfObject.push_blob(data=fat_man_json, digest=manifest_digest)
manifest_size = self.dxfObject.blob_size(manifest_digest)
self.image_manifest.init_cvmfs_layer(gz_digest, layer_size, manifest_digest, manifest_size)
image_man_json = self.image_manifest.as_JSON()
self.dxfObject.set_manifest(push_alias, image_man_json)
def unpack(self, dest_dir):
"""
Unpacks the current version of a layer into the dest_dir directory in order to update it
"""
if not self.fat_manifest.is_cvmfs_prepared():
os.makedirs(dest_dir+"/cvmfs", exist_ok=True)
return
gz_digest = self.fat_manifest.get_gz_digest()
# Write out tar file
decompress_object = zlib.decompressobj(16+zlib.MAX_WBITS)
try:
chunk_it = self.dxfObject.pull_blob(gz_digest)
except HTTPError as e:
if e.response.status_code == 404:
print("ERROR: The hash of the CVMFS layer must have changed.")
print("This is a known issue. Please do not reupload images to other repositories after CVMFS injection!")
else:
raise e
with tempfile.TemporaryFile() as tmp_file:
for chunk in chunk_it:
tmp_file.write(decompress_object.decompress(chunk))
tmp_file.write(decompress_object.flush())
tmp_file.seek(0)
tar = tarfile.TarFile(fileobj=tmp_file)
tar.extractall(dest_dir)
tar.close()
def update(self, src_dir, push_alias):
"""
Packs and uploads the contents of src_dir as a layer and injects the layer into the image.
The new layer version is stored under the tag push_alias
"""
if not self.fat_manifest.is_cvmfs_prepared():
print("Preparing image for CVMFS injection...")
self.setup(push_alias)
with tempfile.NamedTemporaryFile(delete=False) as tmp_file:
print("Bundling file into tar...")
_, error = exec_bash("tar --xattrs -C "+src_dir+" -cvf "+tmp_file.name+" .")
if error:
raise RuntimeError("Failed to tar with error " + str(error))
tar_digest = hash_file(tmp_file.name)
print("Bundling tar into gz...")
gz_dest = tmp_file.name+".gz"
_, error = exec_bash("gzip "+tmp_file.name)
if error:
raise RuntimeError("Failed to tar with error " + str(error))
print("Uploading...")
gz_digest = self.dxfObject.push_blob(gz_dest)
os.unlink(gz_dest)
print("Refreshing manifests...")
old_gz_digest = self.fat_manifest.get_gz_digest()
layer_size = self.dxfObject.blob_size(gz_digest)
self.fat_manifest.inject(tar_digest, gz_digest)
fat_man_json = self.fat_manifest.as_JSON()
manifest_digest = hash_bytes(bytes(fat_man_json, 'utf-8'))
self.dxfObject.push_blob(data=fat_man_json, digest=manifest_digest)
manifest_size = self.dxfObject.blob_size(manifest_digest)
self.image_manifest.inject(old_gz_digest, gz_digest, layer_size, manifest_digest, manifest_size)
image_man_json = self.image_manifest.as_JSON()
self.dxfObject.set_manifest(push_alias, image_man_json)
def _get_manifest(self, alias):
return ImageManifest(self.dxfObject.get_manifest(alias))
def _get_fat_manifest(self, image_manifest):
fat_manifest = ""
(readIter, _) = self.dxfObject.pull_blob(self.image_manifest.get_fat_manif_digest(), size=True, chunk_size=4096)
for chunk in readIter:
fat_manifest += str(chunk)[2:-1]
fat_manifest = fat_manifest.replace("\\\\","\\")
return FatManifest(fat_manifest)
def _build_init_tar(self):
"""
Builds an empty /cvmfs tar and uploads it to the registry
:rtype: tuple
:returns: Tuple containing the tar digest and gz digest
"""
ident = self.image_manifest.get_fat_manif_digest()[5:15]
tmp_name = "/tmp/injector-"+ident
os.makedirs(tmp_name+"/cvmfs", exist_ok=True)
tar_dest = "/tmp/"+ident+".tar"
_, error = exec_bash("tar --xattrs -C "+tmp_name+" -cvf "+tar_dest+" .")
if error:
print("Failed to tar with error " + str(error))
return
tar_digest = hash_file(tar_dest)
_, error = exec_bash("gzip -n "+tar_dest)
if error:
print("Failed to tar with error " + str(error))
return
gz_dest = tar_dest+".gz"
gzip_digest = self.dxfObject.push_blob(tar_dest+".gz")
# Cleanup
os.rmdir(tmp_name+"/cvmfs")
os.rmdir(tmp_name)
os.unlink(gz_dest)
return (tar_digest, gzip_digest)
