def confirm_registry_asset(repo, pointer):
"""
Validates a registry asset by querying the remote registry.
Returns the registry manifest at the given pointer, and a pullable image string.
"""
host, username, api_key = get_registry_connectivity()
image = host + '/' + repo + '@' + pointer
log.debug('Validating image ' + image + '...')
# Authenticate via callable
def auth(dxf, response):
log.debug('Authenticating to registry...')
dxf.authenticate(username, api_key, response=response)
log.debug('Auth to registry successful')
# Connects over internal network with override host and autogenerated TLS
dxf = DXF(host, repo, auth, tlsverify=False)
# Fetch and sanity check the blob size
blob_id = str(dxf.get_digest(pointer))
if dxf.blob_size(blob_id) > (10 * 1000 * 1000): # 10 MB to bytes
raise Exception(
'Manifest is larger than 10 MB. Possible registry error?')
# Pull and assemble the manifest
raw_blob, _ = dxf.pull_blob(blob_id, size=True)
manifest = json.loads(''.join(raw_blob))
# Compatibility checks for the gears platform
if manifest['architecture'] != 'amd64':
raise Exception("Architecture must be amd64")
if manifest['os'] != 'linux':
raise Exception("Os must be linux")
return manifest, image
class DockerInjector:
"""
The main class of the Docker injector which injects new versions of a layer into
OCI images retrieved from an OCI compliant distribution API
"""
def __init__(self, host, repo, alias, user, pw):
"""
Initializes the injector by downloading both the slim and the fat image manifest
"""
def auth(dxf, response):
dxf.authenticate(user, pw, response=response)
self.dxfObject = DXF(host, repo, tlsverify=True, auth=auth)
self.image_manifest = self._get_manifest(alias)
self.fat_manifest = self._get_fat_manifest(self.image_manifest)
def setup(self, push_alias):
"""
Sets an image up for layer injection
"""
tar_digest, gz_digest = self._build_init_tar()
layer_size = self.dxfObject.blob_size(gz_digest)
self.fat_manifest.init_cvmfs_layer(tar_digest, gz_digest)
fat_man_json = self.fat_manifest.as_JSON()
manifest_digest = hash_bytes(bytes(fat_man_json, 'utf-8'))
self.dxfObject.push_blob(data=fat_man_json, digest=manifest_digest)
manifest_size = self.dxfObject.blob_size(manifest_digest)
self.image_manifest.init_cvmfs_layer(gz_digest, layer_size,
manifest_digest, manifest_size)
image_man_json = self.image_manifest.as_JSON()
self.dxfObject.set_manifest(push_alias, image_man_json)
def unpack(self, dest_dir):
"""
Unpacks the current version of a layer into the dest_dir directory in order to update it
"""
if not self.fat_manifest.is_cvmfs_prepared():
os.makedirs(dest_dir + "/cvmfs", exist_ok=True)
return
gz_digest = self.fat_manifest.get_gz_digest()
# Write out tar file
decompress_object = zlib.decompressobj(16 + zlib.MAX_WBITS)
try:
chunk_it = self.dxfObject.pull_blob(gz_digest)
except HTTPError as e:
if e.response.status_code == 404:
print("ERROR: The hash of the CVMFS layer must have changed.")
print(
"This is a known issue. Please do not reupload images to other repositories after CVMFS injection!"
)
else:
raise e
with tempfile.TemporaryFile() as tmp_file:
for chunk in chunk_it:
tmp_file.write(decompress_object.decompress(chunk))
tmp_file.write(decompress_object.flush())
tmp_file.seek(0)
tar = tarfile.TarFile(fileobj=tmp_file)
tar.extractall(dest_dir)
tar.close()
def update(self, src_dir, push_alias):
"""
Packs and uploads the contents of src_dir as a layer and injects the layer into the image.
The new layer version is stored under the tag push_alias
"""
if not self.fat_manifest.is_cvmfs_prepared():
print("Preparing image for CVMFS injection...")
self.setup(push_alias)
with tempfile.NamedTemporaryFile(delete=False) as tmp_file:
print("Bundling file into tar...")
_, error = exec_bash("tar --xattrs -C " + src_dir + " -cvf " +
tmp_file.name + " .")
if error:
raise RuntimeError("Failed to tar with error " + str(error))
tar_digest = hash_file(tmp_file.name)
print("Bundling tar into gz...")
gz_dest = tmp_file.name + ".gz"
_, error = exec_bash("gzip " + tmp_file.name)
if error:
raise RuntimeError("Failed to tar with error " + str(error))
print("Uploading...")
gz_digest = self.dxfObject.push_blob(gz_dest)
os.unlink(gz_dest)
print("Refreshing manifests...")
old_gz_digest = self.fat_manifest.get_gz_digest()
layer_size = self.dxfObject.blob_size(gz_digest)
self.fat_manifest.inject(tar_digest, gz_digest)
fat_man_json = self.fat_manifest.as_JSON()
manifest_digest = hash_bytes(bytes(fat_man_json, 'utf-8'))
self.dxfObject.push_blob(data=fat_man_json, digest=manifest_digest)
manifest_size = self.dxfObject.blob_size(manifest_digest)
self.image_manifest.inject(old_gz_digest, gz_digest, layer_size,
manifest_digest, manifest_size)
image_man_json = self.image_manifest.as_JSON()
self.dxfObject.set_manifest(push_alias, image_man_json)
def _get_manifest(self, alias):
return ImageManifest(self.dxfObject.get_manifest(alias))
def _get_fat_manifest(self, image_manifest):
fat_manifest = ""
(readIter, _) = self.dxfObject.pull_blob(
self.image_manifest.get_fat_manif_digest(),
size=True,
chunk_size=4096)
for chunk in readIter:
fat_manifest += str(chunk)[2:-1]
fat_manifest = fat_manifest.replace("\\\\", "\\")
return FatManifest(fat_manifest)
def _build_init_tar(self):
"""
Builds an empty /cvmfs tar and uploads it to the registry
:rtype: tuple
:returns: Tuple containing the tar digest and gz digest
"""
ident = self.image_manifest.get_fat_manif_digest()[5:15]
tmp_name = "/tmp/injector-" + ident
os.makedirs(tmp_name + "/cvmfs", exist_ok=True)
tar_dest = "/tmp/" + ident + ".tar"
_, error = exec_bash("tar --xattrs -C " + tmp_name + " -cvf " +
tar_dest + " .")
if error:
print("Failed to tar with error " + str(error))
return
tar_digest = hash_file(tar_dest)
_, error = exec_bash("gzip -n " + tar_dest)
if error:
print("Failed to tar with error " + str(error))
return
gz_dest = tar_dest + ".gz"
gzip_digest = self.dxfObject.push_blob(tar_dest + ".gz")
# Cleanup
os.rmdir(tmp_name + "/cvmfs")
os.rmdir(tmp_name)
os.unlink(gz_dest)
return (tar_digest, gzip_digest)
class DockerInjector:
"""
The main class of the Docker injector which injects new versions of a layer into
OCI images retrieved from an OCI compliant distribution API
"""
def __init__(self, host, repo, alias, user, pw):
"""
Initializes the injector by downloading both the slim and the fat image manifest
"""
def auth(dxf, response):
dxf.authenticate(user, pw, response=response)
self.dxfObject = DXF(host, repo, tlsverify=True, auth=auth)
self.image_manifest = self._get_manifest(alias)
self.fat_manifest = self._get_fat_manifest(self.image_manifest)
def setup(self, push_alias):
"""
Sets an image up for layer injection
"""
tar_digest, gz_digest = self._build_init_tar()
layer_size = self.dxfObject.blob_size(gz_digest)
self.fat_manifest.init_cvmfs_layer(tar_digest, gz_digest)
fat_man_json = self.fat_manifest.as_JSON()
manifest_digest = hash_bytes(bytes(fat_man_json, 'utf-8'))
self.dxfObject.push_blob(data=fat_man_json, digest=manifest_digest)
manifest_size = self.dxfObject.blob_size(manifest_digest)
self.image_manifest.init_cvmfs_layer(gz_digest, layer_size, manifest_digest, manifest_size)
image_man_json = self.image_manifest.as_JSON()
self.dxfObject.set_manifest(push_alias, image_man_json)
def unpack(self, dest_dir):
"""
Unpacks the current version of a layer into the dest_dir directory in order to update it
"""
if not self.fat_manifest.is_cvmfs_prepared():
os.makedirs(dest_dir+"/cvmfs", exist_ok=True)
return
gz_digest = self.fat_manifest.get_gz_digest()
# Write out tar file
decompress_object = zlib.decompressobj(16+zlib.MAX_WBITS)
try:
chunk_it = self.dxfObject.pull_blob(gz_digest)
except HTTPError as e:
if e.response.status_code == 404:
print("ERROR: The hash of the CVMFS layer must have changed.")
print("This is a known issue. Please do not reupload images to other repositories after CVMFS injection!")
else:
raise e
with tempfile.TemporaryFile() as tmp_file:
for chunk in chunk_it:
tmp_file.write(decompress_object.decompress(chunk))
tmp_file.write(decompress_object.flush())
tmp_file.seek(0)
tar = tarfile.TarFile(fileobj=tmp_file)
tar.extractall(dest_dir)
tar.close()
def update(self, src_dir, push_alias):
"""
Packs and uploads the contents of src_dir as a layer and injects the layer into the image.
The new layer version is stored under the tag push_alias
"""
if not self.fat_manifest.is_cvmfs_prepared():
print("Preparing image for CVMFS injection...")
self.setup(push_alias)
with tempfile.NamedTemporaryFile(delete=False) as tmp_file:
print("Bundling file into tar...")
_, error = exec_bash("tar --xattrs -C "+src_dir+" -cvf "+tmp_file.name+" .")
if error:
raise RuntimeError("Failed to tar with error " + str(error))
tar_digest = hash_file(tmp_file.name)
print("Bundling tar into gz...")
gz_dest = tmp_file.name+".gz"
_, error = exec_bash("gzip "+tmp_file.name)
if error:
raise RuntimeError("Failed to tar with error " + str(error))
print("Uploading...")
gz_digest = self.dxfObject.push_blob(gz_dest)
os.unlink(gz_dest)
print("Refreshing manifests...")
old_gz_digest = self.fat_manifest.get_gz_digest()
layer_size = self.dxfObject.blob_size(gz_digest)
self.fat_manifest.inject(tar_digest, gz_digest)
fat_man_json = self.fat_manifest.as_JSON()
manifest_digest = hash_bytes(bytes(fat_man_json, 'utf-8'))
self.dxfObject.push_blob(data=fat_man_json, digest=manifest_digest)
manifest_size = self.dxfObject.blob_size(manifest_digest)
self.image_manifest.inject(old_gz_digest, gz_digest, layer_size, manifest_digest, manifest_size)
image_man_json = self.image_manifest.as_JSON()
self.dxfObject.set_manifest(push_alias, image_man_json)
def _get_manifest(self, alias):
return ImageManifest(self.dxfObject.get_manifest(alias))
def _get_fat_manifest(self, image_manifest):
fat_manifest = ""
(readIter, _) = self.dxfObject.pull_blob(self.image_manifest.get_fat_manif_digest(), size=True, chunk_size=4096)
for chunk in readIter:
fat_manifest += str(chunk)[2:-1]
fat_manifest = fat_manifest.replace("\\\\","\\")
return FatManifest(fat_manifest)
def _build_init_tar(self):
"""
Builds an empty /cvmfs tar and uploads it to the registry
:rtype: tuple
:returns: Tuple containing the tar digest and gz digest
"""
ident = self.image_manifest.get_fat_manif_digest()[5:15]
tmp_name = "/tmp/injector-"+ident
os.makedirs(tmp_name+"/cvmfs", exist_ok=True)
tar_dest = "/tmp/"+ident+".tar"
_, error = exec_bash("tar --xattrs -C "+tmp_name+" -cvf "+tar_dest+" .")
if error:
print("Failed to tar with error " + str(error))
return
tar_digest = hash_file(tar_dest)
_, error = exec_bash("gzip -n "+tar_dest)
if error:
print("Failed to tar with error " + str(error))
return
gz_dest = tar_dest+".gz"
gzip_digest = self.dxfObject.push_blob(tar_dest+".gz")
# Cleanup
os.rmdir(tmp_name+"/cvmfs")
os.rmdir(tmp_name)
os.unlink(gz_dest)
return (tar_digest, gzip_digest)
import sys
sys.path.append(path.abspath(path.join(path.dirname(__file__), '..')))
os.chdir('/tmp')
from dxf import DXF
def auth(dxf, response):
dxf.authenticate(os.environ['DOCKER_REG_USERNAME'],
os.environ['DOCKER_REG_PASSWORD'],
response=response)
dxf = DXF(os.environ.get('DOCKER_REG_HOST', 'registry-1.docker.io'),
os.environ['DOCKER_REG_REPO'],
auth)
with open('logger.dat', 'wb') as f:
f.write(b'2015-05 11\n')
dgst = dxf.push_blob('logger.dat')
dxf.set_alias('may15-readings', dgst)
assert dxf.get_alias('may15-readings') == [dgst]
s = b''
for chunk in dxf.pull_blob(dgst):
s += chunk
assert s == b'2015-05 11\n'
print(s)
import sys
sys.path.append(path.abspath(path.join(path.dirname(__file__), '..')))
os.chdir('/tmp')
from dxf import DXF
def auth(dxf, response):
dxf.authenticate(os.environ['DOCKER_REG_USERNAME'],
os.environ['DOCKER_REG_PASSWORD'],
response=response)
dxf = DXF(os.environ.get('DOCKER_REG_HOST', 'registry-1.docker.io'),
os.environ['DOCKER_REG_REPO'],
auth)
with open('logger.dat', 'wb') as f:
f.write(b'2015-05 11\n')
dgst = dxf.push_blob('logger.dat')
dxf.set_alias('may15-readings', dgst)
assert dxf.get_alias('may15-readings') == [dgst]
s = b''
for chunk in dxf.pull_blob(dgst):
s += chunk
assert s == b'2015-05 11\n'
print(s)
