def upload_content_v2(registry: str, remote_image_name: str, local_image: str) -> Tuple[str, str]: """ Push a content to Docker Registry and return the URL to access :return: a tuple (image_link: str, image_digest: str) """ # Replace \\ -> none --> because in command line we can't write # "nginx:latest" without the \\ ---> "nginx\:latest" _image = os.path.abspath(local_image.replace("\\", "")) if not os.path.exists(_image): raise DockerscanNotExitsError("Local image selected do not exits") insecure, registry_without_schema = _get_schema_and_security(registry) d = DXF(registry_without_schema, remote_image_name, insecure=insecure) image_digest = d.push_blob(_image) # Image link img_link = "{schema}://{host}/v2/{repo}/blobs/sha256:{digest}".format( schema="http" if insecure else "https", host=registry_without_schema, repo=remote_image_name, digest=image_digest ) return img_link, image_digest
class DockerInjector: """ The main class of the Docker injector which injects new versions of a layer into OCI images retrieved from an OCI compliant distribution API """ def __init__(self, host, repo, alias, user, pw): """ Initializes the injector by downloading both the slim and the fat image manifest """ def auth(dxf, response): dxf.authenticate(user, pw, response=response) self.dxfObject = DXF(host, repo, tlsverify=True, auth=auth) self.image_manifest = self._get_manifest(alias) self.fat_manifest = self._get_fat_manifest(self.image_manifest) def setup(self, push_alias): """ Sets an image up for layer injection """ tar_digest, gz_digest = self._build_init_tar() layer_size = self.dxfObject.blob_size(gz_digest) self.fat_manifest.init_cvmfs_layer(tar_digest, gz_digest) fat_man_json = self.fat_manifest.as_JSON() manifest_digest = hash_bytes(bytes(fat_man_json, 'utf-8')) self.dxfObject.push_blob(data=fat_man_json, digest=manifest_digest) manifest_size = self.dxfObject.blob_size(manifest_digest) self.image_manifest.init_cvmfs_layer(gz_digest, layer_size, manifest_digest, manifest_size) image_man_json = self.image_manifest.as_JSON() self.dxfObject.set_manifest(push_alias, image_man_json) def unpack(self, dest_dir): """ Unpacks the current version of a layer into the dest_dir directory in order to update it """ if not self.fat_manifest.is_cvmfs_prepared(): os.makedirs(dest_dir + "/cvmfs", exist_ok=True) return gz_digest = self.fat_manifest.get_gz_digest() # Write out tar file decompress_object = zlib.decompressobj(16 + zlib.MAX_WBITS) try: chunk_it = self.dxfObject.pull_blob(gz_digest) except HTTPError as e: if e.response.status_code == 404: print("ERROR: The hash of the CVMFS layer must have changed.") print( "This is a known issue. Please do not reupload images to other repositories after CVMFS injection!" ) else: raise e with tempfile.TemporaryFile() as tmp_file: for chunk in chunk_it: tmp_file.write(decompress_object.decompress(chunk)) tmp_file.write(decompress_object.flush()) tmp_file.seek(0) tar = tarfile.TarFile(fileobj=tmp_file) tar.extractall(dest_dir) tar.close() def update(self, src_dir, push_alias): """ Packs and uploads the contents of src_dir as a layer and injects the layer into the image. The new layer version is stored under the tag push_alias """ if not self.fat_manifest.is_cvmfs_prepared(): print("Preparing image for CVMFS injection...") self.setup(push_alias) with tempfile.NamedTemporaryFile(delete=False) as tmp_file: print("Bundling file into tar...") _, error = exec_bash("tar --xattrs -C " + src_dir + " -cvf " + tmp_file.name + " .") if error: raise RuntimeError("Failed to tar with error " + str(error)) tar_digest = hash_file(tmp_file.name) print("Bundling tar into gz...") gz_dest = tmp_file.name + ".gz" _, error = exec_bash("gzip " + tmp_file.name) if error: raise RuntimeError("Failed to tar with error " + str(error)) print("Uploading...") gz_digest = self.dxfObject.push_blob(gz_dest) os.unlink(gz_dest) print("Refreshing manifests...") old_gz_digest = self.fat_manifest.get_gz_digest() layer_size = self.dxfObject.blob_size(gz_digest) self.fat_manifest.inject(tar_digest, gz_digest) fat_man_json = self.fat_manifest.as_JSON() manifest_digest = hash_bytes(bytes(fat_man_json, 'utf-8')) self.dxfObject.push_blob(data=fat_man_json, digest=manifest_digest) manifest_size = self.dxfObject.blob_size(manifest_digest) self.image_manifest.inject(old_gz_digest, gz_digest, layer_size, manifest_digest, manifest_size) image_man_json = self.image_manifest.as_JSON() self.dxfObject.set_manifest(push_alias, image_man_json) def _get_manifest(self, alias): return ImageManifest(self.dxfObject.get_manifest(alias)) def _get_fat_manifest(self, image_manifest): fat_manifest = "" (readIter, _) = self.dxfObject.pull_blob( self.image_manifest.get_fat_manif_digest(), size=True, chunk_size=4096) for chunk in readIter: fat_manifest += str(chunk)[2:-1] fat_manifest = fat_manifest.replace("\\\\", "\\") return FatManifest(fat_manifest) def _build_init_tar(self): """ Builds an empty /cvmfs tar and uploads it to the registry :rtype: tuple :returns: Tuple containing the tar digest and gz digest """ ident = self.image_manifest.get_fat_manif_digest()[5:15] tmp_name = "/tmp/injector-" + ident os.makedirs(tmp_name + "/cvmfs", exist_ok=True) tar_dest = "/tmp/" + ident + ".tar" _, error = exec_bash("tar --xattrs -C " + tmp_name + " -cvf " + tar_dest + " .") if error: print("Failed to tar with error " + str(error)) return tar_digest = hash_file(tar_dest) _, error = exec_bash("gzip -n " + tar_dest) if error: print("Failed to tar with error " + str(error)) return gz_dest = tar_dest + ".gz" gzip_digest = self.dxfObject.push_blob(tar_dest + ".gz") # Cleanup os.rmdir(tmp_name + "/cvmfs") os.rmdir(tmp_name) os.unlink(gz_dest) return (tar_digest, gzip_digest)
import sys sys.path.append(path.abspath(path.join(path.dirname(__file__), '..'))) os.chdir('/tmp') from dxf import DXF def auth(dxf, response): dxf.authenticate(os.environ['DOCKER_REG_USERNAME'], os.environ['DOCKER_REG_PASSWORD'], response=response) dxf = DXF(os.environ.get('DOCKER_REG_HOST', 'registry-1.docker.io'), os.environ['DOCKER_REG_REPO'], auth) with open('logger.dat', 'wb') as f: f.write(b'2015-05 11\n') dgst = dxf.push_blob('logger.dat') dxf.set_alias('may15-readings', dgst) assert dxf.get_alias('may15-readings') == [dgst] s = b'' for chunk in dxf.pull_blob(dgst): s += chunk assert s == b'2015-05 11\n' print(s)
class DockerInjector: """ The main class of the Docker injector which injects new versions of a layer into OCI images retrieved from an OCI compliant distribution API """ def __init__(self, host, repo, alias, user, pw): """ Initializes the injector by downloading both the slim and the fat image manifest """ def auth(dxf, response): dxf.authenticate(user, pw, response=response) self.dxfObject = DXF(host, repo, tlsverify=True, auth=auth) self.image_manifest = self._get_manifest(alias) self.fat_manifest = self._get_fat_manifest(self.image_manifest) def setup(self, push_alias): """ Sets an image up for layer injection """ tar_digest, gz_digest = self._build_init_tar() layer_size = self.dxfObject.blob_size(gz_digest) self.fat_manifest.init_cvmfs_layer(tar_digest, gz_digest) fat_man_json = self.fat_manifest.as_JSON() manifest_digest = hash_bytes(bytes(fat_man_json, 'utf-8')) self.dxfObject.push_blob(data=fat_man_json, digest=manifest_digest) manifest_size = self.dxfObject.blob_size(manifest_digest) self.image_manifest.init_cvmfs_layer(gz_digest, layer_size, manifest_digest, manifest_size) image_man_json = self.image_manifest.as_JSON() self.dxfObject.set_manifest(push_alias, image_man_json) def unpack(self, dest_dir): """ Unpacks the current version of a layer into the dest_dir directory in order to update it """ if not self.fat_manifest.is_cvmfs_prepared(): os.makedirs(dest_dir+"/cvmfs", exist_ok=True) return gz_digest = self.fat_manifest.get_gz_digest() # Write out tar file decompress_object = zlib.decompressobj(16+zlib.MAX_WBITS) try: chunk_it = self.dxfObject.pull_blob(gz_digest) except HTTPError as e: if e.response.status_code == 404: print("ERROR: The hash of the CVMFS layer must have changed.") print("This is a known issue. Please do not reupload images to other repositories after CVMFS injection!") else: raise e with tempfile.TemporaryFile() as tmp_file: for chunk in chunk_it: tmp_file.write(decompress_object.decompress(chunk)) tmp_file.write(decompress_object.flush()) tmp_file.seek(0) tar = tarfile.TarFile(fileobj=tmp_file) tar.extractall(dest_dir) tar.close() def update(self, src_dir, push_alias): """ Packs and uploads the contents of src_dir as a layer and injects the layer into the image. The new layer version is stored under the tag push_alias """ if not self.fat_manifest.is_cvmfs_prepared(): print("Preparing image for CVMFS injection...") self.setup(push_alias) with tempfile.NamedTemporaryFile(delete=False) as tmp_file: print("Bundling file into tar...") _, error = exec_bash("tar --xattrs -C "+src_dir+" -cvf "+tmp_file.name+" .") if error: raise RuntimeError("Failed to tar with error " + str(error)) tar_digest = hash_file(tmp_file.name) print("Bundling tar into gz...") gz_dest = tmp_file.name+".gz" _, error = exec_bash("gzip "+tmp_file.name) if error: raise RuntimeError("Failed to tar with error " + str(error)) print("Uploading...") gz_digest = self.dxfObject.push_blob(gz_dest) os.unlink(gz_dest) print("Refreshing manifests...") old_gz_digest = self.fat_manifest.get_gz_digest() layer_size = self.dxfObject.blob_size(gz_digest) self.fat_manifest.inject(tar_digest, gz_digest) fat_man_json = self.fat_manifest.as_JSON() manifest_digest = hash_bytes(bytes(fat_man_json, 'utf-8')) self.dxfObject.push_blob(data=fat_man_json, digest=manifest_digest) manifest_size = self.dxfObject.blob_size(manifest_digest) self.image_manifest.inject(old_gz_digest, gz_digest, layer_size, manifest_digest, manifest_size) image_man_json = self.image_manifest.as_JSON() self.dxfObject.set_manifest(push_alias, image_man_json) def _get_manifest(self, alias): return ImageManifest(self.dxfObject.get_manifest(alias)) def _get_fat_manifest(self, image_manifest): fat_manifest = "" (readIter, _) = self.dxfObject.pull_blob(self.image_manifest.get_fat_manif_digest(), size=True, chunk_size=4096) for chunk in readIter: fat_manifest += str(chunk)[2:-1] fat_manifest = fat_manifest.replace("\\\\","\\") return FatManifest(fat_manifest) def _build_init_tar(self): """ Builds an empty /cvmfs tar and uploads it to the registry :rtype: tuple :returns: Tuple containing the tar digest and gz digest """ ident = self.image_manifest.get_fat_manif_digest()[5:15] tmp_name = "/tmp/injector-"+ident os.makedirs(tmp_name+"/cvmfs", exist_ok=True) tar_dest = "/tmp/"+ident+".tar" _, error = exec_bash("tar --xattrs -C "+tmp_name+" -cvf "+tar_dest+" .") if error: print("Failed to tar with error " + str(error)) return tar_digest = hash_file(tar_dest) _, error = exec_bash("gzip -n "+tar_dest) if error: print("Failed to tar with error " + str(error)) return gz_dest = tar_dest+".gz" gzip_digest = self.dxfObject.push_blob(tar_dest+".gz") # Cleanup os.rmdir(tmp_name+"/cvmfs") os.rmdir(tmp_name) os.unlink(gz_dest) return (tar_digest, gzip_digest)