def upload_content_v2(registry: str, remote_image_name: str, local_image: str) -> Tuple[str, str]: """ Push a content to Docker Registry and return the URL to access :return: a tuple (image_link: str, image_digest: str) """ # Replace \\ -> none --> because in command line we can't write # "nginx:latest" without the \\ ---> "nginx\:latest" _image = os.path.abspath(local_image.replace("\\", "")) if not os.path.exists(_image): raise DockerscanNotExitsError("Local image selected do not exits") insecure, registry_without_schema = _get_schema_and_security(registry) d = DXF(registry_without_schema, remote_image_name, insecure=insecure) image_digest = d.push_blob(_image) # Image link img_link = "{schema}://{host}/v2/{repo}/blobs/sha256:{digest}".format( schema="http" if insecure else "https", host=registry_without_schema, repo=remote_image_name, digest=image_digest ) return img_link, image_digest
def do(self, container_image: Optional[str] = None, **kwargs) -> str: # type: ignore [host, repo, alias, digest] = self.parse(container_image) if digest: return digest if host is None: host = "registry-1.docker.io" if repo is None: repo = "stencila/executa-midi" if alias is None: alias = "latest" try: dxf = DXF(host, repo, self.authenticate) # Get the "Docker-Content-Digest" for the alias. This is the SHA256 hash # that can be used with `docker run` and is on the Docker Hub for an image tag, # not the one returned by ``.get_digest()`. digest = dxf._get_dcd(alias) except MissingCredentialsError as exc: # If no credentials were available for the container registry # then log a warning and return an unpinned container image. # This is intended primarily for use during development so it is not # necessary to provide credentials to create snapshots. logger.warning(str(exc)) return self.deparse(host, repo) else: return self.deparse(host, repo, cast(str, digest))
def resolve_tag(image_loc: str): logging.info(f"Resolving {image_loc}") domain, org, repo, tag = parse_image_loc(image_loc) dxf = DXF(domain, f"{org}/{repo}", tag, None) hash = sha256(dxf.get_manifest(tag_to_pin).encode("utf-8")).hexdigest() resolved = f"{domain}/{org}/{repo}@sha256:{hash}" logging.info(f"{resolved}") return resolved
def __init__(self, host, repo, alias, user, pw): """ Initializes the injector by downloading both the slim and the fat image manifest """ def auth(dxf, response): dxf.authenticate(user, pw, response=response) self.dxfObject = DXF(host, repo, tlsverify=True, auth=auth) self.image_manifest = self._get_manifest(alias) self.fat_manifest = self._get_fat_manifest(self.image_manifest)
def list_dockerhub_tags(): reponame = getattr(settings, 'DOCKER_REPO', None) if reponame is None: raise Exception('Unable to determine the current docker repo/image in use.') # we'll assume dockerhub for the time being dxf = DXF('registry-1.docker.io', reponame, auth) return dxf.list_aliases()
def last_build_commit(self, namespace, repo, tag): def auth(_dxf, response): _dxf.authenticate(username=self.username, password=self.password, response=response, actions='*') try: dxf = DXF(host=self.registry, repo='{}/{}'.format(namespace, repo), auth=auth) r = dxf._request('get', 'manifests/' + tag, headers={'Accept': 'application/vnd.docker.distribution.manifest.v1+json'}) metadata = json.loads(r.content.decode('utf-8')) return json.loads(metadata['history'][0]['v1Compatibility'])['config']['Labels']['org.label-schema.vcs-ref'] except Exception as e: print("last_build_commit failed: {}".format(e)) return None
def _get_digest_by_tag(registry: str, remote_image_name: str, tag: str) -> str: insecure, registry_without_schema = _get_schema_and_security(registry) d = DXF(registry_without_schema, remote_image_name, insecure=insecure) try: return d.get_alias(alias=tag)[0] except (IndexError, requests.exceptions.HTTPError): return ""
def get_image_digest(self, repository: str, tag: str) -> str: from dxf import DXF from dxf.exceptions import DXFUnauthorizedError try: repository = f'library/{repository}' if '/' not in repository else repository auth = 'Basic ' + self.auth if self.auth is not None else None dxf = DXF(self.registry_host, repo=repository) dxf.authenticate(authorization=auth, actions=['pull']) except DXFUnauthorizedError: raise RegistryAuthorizationException(f'Authentication with Docker registry {self.registry_host} failed. Run `docker login` first?') try: manifest = json.loads(dxf.get_manifest(tag)) return manifest['config']['digest'] except DXFUnauthorizedError: raise RegistryImageNotFoundException(f'Image {repository}:{tag} not found in registry {self.registry_host}')
def last_build_commit(repo, tag): try: dxf = DXF(host=registry, repo='%s/%s' % (namespace, repo), auth=auth) r = dxf._request( 'get', 'manifests/' + tag, headers={ 'Accept': 'application/vnd.docker.distribution.manifest.v1+json' }) metadata = json.loads(r.content.decode('utf-8')) return json.loads( metadata['history'][0] ['v1Compatibility'])['config']['Labels'][last_commit_label] except: return None
def push_image_v2(registry: str, remote_image_name: str, local_image: str, tag: str) -> str: """Push a content to Docker Registry and return the URL to access""" insecure, registry_without_schema = _get_schema_and_security(registry) download_link, digest = upload_content_v2(registry, remote_image_name, local_image) d = DXF(registry_without_schema, remote_image_name, insecure=insecure) d.set_alias(tag, digest) return download_link
def find_current_tag(image, config): registry = DXF(image.fullhost(), image.fullrepo(), registry_auth) # FIXME: fail if check fails #print(registry.api_version_check()) p = re.compile(config["tag"]) candidates = [] for tag in registry.list_aliases(iterate=True): if p.match(tag): candidates.append(tag) if len(candidates) == 0: return None # more than one? sort by semver, highest 1st if len(candidates) > 1: candidates = sorted(candidates, reverse=True, key=functools.cmp_to_key(semver.compare)) return candidates[0]
def delete_image_v2(registry: str, remote_image_name: str, tag: str = "latest") -> Union[Set[str], DockerscanError]: """ delete selected images from remote repo. remote_image_name can contain regex expressions. :return: return a set() with the images deleted """ insecure, registry_without_schema = _get_schema_and_security(registry) d = DXF(registry_without_schema, remote_image_name, insecure=insecure) removed = set() # Getting remote digest for the tag digest = _get_digest_by_tag(registry, remote_image_name, tag) if not digest: raise DockerscanError("> Can't obtain digest reference for selected " "image / tag") try: if digest: # If digest found -> remote image is not a regex. Then remove it d.del_alias(digest) removed.add(remote_image_name) return removed except requests.exceptions.HTTPError: raise DockerscanError("> Registry does not support delete " "operations. Default Docker Registry does not " "support deletion. For more information see: " "https://docs.docker.com/registry/" "configuration/")
def __init__(self, host, repo, repos_root=None, auth=None, insecure=False, auth_host=None): self._dxf = DXF(host, repo, self._wrap_auth(auth), insecure, auth_host) self._repo_root = path.join( repos_root if repos_root else path.join(getcwd(), 'dtuf_repos'), repo)
def get_tag(self, repo, tag): try: tag_info = {} dxf = DXF(REGISTRY_ADDRESS, repo) digest = dxf.get_digest(tag) response = self.request_registry(self.GET_LAYER_TEMPLATE.format( url=REGISTRY_V2_API_ADDRESS, repo=repo, digest=digest), method='HEAD') if response.status == 200: manifest = self.get_manifest(repo, tag) tag_info['Tag'] = tag tag_info['Created'] = manifest.get_created_date() tag_info['Entrypoint'] = manifest.get_entrypoint() tag_info['DockerVersion'] = manifest.get_docker_version() tag_info['ExposedPorts'] = manifest.get_exposed_ports() tag_info['Volumes'] = manifest.get_volumes() return tag_info else: return None except Exception as ex: LOGGER.error(ex)
def confirm_registry_asset(repo, pointer): """ Validates a registry asset by querying the remote registry. Returns the registry manifest at the given pointer, and a pullable image string. """ host, username, api_key = get_registry_connectivity() image = host + '/' + repo + '@' + pointer log.debug('Validating image ' + image + '...') # Authenticate via callable def auth(dxf, response): log.debug('Authenticating to registry...') dxf.authenticate(username, api_key, response=response) log.debug('Auth to registry successful') # Connects over internal network with override host and autogenerated TLS dxf = DXF(host, repo, auth, tlsverify=False) # Fetch and sanity check the blob size blob_id = str(dxf.get_digest(pointer)) if dxf.blob_size(blob_id) > (10 * 1000 * 1000): # 10 MB to bytes raise Exception( 'Manifest is larger than 10 MB. Possible registry error?') # Pull and assemble the manifest raw_blob, _ = dxf.pull_blob(blob_id, size=True) manifest = json.loads(''.join(raw_blob)) # Compatibility checks for the gears platform if manifest['architecture'] != 'amd64': raise Exception("Architecture must be amd64") if manifest['os'] != 'linux': raise Exception("Os must be linux") return manifest, image
def delete(self, _, **kwargs): """ @api {delete} /registry/repository/<str:repo>/<str:tag>/ Delete an image @apiName DeleteImage @apiGroup RegistryManager @apiVersion 0.1.0 @apiPermission admin @apiSuccess {Object} payload Success payload is empty @apiUse APIHeader @apiUse Success @apiUse OperationFailed @apiUse Unauthorized """ try: repo = kwargs.get('repo') tag = kwargs.get('tag') dxf = DXF(REGISTRY_ADDRESS, repo) digest = dxf.get_digest(tag) dxf.del_blob(digest) return JsonResponse(RESPONSE.SUCCESS) except Exception as ex: LOGGER.exception(ex) return JsonResponse(RESPONSE.OPERATION_FAILED)
def upload(self, filename, md, repo): try: ImageModel.objects.filter(hashid=md).update( status=ImageStatusCode.UPLOADING) dxf = DXF(REGISTRY_ADDRESS, repo) status = DockerTarUploader(dxf).upload_tar(self.basePath + filename) LOGGER.info("upload status") LOGGER.info(status) if os.path.exists(self.basePath + filename): os.remove(self.basePath + filename) LOGGER.info("done upload") ImageModel.objects.filter(hashid=md).update( status=ImageStatusCode.SUCCEEDED) except Exception as ex: LOGGER.exception(ex) ImageModel.objects.filter(hashid=md).update( status=ImageStatusCode.FAILED)
def _clean_repository(repository): logger.info('{0} Processing "{1}" {0}'.format("*" * 2, repository["name"])) dxf = DXF(registry_host, repository["name"], _auth) tags = _fetch_tags(dxf) tags_groups = [] for tag_rule in repository["tags"]: pattern = re.compile(tag_rule["pattern"]) matched = {} for tag, created in list(tags.items()): if pattern.match(tag): matched[tag] = created del tags[tag] if matched: tags_groups.append({"rule": tag_rule, "tags": matched}) for tag_group in tags_groups: _clean_tags(dxf, tag_group["rule"], tag_group["tags"])
def loadDXF(self, filename): try: dxf = DXF(filename,"r") except: return False dxf.readFile() dxf.close() for name in dxf.layers.keys(): layer = dxf.sortLayer(name) path = Path(name) path.fromLayer(layer) path.removeZeroLength() opath = path.order() changed = True while changed: longest = opath[0] for p in opath: if longest.length() > p.length(): longest = p opath.remove(longest) changed = longest.mergeLoops(opath) self.fromPath(longest) self.fromPath(opath) return True
# generate the basis function for this value of t rbasis(k,t,npts,x,h,nbasis) # generate a point on the curve for j in range(1,4): jcount = j p[icount+j] = 0.0 # Do local matrix multiplication for i in range(1,npts+1): p[icount+j] += nbasis[i]*b[jcount] jcount += 3 icount += 3 t += step # ============================================================================= if __name__ == "__main__": SPLINE_SEGMENTS = 20 from dxf import DXF # from dxfwrite.algebra import CubicSpline, CubicBezierCurve dxf = DXF(sys.argv[1],"r") dxf.readFile() dxf.close() for name,layer in dxf.layers.items(): for entity in layer.entities: if entity.type == "SPLINE": xy = zip(entity[10], entity[20]) x,y = spline2Polyline(xy, int(entity[71]), True, SPLINE_SEGMENTS) #for a,b in zip(x,y): # print a,b
def saveDXF(self, filename): try: dxf = DXF(filename,"w") except: return False dxf.writeHeader() for block in self.blocks: name = block.name() for line in block: cmds = self.cnc.parseLine(line) if cmds is None: continue self.cnc.processPath(cmds) if self.cnc.gcode == 1: # line dxf.line(self.cnc.x, self.cnc.y, self.cnc.xval, self.cnc.yval, name) elif self.cnc.gcode in (2,3): # arc xc,yc,zc = self.cnc.motionCenter() sphi = math.atan2(self.cnc.y-yc, self.cnc.x-xc) ephi = math.atan2(self.cnc.yval-yc, self.cnc.xval-xc) if self.cnc.gcode==2: if ephi<=sphi+1e-10: ephi += 2.0*math.pi dxf.arc(xc,yc,self.cnc.rval, math.degrees(ephi), math.degrees(sphi),name) else: if ephi<=sphi+1e-10: ephi += 2.0*math.pi dxf.arc(xc,yc,self.cnc.rval, math.degrees(sphi), math.degrees(ephi),name) self.cnc.motionPathEnd() dxf.writeEOF() dxf.close() return True
from os import path import sys sys.path.append(path.abspath(path.join(path.dirname(__file__), '..'))) os.chdir('/tmp') from dxf import DXF def auth(dxf, response): dxf.authenticate(os.environ['DOCKER_REG_USERNAME'], os.environ['DOCKER_REG_PASSWORD'], response=response) dxf = DXF(os.environ.get('DOCKER_REG_HOST', 'registry-1.docker.io'), os.environ['DOCKER_REG_REPO'], auth) with open('logger.dat', 'wb') as f: f.write(b'2015-05 11\n') dgst = dxf.push_blob('logger.dat') dxf.set_alias('may15-readings', dgst) assert dxf.get_alias('may15-readings') == [dgst] s = b'' for chunk in dxf.pull_blob(dgst): s += chunk assert s == b'2015-05 11\n' print(s)
class DockerInjector: """ The main class of the Docker injector which injects new versions of a layer into OCI images retrieved from an OCI compliant distribution API """ def __init__(self, host, repo, alias, user, pw): """ Initializes the injector by downloading both the slim and the fat image manifest """ def auth(dxf, response): dxf.authenticate(user, pw, response=response) self.dxfObject = DXF(host, repo, tlsverify=True, auth=auth) self.image_manifest = self._get_manifest(alias) self.fat_manifest = self._get_fat_manifest(self.image_manifest) def setup(self, push_alias): """ Sets an image up for layer injection """ tar_digest, gz_digest = self._build_init_tar() layer_size = self.dxfObject.blob_size(gz_digest) self.fat_manifest.init_cvmfs_layer(tar_digest, gz_digest) fat_man_json = self.fat_manifest.as_JSON() manifest_digest = hash_bytes(bytes(fat_man_json, 'utf-8')) self.dxfObject.push_blob(data=fat_man_json, digest=manifest_digest) manifest_size = self.dxfObject.blob_size(manifest_digest) self.image_manifest.init_cvmfs_layer(gz_digest, layer_size, manifest_digest, manifest_size) image_man_json = self.image_manifest.as_JSON() self.dxfObject.set_manifest(push_alias, image_man_json) def unpack(self, dest_dir): """ Unpacks the current version of a layer into the dest_dir directory in order to update it """ if not self.fat_manifest.is_cvmfs_prepared(): os.makedirs(dest_dir + "/cvmfs", exist_ok=True) return gz_digest = self.fat_manifest.get_gz_digest() # Write out tar file decompress_object = zlib.decompressobj(16 + zlib.MAX_WBITS) try: chunk_it = self.dxfObject.pull_blob(gz_digest) except HTTPError as e: if e.response.status_code == 404: print("ERROR: The hash of the CVMFS layer must have changed.") print( "This is a known issue. Please do not reupload images to other repositories after CVMFS injection!" ) else: raise e with tempfile.TemporaryFile() as tmp_file: for chunk in chunk_it: tmp_file.write(decompress_object.decompress(chunk)) tmp_file.write(decompress_object.flush()) tmp_file.seek(0) tar = tarfile.TarFile(fileobj=tmp_file) tar.extractall(dest_dir) tar.close() def update(self, src_dir, push_alias): """ Packs and uploads the contents of src_dir as a layer and injects the layer into the image. The new layer version is stored under the tag push_alias """ if not self.fat_manifest.is_cvmfs_prepared(): print("Preparing image for CVMFS injection...") self.setup(push_alias) with tempfile.NamedTemporaryFile(delete=False) as tmp_file: print("Bundling file into tar...") _, error = exec_bash("tar --xattrs -C " + src_dir + " -cvf " + tmp_file.name + " .") if error: raise RuntimeError("Failed to tar with error " + str(error)) tar_digest = hash_file(tmp_file.name) print("Bundling tar into gz...") gz_dest = tmp_file.name + ".gz" _, error = exec_bash("gzip " + tmp_file.name) if error: raise RuntimeError("Failed to tar with error " + str(error)) print("Uploading...") gz_digest = self.dxfObject.push_blob(gz_dest) os.unlink(gz_dest) print("Refreshing manifests...") old_gz_digest = self.fat_manifest.get_gz_digest() layer_size = self.dxfObject.blob_size(gz_digest) self.fat_manifest.inject(tar_digest, gz_digest) fat_man_json = self.fat_manifest.as_JSON() manifest_digest = hash_bytes(bytes(fat_man_json, 'utf-8')) self.dxfObject.push_blob(data=fat_man_json, digest=manifest_digest) manifest_size = self.dxfObject.blob_size(manifest_digest) self.image_manifest.inject(old_gz_digest, gz_digest, layer_size, manifest_digest, manifest_size) image_man_json = self.image_manifest.as_JSON() self.dxfObject.set_manifest(push_alias, image_man_json) def _get_manifest(self, alias): return ImageManifest(self.dxfObject.get_manifest(alias)) def _get_fat_manifest(self, image_manifest): fat_manifest = "" (readIter, _) = self.dxfObject.pull_blob( self.image_manifest.get_fat_manif_digest(), size=True, chunk_size=4096) for chunk in readIter: fat_manifest += str(chunk)[2:-1] fat_manifest = fat_manifest.replace("\\\\", "\\") return FatManifest(fat_manifest) def _build_init_tar(self): """ Builds an empty /cvmfs tar and uploads it to the registry :rtype: tuple :returns: Tuple containing the tar digest and gz digest """ ident = self.image_manifest.get_fat_manif_digest()[5:15] tmp_name = "/tmp/injector-" + ident os.makedirs(tmp_name + "/cvmfs", exist_ok=True) tar_dest = "/tmp/" + ident + ".tar" _, error = exec_bash("tar --xattrs -C " + tmp_name + " -cvf " + tar_dest + " .") if error: print("Failed to tar with error " + str(error)) return tar_digest = hash_file(tar_dest) _, error = exec_bash("gzip -n " + tar_dest) if error: print("Failed to tar with error " + str(error)) return gz_dest = tar_dest + ".gz" gzip_digest = self.dxfObject.push_blob(tar_dest + ".gz") # Cleanup os.rmdir(tmp_name + "/cvmfs") os.rmdir(tmp_name) os.unlink(gz_dest) return (tar_digest, gzip_digest)
dest='destination') parser.add_argument('-i', '--insecure', action='store_false', dest='tlsverify') parser.set_defaults(tlsverify=True) args = parser.parse_args() source = DXFBase(args.source.address, auth=args.source.auth, tlsverify=args.tlsverify) source_repositories = source.list_repos() destination = DXFBase(args.destination.address, auth=args.destination.auth, tlsverify=args.tlsverify) destination_repositories = destination.list_repos() for repository in source_repositories: source_repository = DXF.from_base(source, repository) destination_repository = DXF.from_base(destination, repository) try: tags = source_repository.list_aliases() except DXFUnauthorizedError: print( f'Skipping {repository} after being denied access to source repository' ) continue for tag in tags: manifest_string, response = source_repository.get_manifest_and_response( tag) manifest = response.json() source_layers = [manifest['config'], *manifest['layers']]
# generate a point on the curve for j in range(1, 4): jcount = j p[icount + j] = 0.0 # Do local matrix multiplication for i in range(1, npts + 1): p[icount + j] += nbasis[i] * b[jcount] jcount += 3 icount += 3 t += step # ============================================================================= if __name__ == "__main__": SPLINE_SEGMENTS = 20 from dxf import DXF # from dxfwrite.algebra import CubicSpline, CubicBezierCurve dxf = DXF(sys.argv[1], "r") dxf.readFile() dxf.close() for name, layer in dxf.layers.items(): for entity in layer.entities: if entity.type == "SPLINE": xy = zip(entity[10], entity[20]) x, y = spline2Polyline(xy, int(entity[71]), True, SPLINE_SEGMENTS) #for a,b in zip(x,y): # print a,b
def cost_part(): # PART NUMBER partNoItems = [{ 'type': 'input', 'name': 'partNumber', 'elements': 'PART NUMBER: ' }] partNo = View(title='PART COST MENU', version='input', items=partNoItems) print(type(partNo.answer[0])) # MATERIAL # materialDF = Server().find('standards')['material'][0] # material = dfToList(materialDF, 'materialName') matItems = [{ 'type': 'list', 'name': 'material', 'message': 'PART MATERIAL', 'elements': ['carbon steel', 'stainless'] }] mat = View(title='PART COST MENU', version='input', items=matItems) # GAUGE # gaugeDF = Server().find('standards')['gauge'][0] # gauge = dfToList(gaugeDF, 'gaugeName') gaugeItems = [{ 'type': 'list', 'name': 'gauge', 'message': '', 'elements': ['18GA', '16GA'] }] ga = View(title='PART COST MENU', version='input', items=gaugeItems) # BLANK dxf = DXF(partNo.answer[0]) blankx, blanky = dxf.blank() length = dxf.laserPath() blankInfo = [ f'BLANK WIDTH: {blankx}', f'BLANK HEIGHT: {blanky}', f'BLANK LASER PATH: {length}' ] blankItems = [{ 'type': 'print', 'name': 'blankInfo', 'elements': blankInfo }, { 'type': 'list', 'name': 'blank', 'message': 'BLANK INFO CORRECT: ', 'elements': ['YES', 'NO'] }] blank = View(title='PART COST MENU', version='input', items=blankItems) if blank.answer[0] == 'YES': blank = [blankx, blanky, length] else: blankItems = [ { 'type': 'input', 'name': 'width', 'elements': 'BLANK WIDTH: ' }, { 'type': 'input', 'name': 'height', 'elements': 'BLANK HEIGHT: ' }, { 'type': 'input', 'name': 'laserPath', 'elements': 'LASER PATH LENGTH: ' }, { 'type': 'input', 'name': 'weight', 'elements': 'BLANK WEIGHT: ' }, ] blank = View(title='PART COST MENU', version='input', items=blankItems) # PROCESS CATEGORY # processCategoryDF = Server().find('standards')['processCategory'][0] # processCategory = dfToList(processCategoryDF, 'processCategoryName') # workCenterIDDF = Server().find('standards')['workCenter'][0] # workCenterID = dfToList(workCenterIDDF, 'workCenterID') addProcess = True processCount = 0 processes = [] while addProcess: processItems = [{ 'type': 'input', 'name': 'operationNumber', 'elements': 'OPERATION NUMBER: ' }, { 'type': 'input', 'name': 'operationName', 'elements': 'OPERATION NAME: ' }, { 'type': 'list', 'name': 'processCategoryName', 'message': 'PROCESS CATEGORY: ', 'elements': ['PRESS BRAKE', 'LASER'] }, { 'type': 'list', 'name': 'workCenterID', 'message': 'WORK CENTER ID: ', 'elements': ['1002', '25309'] }, { 'type': 'input', 'name': 'setup', 'elements': 'SETUP TIME: ' }, { 'type': 'input', 'name': 'operationTime', 'elements': 'TIME PER OPERATION: ' }, { 'type': 'input', 'name': 'operationQuantity', 'elements': 'OPERATION QUANTITY: ' }, { 'type': 'list', 'name': 'addProcess', 'message': 'ADD ANOTHER PROCESS: ', 'elements': ['YES', 'NO'] }] process = View(title='PART COST MENU', version='input', items=processItems) processes.append(process.answer) if process.answer[7] == 'YES': addProcess = True processCount += 1 else: addProcess = False print(partNo.answer) print(mat.answer) print(ga.answer) print(blank) print(processes)
def registry(self, repository=''): return DXF(self._registry_host, repository, auth=self._auth, insecure=self._registry_url.scheme == 'http', tlsverify=not self._registry_no_verify)
class DockerInjector: """ The main class of the Docker injector which injects new versions of a layer into OCI images retrieved from an OCI compliant distribution API """ def __init__(self, host, repo, alias, user, pw): """ Initializes the injector by downloading both the slim and the fat image manifest """ def auth(dxf, response): dxf.authenticate(user, pw, response=response) self.dxfObject = DXF(host, repo, tlsverify=True, auth=auth) self.image_manifest = self._get_manifest(alias) self.fat_manifest = self._get_fat_manifest(self.image_manifest) def setup(self, push_alias): """ Sets an image up for layer injection """ tar_digest, gz_digest = self._build_init_tar() layer_size = self.dxfObject.blob_size(gz_digest) self.fat_manifest.init_cvmfs_layer(tar_digest, gz_digest) fat_man_json = self.fat_manifest.as_JSON() manifest_digest = hash_bytes(bytes(fat_man_json, 'utf-8')) self.dxfObject.push_blob(data=fat_man_json, digest=manifest_digest) manifest_size = self.dxfObject.blob_size(manifest_digest) self.image_manifest.init_cvmfs_layer(gz_digest, layer_size, manifest_digest, manifest_size) image_man_json = self.image_manifest.as_JSON() self.dxfObject.set_manifest(push_alias, image_man_json) def unpack(self, dest_dir): """ Unpacks the current version of a layer into the dest_dir directory in order to update it """ if not self.fat_manifest.is_cvmfs_prepared(): os.makedirs(dest_dir+"/cvmfs", exist_ok=True) return gz_digest = self.fat_manifest.get_gz_digest() # Write out tar file decompress_object = zlib.decompressobj(16+zlib.MAX_WBITS) try: chunk_it = self.dxfObject.pull_blob(gz_digest) except HTTPError as e: if e.response.status_code == 404: print("ERROR: The hash of the CVMFS layer must have changed.") print("This is a known issue. Please do not reupload images to other repositories after CVMFS injection!") else: raise e with tempfile.TemporaryFile() as tmp_file: for chunk in chunk_it: tmp_file.write(decompress_object.decompress(chunk)) tmp_file.write(decompress_object.flush()) tmp_file.seek(0) tar = tarfile.TarFile(fileobj=tmp_file) tar.extractall(dest_dir) tar.close() def update(self, src_dir, push_alias): """ Packs and uploads the contents of src_dir as a layer and injects the layer into the image. The new layer version is stored under the tag push_alias """ if not self.fat_manifest.is_cvmfs_prepared(): print("Preparing image for CVMFS injection...") self.setup(push_alias) with tempfile.NamedTemporaryFile(delete=False) as tmp_file: print("Bundling file into tar...") _, error = exec_bash("tar --xattrs -C "+src_dir+" -cvf "+tmp_file.name+" .") if error: raise RuntimeError("Failed to tar with error " + str(error)) tar_digest = hash_file(tmp_file.name) print("Bundling tar into gz...") gz_dest = tmp_file.name+".gz" _, error = exec_bash("gzip "+tmp_file.name) if error: raise RuntimeError("Failed to tar with error " + str(error)) print("Uploading...") gz_digest = self.dxfObject.push_blob(gz_dest) os.unlink(gz_dest) print("Refreshing manifests...") old_gz_digest = self.fat_manifest.get_gz_digest() layer_size = self.dxfObject.blob_size(gz_digest) self.fat_manifest.inject(tar_digest, gz_digest) fat_man_json = self.fat_manifest.as_JSON() manifest_digest = hash_bytes(bytes(fat_man_json, 'utf-8')) self.dxfObject.push_blob(data=fat_man_json, digest=manifest_digest) manifest_size = self.dxfObject.blob_size(manifest_digest) self.image_manifest.inject(old_gz_digest, gz_digest, layer_size, manifest_digest, manifest_size) image_man_json = self.image_manifest.as_JSON() self.dxfObject.set_manifest(push_alias, image_man_json) def _get_manifest(self, alias): return ImageManifest(self.dxfObject.get_manifest(alias)) def _get_fat_manifest(self, image_manifest): fat_manifest = "" (readIter, _) = self.dxfObject.pull_blob(self.image_manifest.get_fat_manif_digest(), size=True, chunk_size=4096) for chunk in readIter: fat_manifest += str(chunk)[2:-1] fat_manifest = fat_manifest.replace("\\\\","\\") return FatManifest(fat_manifest) def _build_init_tar(self): """ Builds an empty /cvmfs tar and uploads it to the registry :rtype: tuple :returns: Tuple containing the tar digest and gz digest """ ident = self.image_manifest.get_fat_manif_digest()[5:15] tmp_name = "/tmp/injector-"+ident os.makedirs(tmp_name+"/cvmfs", exist_ok=True) tar_dest = "/tmp/"+ident+".tar" _, error = exec_bash("tar --xattrs -C "+tmp_name+" -cvf "+tar_dest+" .") if error: print("Failed to tar with error " + str(error)) return tar_digest = hash_file(tar_dest) _, error = exec_bash("gzip -n "+tar_dest) if error: print("Failed to tar with error " + str(error)) return gz_dest = tar_dest+".gz" gzip_digest = self.dxfObject.push_blob(tar_dest+".gz") # Cleanup os.rmdir(tmp_name+"/cvmfs") os.rmdir(tmp_name) os.unlink(gz_dest) return (tar_digest, gzip_digest)
sys.stdout.write("Please respond with 'yes' or 'no' " "(or 'y' or 'n').\n") def auth(dxf, response): dxf.authenticate(os.environ.get("DOCKER_USERNAME"), os.environ.get("DOCKER_TOKEN"), response=response) if __name__ == '__main__': try: logging.basicConfig(level=logging.INFO) parsedArgs = parseArguments() logging.info('Call arguments given: %s' % sys.argv[1:]) dxf = DXF('registry-1.docker.io', 'schulcloud/{}'.format(parsedArgs.repo), auth) if hasattr(parsedArgs, 'add_tag') and parsedArgs.add_tag == True: logging.info("Adding tag '{}' to '{}' in repository '{}'".format( parsedArgs.new_tag, parsedArgs.exist_tag, parsedArgs.repo)) manifest = dxf.get_manifest('{}'.format(parsedArgs.exist_tag)) dxf.set_manifest('{}'.format(parsedArgs.new_tag), manifest) else: logging.info("Deleting tag '{}' in repository '{}'".format( parsedArgs.exist_tag, parsedArgs.repo)) suretodel = query_yes_no( "Are you sure to delete tag '{}' from '{}'".format( parsedArgs.exist_tag, parsedArgs.repo)) if suretodel: dxf.del_alias(parsedArgs.exist_tag) exit(0)