def _create_manifest(conf_path, layer_paths): def _get_size_of(path): return os.path.getsize(path) result = dict() result["schemaVersion"] = 2 result[ "mediaType"] = "application/vnd.docker.distribution.manifest.v2+json" result["config"] = dict() result["config"][ "mediaType"] = "application/vnd.docker.container.image.v1+json" result["config"]["size"] = _get_size_of(conf_path) result["config"]["digest"] = hash_file(conf_path) result["layers"] = [] for layer in layer_paths: layer_dict = dict() layer_dict[ "mediaType"] = "application/vnd.docker.image.rootfs.diff.tar" layer_dict["size"] = _get_size_of(layer) layer_dict["digest"] = hash_file(layer) result["layers"].append(layer_dict) return json.dumps(result)
def _build_init_tar(self): """ Builds an empty /cvmfs tar and uploads it to the registry :rtype: tuple :returns: Tuple containing the tar digest and gz digest """ ident = self.image_manifest.get_fat_manif_digest()[5:15] tmp_name = "/tmp/injector-" + ident os.makedirs(tmp_name + "/cvmfs", exist_ok=True) tar_dest = "/tmp/" + ident + ".tar" _, error = exec_bash("tar --xattrs -C " + tmp_name + " -cvf " + tar_dest + " .") if error: print("Failed to tar with error " + str(error)) return tar_digest = hash_file(tar_dest) _, error = exec_bash("gzip -n " + tar_dest) if error: print("Failed to tar with error " + str(error)) return gz_dest = tar_dest + ".gz" gzip_digest = self.dxfObject.push_blob(tar_dest + ".gz") # Cleanup os.rmdir(tmp_name + "/cvmfs") os.rmdir(tmp_name) os.unlink(gz_dest) return (tar_digest, gzip_digest)
def _build_init_tar(self): """ Builds an empty /cvmfs tar and uploads it to the registry :rtype: tuple :returns: Tuple containing the tar digest and gz digest """ ident = self.image_manifest.get_fat_manif_digest()[5:15] tmp_name = "/tmp/injector-"+ident os.makedirs(tmp_name+"/cvmfs", exist_ok=True) tar_dest = "/tmp/"+ident+".tar" _, error = exec_bash("tar --xattrs -C "+tmp_name+" -cvf "+tar_dest+" .") if error: print("Failed to tar with error " + str(error)) return tar_digest = hash_file(tar_dest) _, error = exec_bash("gzip -n "+tar_dest) if error: print("Failed to tar with error " + str(error)) return gz_dest = tar_dest+".gz" gzip_digest = self.dxfObject.push_blob(tar_dest+".gz") # Cleanup os.rmdir(tmp_name+"/cvmfs") os.rmdir(tmp_name) os.unlink(gz_dest) return (tar_digest, gzip_digest)
def update(self, src_dir, push_alias): """ Packs and uploads the contents of src_dir as a layer and injects the layer into the image. The new layer version is stored under the tag push_alias """ if not self.fat_manifest.is_cvmfs_prepared(): print("Preparing image for CVMFS injection...") self.setup(push_alias) with tempfile.NamedTemporaryFile(delete=False) as tmp_file: print("Bundling file into tar...") _, error = exec_bash("tar --xattrs -C "+src_dir+" -cvf "+tmp_file.name+" .") if error: raise RuntimeError("Failed to tar with error " + str(error)) tar_digest = hash_file(tmp_file.name) print("Bundling tar into gz...") gz_dest = tmp_file.name+".gz" _, error = exec_bash("gzip "+tmp_file.name) if error: raise RuntimeError("Failed to tar with error " + str(error)) print("Uploading...") gz_digest = self.dxfObject.push_blob(gz_dest) os.unlink(gz_dest) print("Refreshing manifests...") old_gz_digest = self.fat_manifest.get_gz_digest() layer_size = self.dxfObject.blob_size(gz_digest) self.fat_manifest.inject(tar_digest, gz_digest) fat_man_json = self.fat_manifest.as_JSON() manifest_digest = hash_bytes(bytes(fat_man_json, 'utf-8')) self.dxfObject.push_blob(data=fat_man_json, digest=manifest_digest) manifest_size = self.dxfObject.blob_size(manifest_digest) self.image_manifest.inject(old_gz_digest, gz_digest, layer_size, manifest_digest, manifest_size) image_man_json = self.image_manifest.as_JSON() self.dxfObject.set_manifest(push_alias, image_man_json)
def check_target(self, target, *filenames): """ Check whether the hashes of a target's blobs match the hashes of a given list of filenames. Raises `dxf.exceptions.DXFDigestMismatchError` if they don't. :param target: Name of target to check. :type target: str :param filenames: Names of files to check against. :type filenames: list """ blob_dgsts = self._get_digests(target) if len(blob_dgsts) != len(filenames): raise dxf.exceptions.DXFDigestMismatchError(filenames, blob_dgsts) for i, filename in enumerate(filenames): file_dgst = hash_file(filename) if file_dgst != blob_dgsts[i]: raise dxf.exceptions.DXFDigestMismatchError(file_dgst, blob_dgsts[i])
def update(self, src_dir, push_alias): """ Packs and uploads the contents of src_dir as a layer and injects the layer into the image. The new layer version is stored under the tag push_alias """ if not self.fat_manifest.is_cvmfs_prepared(): print("Preparing image for CVMFS injection...") self.setup(push_alias) with tempfile.NamedTemporaryFile(delete=False) as tmp_file: print("Bundling file into tar...") _, error = exec_bash("tar --xattrs -C " + src_dir + " -cvf " + tmp_file.name + " .") if error: raise RuntimeError("Failed to tar with error " + str(error)) tar_digest = hash_file(tmp_file.name) print("Bundling tar into gz...") gz_dest = tmp_file.name + ".gz" _, error = exec_bash("gzip " + tmp_file.name) if error: raise RuntimeError("Failed to tar with error " + str(error)) print("Uploading...") gz_digest = self.dxfObject.push_blob(gz_dest) os.unlink(gz_dest) print("Refreshing manifests...") old_gz_digest = self.fat_manifest.get_gz_digest() layer_size = self.dxfObject.blob_size(gz_digest) self.fat_manifest.inject(tar_digest, gz_digest) fat_man_json = self.fat_manifest.as_JSON() manifest_digest = hash_bytes(bytes(fat_man_json, 'utf-8')) self.dxfObject.push_blob(data=fat_man_json, digest=manifest_digest) manifest_size = self.dxfObject.blob_size(manifest_digest) self.image_manifest.inject(old_gz_digest, gz_digest, layer_size, manifest_digest, manifest_size) image_man_json = self.image_manifest.as_JSON() self.dxfObject.set_manifest(push_alias, image_man_json)
def _doit(): # pylint: disable=too-many-branches if args.op == "auth": username = environ.get('DXF_USERNAME') password = environ.get('DXF_PASSWORD') authorization = environ.get('DXF_AUTHORIZATION') token = dxf_obj.authenticate(username, password, actions=args.args, authorization=authorization) if token: print(token) return token = environ.get('DXF_TOKEN') if token: dxf_obj.token = token if args.op == "push-blob": if len(args.args) < 1: _parser.error('too few arguments') if len(args.args) > 2: _parser.error('too many arguments') if len(args.args) == 2 and not args.args[1].startswith('@'): _parser.error('invalid alias') dgst = dxf_obj.push_blob(args.args[0], progress) if len(args.args) == 2: dxf_obj.set_alias(args.args[1][1:], dgst) print(dgst) elif args.op == "pull-blob": _stdout = getattr(sys.stdout, 'buffer', sys.stdout) if args.args: dgsts = _flatten([ dxf_obj.get_alias(name[1:]) if name.startswith('@') else [name] for name in args.args ]) else: dgsts = dxf_obj.get_alias(manifest=sys.stdin.read()) for dgst in dgsts: it, size = dxf_obj.pull_blob( dgst, size=True, chunk_size=environ.get('DXF_CHUNK_SIZE')) if environ.get('DXF_BLOB_INFO') == '1': print(dgst + ' ' + str(size)) if progress: progress(dgst, b'', size) for chunk in it: if progress: progress(dgst, chunk, size) _stdout.write(chunk) elif args.op == 'blob-size': if args.args: sizes = [ dxf_obj.get_alias(name[1:], sizes=True) if name.startswith('@') else [(name, dxf_obj.blob_size(name))] for name in args.args ] else: sizes = [ dxf_obj.get_alias(manifest=sys.stdin.read(), sizes=True) ] for tuples in sizes: print(sum([size for _, size in tuples])) elif args.op == 'del-blob': if args.args: dgsts = _flatten([ dxf_obj.del_alias(name[1:]) if name.startswith('@') else [name] for name in args.args ]) else: dgsts = dxf_obj.get_alias(manifest=sys.stdin.read()) for dgst in dgsts: dxf_obj.del_blob(dgst) elif args.op == "set-alias": if len(args.args) < 2: _parser.error('too few arguments') dgsts = [ dxf.hash_file(dgst) if os.sep in dgst else dgst for dgst in args.args[1:] ] sys.stdout.write(dxf_obj.set_alias(args.args[0], *dgsts)) elif args.op == "get-alias": if args.args: dgsts = _flatten( [dxf_obj.get_alias(name) for name in args.args]) else: dgsts = dxf_obj.get_alias(manifest=sys.stdin.read()) for dgst in dgsts: print(dgst) elif args.op == "del-alias": for name in args.args: for dgst in dxf_obj.del_alias(name): print(dgst) elif args.op == 'list-aliases': if args.args: _parser.error('too many arguments') for name in dxf_obj.list_aliases(): print(name) elif args.op == 'list-repos': for name in dxf_obj.list_repos(): print(name)
def _doit(): # pylint: disable=too-many-branches if args.op == "auth": token = dxf_obj.authenticate(environ['DXF_USERNAME'], environ['DXF_PASSWORD'], actions=args.args) if token: print(token) return token = environ.get('DXF_TOKEN') if token: dxf_obj.token = token if args.op == "push-blob": if len(args.args) < 1: _parser.error('too few arguments') if len(args.args) > 2: _parser.error('too many arguments') if len(args.args) == 2 and not args.args[1].startswith('@'): _parser.error('invalid alias') dgst = dxf_obj.push_blob(args.args[0], progress) if len(args.args) == 2: dxf_obj.set_alias(args.args[1][1:], dgst) print(dgst) elif args.op == "pull-blob": _stdout = getattr(sys.stdout, 'buffer', sys.stdout) if len(args.args) == 0: dgsts = dxf_obj.get_alias(manifest=sys.stdin.read()) else: dgsts = _flatten([dxf_obj.get_alias(name[1:]) if name.startswith('@') else [name] for name in args.args]) for dgst in dgsts: it, size = dxf_obj.pull_blob( dgst, size=True, chunk_size=environ.get('DXF_CHUNK_SIZE')) if environ.get('DXF_BLOB_INFO') == '1': print(dgst + ' ' + str(size)) if progress: progress(dgst, b'', size) for chunk in it: if progress: progress(dgst, chunk, size) _stdout.write(chunk) elif args.op == 'blob-size': if len(args.args) == 0: sizes = [dxf_obj.get_alias(manifest=sys.stdin.read(), sizes=True)] else: sizes = [dxf_obj.get_alias(name[1:], sizes=True) if name.startswith('@') else [(name, dxf_obj.blob_size(name))] for name in args.args] for tuples in sizes: print(sum([size for _, size in tuples])) elif args.op == 'del-blob': if len(args.args) == 0: dgsts = dxf_obj.get_alias(manifest=sys.stdin.read()) else: dgsts = _flatten([dxf_obj.del_alias(name[1:]) if name.startswith('@') else [name] for name in args.args]) for dgst in dgsts: dxf_obj.del_blob(dgst) elif args.op == "set-alias": if len(args.args) < 2: _parser.error('too few arguments') dgsts = [dxf.hash_file(dgst) if os.sep in dgst else dgst for dgst in args.args[1:]] sys.stdout.write(dxf_obj.set_alias(args.args[0], *dgsts)) elif args.op == "get-alias": if len(args.args) == 0: dgsts = dxf_obj.get_alias(manifest=sys.stdin.read()) else: dgsts = _flatten([dxf_obj.get_alias(name) for name in args.args]) for dgst in dgsts: print(dgst) elif args.op == "del-alias": for name in args.args: for dgst in dxf_obj.del_alias(name): print(dgst) elif args.op == 'list-aliases': if len(args.args) > 0: _parser.error('too many arguments') for name in dxf_obj.list_aliases(): print(name) elif args.op == 'list-repos': for name in dxf_obj.list_repos(): print(name)