def _create_manifest(conf_path, layer_paths):
def _get_size_of(path):
return os.path.getsize(path)
result = dict()
result["schemaVersion"] = 2
result[
"mediaType"] = "application/vnd.docker.distribution.manifest.v2+json"
result["config"] = dict()
result["config"][
"mediaType"] = "application/vnd.docker.container.image.v1+json"
result["config"]["size"] = _get_size_of(conf_path)
result["config"]["digest"] = hash_file(conf_path)
result["layers"] = []
for layer in layer_paths:
layer_dict = dict()
layer_dict[
"mediaType"] = "application/vnd.docker.image.rootfs.diff.tar"
layer_dict["size"] = _get_size_of(layer)
layer_dict["digest"] = hash_file(layer)
result["layers"].append(layer_dict)
return json.dumps(result)
def _build_init_tar(self):
"""
Builds an empty /cvmfs tar and uploads it to the registry
:rtype: tuple
:returns: Tuple containing the tar digest and gz digest
"""
ident = self.image_manifest.get_fat_manif_digest()[5:15]
tmp_name = "/tmp/injector-" + ident
os.makedirs(tmp_name + "/cvmfs", exist_ok=True)
tar_dest = "/tmp/" + ident + ".tar"
_, error = exec_bash("tar --xattrs -C " + tmp_name + " -cvf " +
tar_dest + " .")
if error:
print("Failed to tar with error " + str(error))
return
tar_digest = hash_file(tar_dest)
_, error = exec_bash("gzip -n " + tar_dest)
if error:
print("Failed to tar with error " + str(error))
return
gz_dest = tar_dest + ".gz"
gzip_digest = self.dxfObject.push_blob(tar_dest + ".gz")
# Cleanup
os.rmdir(tmp_name + "/cvmfs")
os.rmdir(tmp_name)
os.unlink(gz_dest)
return (tar_digest, gzip_digest)
def _build_init_tar(self):
"""
Builds an empty /cvmfs tar and uploads it to the registry
:rtype: tuple
:returns: Tuple containing the tar digest and gz digest
"""
ident = self.image_manifest.get_fat_manif_digest()[5:15]
tmp_name = "/tmp/injector-"+ident
os.makedirs(tmp_name+"/cvmfs", exist_ok=True)
tar_dest = "/tmp/"+ident+".tar"
_, error = exec_bash("tar --xattrs -C "+tmp_name+" -cvf "+tar_dest+" .")
if error:
print("Failed to tar with error " + str(error))
return
tar_digest = hash_file(tar_dest)
_, error = exec_bash("gzip -n "+tar_dest)
if error:
print("Failed to tar with error " + str(error))
return
gz_dest = tar_dest+".gz"
gzip_digest = self.dxfObject.push_blob(tar_dest+".gz")
# Cleanup
os.rmdir(tmp_name+"/cvmfs")
os.rmdir(tmp_name)
os.unlink(gz_dest)
return (tar_digest, gzip_digest)
def update(self, src_dir, push_alias):
"""
Packs and uploads the contents of src_dir as a layer and injects the layer into the image.
The new layer version is stored under the tag push_alias
"""
if not self.fat_manifest.is_cvmfs_prepared():
print("Preparing image for CVMFS injection...")
self.setup(push_alias)
with tempfile.NamedTemporaryFile(delete=False) as tmp_file:
print("Bundling file into tar...")
_, error = exec_bash("tar --xattrs -C "+src_dir+" -cvf "+tmp_file.name+" .")
if error:
raise RuntimeError("Failed to tar with error " + str(error))
tar_digest = hash_file(tmp_file.name)
print("Bundling tar into gz...")
gz_dest = tmp_file.name+".gz"
_, error = exec_bash("gzip "+tmp_file.name)
if error:
raise RuntimeError("Failed to tar with error " + str(error))
print("Uploading...")
gz_digest = self.dxfObject.push_blob(gz_dest)
os.unlink(gz_dest)
print("Refreshing manifests...")
old_gz_digest = self.fat_manifest.get_gz_digest()
layer_size = self.dxfObject.blob_size(gz_digest)
self.fat_manifest.inject(tar_digest, gz_digest)
fat_man_json = self.fat_manifest.as_JSON()
manifest_digest = hash_bytes(bytes(fat_man_json, 'utf-8'))
self.dxfObject.push_blob(data=fat_man_json, digest=manifest_digest)
manifest_size = self.dxfObject.blob_size(manifest_digest)
self.image_manifest.inject(old_gz_digest, gz_digest, layer_size, manifest_digest, manifest_size)
image_man_json = self.image_manifest.as_JSON()
self.dxfObject.set_manifest(push_alias, image_man_json)
def check_target(self, target, *filenames):
"""
Check whether the hashes of a target's blobs match the hashes of a
given list of filenames.
Raises `dxf.exceptions.DXFDigestMismatchError` if they don't.
:param target: Name of target to check.
:type target: str
:param filenames: Names of files to check against.
:type filenames: list
"""
blob_dgsts = self._get_digests(target)
if len(blob_dgsts) != len(filenames):
raise dxf.exceptions.DXFDigestMismatchError(filenames, blob_dgsts)
for i, filename in enumerate(filenames):
file_dgst = hash_file(filename)
if file_dgst != blob_dgsts[i]:
raise dxf.exceptions.DXFDigestMismatchError(file_dgst, blob_dgsts[i])
def check_target(self, target, *filenames):
"""
Check whether the hashes of a target's blobs match the hashes of a
given list of filenames.
Raises `dxf.exceptions.DXFDigestMismatchError` if they don't.
:param target: Name of target to check.
:type target: str
:param filenames: Names of files to check against.
:type filenames: list
"""
blob_dgsts = self._get_digests(target)
if len(blob_dgsts) != len(filenames):
raise dxf.exceptions.DXFDigestMismatchError(filenames, blob_dgsts)
for i, filename in enumerate(filenames):
file_dgst = hash_file(filename)
if file_dgst != blob_dgsts[i]:
raise dxf.exceptions.DXFDigestMismatchError(file_dgst, blob_dgsts[i])
def update(self, src_dir, push_alias):
"""
Packs and uploads the contents of src_dir as a layer and injects the layer into the image.
The new layer version is stored under the tag push_alias
"""
if not self.fat_manifest.is_cvmfs_prepared():
print("Preparing image for CVMFS injection...")
self.setup(push_alias)
with tempfile.NamedTemporaryFile(delete=False) as tmp_file:
print("Bundling file into tar...")
_, error = exec_bash("tar --xattrs -C " + src_dir + " -cvf " +
tmp_file.name + " .")
if error:
raise RuntimeError("Failed to tar with error " + str(error))
tar_digest = hash_file(tmp_file.name)
print("Bundling tar into gz...")
gz_dest = tmp_file.name + ".gz"
_, error = exec_bash("gzip " + tmp_file.name)
if error:
raise RuntimeError("Failed to tar with error " + str(error))
print("Uploading...")
gz_digest = self.dxfObject.push_blob(gz_dest)
os.unlink(gz_dest)
print("Refreshing manifests...")
old_gz_digest = self.fat_manifest.get_gz_digest()
layer_size = self.dxfObject.blob_size(gz_digest)
self.fat_manifest.inject(tar_digest, gz_digest)
fat_man_json = self.fat_manifest.as_JSON()
manifest_digest = hash_bytes(bytes(fat_man_json, 'utf-8'))
self.dxfObject.push_blob(data=fat_man_json, digest=manifest_digest)
manifest_size = self.dxfObject.blob_size(manifest_digest)
self.image_manifest.inject(old_gz_digest, gz_digest, layer_size,
manifest_digest, manifest_size)
image_man_json = self.image_manifest.as_JSON()
self.dxfObject.set_manifest(push_alias, image_man_json)
def _doit():
# pylint: disable=too-many-branches
if args.op == "auth":
username = environ.get('DXF_USERNAME')
password = environ.get('DXF_PASSWORD')
authorization = environ.get('DXF_AUTHORIZATION')
token = dxf_obj.authenticate(username,
password,
actions=args.args,
authorization=authorization)
if token:
print(token)
return
token = environ.get('DXF_TOKEN')
if token:
dxf_obj.token = token
if args.op == "push-blob":
if len(args.args) < 1:
_parser.error('too few arguments')
if len(args.args) > 2:
_parser.error('too many arguments')
if len(args.args) == 2 and not args.args[1].startswith('@'):
_parser.error('invalid alias')
dgst = dxf_obj.push_blob(args.args[0], progress)
if len(args.args) == 2:
dxf_obj.set_alias(args.args[1][1:], dgst)
print(dgst)
elif args.op == "pull-blob":
_stdout = getattr(sys.stdout, 'buffer', sys.stdout)
if args.args:
dgsts = _flatten([
dxf_obj.get_alias(name[1:])
if name.startswith('@') else [name] for name in args.args
])
else:
dgsts = dxf_obj.get_alias(manifest=sys.stdin.read())
for dgst in dgsts:
it, size = dxf_obj.pull_blob(
dgst, size=True, chunk_size=environ.get('DXF_CHUNK_SIZE'))
if environ.get('DXF_BLOB_INFO') == '1':
print(dgst + ' ' + str(size))
if progress:
progress(dgst, b'', size)
for chunk in it:
if progress:
progress(dgst, chunk, size)
_stdout.write(chunk)
elif args.op == 'blob-size':
if args.args:
sizes = [
dxf_obj.get_alias(name[1:], sizes=True) if
name.startswith('@') else [(name, dxf_obj.blob_size(name))]
for name in args.args
]
else:
sizes = [
dxf_obj.get_alias(manifest=sys.stdin.read(), sizes=True)
]
for tuples in sizes:
print(sum([size for _, size in tuples]))
elif args.op == 'del-blob':
if args.args:
dgsts = _flatten([
dxf_obj.del_alias(name[1:])
if name.startswith('@') else [name] for name in args.args
])
else:
dgsts = dxf_obj.get_alias(manifest=sys.stdin.read())
for dgst in dgsts:
dxf_obj.del_blob(dgst)
elif args.op == "set-alias":
if len(args.args) < 2:
_parser.error('too few arguments')
dgsts = [
dxf.hash_file(dgst) if os.sep in dgst else dgst
for dgst in args.args[1:]
]
sys.stdout.write(dxf_obj.set_alias(args.args[0], *dgsts))
elif args.op == "get-alias":
if args.args:
dgsts = _flatten(
[dxf_obj.get_alias(name) for name in args.args])
else:
dgsts = dxf_obj.get_alias(manifest=sys.stdin.read())
for dgst in dgsts:
print(dgst)
elif args.op == "del-alias":
for name in args.args:
for dgst in dxf_obj.del_alias(name):
print(dgst)
elif args.op == 'list-aliases':
if args.args:
_parser.error('too many arguments')
for name in dxf_obj.list_aliases():
print(name)
elif args.op == 'list-repos':
for name in dxf_obj.list_repos():
print(name)
def _doit():
# pylint: disable=too-many-branches
if args.op == "auth":
token = dxf_obj.authenticate(environ['DXF_USERNAME'],
environ['DXF_PASSWORD'],
actions=args.args)
if token:
print(token)
return
token = environ.get('DXF_TOKEN')
if token:
dxf_obj.token = token
if args.op == "push-blob":
if len(args.args) < 1:
_parser.error('too few arguments')
if len(args.args) > 2:
_parser.error('too many arguments')
if len(args.args) == 2 and not args.args[1].startswith('@'):
_parser.error('invalid alias')
dgst = dxf_obj.push_blob(args.args[0], progress)
if len(args.args) == 2:
dxf_obj.set_alias(args.args[1][1:], dgst)
print(dgst)
elif args.op == "pull-blob":
_stdout = getattr(sys.stdout, 'buffer', sys.stdout)
if len(args.args) == 0:
dgsts = dxf_obj.get_alias(manifest=sys.stdin.read())
else:
dgsts = _flatten([dxf_obj.get_alias(name[1:])
if name.startswith('@') else [name]
for name in args.args])
for dgst in dgsts:
it, size = dxf_obj.pull_blob(
dgst, size=True, chunk_size=environ.get('DXF_CHUNK_SIZE'))
if environ.get('DXF_BLOB_INFO') == '1':
print(dgst + ' ' + str(size))
if progress:
progress(dgst, b'', size)
for chunk in it:
if progress:
progress(dgst, chunk, size)
_stdout.write(chunk)
elif args.op == 'blob-size':
if len(args.args) == 0:
sizes = [dxf_obj.get_alias(manifest=sys.stdin.read(),
sizes=True)]
else:
sizes = [dxf_obj.get_alias(name[1:], sizes=True)
if name.startswith('@') else
[(name, dxf_obj.blob_size(name))]
for name in args.args]
for tuples in sizes:
print(sum([size for _, size in tuples]))
elif args.op == 'del-blob':
if len(args.args) == 0:
dgsts = dxf_obj.get_alias(manifest=sys.stdin.read())
else:
dgsts = _flatten([dxf_obj.del_alias(name[1:])
if name.startswith('@') else [name]
for name in args.args])
for dgst in dgsts:
dxf_obj.del_blob(dgst)
elif args.op == "set-alias":
if len(args.args) < 2:
_parser.error('too few arguments')
dgsts = [dxf.hash_file(dgst) if os.sep in dgst else dgst
for dgst in args.args[1:]]
sys.stdout.write(dxf_obj.set_alias(args.args[0], *dgsts))
elif args.op == "get-alias":
if len(args.args) == 0:
dgsts = dxf_obj.get_alias(manifest=sys.stdin.read())
else:
dgsts = _flatten([dxf_obj.get_alias(name) for name in args.args])
for dgst in dgsts:
print(dgst)
elif args.op == "del-alias":
for name in args.args:
for dgst in dxf_obj.del_alias(name):
print(dgst)
elif args.op == 'list-aliases':
if len(args.args) > 0:
_parser.error('too many arguments')
for name in dxf_obj.list_aliases():
print(name)
elif args.op == 'list-repos':
for name in dxf_obj.list_repos():
print(name)
