def writeDataFile(self, filename, text, content_type, subdir=None):
""" See IExportContext.
"""
mod_time = time.time()
if subdir is not None:
elements = subdir.split('/')
parents = filter(None, elements)
while parents:
dirname = os.path.join(*parents)
try:
self._archive.getmember(dirname + '/')
except KeyError:
info = TarInfo(dirname)
info.size = 0
info.mode = 509
info.mtime = mod_time
info.type = DIRTYPE
self._archive.addfile(info, StringIO())
parents = parents[:-1]
filename = '/'.join((subdir, filename))
stream = StringIO(text)
info = TarInfo(filename)
info.size = len(text)
info.mode = 436
info.mtime = mod_time
self._archive.addfile(info, stream)
def writeDataFile( self, filename, text, content_type, subdir=None ):
""" See IExportContext.
"""
mod_time = time.time()
if subdir is not None:
elements = subdir.split('/')
parents = filter(None, elements)
while parents:
dirname = os.path.join(*parents)
try:
self._archive.getmember(dirname+'/')
except KeyError:
info = TarInfo(dirname)
info.size = 0
info.mode = 509
info.mtime = mod_time
info.type = DIRTYPE
self._archive.addfile(info, StringIO())
parents = parents[:-1]
filename = '/'.join( ( subdir, filename ) )
stream = StringIO( text )
info = TarInfo( filename )
info.size = len( text )
info.mode = 436
info.mtime = mod_time
self._archive.addfile( info, stream )
def _add_entry(self, name, type, mode, size, data):
info = TarInfo(name)
info.type = type
info.mode = mode
info.size = size
info.mtime = time.time()
self._tarfile.addfile(info, data)
def writeDataFile(self, filename, text, content_type, subdir=None):
""" See IExportContext.
"""
if subdir is not None:
filename = '/'.join((subdir, filename))
parents = filename.split('/')[:-1]
while parents:
path = '/'.join(parents) + '/'
if path not in self._archive.getnames():
info = TarInfo(path)
info.type = DIRTYPE
# tarfile.filemode(0755) == '-rwxr-xr-x'
info.mode = 0755
info.mtime = time.time()
self._archive.addfile(info)
parents.pop()
info = TarInfo(filename)
if isinstance(text, str):
stream = StringIO(text)
info.size = len(text)
elif isinstance(text, unicode):
raise ValueError("Unicode text is not supported, even if it only "
"contains ascii. Please encode your data")
else:
# Assume text is a an instance of a class like
# Products.Archetypes.WebDAVSupport.PdataStreamIterator,
# as in the case of ATFile
stream = text.file
info.size = text.size
info.mtime = time.time()
self._archive.addfile(info, stream)
def upload(self):
logger.info("Loading artifacts")
t = ts()
tar_data = io.BytesIO()
with tarfile.open(fileobj=tar_data, mode="w|") as tar:
for root, _, files in os.walk(self._artifact_directory):
for af in files:
full_path = os.path.join(root, af)
relpath = os.path.relpath(full_path, self._artifact_directory)
ti = TarInfo(relpath)
stat = os.stat(full_path)
ti.size = stat.st_size
ti.mode = stat.st_mode
with open(full_path, "rb") as f:
tar.addfile(ti, f)
res = self._container.put_archive(config.build_dir, tar_data.getvalue())
if not res:
raise Exception(f"Error loading artifact: {af}")
t = ts() - t
logger.info("Artifacts loaded in %.3fs", t)
def writeDataFile( self, filename, text, content_type, subdir=None ):
""" See IExportContext.
"""
if subdir is not None:
filename = '/'.join( ( subdir, filename ) )
parents = filename.split('/')[:-1]
while parents:
path = '/'.join(parents) + '/'
if path not in self._archive.getnames():
info = TarInfo(path)
info.type = DIRTYPE
# tarfile.filemode(0755) == '-rwxr-xr-x'
info.mode = 0755
info.mtime = time.time()
self._archive.addfile(info)
parents.pop()
info = TarInfo(filename)
if isinstance(text, basestring):
stream = StringIO(text)
info.size = len(text)
else:
# Assume text is a an instance of a class like
# Products.Archetypes.WebDAVSupport.PdataStreamIterator,
# as in the case of ATFile
stream = text.file
info.size = text.size
info.mtime = time.time()
self._archive.addfile( info, stream )
def writeDataFile( self, filename, text, content_type, subdir=None ):
""" See IExportContext.
"""
if subdir is not None:
filename = '/'.join( ( subdir, filename ) )
parents = filename.split('/')[:-1]
while parents:
path = '/'.join(parents) + '/'
if path not in self._archive.getnames():
info = TarInfo(path)
info.type = DIRTYPE
# tarfile.filemode(0755) == '-rwxr-xr-x'
info.mode = 0755
info.mtime = time.time()
self._archive.addfile(info)
parents.pop()
info = TarInfo(filename)
if isinstance(text, str):
stream = StringIO(text)
info.size = len(text)
elif isinstance(text, unicode):
raise ValueError("Unicode text is not supported, even if it only "
"contains ascii. Please encode your data. See "
"GS 1.7.0 changes for more")
else:
# Assume text is a an instance of a class like
# Products.Archetypes.WebDAVSupport.PdataStreamIterator,
# as in the case of ATFile
stream = text.file
info.size = text.size
info.mtime = time.time()
self._archive.addfile( info, stream )
def GetTarInfo(filename, filetype=DIRTYPE, mode=0755):
"""Create information for tar files"""
tarinfo = TarInfo(path.basename(filename))
tarinfo.type = filetype
tarinfo.mode = mode
tarinfo.mtime = time()
return tarinfo
def add_filter(info: tarfile.TarInfo) -> Optional[tarfile.TarInfo]:
"""Filter files targeted to be added to tarfile.
Args:
info: Information on the file targeted to be added
Returns:
None: if file is not to be added
TarInfo: when file is to be added. Modified as needed.
Notes:
exclude is captured from parent
"""
if not (info.isfile() or info.isdir() or info.issym()):
return None
if _exclude_matcher(info.name, exclude):
return None
# Workaround https://bugs.python.org/issue32713. Fixed in Python 3.7
if info.mtime < 0 or info.mtime > 8 ** 11 - 1:
info.mtime = int(info.mtime)
# do not leak client information to service
info.uid = 0
info.uname = info.gname = "root"
if sys.platform == "win32":
info.mode = info.mode & 0o755 | 0o111
return info
def zip2tar(zip_file: str, tar_file, tar_mode: Optional[str] = 'w:gz'):
"""
:param zip_file: zip file path
:param tar_file:
IO(_io.IOBase): file obj
:param tar_mode: ref `tarfile.TarFile.open`
:return:
"""
zip_file = ZipFile(file=zip_file, mode='r')
tar_file = TarFile.open(fileobj=tar_file, mode=tar_mode)
try:
for zip_info in zip_file.infolist():
tar_info = TarInfo(name=zip_info.filename)
tar_info.size = zip_info.file_size
tar_info.mtime = datetime.now().timestamp()
# https://stackoverflow.com/a/434689/11722440
tar_info.mode = zip_info.external_attr >> 16
# https://stackoverflow.com/a/18432983/11722440
# https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT
# TODO whg fix other file (like symbolic link) in zip to regular file in tar file
if zip_info.filename.endswith('/'):
tar_info.type = tarfile.DIRTYPE
else:
tar_info.type = tarfile.REGTYPE
infile = zip_file.open(zip_info.filename)
tar_file.addfile(tar_info, infile)
except Exception as e:
raise
finally:
tar_file.close()
zip_file.close()
def writeDataFile(self, filename, text, content_type, subdir=None):
""" See IExportContext.
"""
if subdir is not None:
filename = '/'.join((subdir, filename))
parents = filename.split('/')[:-1]
while parents:
path = '/'.join(parents) + '/'
if path not in self._archive.getnames():
info = TarInfo(path)
info.type = DIRTYPE
# tarfile.filemode(0o755) == '-rwxr-xr-x'
info.mode = 0o755
info.mtime = time.time()
self._archive.addfile(info)
parents.pop()
info = TarInfo(filename)
if isinstance(text, six.text_type):
encoding = self.getEncoding() or 'utf-8'
text = text.encode(encoding)
if isinstance(text, six.binary_type):
stream = BytesIO(text)
info.size = len(text)
else:
# Assume text is a an instance of a class like
# Products.Archetypes.WebDAVSupport.PdataStreamIterator,
# as in the case of ATFile
stream = text.file
info.size = text.size
info.mtime = time.time()
self._archive.addfile(info, stream)
def move_certs(self, paths):
self.log.info("Staging internal ssl certs for %s", self._log_name)
yield self.pull_image(self.move_certs_image)
# create the volume
volume_name = self.format_volume_name(self.certs_volume_name, self)
# create volume passes even if it already exists
self.log.info("Creating ssl volume %s for %s", volume_name, self._log_name)
yield self.docker('create_volume', volume_name)
# create a tar archive of the internal cert files
# docker.put_archive takes a tarfile and a running container
# and unpacks the archive into the container
nb_paths = {}
tar_buf = BytesIO()
archive = TarFile(fileobj=tar_buf, mode='w')
for key, hub_path in paths.items():
fname = os.path.basename(hub_path)
nb_paths[key] = '/certs/' + fname
with open(hub_path, 'rb') as f:
content = f.read()
tarinfo = TarInfo(name=fname)
tarinfo.size = len(content)
tarinfo.mtime = os.stat(hub_path).st_mtime
tarinfo.mode = 0o644
archive.addfile(tarinfo, BytesIO(content))
archive.close()
tar_buf.seek(0)
# run a container to stage the certs,
# mounting the volume at /certs/
host_config = self.client.create_host_config(
binds={
volume_name: {"bind": "/certs", "mode": "rw"},
},
)
container = yield self.docker('create_container',
self.move_certs_image,
volumes=["/certs"],
host_config=host_config,
)
container_id = container['Id']
self.log.debug(
"Container %s is creating ssl certs for %s",
container_id[:12], self._log_name,
)
# start the container
yield self.docker('start', container_id)
# stage the archive to the container
try:
yield self.docker(
'put_archive',
container=container_id,
path='/certs',
data=tar_buf,
)
finally:
yield self.docker('remove_container', container_id)
return nb_paths
def move_certs(self, paths):
self.log.info("Staging internal ssl certs for %s", self._log_name)
yield self.pull_image(self.move_certs_image)
# create the volume
volume_name = self.format_volume_name(self.certs_volume_name, self)
# create volume passes even if it already exists
self.log.info("Creating ssl volume %s for %s", volume_name, self._log_name)
yield self.docker('create_volume', volume_name)
# create a tar archive of the internal cert files
# docker.put_archive takes a tarfile and a running container
# and unpacks the archive into the container
nb_paths = {}
tar_buf = BytesIO()
archive = TarFile(fileobj=tar_buf, mode='w')
for key, hub_path in paths.items():
fname = os.path.basename(hub_path)
nb_paths[key] = '/certs/' + fname
with open(hub_path, 'rb') as f:
content = f.read()
tarinfo = TarInfo(name=fname)
tarinfo.size = len(content)
tarinfo.mtime = os.stat(hub_path).st_mtime
tarinfo.mode = 0o644
archive.addfile(tarinfo, BytesIO(content))
archive.close()
tar_buf.seek(0)
# run a container to stage the certs,
# mounting the volume at /certs/
host_config = self.client.create_host_config(
binds={
volume_name: {"bind": "/certs", "mode": "rw"},
},
)
container = yield self.docker('create_container',
self.move_certs_image,
volumes=["/certs"],
host_config=host_config,
)
container_id = container['Id']
self.log.debug(
"Container %s is creating ssl certs for %s",
container_id[:12], self._log_name,
)
# start the container
yield self.docker('start', container_id)
# stage the archive to the container
try:
yield self.docker(
'put_archive',
container=container_id,
path='/certs',
data=tar_buf,
)
finally:
yield self.docker('remove_container', container_id)
return nb_paths
def addFile(tar, dest, file, file_size):
if dest not in written_files:
info = TarInfo(dest)
info.size = file_size
info.mtime = now
info.mode = 0o777
tar.addfile(info, fileobj=file)
written_files.add(dest)
def file_filter(info: tarfile.TarInfo):
info.mode = 0o00777 if executable else 0o00666
info.mtime = 0
info.type = tarfile.REGTYPE
info.uid = info.gid = 0
info.uname = info.gname = "root"
info.pax_headers = {}
return info
def addFile(tar, dest, file, file_size):
if dest not in written_files:
info = TarInfo(dest)
info.size = file_size
info.mtime = now
info.mode = 0777
tar.addfile(info, fileobj=file)
written_files.add(dest)
def create_dir(self, path):
"""Create a directory within the tarfile.
:param path: the path to put the directory at.
"""
tarinfo = TarInfo(name=path)
tarinfo.type = DIRTYPE
tarinfo.mode = 0755
self._set_defaults(tarinfo)
self.addfile(tarinfo)
def add_to_tar(tar: TarFile, data: bytes, filename: str):
tarinfo = TarInfo(name=filename)
tarinfo.size = len(data)
tarinfo.mtime = int(datetime.timestamp(datetime.utcnow()))
tarinfo.mode = 436
tarinfo.type = b'0'
tarinfo.uid = tarinfo.gid = 0
tarinfo.uname = tarinfo.gname = "0"
tar.addfile(tarinfo, BytesIO(data))
def create_dir(self, path):
"""Create a directory within the tarfile.
:param path: the path to put the directory at.
"""
tarinfo = TarInfo(name=path)
tarinfo.type = DIRTYPE
tarinfo.mode = 0755
self._set_defaults(tarinfo)
self.addfile(tarinfo)
def addString(tar, dest, string):
if dest not in written_files:
print dest, string
info = TarInfo(dest)
info.size = len(string)
info.mtime = now
info.mode = 0777
file = StringIO(string)
tar.addfile(info, fileobj=file)
file.close()
written_files.add(dest)
def addBytes(tar, dest, bytes):
if dest not in written_files:
# print dest, string
info = TarInfo(dest)
info.size = len(bytes)
info.mtime = now
info.mode = 0o777
file = BytesIO(bytes)
tar.addfile(info, fileobj=file)
file.close()
written_files.add(dest)
def addString(tar, dest, string):
if dest not in written_files:
print dest, string
info = TarInfo(dest)
info.size = len(string)
info.mtime = now
info.mode = 0777
file = StringIO(string)
tar.addfile(info, fileobj=file)
file.close()
written_files.add(dest)
def _set_uid_gid(tarinfo: TarInfo) -> TarInfo:
tarinfo.uid = tarinfo.gid = 0
tarinfo.uname = tarinfo.gname = ''
# Set 644 permissions, leaving higher bits of st_mode unchanged
new_mode = (tarinfo.mode | 0o644) & ~0o133
if tarinfo.mode & 0o100:
new_mode |= 0o111 # Executable: 644 -> 755
tarinfo.mode = new_mode
return tarinfo
def write_package():
tbs = ForgeClient.UPLOAD_TAR_BUFFER_SIZE
with TarFile.open(mode="w|gz", fileobj=body, bufsize=tbs,
dereference=True) as tar:
for file in files:
self.debug("Sending %s", file)
ti = TarInfo(file)
fp = os.path.join(self.path, file)
ti.size = os.path.getsize(fp)
ti.mode = 0o666
with open(fp, "rb") as fd:
tar.addfile(ti, fileobj=fd)
body.close()
def _sanitize_tar(tarinfo: TarInfo) -> TarInfo or None:
if '__pycache__' in tarinfo.name:
return None
tarinfo.uid = tarinfo.gid = 0
tarinfo.uname = tarinfo.gname = ''
# Set 644 permissions, leaving higher bits of st_mode unchanged
new_mode = (tarinfo.mode | 0o644) & ~0o133
if tarinfo.mode & 0o100:
new_mode |= 0o111 # Executable: 644 -> 755
tarinfo.mode = new_mode
return tarinfo
def uploadDF(dataflowName):
dataflowStr = None
udfs = {}
dataflowPath = os.path.join(path, "dataflows", dataflowName)
with open(os.path.join(dataflowPath, "dataflowInfo.json"), 'r') as df:
dataflowStr = df.read()
if os.path.exists(dataflowPath + "/udfs/"):
for udf in os.listdir(os.path.join(dataflowPath, "udfs")):
with open(os.path.join(dataflowPath, "udfs", udf), 'r') as udfFile:
udfs[udf] = udfFile.read()
retinaBuf = io.BytesIO()
with tarfile.open(fileobj=retinaBuf, mode="w:gz") as tar:
info = TarInfo("dataflowInfo.json")
info.size = len(dataflowStr)
tar.addfile(info, io.BytesIO(bytearray(dataflowStr, "utf-8")))
# # ##udfs directory
if udfs:
info = TarInfo("udfs")
info.type = tarfile.DIRTYPE
info.mode = 0o755
tar.addfile(info)
# ##Add udf to the above dir
for udfName, udfCode in udfs.items():
info = TarInfo(name="udfs/" + udfName)
info.size = len(udfCode)
info.mode = 0o755
tar.addfile(info, io.BytesIO(bytearray(udfCode, "utf-8")))
try:
retina.delete(dataflowName)
except:
print("Dataflow deletion failed!", dataflowName, availableRetinas)
retina.add(dataflowName, retinaBuf.getvalue())
def _add_entry(
self,
name: str,
type: bytes,
mode: int,
mtime: int,
size: int,
data: Optional[IO[bytes]],
linkname: str = "",
) -> None:
info = TarInfo(name)
info.type = type
info.mode = mode
info.size = size
info.mtime = mtime
info.linkname = linkname
return self._inner.addfile(info, data)
def file_write(self,
path,
content,
mode=None,
owner=None,
group=None,
append=False,
hide=False,
sudo=False):
"""
Writes a file to the container
@param path: path of the file
@param content: content to be put in the file
@param mode: file mode
@param owner: owner of the file
@param group: group of the file
@param append: append content to the file
@param hide: hide (debug) logs
@raise runtimeError: path for file couldn't be created
"""
if append and self.exists(path):
content = self.file_read(path) + content
file_name = os.path.basename(path)
dir_name = os.path.dirname(path)
buf = BytesIO()
with TarFile("write_file", mode='w', fileobj=buf) as tarf:
f = BytesIO()
length = f.write(content.encode('utf8'))
f.seek(0)
tari = TarInfo(name=file_name)
tari.size = length
if not mode is None:
tari.mode = mode
if not owner is None:
tari.uname = owner
if not group is None:
tari.gname = group
tarf.addfile(tari, f)
if not self.exists(dir_name):
result = self.container.exec_run("mkdir -p %s" % dir_name)
if result.exit_code != 0:
raise RuntimeError("Could not create path %s!\n%s" %
(dir_name, result.output))
self.container.put_archive(dir_name, buf.getvalue())
def get_image(self, image):
if not image:
raise APIError(HTTPError('500 Server Error'),
None,
explanation='Usage: image_export IMAGE [IMAGE...]')
layers = []
next_layer_id = image
while next_layer_id:
layer = normalizeimage(self._findlayer(next_layer_id), copy=True)
layers.append(layer)
next_layer_id = layers[-1][':parent_id']
image_file = BytesIO()
mtime = time()
with tarfile_open(mode='w', fileobj=image_file) as image_tar_file:
for layer in layers:
ti_dir = TarInfo(layer[':id'])
ti_dir.mtime = mtime
ti_dir.mode = 0o755
ti_dir.type = DIRTYPE
image_tar_file.addfile(ti_dir)
layer_tar_src_path = ospath_join(self._my_dir, 'data',
layer[':short_id'],
'layer.tar')
with open(layer_tar_src_path, 'rb') as layer_tar_src_file:
layer_tar_dst_path = '{}/layer.tar'.format(layer[':id'])
ti_layer = image_tar_file.gettarinfo(
layer_tar_src_path, layer_tar_dst_path)
ti_layer.mtime = mtime
ti_layer.mode = 0o644
ti_layer.uid = ti_layer.gid = 0
ti_layer.uname = ti_layer.gname = ''
image_tar_file.addfile(ti_layer,
fileobj=layer_tar_src_file)
image_file.seek(0)
return image_file
def _reset_entry_attrs(tarinfo: tarfile.TarInfo):
"""
filter function for tar creation that will remove all file attributes
(uid, gid, mtime) from the file added to tar would can make the build
of package not reproducible.
Args:
tarinfo: entry being added to the tar
Returns:
the modified tarinfo that will be actually added to tar
"""
tarinfo.uid = tarinfo.gid = 0
tarinfo.uname = tarinfo.gname = 'root'
tarinfo.mtime = 0
if tarinfo.name.lower().endswith('.dll'):
tarinfo.mode = 0o755
return tarinfo
def writeDataFile(self, filename, text, content_type, subdir=None):
""" See IExportContext.
"""
if subdir is not None:
filename = '/'.join((subdir, filename))
parents = filename.split('/')[:-1]
while parents:
path = '/'.join(parents) + '/'
if path not in self._archive.getnames():
info = TarInfo(path)
info.type = DIRTYPE
# tarfile.filemode(0755) == '-rwxr-xr-x'
info.mode = 0755
info.mtime = time.time()
self._archive.addfile(info)
parents.pop()
stream = StringIO(text)
info = TarInfo(filename)
info.size = len(text)
info.mtime = time.time()
self._archive.addfile(info, stream)
def get_image(self, image):
if not image:
raise APIError(HTTPError('500 Server Error'), None, explanation='Usage: image_export IMAGE [IMAGE...]')
layers = []
next_layer_id = image
while next_layer_id:
layer = normalizeimage(self._findlayer(next_layer_id), copy=True)
layers.append(layer)
next_layer_id = layers[-1][':parent_id']
image_file = BytesIO()
mtime = time()
with tarfile_open(mode='w', fileobj=image_file) as image_tar_file:
for layer in layers:
ti_dir = TarInfo(layer[':id'])
ti_dir.mtime = mtime
ti_dir.mode = 0o755
ti_dir.type = DIRTYPE
image_tar_file.addfile(ti_dir)
layer_tar_src_path = ospath_join(self._my_dir, 'data', layer[':short_id'], 'layer.tar')
with open(layer_tar_src_path, 'rb') as layer_tar_src_file:
layer_tar_dst_path = '{}/layer.tar'.format(layer[':id'])
ti_layer = image_tar_file.gettarinfo(layer_tar_src_path, layer_tar_dst_path)
ti_layer.mtime = mtime
ti_layer.mode = 0o644
ti_layer.uid = ti_layer.gid = 0
ti_layer.uname = ti_layer.gname = ''
image_tar_file.addfile(ti_layer, fileobj=layer_tar_src_file)
image_file.seek(0)
return image_file
def tar(host, backup, share, path):
binary_stdout = stdout.buffer
fbak = Fruitbak(confdir = Path('/dev/shm/conf'))
backup = fbak[host][backup]
if path is None:
share, path = backup.locate_path(share)
else:
share = backup[share]
def iterator():
for dentry in share.find(path):
if dentry.is_file and not dentry.is_hardlink:
yield from dentry.hashes
with fbak.pool.agent().readahead(iterator()) as reader:
for dentry in share.find(path):
name = dentry.name or b'.'
i = TarInfo(fsdecode(bytes(name)))
i.mode = dentry.mode & 0o7777
i.uid = dentry.uid
i.gid = dentry.gid
i.mtime = dentry.mtime // 1000000000
if dentry.is_hardlink:
i.type = LNKTYPE
hardlink = dentry.hardlink or b'.'
i.linkname = fsdecode(bytes(hardlink))
elif dentry.is_file:
i.type = REGTYPE
i.size = dentry.size
elif dentry.is_symlink:
i.type = SYMTYPE
i.linkname = fsdecode(bytes(dentry.symlink))
elif dentry.is_chardev:
i.type = CHRTYPE
i.devmajor = dentry.major
i.devminor = dentry.minor
elif dentry.is_blockdev:
i.type = BLKTYPE
i.devmajor = dentry.major
i.devminor = dentry.minor
elif dentry.is_directory:
i.type = DIRTYPE
elif dentry.is_fifo:
i.type = FIFOTYPE
else:
continue
binary_stdout.write(i.tobuf(GNU_FORMAT))
if dentry.is_file and not dentry.is_hardlink:
for hash in dentry.hashes:
action = next(reader)
if action.exception:
raise action.exception[1]
binary_stdout.write(action.value)
padding = -i.size % BLOCKSIZE
if padding:
binary_stdout.write(bytes(padding))
binary_stdout.write(b'\0' * (BLOCKSIZE*2))
def compute(self, conn, data=None):
tarinfo = TarInfo()
tarinfo.name = self.name
tarinfo.mode = 0o700
tarinfo.uid = 0
tarinfo.gid = 0
tarinfo.type = REGTYPE
tarinfo.linkname = ""
if self.name == CONTAINER_PROPERTIES:
meta = data or conn.container_get_properties(self.acct, self.ref)
tarinfo.size = len(json.dumps(meta['properties'], sort_keys=True))
self._filesize = tarinfo.size
self._buf = tarinfo.tobuf(format=PAX_FORMAT, encoding='utf-8')
return
elif self.name == CONTAINER_MANIFEST:
tarinfo.size = len(json.dumps(data, sort_keys=True))
self._filesize = tarinfo.size
self._buf = tarinfo.tobuf(format=PAX_FORMAT, encoding='utf-8')
return
entry = conn.object_get_properties(self.acct, self.ref, self.name)
properties = entry['properties']
# x-static-large-object
if properties.get(SLO, False):
tarinfo.size = int(properties.get(SLO_SIZE))
_, slo = conn.object_fetch(self.acct, self.ref, self.name,
properties=False)
self._slo = json.loads(b''.join(slo).decode('utf-8'),
object_pairs_hook=OrderedDict)
self._checksums = {}
# format MD5 to share same format as multi chunks object
offset = 0
for idx, ck in enumerate(self._slo):
self._checksums[idx] = {
'hash': ck['hash'].upper(),
'size': ck['bytes'],
'offset': offset
}
offset += ck['bytes']
else:
tarinfo.size = int(entry['length'])
meta, chunks = conn.object_locate(self.acct, self.ref, self.name,
properties=False)
storage_method = STORAGE_METHODS.load(meta['chunk_method'])
chunks = _sort_chunks(chunks, storage_method.ec)
for idx in chunks:
chunks[idx] = chunks[idx][0]
del chunks[idx]['url']
del chunks[idx]['score']
del chunks[idx]['pos']
self._checksums = chunks
self._filesize = tarinfo.size
# XATTR
# do we have to store basic properties like policy, ... ?
for key, val in properties.items():
assert isinstance(val, string_types), \
"Invalid type for %s:%s:%s" % (self.acct, self.name, key)
if self.slo and key in SLO_HEADERS:
continue
tarinfo.pax_headers[SCHILY + key] = val
tarinfo.pax_headers['mime_type'] = entry['mime_type']
self._buf = tarinfo.tobuf(format=PAX_FORMAT, encoding='utf-8')
