def writeDataFile(self, filename, text, content_type, subdir=None):
""" See IExportContext.
"""
if subdir is not None:
filename = '/'.join((subdir, filename))
parents = filename.split('/')[:-1]
while parents:
path = '/'.join(parents) + '/'
if path not in self._archive.getnames():
info = TarInfo(path)
info.type = DIRTYPE
# tarfile.filemode(0o755) == '-rwxr-xr-x'
info.mode = 0o755
info.mtime = time.time()
self._archive.addfile(info)
parents.pop()
info = TarInfo(filename)
if isinstance(text, six.text_type):
encoding = self.getEncoding() or 'utf-8'
text = text.encode(encoding)
if isinstance(text, six.binary_type):
stream = BytesIO(text)
info.size = len(text)
else:
# Assume text is a an instance of a class like
# Products.Archetypes.WebDAVSupport.PdataStreamIterator,
# as in the case of ATFile
stream = text.file
info.size = text.size
info.mtime = time.time()
self._archive.addfile(info, stream)
def writeDataFile( self, filename, text, content_type, subdir=None ):
""" See IExportContext.
"""
if subdir is not None:
filename = '/'.join( ( subdir, filename ) )
parents = filename.split('/')[:-1]
while parents:
path = '/'.join(parents) + '/'
if path not in self._archive.getnames():
info = TarInfo(path)
info.type = DIRTYPE
# tarfile.filemode(0755) == '-rwxr-xr-x'
info.mode = 0755
info.mtime = time.time()
self._archive.addfile(info)
parents.pop()
info = TarInfo(filename)
if isinstance(text, basestring):
stream = StringIO(text)
info.size = len(text)
else:
# Assume text is a an instance of a class like
# Products.Archetypes.WebDAVSupport.PdataStreamIterator,
# as in the case of ATFile
stream = text.file
info.size = text.size
info.mtime = time.time()
self._archive.addfile( info, stream )
def run_csvlint(context):
client = docker.from_env()
csvlint = client.containers.create(
'gsscogs/csvlint',
command=f'csvlint -s /tmp/{context.metadata_filename}')
archive = BytesIO()
context.metadata_io.seek(0, SEEK_END)
metadata_size = context.metadata_io.tell()
context.metadata_io.seek(0)
context.csv_io.seek(0, SEEK_END)
csv_size = context.csv_io.tell()
context.csv_io.seek(0)
with TarFile(fileobj=archive, mode='w') as t:
tis = TarInfo(str(context.metadata_filename))
tis.size = metadata_size
tis.mtime = time.time()
t.addfile(tis, BytesIO(context.metadata_io.read().encode('utf-8')))
tic = TarInfo(str(context.csv_filename))
tic.size = csv_size
tic.mtime = time.time()
t.addfile(tic, BytesIO(context.csv_io.read().encode('utf-8')))
if hasattr(context, 'codelists'):
t.add(Path('features') / 'fixtures' / context.codelists,
arcname=context.codelists)
archive.seek(0)
csvlint.put_archive('/tmp/', archive)
csvlint.start()
response = csvlint.wait()
sys.stdout.write(csvlint.logs().decode('utf-8'))
return (response, csvlint.logs().decode('utf-8'))
def writeDataFile( self, filename, text, content_type, subdir=None ):
""" See IExportContext.
"""
mod_time = time.time()
if subdir is not None:
elements = subdir.split('/')
parents = filter(None, elements)
while parents:
dirname = os.path.join(*parents)
try:
self._archive.getmember(dirname+'/')
except KeyError:
info = TarInfo(dirname)
info.size = 0
info.mode = 509
info.mtime = mod_time
info.type = DIRTYPE
self._archive.addfile(info, StringIO())
parents = parents[:-1]
filename = '/'.join( ( subdir, filename ) )
stream = StringIO( text )
info = TarInfo( filename )
info.size = len( text )
info.mode = 436
info.mtime = mod_time
self._archive.addfile( info, stream )
def run_ics(group: str, turtle: bytes, extra_files: List[str] = (), extra_data: List[str] = ()):
client = docker.from_env()
files = ['data.ttl']
if len(extra_files) > 0:
files.extend(extra_files)
tests = client.containers.create(
'gsscogs/gdp-sparql-tests',
command=f'''sparql-test-runner -t /usr/local/tests/{group} -p dsgraph='<urn:x-arq:DefaultGraph>' '''
f'''{" ".join('/tmp/' + f for f in files)}'''
)
archive = BytesIO()
with TarFile(fileobj=archive, mode='w') as t:
ttl = TarInfo('data.ttl')
ttl.size = len(turtle)
ttl.mtime = time.time()
t.addfile(ttl, BytesIO(turtle))
for filename in extra_files:
actual_path = Path('features') / 'fixtures' / 'extra' / filename
with actual_path.open('rb') as actual_file:
extra_file = t.gettarinfo(arcname=filename, fileobj=actual_file)
t.addfile(extra_file, actual_file)
for i, add_turtle in enumerate(extra_data):
filename = f'extra_{i}.ttl'
add_ttl = TarInfo(filename)
add_ttl.size = len(add_turtle)
add_ttl.mtime = time.time()
t.addfile(add_ttl, BytesIO(add_turtle.encode('utf-8')))
files.append(filename)
archive.seek(0)
tests.put_archive('/tmp/', archive)
tests.start()
response = tests.wait()
sys.stdout.write(tests.logs().decode('utf-8'))
return response['StatusCode']
def writeDataFile( self, filename, text, content_type, subdir=None ):
""" See IExportContext.
"""
if subdir is not None:
filename = '/'.join( ( subdir, filename ) )
parents = filename.split('/')[:-1]
while parents:
path = '/'.join(parents) + '/'
if path not in self._archive.getnames():
info = TarInfo(path)
info.type = DIRTYPE
# tarfile.filemode(0755) == '-rwxr-xr-x'
info.mode = 0755
info.mtime = time.time()
self._archive.addfile(info)
parents.pop()
info = TarInfo(filename)
if isinstance(text, str):
stream = StringIO(text)
info.size = len(text)
elif isinstance(text, unicode):
raise ValueError("Unicode text is not supported, even if it only "
"contains ascii. Please encode your data. See "
"GS 1.7.0 changes for more")
else:
# Assume text is a an instance of a class like
# Products.Archetypes.WebDAVSupport.PdataStreamIterator,
# as in the case of ATFile
stream = text.file
info.size = text.size
info.mtime = time.time()
self._archive.addfile( info, stream )
def writeDataFile(self, filename, text, content_type, subdir=None):
""" See IExportContext.
"""
if subdir is not None:
filename = '/'.join((subdir, filename))
parents = filename.split('/')[:-1]
while parents:
path = '/'.join(parents) + '/'
if path not in self._archive.getnames():
info = TarInfo(path)
info.type = DIRTYPE
# tarfile.filemode(0755) == '-rwxr-xr-x'
info.mode = 0755
info.mtime = time.time()
self._archive.addfile(info)
parents.pop()
info = TarInfo(filename)
if isinstance(text, str):
stream = StringIO(text)
info.size = len(text)
elif isinstance(text, unicode):
raise ValueError("Unicode text is not supported, even if it only "
"contains ascii. Please encode your data")
else:
# Assume text is a an instance of a class like
# Products.Archetypes.WebDAVSupport.PdataStreamIterator,
# as in the case of ATFile
stream = text.file
info.size = text.size
info.mtime = time.time()
self._archive.addfile(info, stream)
def writeDataFile(self, filename, text, content_type, subdir=None):
""" See IExportContext.
"""
mod_time = time.time()
if subdir is not None:
elements = subdir.split('/')
parents = filter(None, elements)
while parents:
dirname = os.path.join(*parents)
try:
self._archive.getmember(dirname + '/')
except KeyError:
info = TarInfo(dirname)
info.size = 0
info.mode = 509
info.mtime = mod_time
info.type = DIRTYPE
self._archive.addfile(info, StringIO())
parents = parents[:-1]
filename = '/'.join((subdir, filename))
stream = StringIO(text)
info = TarInfo(filename)
info.size = len(text)
info.mode = 436
info.mtime = mod_time
self._archive.addfile(info, stream)
def step_impl(context):
client = docker.from_env()
csvlint = client.containers.create(
'gsscogs/csvlint',
command=f'csvlint -s /tmp/{context.schema_filename}'
)
archive = BytesIO()
context.schema_io.seek(0, SEEK_END)
schema_size = context.schema_io.tell()
context.schema_io.seek(0)
context.csv_io.seek(0, SEEK_END)
csv_size = context.csv_io.tell()
context.csv_io.seek(0)
with TarFile(fileobj=archive, mode='w') as t:
tis = TarInfo(str(context.schema_filename))
tis.size = schema_size
tis.mtime = time.time()
t.addfile(tis, BytesIO(context.schema_io.getvalue().encode('utf-8')))
tic = TarInfo(str(context.csv_filename))
tic.size = csv_size
tic.mtime = time.time()
t.addfile(tic, BytesIO(context.csv_io.getvalue().encode('utf-8')))
archive.seek(0)
csvlint.put_archive('/tmp/', archive)
csvlint.start()
response = csvlint.wait()
sys.stdout.write(csvlint.logs().decode('utf-8'))
assert_equal(response['StatusCode'], 0)
def _add_entry(self, name, type, mode, size, data):
info = TarInfo(name)
info.type = type
info.mode = mode
info.size = size
info.mtime = time.time()
self._tarfile.addfile(info, data)
def GetTarInfo(filename, filetype=DIRTYPE, mode=0755):
"""Create information for tar files"""
tarinfo = TarInfo(path.basename(filename))
tarinfo.type = filetype
tarinfo.mode = mode
tarinfo.mtime = time()
return tarinfo
def add_filter(info: tarfile.TarInfo) -> Optional[tarfile.TarInfo]:
"""Filter files targeted to be added to tarfile.
Args:
info: Information on the file targeted to be added
Returns:
None: if file is not to be added
TarInfo: when file is to be added. Modified as needed.
Notes:
exclude is captured from parent
"""
if not (info.isfile() or info.isdir() or info.issym()):
return None
if _exclude_matcher(info.name, exclude):
return None
# Workaround https://bugs.python.org/issue32713. Fixed in Python 3.7
if info.mtime < 0 or info.mtime > 8 ** 11 - 1:
info.mtime = int(info.mtime)
# do not leak client information to service
info.uid = 0
info.uname = info.gname = "root"
if sys.platform == "win32":
info.mode = info.mode & 0o755 | 0o111
return info
def _addToArchive(output, filename, archive):
output.seek(0)
xmlInfo = TarInfo(filename)
xmlInfo.size = len(output.getbuffer())
xmlInfo.mtime = time.time()
archive.addfile(xmlInfo,output)
output.close()
def copy_to_container(container: "Container", source_path: str,
target_path: str) -> None:
"""
Copy a file into a Docker container
:param container: Container object
:param source_path: Source file path
:param target_path: Target file path (in the container)
:return:
"""
# https://github.com/docker/docker-py/issues/1771
with open(source_path, "rb") as f:
data = f.read()
tarinfo = TarInfo(name=os.path.basename(target_path))
tarinfo.size = len(data)
tarinfo.mtime = int(time.time())
stream = BytesIO()
tar = TarFile(fileobj=stream, mode="w")
tar.addfile(tarinfo, BytesIO(data))
tar.close()
stream.seek(0)
container.put_archive(path=os.path.dirname(target_path),
data=stream.read())
def zip2tar(zip_file: str, tar_file, tar_mode: Optional[str] = 'w:gz'):
"""
:param zip_file: zip file path
:param tar_file:
IO(_io.IOBase): file obj
:param tar_mode: ref `tarfile.TarFile.open`
:return:
"""
zip_file = ZipFile(file=zip_file, mode='r')
tar_file = TarFile.open(fileobj=tar_file, mode=tar_mode)
try:
for zip_info in zip_file.infolist():
tar_info = TarInfo(name=zip_info.filename)
tar_info.size = zip_info.file_size
tar_info.mtime = datetime.now().timestamp()
# https://stackoverflow.com/a/434689/11722440
tar_info.mode = zip_info.external_attr >> 16
# https://stackoverflow.com/a/18432983/11722440
# https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT
# TODO whg fix other file (like symbolic link) in zip to regular file in tar file
if zip_info.filename.endswith('/'):
tar_info.type = tarfile.DIRTYPE
else:
tar_info.type = tarfile.REGTYPE
infile = zip_file.open(zip_info.filename)
tar_file.addfile(tar_info, infile)
except Exception as e:
raise
finally:
tar_file.close()
zip_file.close()
def _clean_info(
root: Optional[str], epoch: int, info: tarfile.TarInfo
) -> tarfile.TarInfo:
"""
Remove variable data from an archive entry.
:param root: absolute path to the root directory from which the
entry was added, or None to disable turning the name into a
relative path
:param epoch: fixed modification time to set
:param info: tarinfo object to set
:returns: changed tarinfo
"""
if root is not None:
info.name = os.path.relpath("/" + info.name, root)
if not info.name.startswith("."):
info.name = "./" + info.name
info.uid = 0
info.gid = 0
info.uname = ""
info.gname = ""
info.mtime = epoch
return info
def put(self, content, filename="", file_hash=""):
""" Store file information in hashed tree """
if not filename and not file_hash:
raise ValueError('Filename or FileHash is mandatory')
if filename:
# File accesibility
if not os.path.exists(filename):
raise IOError('Unaccesible file %s', filename)
# Calc hash
file_hash = self._get_hash(filename)
if not file_hash:
raise ValueError('Hash of file is mandatory')
# Get file path for hash
path, tarfile, hashed_filename = self._get_path(file_hash)
# Create file path
try:
os.makedirs(path)
except WindowsError:
pass
except OSError:
pass
# Open tarfile
if self.external_compressor:
# External compressor is not suited for adding files.
raise ValueError('You cannot use external compressor for write files')
with TarFile.open(name=os.path.join(path, tarfile), mode='a') as tar:
with FileLock(os.path.join(path, tarfile)) as lock:
# Test if file already exists into tarfile
try:
tar.getmember(hashed_filename)
raise ValueError('Member already exists')
except KeyError:
pass
except:
raise
data = self.encoder.encode(content)
if self.internal_compressor:
data = self.internal_compressor.compress(data)
data_file = StringIO(data)
mtime = time.time()
ti = TarInfo(hashed_filename)
ti.size = data_file.len
ti.mtime = mtime
tar.addfile(tarinfo=ti, fileobj=data_file)
tar.close()
return file_hash
def move_certs(self, paths):
self.log.info("Staging internal ssl certs for %s", self._log_name)
yield self.pull_image(self.move_certs_image)
# create the volume
volume_name = self.format_volume_name(self.certs_volume_name, self)
# create volume passes even if it already exists
self.log.info("Creating ssl volume %s for %s", volume_name, self._log_name)
yield self.docker('create_volume', volume_name)
# create a tar archive of the internal cert files
# docker.put_archive takes a tarfile and a running container
# and unpacks the archive into the container
nb_paths = {}
tar_buf = BytesIO()
archive = TarFile(fileobj=tar_buf, mode='w')
for key, hub_path in paths.items():
fname = os.path.basename(hub_path)
nb_paths[key] = '/certs/' + fname
with open(hub_path, 'rb') as f:
content = f.read()
tarinfo = TarInfo(name=fname)
tarinfo.size = len(content)
tarinfo.mtime = os.stat(hub_path).st_mtime
tarinfo.mode = 0o644
archive.addfile(tarinfo, BytesIO(content))
archive.close()
tar_buf.seek(0)
# run a container to stage the certs,
# mounting the volume at /certs/
host_config = self.client.create_host_config(
binds={
volume_name: {"bind": "/certs", "mode": "rw"},
},
)
container = yield self.docker('create_container',
self.move_certs_image,
volumes=["/certs"],
host_config=host_config,
)
container_id = container['Id']
self.log.debug(
"Container %s is creating ssl certs for %s",
container_id[:12], self._log_name,
)
# start the container
yield self.docker('start', container_id)
# stage the archive to the container
try:
yield self.docker(
'put_archive',
container=container_id,
path='/certs',
data=tar_buf,
)
finally:
yield self.docker('remove_container', container_id)
return nb_paths
def move_certs(self, paths):
self.log.info("Staging internal ssl certs for %s", self._log_name)
yield self.pull_image(self.move_certs_image)
# create the volume
volume_name = self.format_volume_name(self.certs_volume_name, self)
# create volume passes even if it already exists
self.log.info("Creating ssl volume %s for %s", volume_name, self._log_name)
yield self.docker('create_volume', volume_name)
# create a tar archive of the internal cert files
# docker.put_archive takes a tarfile and a running container
# and unpacks the archive into the container
nb_paths = {}
tar_buf = BytesIO()
archive = TarFile(fileobj=tar_buf, mode='w')
for key, hub_path in paths.items():
fname = os.path.basename(hub_path)
nb_paths[key] = '/certs/' + fname
with open(hub_path, 'rb') as f:
content = f.read()
tarinfo = TarInfo(name=fname)
tarinfo.size = len(content)
tarinfo.mtime = os.stat(hub_path).st_mtime
tarinfo.mode = 0o644
archive.addfile(tarinfo, BytesIO(content))
archive.close()
tar_buf.seek(0)
# run a container to stage the certs,
# mounting the volume at /certs/
host_config = self.client.create_host_config(
binds={
volume_name: {"bind": "/certs", "mode": "rw"},
},
)
container = yield self.docker('create_container',
self.move_certs_image,
volumes=["/certs"],
host_config=host_config,
)
container_id = container['Id']
self.log.debug(
"Container %s is creating ssl certs for %s",
container_id[:12], self._log_name,
)
# start the container
yield self.docker('start', container_id)
# stage the archive to the container
try:
yield self.docker(
'put_archive',
container=container_id,
path='/certs',
data=tar_buf,
)
finally:
yield self.docker('remove_container', container_id)
return nb_paths
def addFile(tar, dest, file, file_size):
if dest not in written_files:
info = TarInfo(dest)
info.size = file_size
info.mtime = now
info.mode = 0777
tar.addfile(info, fileobj=file)
written_files.add(dest)
def file_filter(info: tarfile.TarInfo):
info.mode = 0o00777 if executable else 0o00666
info.mtime = 0
info.type = tarfile.REGTYPE
info.uid = info.gid = 0
info.uname = info.gname = "root"
info.pax_headers = {}
return info
def addFile(tar, dest, file, file_size):
if dest not in written_files:
info = TarInfo(dest)
info.size = file_size
info.mtime = now
info.mode = 0o777
tar.addfile(info, fileobj=file)
written_files.add(dest)
def run_csv2rdf(csv_filename: str,
metadata_filename: str,
csv_io: TextIO,
metadata_io: TextIO,
codelists_base: Optional[str] = None):
client = docker.from_env()
csv2rdf = client.containers.create(
'gsscogs/csv2rdf',
command=
f'csv2rdf -m annotated -o /tmp/output.ttl -t /tmp/{csv_filename} -u /tmp/{metadata_filename}'
)
archive = BytesIO()
metadata_io.seek(0, SEEK_END)
metadata_size = metadata_io.tell()
metadata_io.seek(0)
csv_io.seek(0, SEEK_END)
csv_size = csv_io.tell()
csv_io.seek(0)
with TarFile(fileobj=archive, mode='w') as t:
tis = TarInfo(str(metadata_filename))
tis.size = metadata_size
tis.mtime = time.time()
t.addfile(tis, BytesIO(metadata_io.read().encode('utf-8')))
tic = TarInfo(str(csv_filename))
tic.size = csv_size
tic.mtime = time.time()
t.addfile(tic, BytesIO(csv_io.read().encode('utf-8')))
if codelists_base is not None:
t.add(Path('features') / 'fixtures' / codelists_base,
arcname=codelists_base)
archive.seek(0)
csv2rdf.put_archive('/tmp/', archive)
csv2rdf.start()
response = csv2rdf.wait()
sys.stdout.write(csv2rdf.logs().decode('utf-8'))
assert_equal(response['StatusCode'], 0)
output_stream, output_stat = csv2rdf.get_archive('/tmp/output.ttl')
output_archive = BytesIO()
for line in output_stream:
output_archive.write(line)
output_archive.seek(0)
with TarFile(fileobj=output_archive, mode='r') as t:
output_ttl = t.extractfile('output.ttl')
return output_ttl.read()
def get_tarinfo(name, buffer: typing.Union[BytesIO, StringIO]):
tar_info = TarInfo(name=name)
buffer.seek(0)
if isinstance(buffer, BytesIO):
tar_info.size = len(buffer.getbuffer())
else:
tar_info.size = len(buffer.getvalue())
tar_info.mtime = datetime.now().timestamp()
return tar_info
def add_to_tar(tar: TarFile, data: bytes, filename: str):
tarinfo = TarInfo(name=filename)
tarinfo.size = len(data)
tarinfo.mtime = int(datetime.timestamp(datetime.utcnow()))
tarinfo.mode = 436
tarinfo.type = b'0'
tarinfo.uid = tarinfo.gid = 0
tarinfo.uname = tarinfo.gname = "0"
tar.addfile(tarinfo, BytesIO(data))
def addBytes(tar, dest, bytes):
if dest not in written_files:
# print dest, string
info = TarInfo(dest)
info.size = len(bytes)
info.mtime = now
info.mode = 0o777
file = BytesIO(bytes)
tar.addfile(info, fileobj=file)
file.close()
written_files.add(dest)
def addString(tar, dest, string):
if dest not in written_files:
print dest, string
info = TarInfo(dest)
info.size = len(string)
info.mtime = now
info.mode = 0777
file = StringIO(string)
tar.addfile(info, fileobj=file)
file.close()
written_files.add(dest)
def writeDataFile(self, filename, text, content_type, subdir=None):
""" See IExportContext.
"""
if subdir is not None:
filename = '/'.join((subdir, filename))
stream = StringIO(text)
info = TarInfo(filename)
info.size = len(text)
info.mtime = time.time()
self._archive.addfile(info, stream)
def addString(tar, dest, string):
if dest not in written_files:
print dest, string
info = TarInfo(dest)
info.size = len(string)
info.mtime = now
info.mode = 0777
file = StringIO(string)
tar.addfile(info, fileobj=file)
file.close()
written_files.add(dest)
def _addMember(path, data, modtime):
from tarfile import DIRTYPE
elements = path.split('/')
parents = filter(None, [elements[x] for x in range(len(elements))])
for parent in parents:
info = TarInfo()
info.name = parent
info.size = 0
info.mtime = mod_time
info.type = DIRTYPE
archive.addfile(info, StringIO())
_addOneMember(path, data, modtime)
def _addMember(path, data, modtime):
from tarfile import DIRTYPE
elements = path.split('/')
parents = filter(None, [elements[x] for x in range(len(elements))])
for parent in parents:
info = TarInfo()
info.name = parent
info.size = 0
info.mtime = mod_time
info.type = DIRTYPE
archive.addfile(info, StringIO())
_addOneMember(path, data, modtime)
def _addMember(filename, data, modtime):
from tarfile import DIRTYPE
parents = filename.split('/')[:-1]
while parents:
path = '/'.join(parents) + '/'
if path not in archive.getnames():
info = TarInfo(path)
info.type = DIRTYPE
info.mtime = modtime
archive.addfile(info)
parents.pop()
_addOneMember(filename, data, modtime)
def writeDataFile( self, filename, text, content_type, subdir=None ):
""" See IExportContext.
"""
if subdir is not None:
filename = '/'.join( ( subdir, filename ) )
stream = StringIO( text )
info = TarInfo( filename )
info.size = len( text )
info.mtime = time.time()
self._archive.addfile( info, stream )
def writeDataFile(self, filename, text, content_type, subdir=None):
""" See IExportContext.
"""
if subdir is not None:
filename = '/'.join((subdir, filename))
parents = filename.split('/')[:-1]
while parents:
path = '/'.join(parents) + '/'
if path not in self._archive.getnames():
info = TarInfo(path)
info.type = DIRTYPE
info.mtime = time.time()
self._archive.addfile(info)
parents.pop()
stream = StringIO(text)
info = TarInfo(filename)
info.size = len(text)
info.mtime = time.time()
self._archive.addfile(info, stream)
def _addMember(filename, data, modtime):
from tarfile import DIRTYPE
parents = filename.split('/')[:-1]
while parents:
path = '/'.join(parents) + '/'
if path not in archive.getnames():
info = TarInfo(path)
info.type = DIRTYPE
info.mtime = modtime
archive.addfile(info)
parents.pop()
_addOneMember(filename, data, modtime)
def test_change_info(self):
# writing
db = TarDB(dirname).open('r+')
data_foo = '123'
mtime = 12345
info = TarInfo('foo')
info.mtime = mtime
db.add_record(info, data_foo)
db.close()
# reading
db = TarDB(dirname).open('r+')
info = db[0]
self.assertEqual(info.mtime, mtime)
db[0] = info
db.close()
return
def tar_strip_file_attributes(tar_info: tarfile.TarInfo) -> tarfile.TarInfo:
# set time to epoch timestamp 0, aka 00:00:00 UTC on 1 January 1970
# note that when extracting this tarfile, this time will be shown as the modified date
tar_info.mtime = 0
# user/group info
tar_info.uid = 0
tar_info.uname = ""
tar_info.gid = 0
tar_info.gname = ""
# stripping paxheaders may not be required
# see https://stackoverflow.com/questions/34688392/paxheaders-in-tarball
tar_info.pax_headers = {}
return tar_info
def test_change_info(self):
# writing
db = TarDB(dirname).open(mode='w')
data_foo = '123'
mtime = 12345
info = TarInfo('foo')
info.mtime = mtime
db.add_record(info, data_foo)
db.close()
# reading
db = TarDB(dirname).open(mode='w')
info = db[0]
self.assertEqual(info.mtime, mtime)
db[0] = info
db.close()
return
def create_archive(filepaths):
tarstream = BytesIO()
tarfile = TarFile(fileobj=tarstream, mode='w')
for filepath in filepaths:
file = open(filepath, 'r')
file_data = file.read()
tarinfo = TarInfo(name=basename(file.name))
tarinfo.size = len(file_data)
tarinfo.mtime = time()
tarfile.addfile(tarinfo, BytesIO(file_data))
tarfile.close()
tarstream.seek(0)
return tarstream
def _add_entry(
self,
name: str,
type: bytes,
mode: int,
mtime: int,
size: int,
data: Optional[IO[bytes]],
linkname: str = "",
) -> None:
info = TarInfo(name)
info.type = type
info.mode = mode
info.size = size
info.mtime = mtime
info.linkname = linkname
return self._inner.addfile(info, data)
def get_image(self, image):
if not image:
raise APIError(HTTPError('500 Server Error'),
None,
explanation='Usage: image_export IMAGE [IMAGE...]')
layers = []
next_layer_id = image
while next_layer_id:
layer = normalizeimage(self._findlayer(next_layer_id), copy=True)
layers.append(layer)
next_layer_id = layers[-1][':parent_id']
image_file = BytesIO()
mtime = time()
with tarfile_open(mode='w', fileobj=image_file) as image_tar_file:
for layer in layers:
ti_dir = TarInfo(layer[':id'])
ti_dir.mtime = mtime
ti_dir.mode = 0o755
ti_dir.type = DIRTYPE
image_tar_file.addfile(ti_dir)
layer_tar_src_path = ospath_join(self._my_dir, 'data',
layer[':short_id'],
'layer.tar')
with open(layer_tar_src_path, 'rb') as layer_tar_src_file:
layer_tar_dst_path = '{}/layer.tar'.format(layer[':id'])
ti_layer = image_tar_file.gettarinfo(
layer_tar_src_path, layer_tar_dst_path)
ti_layer.mtime = mtime
ti_layer.mode = 0o644
ti_layer.uid = ti_layer.gid = 0
ti_layer.uname = ti_layer.gname = ''
image_tar_file.addfile(ti_layer,
fileobj=layer_tar_src_file)
image_file.seek(0)
return image_file
def test_tar_compatibility(self):
# writing
db = TarDB(dirname).open('r+')
data_foo = '123'
mtime = 12345
info = TarInfo('foo')
info.mtime = mtime
db.add_record(info, data_foo)
db.close()
# reading with tarfile
import tarfile
tar = tarfile.TarFile(os.path.join(dirname, 'db00000.tar'))
info = tar.next()
data = tar.extractfile(info).read()
self.assertEqual(data, data_foo)
self.assertEqual(len(data), info.size)
self.assertEqual(info.mtime, mtime)
tar.close()
return
def test_tar_compatibility(self):
# writing
db = TarDB(dirname).open(mode='w')
data_foo = '123'
mtime = 12345
info = TarInfo('foo')
info.mtime = mtime
db.add_record(info, data_foo)
db.close()
# reading with tarfile
import tarfile
tar = tarfile.TarFile(os.path.join(dirname, 'db00000.tar'))
info = tar.next()
data = tar.extractfile(info).read()
self.assertEqual(data, data_foo)
self.assertEqual(len(data), info.size)
self.assertEqual(info.mtime, mtime)
tar.close()
return
def add_str(self, name, content, ftype, mode, mtime=None,
uid=None, gid=None, uname=None, gname=None):
'''
Add a string in memory as a file in tarball
'''
if isinstance(name, unicode):
name = name.encode("UTF-8")
ti = TarInfo(name)
# set tarinfo attribute
for v in ("name", "ftype", "mode", "mtime", "uid", "gid", "uname", "gname"):
if vars()[v] is not None:
vars(ti)[v] = vars()[v]
# set mtime to current if not specified
if mtime is None:
ti.mtime = int(time())
# unicode char is encoded in UTF-8, has changelog must be in UTF-8
if isinstance(content, unicode):
content = content.encode("UTF-8")
ti.size = len(content) if content is not None else 0
self.addfile(ti, StringIO(content))
def _reset_tarinfo(
self,
info: TarInfo,
predicate: Optional[ArchiveAdapter.FileFilter],
mtime: Optional[int],
) -> Optional[TarInfo]:
if predicate is not None and not predicate(info.name):
return None
# remove user and group IDs as they are irrelevant for distribution and
# may require subsequent `chown`ing on multi-tenant systems
info.uid = 0
info.uname = ""
info.gid = 0
info.gname = ""
if mtime is not None:
info.mtime = mtime
return info
def get_image(self, image):
if not image:
raise APIError(HTTPError('500 Server Error'), None, explanation='Usage: image_export IMAGE [IMAGE...]')
layers = []
next_layer_id = image
while next_layer_id:
layer = normalizeimage(self._findlayer(next_layer_id), copy=True)
layers.append(layer)
next_layer_id = layers[-1][':parent_id']
image_file = BytesIO()
mtime = time()
with tarfile_open(mode='w', fileobj=image_file) as image_tar_file:
for layer in layers:
ti_dir = TarInfo(layer[':id'])
ti_dir.mtime = mtime
ti_dir.mode = 0o755
ti_dir.type = DIRTYPE
image_tar_file.addfile(ti_dir)
layer_tar_src_path = ospath_join(self._my_dir, 'data', layer[':short_id'], 'layer.tar')
with open(layer_tar_src_path, 'rb') as layer_tar_src_file:
layer_tar_dst_path = '{}/layer.tar'.format(layer[':id'])
ti_layer = image_tar_file.gettarinfo(layer_tar_src_path, layer_tar_dst_path)
ti_layer.mtime = mtime
ti_layer.mode = 0o644
ti_layer.uid = ti_layer.gid = 0
ti_layer.uname = ti_layer.gname = ''
image_tar_file.addfile(ti_layer, fileobj=layer_tar_src_file)
image_file.seek(0)
return image_file
def _addOneMember(path, data, modtime):
stream = StringIO(v)
info = TarInfo(k)
info.size = len(v)
info.mtime = modtime
archive.addfile(info, stream)
def tar(host, backup, share, path):
binary_stdout = stdout.buffer
fbak = Fruitbak(confdir = Path('/dev/shm/conf'))
backup = fbak[host][backup]
if path is None:
share, path = backup.locate_path(share)
else:
share = backup[share]
def iterator():
for dentry in share.find(path):
if dentry.is_file and not dentry.is_hardlink:
yield from dentry.hashes
with fbak.pool.agent().readahead(iterator()) as reader:
for dentry in share.find(path):
name = dentry.name or b'.'
i = TarInfo(fsdecode(bytes(name)))
i.mode = dentry.mode & 0o7777
i.uid = dentry.uid
i.gid = dentry.gid
i.mtime = dentry.mtime // 1000000000
if dentry.is_hardlink:
i.type = LNKTYPE
hardlink = dentry.hardlink or b'.'
i.linkname = fsdecode(bytes(hardlink))
elif dentry.is_file:
i.type = REGTYPE
i.size = dentry.size
elif dentry.is_symlink:
i.type = SYMTYPE
i.linkname = fsdecode(bytes(dentry.symlink))
elif dentry.is_chardev:
i.type = CHRTYPE
i.devmajor = dentry.major
i.devminor = dentry.minor
elif dentry.is_blockdev:
i.type = BLKTYPE
i.devmajor = dentry.major
i.devminor = dentry.minor
elif dentry.is_directory:
i.type = DIRTYPE
elif dentry.is_fifo:
i.type = FIFOTYPE
else:
continue
binary_stdout.write(i.tobuf(GNU_FORMAT))
if dentry.is_file and not dentry.is_hardlink:
for hash in dentry.hashes:
action = next(reader)
if action.exception:
raise action.exception[1]
binary_stdout.write(action.value)
padding = -i.size % BLOCKSIZE
if padding:
binary_stdout.write(bytes(padding))
binary_stdout.write(b'\0' * (BLOCKSIZE*2))
